Integrate encrypted internal metadata in LiveSync

test/e2e-obsidian/README.md

@@ -200,6 +200,8 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms
 
 `test:e2e:obsidian:customisation-sync` runs a two-vault Customisation Sync workflow. It scans a real snippet CSS file, config JSON file, and sample plug-in fixture into per-file Customisation Sync data, synchronises the entries through CouchDB, applies them on the second vault, verifies the resulting `.obsidian` files, propagates a snippet update, and verifies deletion of the source-vault snippet sync data without confusing it with the target vault's own applied copy.
 
+`test:e2e:obsidian:remote-feature-change` starts real Obsidian with continuous CouchDB replication, then changes the remote version document from generation 12 to generation 13 with an unknown feature. It waits for the control document to reach the local database and the active Replicator to retire, checks that another replication is refused, and verifies that an already accepted Vault note remains intact. It is a focused test outside `test:e2e:obsidian:local-suite`; pending-work recovery with a future compatible client remains a separate validation boundary.
+
 `test:e2e:obsidian:setting-markdown-export` enables setting Markdown export, waits for the generated Markdown file in the vault, and verifies that credentials are omitted when `writeCredentialsForSettingSync=false`.
 
 `test:e2e:obsidian:upgrade-from-stable` is the release-acceptance upgrade workflow. It installs the exact published 0.25.83 artefacts into an isolated Vault, verifies their pinned SHA-256 values, and then replaces only the plug-in artefacts with the current target while retaining the same Vault and isolated Obsidian profile. The first run downloads the old release into the ignored `_testdata/releases` cache; every later run verifies the cached bytes before use.

test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts

@@ -210,6 +210,7 @@ async function configureLiveSyncCli(
         encrypt: true,
         passphrase: e2eePassphrase,
         usePathObfuscation: true,
+        encryptInternalMetadata: true,
         doctorProcessedVersion: "0.25.27",
         isConfigured: true,
     });
@@ -323,6 +324,7 @@ async function main(): Promise<void> {
                     encrypt: true,
                     passphrase: e2eePassphrase,
                     usePathObfuscation: true,
+                    encryptInternalMetadata: true,
                     E2EEAlgorithm: "v2",
                 }
             ),

test/e2e-obsidian/scripts/customisation-sync.ts

@@ -1,10 +1,13 @@
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
+import { VERSIONING_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
 import { evalObsidianJson } from "../runner/cli.ts";
 import {
     assertCouchDbReachable,
     createCouchDbDatabase,
     deleteCouchDbDatabase,
+    fetchCouchDbDocument,
     loadCouchDbConfig,
     makeUniqueDatabaseName,
     waitForCouchDbDocs,
@@ -159,6 +162,10 @@ async function startConfiguredSession(
         dbName: context.dbName,
     };
     const customisationSettings = {
+        encrypt: true,
+        passphrase: "internal-metadata-e2e-secret",
+        usePathObfuscation: true,
+        encryptInternalMetadata: true,
         deviceAndVaultName: deviceName,
         usePluginSync: true,
         usePluginSyncV2: true,
@@ -486,6 +493,18 @@ async function main(): Promise<void> {
                 (target) => ids.has(target.id) && target.children.every((childId) => ids.has(childId))
             );
         });
+        for (const target of [entry, configEntry, ...pluginEntries]) {
+            const remoteEntry = await fetchCouchDbDocument(context.couchDb, context.dbName, target.id);
+            if (!remoteEntry.path?.startsWith("/\\:") || remoteEntry.children?.length !== 0 ||
+                remoteEntry.ctime !== 0 || remoteEntry.mtime !== 0 || remoteEntry.size !== 0) {
+                throw new Error(`Customisation Sync Metadata was not encrypted for ${target.id}.`);
+            }
+        }
+        const versionInfo = await fetchCouchDbDocument(context.couchDb, context.dbName, VERSIONING_DOCID);
+        if (versionInfo.version !== REMOTE_FEATURE_GENERATION ||
+            !(versionInfo.used_features as unknown[] | undefined)?.includes(ENCRYPTED_INTERNAL_METADATA_FEATURE)) {
+            throw new Error("The remote feature list does not declare encrypted internal Metadata.");
+        }
         await session.app.stop();
 
         session = await startConfiguredSession(context, vaultB, targetDeviceName);

test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts

@@ -1,5 +1,7 @@
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
+import { VERSIONING_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
 import {
     assertLocatorHasMinimumTouchTarget,
     assertLocatorWithinSafeArea,
@@ -11,6 +13,7 @@ import {
     assertCouchDbReachable,
     createCouchDbDatabase,
     deleteCouchDbDatabase,
+    fetchCouchDbDocument,
     loadCouchDbConfig,
     makeUniqueDatabaseName,
     waitForCouchDbDocs,
@@ -324,6 +327,10 @@ async function startConfiguredSession(
         dbName: context.dbName,
     };
     const hiddenFileSettings = {
+        encrypt: true,
+        passphrase: "internal-metadata-e2e-secret",
+        usePathObfuscation: true,
+        encryptInternalMetadata: true,
         syncInternalFiles: true,
         syncInternalFilesBeforeReplication: true,
         watchInternalFileChanges: false,
@@ -360,6 +367,16 @@ async function uploadHiddenFile(
         const ids = new Set(docs.map((doc) => doc._id));
         return ids.has(entry.id) && entry.children.every((childId) => ids.has(childId));
     });
+    const remoteEntry = await fetchCouchDbDocument(context.couchDb, context.dbName, entry.id);
+    if (!remoteEntry.path?.startsWith("/\\:") || remoteEntry.children?.length !== 0 ||
+        remoteEntry.ctime !== 0 || remoteEntry.mtime !== 0 || remoteEntry.size !== 0) {
+        throw new Error(`Hidden File Sync Metadata was not encrypted for ${entry.id}.`);
+    }
+    const versionInfo = await fetchCouchDbDocument(context.couchDb, context.dbName, VERSIONING_DOCID);
+    if (versionInfo.version !== REMOTE_FEATURE_GENERATION ||
+        !(versionInfo.used_features as unknown[] | undefined)?.includes(ENCRYPTED_INTERNAL_METADATA_FEATURE)) {
+        throw new Error("The remote feature list does not declare encrypted internal Metadata.");
+    }
     return entry;
 }
 

test/e2e-obsidian/scripts/remote-feature-change.ts

@@ -0,0 +1,178 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { VERSIONING_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { evalObsidianJson } from "../runner/cli.ts";
+import {
+    assertCouchDbReachable,
+    createCouchDbDatabase,
+    deleteCouchDbDatabase,
+    fetchCouchDbDocument,
+    loadCouchDbConfig,
+    makeUniqueDatabaseName,
+    putCouchDbDocument,
+} from "../runner/couchdb.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    assertE2eCompatibilityMarker,
+    configureCouchDb,
+    createE2eCouchDbPluginData,
+    createE2eObsidianDeviceLocalState,
+    prepareRemote,
+    pushLocalChanges,
+    waitForLiveSyncCoreReady,
+    waitForLocalDatabaseEntry,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+const acceptedPath = "E2E/remote-feature/accepted.md";
+const acceptedContent = "Accepted before the remote feature changed.\n";
+const unknownFeature = "future-format-v7";
+
+type FeatureState = {
+    version: number | null;
+    features: string[];
+    hasActiveReplicator: boolean;
+};
+
+async function readFeatureState(cliBinary: string, env: NodeJS.ProcessEnv): Promise<FeatureState> {
+    return await evalObsidianJson<FeatureState>(
+        cliBinary,
+        [
+            "(async()=>{",
+            "const core=app.plugins.plugins['obsidian-livesync'].core;",
+            `const id=${JSON.stringify(VERSIONING_DOCID)};`,
+            "const info=await core.localDatabase.getRaw(id).catch(()=>null);",
+            "return JSON.stringify({",
+            "version:typeof info?.version==='number'?info.version:null,",
+            "features:Array.isArray(info?.used_features)?info.used_features:[],",
+            "hasActiveReplicator:!!core.services.replicator.getActiveReplicator(),",
+            "});",
+            "})()",
+        ].join(""),
+        env
+    );
+}
+
+async function waitForState(
+    cliBinary: string,
+    env: NodeJS.ProcessEnv,
+    predicate: (state: FeatureState) => boolean,
+    description: string
+): Promise<FeatureState> {
+    const deadline = Date.now() + 20_000;
+    let state = await readFeatureState(cliBinary, env);
+    while (!predicate(state) && Date.now() < deadline) {
+        await new Promise((resolve) => setTimeout(resolve, 250));
+        state = await readFeatureState(cliBinary, env);
+    }
+    if (!predicate(state)) throw new Error(`Timed out waiting for ${description}: ${JSON.stringify(state)}`);
+    return state;
+}
+
+async function main(): Promise<void> {
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`);
+    const couchDb = await loadCouchDbConfig();
+    const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "remote-feature-change");
+    const vault = await createTemporaryVault();
+    let session: ObsidianLiveSyncSession | undefined;
+
+    try {
+        await assertCouchDbReachable(couchDb);
+        await createCouchDbDatabase(couchDb, dbName);
+        const couchDbSettings = {
+            uri: couchDb.uri,
+            username: couchDb.username,
+            password: couchDb.password,
+            dbName,
+        };
+        const settings = {
+            encrypt: false,
+            usePathObfuscation: false,
+            encryptInternalMetadata: false,
+            liveSync: false,
+        };
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary: cli.binary,
+            vault,
+            startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
+            pluginData: createE2eCouchDbPluginData(couchDbSettings, settings),
+            localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
+        });
+        await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
+        await assertE2eCompatibilityMarker(cli.binary, session.cliEnv);
+        await configureCouchDb(cli.binary, session.cliEnv, couchDbSettings, settings);
+        await prepareRemote(cli.binary, session.cliEnv);
+
+        const fullPath = join(vault.path, acceptedPath);
+        await mkdir(dirname(fullPath), { recursive: true });
+        await writeFile(fullPath, acceptedContent, "utf-8");
+        await waitForLocalDatabaseEntry(cli.binary, session.cliEnv, acceptedPath);
+        await pushLocalChanges(cli.binary, session.cliEnv);
+
+        const initialVersion = await fetchCouchDbDocument(couchDb, dbName, VERSIONING_DOCID);
+        if (initialVersion.version !== 12 || "used_features" in initialVersion) {
+            throw new Error(
+                `An inactive feature unexpectedly changed the remote contract: ${JSON.stringify(initialVersion)}`
+            );
+        }
+
+        const start = await evalObsidianJson<{ status: string }>(
+            cli.binary,
+            [
+                "(async()=>{",
+                "const core=app.plugins.plugins['obsidian-livesync'].core;",
+                "await core.services.setting.applyExternalSettings({liveSync:true},true);",
+                "await core.services.control.applySettings();",
+                "const result=await core.services.replication.startContinuous({trigger:'daemon',interaction:{kind:'forbidden'}});",
+                "return JSON.stringify(result);",
+                "})()",
+            ].join(""),
+            session.cliEnv
+        );
+        if (start.status !== "completed")
+            throw new Error(`Continuous replication did not start: ${JSON.stringify(start)}`);
+        await waitForState(cli.binary, session.cliEnv, (state) => state.hasActiveReplicator, "an active Replicator");
+
+        await putCouchDbDocument(couchDb, dbName, {
+            ...initialVersion,
+            version: 13,
+            used_features: [unknownFeature],
+        });
+        const observed = await waitForState(
+            cli.binary,
+            session.cliEnv,
+            (state) => state.version === 13 && state.features.includes(unknownFeature) && !state.hasActiveReplicator,
+            "the live feature change and Replicator retirement"
+        );
+
+        const replicated = await evalObsidianJson<boolean>(
+            cli.binary,
+            "(async()=>JSON.stringify(!!(await app.plugins.plugins['obsidian-livesync'].core.services.replication.replicate(true))))()",
+            session.cliEnv
+        );
+        if (replicated) throw new Error("An unknown remote feature was admitted for another replication.");
+        const acceptedAfterStop = await readFile(fullPath, "utf-8");
+        if (acceptedAfterStop !== acceptedContent)
+            throw new Error("Previously accepted Vault content changed on stop.");
+        console.log(
+            `Active feature change retired the Replicator and kept accepted content: ${JSON.stringify(observed)}`
+        );
+    } finally {
+        await session?.app.stop();
+        await vault.dispose();
+        if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") {
+            await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => {
+                console.warn(error instanceof Error ? error.message : error);
+            });
+        }
+    }
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exit(1);
+});