Integrate encrypted internal metadata in LiveSync

test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts

@@ -210,6 +210,7 @@ async function configureLiveSyncCli(
         encrypt: true,
         passphrase: e2eePassphrase,
         usePathObfuscation: true,
+        encryptInternalMetadata: true,
         doctorProcessedVersion: "0.25.27",
         isConfigured: true,
     });
@@ -323,6 +324,7 @@ async function main(): Promise<void> {
                     encrypt: true,
                     passphrase: e2eePassphrase,
                     usePathObfuscation: true,
+                    encryptInternalMetadata: true,
                     E2EEAlgorithm: "v2",
                 }
             ),

test/e2e-obsidian/scripts/customisation-sync.ts

@@ -1,10 +1,13 @@
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
+import { VERSIONING_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
 import { evalObsidianJson } from "../runner/cli.ts";
 import {
     assertCouchDbReachable,
     createCouchDbDatabase,
     deleteCouchDbDatabase,
+    fetchCouchDbDocument,
     loadCouchDbConfig,
     makeUniqueDatabaseName,
     waitForCouchDbDocs,
@@ -159,6 +162,10 @@ async function startConfiguredSession(
         dbName: context.dbName,
     };
     const customisationSettings = {
+        encrypt: true,
+        passphrase: "internal-metadata-e2e-secret",
+        usePathObfuscation: true,
+        encryptInternalMetadata: true,
         deviceAndVaultName: deviceName,
         usePluginSync: true,
         usePluginSyncV2: true,
@@ -486,6 +493,18 @@ async function main(): Promise<void> {
                 (target) => ids.has(target.id) && target.children.every((childId) => ids.has(childId))
             );
         });
+        for (const target of [entry, configEntry, ...pluginEntries]) {
+            const remoteEntry = await fetchCouchDbDocument(context.couchDb, context.dbName, target.id);
+            if (!remoteEntry.path?.startsWith("/\\:") || remoteEntry.children?.length !== 0 ||
+                remoteEntry.ctime !== 0 || remoteEntry.mtime !== 0 || remoteEntry.size !== 0) {
+                throw new Error(`Customisation Sync Metadata was not encrypted for ${target.id}.`);
+            }
+        }
+        const versionInfo = await fetchCouchDbDocument(context.couchDb, context.dbName, VERSIONING_DOCID);
+        if (versionInfo.version !== REMOTE_FEATURE_GENERATION ||
+            !(versionInfo.used_features as unknown[] | undefined)?.includes(ENCRYPTED_INTERNAL_METADATA_FEATURE)) {
+            throw new Error("The remote feature list does not declare encrypted internal Metadata.");
+        }
         await session.app.stop();
 
         session = await startConfiguredSession(context, vaultB, targetDeviceName);

test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts

@@ -1,5 +1,7 @@
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
+import { VERSIONING_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
 import {
     assertLocatorHasMinimumTouchTarget,
     assertLocatorWithinSafeArea,
@@ -11,6 +13,7 @@ import {
     assertCouchDbReachable,
     createCouchDbDatabase,
     deleteCouchDbDatabase,
+    fetchCouchDbDocument,
     loadCouchDbConfig,
     makeUniqueDatabaseName,
     waitForCouchDbDocs,
@@ -324,6 +327,10 @@ async function startConfiguredSession(
         dbName: context.dbName,
     };
     const hiddenFileSettings = {
+        encrypt: true,
+        passphrase: "internal-metadata-e2e-secret",
+        usePathObfuscation: true,
+        encryptInternalMetadata: true,
         syncInternalFiles: true,
         syncInternalFilesBeforeReplication: true,
         watchInternalFileChanges: false,
@@ -360,6 +367,16 @@ async function uploadHiddenFile(
         const ids = new Set(docs.map((doc) => doc._id));
         return ids.has(entry.id) && entry.children.every((childId) => ids.has(childId));
     });
+    const remoteEntry = await fetchCouchDbDocument(context.couchDb, context.dbName, entry.id);
+    if (!remoteEntry.path?.startsWith("/\\:") || remoteEntry.children?.length !== 0 ||
+        remoteEntry.ctime !== 0 || remoteEntry.mtime !== 0 || remoteEntry.size !== 0) {
+        throw new Error(`Hidden File Sync Metadata was not encrypted for ${entry.id}.`);
+    }
+    const versionInfo = await fetchCouchDbDocument(context.couchDb, context.dbName, VERSIONING_DOCID);
+    if (versionInfo.version !== REMOTE_FEATURE_GENERATION ||
+        !(versionInfo.used_features as unknown[] | undefined)?.includes(ENCRYPTED_INTERNAL_METADATA_FEATURE)) {
+        throw new Error("The remote feature list does not declare encrypted internal Metadata.");
+    }
     return entry;
 }
 

test/e2e-obsidian/scripts/remote-feature-change.ts

@@ -0,0 +1,178 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { VERSIONING_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { evalObsidianJson } from "../runner/cli.ts";
+import {
+    assertCouchDbReachable,
+    createCouchDbDatabase,
+    deleteCouchDbDatabase,
+    fetchCouchDbDocument,
+    loadCouchDbConfig,
+    makeUniqueDatabaseName,
+    putCouchDbDocument,
+} from "../runner/couchdb.ts";
+import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
+import {
+    assertE2eCompatibilityMarker,
+    configureCouchDb,
+    createE2eCouchDbPluginData,
+    createE2eObsidianDeviceLocalState,
+    prepareRemote,
+    pushLocalChanges,
+    waitForLiveSyncCoreReady,
+    waitForLocalDatabaseEntry,
+} from "../runner/liveSyncWorkflow.ts";
+import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
+import { createTemporaryVault } from "../runner/vault.ts";
+
+const acceptedPath = "E2E/remote-feature/accepted.md";
+const acceptedContent = "Accepted before the remote feature changed.\n";
+const unknownFeature = "future-format-v7";
+
+type FeatureState = {
+    version: number | null;
+    features: string[];
+    hasActiveReplicator: boolean;
+};
+
+async function readFeatureState(cliBinary: string, env: NodeJS.ProcessEnv): Promise<FeatureState> {
+    return await evalObsidianJson<FeatureState>(
+        cliBinary,
+        [
+            "(async()=>{",
+            "const core=app.plugins.plugins['obsidian-livesync'].core;",
+            `const id=${JSON.stringify(VERSIONING_DOCID)};`,
+            "const info=await core.localDatabase.getRaw(id).catch(()=>null);",
+            "return JSON.stringify({",
+            "version:typeof info?.version==='number'?info.version:null,",
+            "features:Array.isArray(info?.used_features)?info.used_features:[],",
+            "hasActiveReplicator:!!core.services.replicator.getActiveReplicator(),",
+            "});",
+            "})()",
+        ].join(""),
+        env
+    );
+}
+
+async function waitForState(
+    cliBinary: string,
+    env: NodeJS.ProcessEnv,
+    predicate: (state: FeatureState) => boolean,
+    description: string
+): Promise<FeatureState> {
+    const deadline = Date.now() + 20_000;
+    let state = await readFeatureState(cliBinary, env);
+    while (!predicate(state) && Date.now() < deadline) {
+        await new Promise((resolve) => setTimeout(resolve, 250));
+        state = await readFeatureState(cliBinary, env);
+    }
+    if (!predicate(state)) throw new Error(`Timed out waiting for ${description}: ${JSON.stringify(state)}`);
+    return state;
+}
+
+async function main(): Promise<void> {
+    const binary = requireObsidianBinary();
+    const cli = discoverObsidianCli();
+    if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`);
+    const couchDb = await loadCouchDbConfig();
+    const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "remote-feature-change");
+    const vault = await createTemporaryVault();
+    let session: ObsidianLiveSyncSession | undefined;
+
+    try {
+        await assertCouchDbReachable(couchDb);
+        await createCouchDbDatabase(couchDb, dbName);
+        const couchDbSettings = {
+            uri: couchDb.uri,
+            username: couchDb.username,
+            password: couchDb.password,
+            dbName,
+        };
+        const settings = {
+            encrypt: false,
+            usePathObfuscation: false,
+            encryptInternalMetadata: false,
+            liveSync: false,
+        };
+        session = await startObsidianLiveSyncSession({
+            binary,
+            cliBinary: cli.binary,
+            vault,
+            startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
+            pluginData: createE2eCouchDbPluginData(couchDbSettings, settings),
+            localStorageEntries: createE2eObsidianDeviceLocalState(vault.name),
+        });
+        await waitForLiveSyncCoreReady(cli.binary, session.cliEnv);
+        await assertE2eCompatibilityMarker(cli.binary, session.cliEnv);
+        await configureCouchDb(cli.binary, session.cliEnv, couchDbSettings, settings);
+        await prepareRemote(cli.binary, session.cliEnv);
+
+        const fullPath = join(vault.path, acceptedPath);
+        await mkdir(dirname(fullPath), { recursive: true });
+        await writeFile(fullPath, acceptedContent, "utf-8");
+        await waitForLocalDatabaseEntry(cli.binary, session.cliEnv, acceptedPath);
+        await pushLocalChanges(cli.binary, session.cliEnv);
+
+        const initialVersion = await fetchCouchDbDocument(couchDb, dbName, VERSIONING_DOCID);
+        if (initialVersion.version !== 12 || "used_features" in initialVersion) {
+            throw new Error(
+                `An inactive feature unexpectedly changed the remote contract: ${JSON.stringify(initialVersion)}`
+            );
+        }
+
+        const start = await evalObsidianJson<{ status: string }>(
+            cli.binary,
+            [
+                "(async()=>{",
+                "const core=app.plugins.plugins['obsidian-livesync'].core;",
+                "await core.services.setting.applyExternalSettings({liveSync:true},true);",
+                "await core.services.control.applySettings();",
+                "const result=await core.services.replication.startContinuous({trigger:'daemon',interaction:{kind:'forbidden'}});",
+                "return JSON.stringify(result);",
+                "})()",
+            ].join(""),
+            session.cliEnv
+        );
+        if (start.status !== "completed")
+            throw new Error(`Continuous replication did not start: ${JSON.stringify(start)}`);
+        await waitForState(cli.binary, session.cliEnv, (state) => state.hasActiveReplicator, "an active Replicator");
+
+        await putCouchDbDocument(couchDb, dbName, {
+            ...initialVersion,
+            version: 13,
+            used_features: [unknownFeature],
+        });
+        const observed = await waitForState(
+            cli.binary,
+            session.cliEnv,
+            (state) => state.version === 13 && state.features.includes(unknownFeature) && !state.hasActiveReplicator,
+            "the live feature change and Replicator retirement"
+        );
+
+        const replicated = await evalObsidianJson<boolean>(
+            cli.binary,
+            "(async()=>JSON.stringify(!!(await app.plugins.plugins['obsidian-livesync'].core.services.replication.replicate(true))))()",
+            session.cliEnv
+        );
+        if (replicated) throw new Error("An unknown remote feature was admitted for another replication.");
+        const acceptedAfterStop = await readFile(fullPath, "utf-8");
+        if (acceptedAfterStop !== acceptedContent)
+            throw new Error("Previously accepted Vault content changed on stop.");
+        console.log(
+            `Active feature change retired the Replicator and kept accepted content: ${JSON.stringify(observed)}`
+        );
+    } finally {
+        await session?.app.stop();
+        await vault.dispose();
+        if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") {
+            await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => {
+                console.warn(error instanceof Error ? error.message : error);
+            });
+        }
+    }
+}
+
+main().catch((error: unknown) => {
+    console.error(error instanceof Error ? error.stack : error);
+    process.exit(1);
+});