Merge internal Metadata integration for stacked Setup URI validation

src/serviceFeatures/replication/ReplicateResultProcessor.ts

@@ -1,6 +1,7 @@
+import { assessRemoteFeatureDocument, describeRemoteFeatureRejection } from "@vrtmrz/livesync-commonlib/replication";
 import {
     SYNCINFO_ID,
-    VER,
+    VERSIONING_DOCID,
     type AnyEntry,
     type EntryDoc,
     type EntryLeaf,
@@ -274,13 +275,14 @@ export class ReplicateResultProcessor {
             this.log(`Processed chunk: ${shortenId(change._id)}`, LOG_LEVEL_DEBUG);
             return true;
         }
-        if (change.type == "versioninfo") {
+        if (change._id === VERSIONING_DOCID || change.type === "versioninfo") {
             this.log(`Version info document received: ${change._id}`, LOG_LEVEL_VERBOSE);
-            if (change.version > VER) {
+            const assessment = assessRemoteFeatureDocument(change);
+            if (assessment.status !== "supported" && assessment.status !== "older-generation") {
                 // Fence and retire the active publication through its owner.
                 this.context.requestActiveReplicatorRetirement();
                 this.log(
-                    `Remote database updated to incompatible version. update your Self-hosted LiveSync plugin.`,
+                    `${describeRemoteFeatureRejection(assessment)} Update Self-hosted LiveSync before synchronising.`,
                     LOG_LEVEL_NOTICE
                 );
             }

src/serviceFeatures/replication/ReplicateResultProcessor.unit.spec.ts

@@ -1,7 +1,12 @@
 import { promiseWithResolvers } from "octagonal-wheels/promises";
 import { reactiveSource } from "octagonal-wheels/dataobject/reactive";
 import { describe, expect, it, vi } from "vitest";
-import { VER, type EntryDoc, type FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import {
+    VERSIONING_DOCID,
+    type EntryDoc,
+    type FilePathWithPrefix,
+} from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
 import {
     isValidFilenameInAndroid,
     isValidFilenameInWidows,
@@ -37,6 +42,10 @@ type SetupOptions = {
     isValidPath?: (path: string) => boolean;
     processSynchroniseResult?: (entry: unknown) => Promise<boolean>;
     setSnapshot?: (key: string, value: unknown) => Promise<unknown>;
+    getSnapshot?: (key: string) => Promise<unknown>;
+    localVersionInfo?: unknown;
+    isTargetFile?: (path: string) => Promise<boolean>;
+    databaseId?: string;
 };
 
 function setup(options: SetupOptions = {}) {
@@ -47,6 +56,9 @@ function setup(options: SetupOptions = {}) {
     const isReady = vi.fn(() => options.applicationReady ?? true);
     const isValidPath = vi.fn(options.isValidPath ?? (() => true));
     const getDBEntryFromMeta = vi.fn(async (entry: object) => ({ ...entry, data: "x" }));
+    const localPhysicalDatabase = {
+        ...(options.databaseId ? { id: vi.fn(async () => options.databaseId) } : {}),
+    } as PouchDB.Database<EntryDoc>;
     const core = {
         services: {
             appLifecycle: { isReady, isSuspended: () => false },
@@ -62,14 +74,21 @@ function setup(options: SetupOptions = {}) {
             },
             replicator: { onCloseActiveReplication, runBoundedLocalApplicationActivity },
             vault: {
-                isTargetFile: vi.fn(async () => true),
+                isTargetFile: vi.fn(options.isTargetFile ?? (async () => true)),
                 isFileSizeTooLarge: vi.fn(() => false),
                 isValidPath,
             },
         },
-        kvDB: { set: setSnapshot },
+        kvDB: { set: setSnapshot, get: vi.fn(options.getSnapshot ?? (async () => undefined)) },
         localDatabase: {
-            getRaw: vi.fn(async (id: string) => ({ _id: id, _rev: "1-test" })),
+            localDatabase: localPhysicalDatabase,
+            getRaw: vi.fn(async (id: string) => {
+                if (id === VERSIONING_DOCID) {
+                    if (options.localVersionInfo === undefined) throw { status: 404 };
+                    return options.localVersionInfo;
+                }
+                return { _id: id, _rev: "1-test" };
+            }),
             getDBEntryFromMeta,
         },
     };
@@ -88,6 +107,9 @@ function setup(options: SetupOptions = {}) {
     } as never);
     return {
         getDBEntryFromMeta,
+        isTargetFile: core.services.vault.isTargetFile,
+        localPhysicalDatabase,
+        localDatabase: core.localDatabase,
         isReady,
         isValidPath,
         onCloseActiveReplication,
@@ -98,6 +120,34 @@ function setup(options: SetupOptions = {}) {
 }
 
 describe("ReplicateResultProcessor", () => {
+    it("does not add a permanent application block when snapshot recovery fails", async () => {
+        const { processor } = setup({
+            getSnapshot: async () => {
+                throw new Error("KV unavailable");
+            },
+        });
+        await expect(processor.restoreFromSnapshotOnce()).rejects.toThrow("KV unavailable");
+        expect(processor.isSuspended).toBe(false);
+    });
+
+    it("restores pending notes without retaining a past feature rejection in KV", async () => {
+        const { processor, processSynchroniseResult, onCloseActiveReplication } = setup({
+            databaseId: "same-database",
+            getSnapshot: async () => ({
+                databaseId: "same-database",
+                invalidControlObserved: true,
+                observedFeatures: ["future-format-v7"],
+                observedGeneration: 14,
+                queued: [note("recovered-note")],
+                processing: [],
+            }),
+        });
+        await processor.restoreFromSnapshotOnce();
+        expect(processor.isSuspended).toBe(false);
+        await vi.waitFor(() => expect(processSynchroniseResult).toHaveBeenCalledOnce());
+        expect(onCloseActiveReplication).not.toHaveBeenCalled();
+    });
+
     it.each([
         ["Windows", isValidFilenameInWidows],
         ["Android", isValidFilenameInAndroid],
@@ -145,9 +195,9 @@ describe("ReplicateResultProcessor", () => {
             });
         }
         expect(processSynchroniseResult).toHaveBeenCalledTimes(11);
-        expect(processSynchroniseResult.mock.calls.some(([entry]) =>
-            (entry as { _id: string })._id === "unrelated-queue"
-        )).toBe(true);
+        expect(
+            processSynchroniseResult.mock.calls.some(([entry]) => (entry as { _id: string })._id === "unrelated-queue")
+        ).toBe(true);
     });
 
     it("suspends result application while the application is not ready", () => {
@@ -199,10 +249,10 @@ describe("ReplicateResultProcessor", () => {
     it("retires active ownership when a newer remote version is observed", async () => {
         const { onCloseActiveReplication, processor } = setup();
         const versionInfo = {
-            _id: "versioninfo",
+            _id: VERSIONING_DOCID,
             _rev: "1-test",
             type: "versioninfo",
-            version: VER + 1,
+            version: REMOTE_FEATURE_GENERATION + 1,
         } as unknown as PouchDB.Core.ExistingDocument<EntryDoc>;
 
         processor.enqueueAll([versionInfo]);
@@ -210,6 +260,64 @@ describe("ReplicateResultProcessor", () => {
         await vi.waitFor(() => expect(onCloseActiveReplication).toHaveBeenCalledOnce());
     });
 
+    it("continues applying documents after restoring a legacy local version document", async () => {
+        const { onCloseActiveReplication, processor, processSynchroniseResult } = setup({
+            localVersionInfo: {
+                _id: VERSIONING_DOCID,
+                type: "versioninfo",
+                version: 11,
+            },
+        });
+
+        await processor.restoreFromSnapshotOnce();
+        processor.enqueueAll([note("legacy-database-note")]);
+
+        await vi.waitFor(() => expect(processSynchroniseResult).toHaveBeenCalledOnce());
+        expect(processor.isSuspended).toBe(false);
+        expect(onCloseActiveReplication).not.toHaveBeenCalled();
+    });
+
+    it("continues when a newly received feature is supported", async () => {
+        const { onCloseActiveReplication, processor, processSynchroniseResult } = setup();
+        const versionInfo = {
+            _id: VERSIONING_DOCID,
+            _rev: "2-supported",
+            type: "versioninfo",
+            version: REMOTE_FEATURE_GENERATION,
+            used_features: [ENCRYPTED_INTERNAL_METADATA_FEATURE],
+        } as PouchDB.Core.ExistingDocument<EntryDoc>;
+
+        processor.enqueueAll([versionInfo, note("supported-update")]);
+
+        await vi.waitFor(() => expect(processSynchroniseResult).toHaveBeenCalledOnce());
+        expect(onCloseActiveReplication).not.toHaveBeenCalled();
+    });
+
+    it("reports unknown feature identifiers and requests Replicator retirement", () => {
+        const logger = vi.fn();
+        setGlobalLogFunction(logger);
+        try {
+            const { processor, onCloseActiveReplication } = setup();
+            processor.enqueueAll([
+                {
+                    _id: VERSIONING_DOCID,
+                    _rev: "1-unknown",
+                    type: "versioninfo",
+                    version: REMOTE_FEATURE_GENERATION,
+                    used_features: ["future-format-v7"],
+                } as PouchDB.Core.ExistingDocument<EntryDoc>,
+            ]);
+            expect(onCloseActiveReplication).toHaveBeenCalledOnce();
+            expect(logger).toHaveBeenCalledWith(
+                expect.stringContaining("future-format-v7"),
+                LOG_LEVEL_NOTICE,
+                undefined
+            );
+        } finally {
+            setGlobalLogFunction(defaultLogger);
+        }
+    });
+
     it("scans normal-file metadata without loading chunk documents and requeues it", async () => {
         const documents = [
             { _id: "first", _rev: "1-a", type: "plain", path: "first.md" },

src/serviceFeatures/replication/centralCompatibilityRecovery.ts

@@ -1,4 +1,8 @@
-import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import {
+    VERSIONING_DOCID,
+    type EntryDoc,
+    type ObsidianLiveSyncSettings,
+} from "@vrtmrz/livesync-commonlib/compat/common/types";
 import { assessTweakCompatibility } from "@vrtmrz/livesync-commonlib/settings";
 import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger";
 import { skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
@@ -7,12 +11,27 @@ import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/rep
 import {
     CENTRAL_COMPATIBILITY_REJECTION_REASONS,
     REPLICATION_PROGRESS_PRESENTATIONS,
+    assessRemoteFeatureDocument,
+    describeRemoteFeatureRejection,
     type ReplicatorInstance,
     type ReplicationFailureRequest,
 } from "@vrtmrz/livesync-commonlib/replication";
 import { $msg } from "@/common/translation";
 import { usesLegacyIndexedDBAdapter } from "@/common/compatibilitySettings";
 import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
+import type PouchDB from "pouchdb-core";
+
+async function canInterpretCleanupDatabase(db: PouchDB.Database<EntryDoc>): Promise<boolean> {
+    try {
+        const assessment = assessRemoteFeatureDocument(await db.get(VERSIONING_DOCID));
+        if (assessment.status === "supported" || assessment.status === "older-generation") return true;
+        Logger(`Database cleanup cancelled: ${describeRemoteFeatureRejection(assessment)}`, LOG_LEVEL_NOTICE);
+    } catch (error) {
+        Logger("Database cleanup cancelled: feature compatibility could not be checked.", LOG_LEVEL_NOTICE);
+        Logger(error, LOG_LEVEL_INFO);
+    }
+    return false;
+}
 
 type CentralCompatibilityRecoveryServices = Pick<
     LiveSyncBaseCore["services"],
@@ -60,6 +79,7 @@ export function createCentralCompatibilityRecovery(context: CentralCompatibility
     ) {
         Logger("The remote database has been cleaned.", showProgress ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
         await skipIfDuplicated("cleanup", async () => {
+            if (!(await canInterpretCleanupDatabase(context.getLocalDatabase().localDatabase))) return;
             const count = await purgeUnreferencedChunks(context.getLocalDatabase().localDatabase, true);
             const message = `The remote database has been cleaned up.
 To synchronize, this device must be also cleaned up. ${count} chunk(s) will be erased from this device.

src/serviceFeatures/replication/centralCompatibilityRecovery.unit.spec.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it, vi } from "vitest";
-import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { VERSIONING_DOCID, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
 import { assessTweakCompatibility } from "@vrtmrz/livesync-commonlib/settings";
 import { defaultLogger, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, setGlobalLogFunction } from "octagonal-wheels/common/logger";
 import {
@@ -24,6 +24,49 @@ import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/rep
 import { createCentralCompatibilityRecovery } from "./centralCompatibilityRecovery";
 
 describe("central compatibility recovery", () => {
+    it("does not count chunks for cleanup when local feature requirements are unknown", async () => {
+        chunkMocks.purgeUnreferencedChunks.mockClear();
+        const confirmWithMessage = vi.fn(async () => "Dismiss");
+        const recovery = createCentralCompatibilityRecovery({
+            confirm: { confirmWithMessage },
+            getLocalDatabase: () => ({
+                localDatabase: {
+                    get: vi.fn(async (id: string) => ({
+                        _id: id,
+                        type: "versioninfo",
+                        version: 13,
+                        used_features: ["future-format-v7"],
+                    })),
+                },
+            }),
+            services: { replicator: {} },
+        } as never);
+
+        await recovery.reconcileCleanedRemote(true, {} as ObsidianLiveSyncSettings, {} as never);
+
+        expect(chunkMocks.purgeUnreferencedChunks).not.toHaveBeenCalled();
+        expect(confirmWithMessage).not.toHaveBeenCalled();
+    });
+
+    it("allows cleanup counting for a legacy local version document", async () => {
+        chunkMocks.purgeUnreferencedChunks.mockClear();
+        const confirmWithMessage = vi.fn(async () => "Dismiss");
+        const recovery = createCentralCompatibilityRecovery({
+            confirm: { confirmWithMessage },
+            getLocalDatabase: () => ({
+                localDatabase: {
+                    get: vi.fn(async (id: string) => ({ _id: id, type: "versioninfo", version: 11 })),
+                },
+            }),
+            services: { replicator: {} },
+        } as never);
+
+        await recovery.reconcileCleanedRemote(true, {} as ObsidianLiveSyncSettings, {} as never);
+
+        expect(chunkMocks.purgeUnreferencedChunks).toHaveBeenCalledWith(expect.anything(), true);
+        expect(confirmWithMessage).toHaveBeenCalledOnce();
+    });
+
     it("passes the failed attempt's exact tweak assessment to mismatch resolution", async () => {
         const setting = { customChunkSize: 0 };
         const preferredTweakValue = { customChunkSize: 60 };
@@ -292,7 +335,9 @@ describe("central compatibility recovery", () => {
         });
         const runFiniteReplicationActivity = vi.fn(async (task: () => unknown) => await task());
         const openOneShotReplication = vi.fn(async () => true);
-        const remoteDatabase = { close: vi.fn(async () => undefined) };
+        const remoteDatabase = {
+            close: vi.fn(async () => undefined),
+        };
         const close = vi.fn(async () => undefined);
         const activeReplicator = Object.assign(new LiveSyncCouchDBReplicator({} as never), {
             connectRemoteCouchDBWithSetting: vi.fn(async () => ({ db: remoteDatabase, close })),
@@ -303,7 +348,12 @@ describe("central compatibility recovery", () => {
         const runWithActiveReplicatorContext = vi.fn(async (task: (context: unknown) => unknown) =>
             task(expectedContext)
         );
-        const localDatabase = { localDatabase: {}, clearCaches: vi.fn() };
+        const localDatabase = {
+            localDatabase: {
+                get: vi.fn(async () => ({ _id: VERSIONING_DOCID, type: "versioninfo", version: 12 })),
+            },
+            clearCaches: vi.fn(),
+        };
         const getLocalDatabase = vi.fn(() => localDatabase);
         const recovery = createCentralCompatibilityRecovery({
             confirm: { confirmWithMessage: vi.fn(async () => "Cleanup") },
@@ -335,7 +385,7 @@ describe("central compatibility recovery", () => {
             activityFinished.mock.invocationCallOrder[0]
         );
         expect(chunkMocks.balanceChunkPurgedDBs).toHaveBeenCalledOnce();
-        expect(getLocalDatabase).toHaveBeenCalledTimes(2);
+        expect(getLocalDatabase).toHaveBeenCalled();
         expect(close).toHaveBeenCalledOnce();
         expect(close.mock.invocationCallOrder[0]).toBeLessThan(activityFinished.mock.invocationCallOrder[0]);
     });

src/serviceFeatures/replication/replicationFeature.unit.spec.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it, vi } from "vitest";
 import { createServiceContext } from "@vrtmrz/livesync-commonlib/context";
-import { VER, type EntryDoc } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { VERSIONING_DOCID, type EntryDoc } from "@vrtmrz/livesync-commonlib/compat/common/types";
+import { REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
 import { promiseWithResolvers } from "octagonal-wheels/promises";
 import { useReplicationFeature } from "./index";
 
@@ -19,8 +20,14 @@ type SetupOptions = {
 };
 
 function setup(options: SetupOptions = {}) {
+    const defaultLocalDatabase = {
+        localDatabase: {},
+        getRaw: vi.fn(async () => {
+            throw { status: 404 };
+        }),
+    };
     const {
-        getLocalDatabase = () => ({}),
+        getLocalDatabase = () => defaultLocalDatabase,
         keyValueDB = {
             kvDB: {
                 get: vi.fn(async () => undefined),
@@ -39,7 +46,7 @@ function setup(options: SetupOptions = {}) {
         API: { isMobile: vi.fn(() => false), isOnline: true },
         appLifecycle: {
             getUnresolvedMessages: { addHandler: vi.fn() },
-            isReady: true,
+            isReady: vi.fn(() => true),
             isSuspended: vi.fn(() => false),
             onSettingLoaded: { addHandler: vi.fn() },
         },
@@ -48,6 +55,7 @@ function setup(options: SetupOptions = {}) {
         keyValueDB,
         path: { getPath: vi.fn((entry: { path: string }) => entry.path) },
         replication: {
+            replicationResultCount: { value: 0 },
             onBeforeReplicate: {
                 addHandler: vi.fn((handler: BooleanHandler, priority = 0) => {
                     beforeReplicateHandlers.set(priority, handler);
@@ -165,10 +173,10 @@ describe("replication serviceFeature composition", () => {
         const onCloseActiveReplication = vi.fn(() => retirement.promise);
         const harness = setup({ onCloseActiveReplication });
         const versionInfo = {
-            _id: "versioninfo",
+            _id: VERSIONING_DOCID,
             _rev: "1-test",
             type: "versioninfo",
-            version: VER + 1,
+            version: REMOTE_FEATURE_GENERATION + 1,
         } as unknown as PouchDB.Core.ExistingDocument<EntryDoc>;
 
         expect(harness.parseHandler).toBeDefined();