Integrate encrypted internal metadata in LiveSync

This commit is contained in:
vorotamoroz
2026-09-27 09:51:53 +00:00
parent 7b3b6ff854
commit 6c50505096
23 changed files with 1220 additions and 61 deletions
@@ -1,4 +1,5 @@
import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { usesEncryptedInternalMetadata } from "@vrtmrz/livesync-commonlib/replication";
type EndpointProjection = readonly [kind: "url" | "invalid-url", value: string];
@@ -77,6 +78,7 @@ export function getCouchDBReplicatorConfigurationIdentity(settings: RemoteDBSett
settings.useRequestAPI,
settings.disableRequestURI,
projectRemoteSecurity(settings),
usesEncryptedInternalMetadata(settings),
settings.enableCompression,
]);
}
@@ -67,6 +67,22 @@ describe("active Replicator configuration identity", () => {
);
});
it("recreates the CouchDB connection when internal Metadata encryption becomes effective", () => {
const active = configuredSettings({ usePathObfuscation: true, encryptInternalMetadata: false });
const enabled = { ...active, encryptInternalMetadata: true };
expect(getCouchDBReplicatorConfigurationIdentity(enabled)).not.toBe(
getCouchDBReplicatorConfigurationIdentity(active)
);
const inactive = { ...active, usePathObfuscation: false };
expect(getCouchDBReplicatorConfigurationIdentity({ ...inactive, encryptInternalMetadata: true })).toBe(
getCouchDBReplicatorConfigurationIdentity(inactive)
);
expect(getObjectStorageReplicatorConfigurationIdentity(enabled)).toBe(
getObjectStorageReplicatorConfigurationIdentity(active)
);
});
it("projects only the active CouchDB authentication mode", () => {
const basic = configuredSettings({ useJWT: false, jwtKey: "inactive-a" });
expect(getCouchDBReplicatorConfigurationIdentity({ ...basic, jwtKey: "inactive-b" })).toBe(
@@ -569,6 +569,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}
}
// Internal Metadata encryption affects future Metadata writes and is not a rebuild requirement.
isNeedRebuildLocal() {
return this.isSomeDirty([
"useIndexedDBAdapter",
@@ -5,6 +5,7 @@ import {
DEFAULT_SETTINGS,
LOG_LEVEL_NOTICE,
type ObsidianLiveSyncSettings,
type EncryptionSettings,
LOG_LEVEL_VERBOSE,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { Menu, type ButtonComponent } from "@/deps.ts";
@@ -31,11 +32,13 @@ import {
import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
import type { RemoteConfigurationResult } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
import SetupRemote from "@/modules/features/SetupWizard/dialogs/SetupRemote.svelte";
import SetupRemoteE2EE from "@/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte";
import SetupRemoteCouchDB from "@/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte";
import SetupRemoteBucket from "@/modules/features/SetupWizard/dialogs/SetupRemoteBucket.svelte";
import type {
SetupRemoteCouchDBInitialData,
SetupRemoteCouchDBResultType,
SetupRemoteE2EEResultType,
} from "@/modules/features/SetupWizard/dialogs/setupDialogTypes.ts";
import { syncActivatedRemoteSettings } from "./remoteConfigBuffer.ts";
@@ -116,7 +119,35 @@ export function paneRemoteConfig(
.onClick(async () => {
const setupManager = this.core.getModule(SetupManager);
const originalSettings = getSettingsFromEditingSettings(this.editingSettings);
await setupManager.onlyE2EEConfiguration(UserMode.Update, originalSettings);
const e2eeConf = await setupManager.dialogManager.openWithExplicitCancel<
SetupRemoteE2EEResultType,
EncryptionSettings
>(SetupRemoteE2EE, originalSettings);
if (e2eeConf === "cancelled") {
return;
}
const onlyInternalMetadataPreferenceChanged =
originalSettings.encryptInternalMetadata !== e2eeConf.encryptInternalMetadata &&
originalSettings.encrypt === e2eeConf.encrypt &&
originalSettings.passphrase === e2eeConf.passphrase &&
originalSettings.E2EEAlgorithm === e2eeConf.E2EEAlgorithm &&
originalSettings.usePathObfuscation === e2eeConf.usePathObfuscation;
if (onlyInternalMetadataPreferenceChanged) {
await this.services.setting.applyPartial(
{ encryptInternalMetadata: e2eeConf.encryptInternalMetadata },
true
);
this.editingSettings.encryptInternalMetadata = e2eeConf.encryptInternalMetadata;
if (this.initialSettings) {
this.initialSettings.encryptInternalMetadata = e2eeConf.encryptInternalMetadata;
}
this.requestUpdate();
} else {
await setupManager.onConfirmApplySettingsFromWizard(
{ ...originalSettings, ...e2eeConf },
UserMode.Update
);
}
updateE2EESummary();
})
.setButtonText("Configure")
@@ -243,6 +274,7 @@ export function paneRemoteConfig(
...DEFAULT_SETTINGS,
encrypt: this.editingSettings.encrypt,
usePathObfuscation: this.editingSettings.usePathObfuscation,
encryptInternalMetadata: this.editingSettings.encryptInternalMetadata,
passphrase: this.editingSettings.passphrase,
configPassphraseStore: this.editingSettings.configPassphraseStore,
});
@@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
const runtime = vi.hoisted(() => ({
buttonClasses: [] as string[],
clickHandlers: [] as Array<() => Promise<void> | void>,
panels: [] as Array<{ destroy: ReturnType<typeof vi.fn> }>,
settingClasses: [] as string[],
}));
@@ -51,7 +52,8 @@ vi.mock("./LiveSyncSetting.ts", () => ({
setDestructive() {
return this;
},
onClick() {
onClick(callback: () => Promise<void> | void) {
runtime.clickHandlers.push(callback);
return this;
},
setButtonText() {
@@ -97,6 +99,7 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/common/ConnectionString", () => ({
},
}));
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemote.svelte", () => ({ default: {} }));
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte", () => ({ default: {} }));
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte", () => ({ default: {} }));
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteBucket.svelte", () => ({ default: {} }));
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte", () => ({ default: {} }));
@@ -114,6 +117,7 @@ function createPanelElement(): HTMLElement {
afterEach(() => {
runtime.buttonClasses.length = 0;
runtime.clickHandlers.length = 0;
runtime.panels.length = 0;
runtime.settingClasses.length = 0;
vi.clearAllMocks();
@@ -148,4 +152,56 @@ describe("paneRemoteConfig", () => {
expect(runtime.panels[0].destroy).toHaveBeenCalledOnce();
});
it("applies an internal Metadata preference change without scheduling setup initialisation", async () => {
const originalSettings = {
encrypt: true,
passphrase: "passphrase",
E2EEAlgorithm: "v2",
usePathObfuscation: true,
encryptInternalMetadata: false,
remoteConfigurations: {},
};
const applyPartial = vi.fn(async () => {});
const onConfirmApplySettingsFromWizard = vi.fn(async () => {});
const setupManager = {
dialogManager: {
openWithExplicitCancel: vi.fn(async () => ({
encrypt: true,
passphrase: "passphrase",
E2EEAlgorithm: "v2",
usePathObfuscation: true,
encryptInternalMetadata: true,
})),
},
onConfirmApplySettingsFromWizard,
};
const host = {
editingSettings: { ...originalSettings },
initialSettings: { ...originalSettings },
services: { setting: { applyPartial } },
core: {
settings: { ...originalSettings },
getModule: vi.fn(() => setupManager),
},
lifetimeComponent: { register: vi.fn() },
requestUpdate: vi.fn(),
};
const addPanel = vi.fn((_parent: HTMLElement, heading: string) => ({
then(callback: (paneEl: HTMLElement) => void) {
if (heading === "E2EE Configuration") {
callback(createPanelElement());
}
},
}));
paneRemoteConfig.call(host as never, {} as HTMLElement, { addPanel } as never);
await runtime.clickHandlers[0]();
expect(applyPartial).toHaveBeenCalledWith({ encryptInternalMetadata: true }, true);
expect(onConfirmApplySettingsFromWizard).not.toHaveBeenCalled();
expect(host.editingSettings.encryptInternalMetadata).toBe(true);
expect(host.initialSettings.encryptInternalMetadata).toBe(true);
expect(host.requestUpdate).toHaveBeenCalledOnce();
});
});
@@ -26,7 +26,8 @@
passphrase: "",
E2EEAlgorithm: DEFAULT_SETTINGS.E2EEAlgorithm,
usePathObfuscation: true,
} as EncryptionSettings;
encryptInternalMetadata: true,
};
let encryptionSettings = $state<EncryptionSettings>({ ...default_encryption });
@@ -42,6 +43,11 @@
if (!encryptionSettings.encrypt) return true;
return encryptionSettings.passphrase.trim().length >= 1;
});
let canEncryptInternalMetadata = $derived(
encryptionSettings.encrypt &&
encryptionSettings.E2EEAlgorithm === E2EEAlgorithms.V2 &&
encryptionSettings.usePathObfuscation
);
function commit() {
setResult(pickEncryptionSettings(encryptionSettings));
@@ -87,6 +93,21 @@
</InfoNote>
{/if}
<InputRow label="Encrypt internal file Metadata">
<input
type="checkbox"
bind:checked={encryptionSettings.encryptInternalMetadata}
disabled={!canEncryptInternalMetadata}
/>
</InputRow>
<InfoNote>
This option applies only to CouchDB and requires End-to-End Encryption, the V2 algorithm, and Property Encryption
(Obfuscate Properties). The remote type is selected later in this setup wizard.
<br />
It protects Metadata written after the option is enabled; existing Metadata is not rewritten. A manual remote
Rebuild is strongly recommended to protect existing Metadata.
</InfoNote>
<ExtraItems title={translateMessage("Advanced")}>
<InputRow label={translateMessage("Encryption Algorithm")}>
<select bind:value={encryptionSettings.E2EEAlgorithm} disabled={!encryptionSettings.encrypt}>
@@ -1,6 +1,7 @@
import {
SYNCINFO_ID,
VER,
VERSIONING_DOCID,
type EntryVersionInfo,
type AnyEntry,
type EntryDoc,
type EntryLeaf,
@@ -8,6 +9,11 @@ import {
type MetaEntry,
type ObsidianLiveSyncSettings,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import {
assessRemoteFeatureDocument,
describeRemoteFeatureRejection,
REMOTE_FEATURE_GENERATION,
} from "@vrtmrz/livesync-commonlib/replication";
import { isChunk } from "@vrtmrz/livesync-commonlib/compat/common/typeUtils";
import {
LOG_LEVEL_DEBUG,
@@ -63,6 +69,10 @@ interface ReplicateResultProcessorContext {
readonly services: ReplicateResultProcessorServices;
}
type ReplicateResultProcessorState = {
databaseId?: string;
observedFeatures?: string[];
highestObservedVersion?: number;
invalidControlObserved?: boolean;
queued: PouchDB.Core.ExistingDocument<EntryDoc>[];
processing: PouchDB.Core.ExistingDocument<EntryDoc>[];
};
@@ -73,6 +83,10 @@ function shortenRev(rev: string | undefined): string {
if (!rev) return "undefined";
return rev.length > 10 ? rev.substring(0, 10) : rev;
}
function getPhysicalDatabaseId(database: PouchDB.Database<EntryDoc>): Promise<string | undefined> {
const identified = database as PouchDB.Database<EntryDoc> & { id?: () => Promise<string> };
return typeof identified.id === "function" ? identified.id() : Promise.resolve(undefined);
}
export class ReplicateResultProcessor {
private log(message: string, level: LOG_LEVEL = LOG_LEVEL_INFO) {
Logger(`[ReplicateResultProcessor] ${message}`, level);
@@ -115,6 +129,89 @@ export class ReplicateResultProcessor {
// If true, the processing queue processor bails the loop.
private _suspended: boolean = false;
// A temporary lifecycle resume cannot make an unknown remote format safe to apply.
private _compatibilityBlocked = false;
private _assessingDatabase = false;
private _physicalDatabase?: PouchDB.Database<EntryDoc>;
private _observedFeatures = new Set<string>();
private _highestObservedVersion = 0;
private _invalidControlObserved = false;
public get isCompatibilityBlocked() {
return this._compatibilityBlocked;
}
private refreshPhysicalDatabase() {
const current = this.localDatabase.localDatabase;
if (current === this._physicalDatabase) return;
if (this._physicalDatabase) {
this._compatibilityBlocked = false;
this._assessingDatabase = true;
this._observedFeatures.clear();
this._highestObservedVersion = 0;
this._invalidControlObserved = false;
this._queuedChanges = [];
this._processingChanges = [];
this._restoreFromSnapshot = undefined;
this.updateProcessingActivity();
}
this._physicalDatabase = current;
}
private shouldStopApplication(sourceDatabase: PouchDB.Database<EntryDoc>) {
return (
this._compatibilityBlocked || this._assessingDatabase || sourceDatabase !== this.localDatabase.localDatabase
);
}
private blockForIncompatibleVersion(document: unknown, recordObservation = true) {
const assessment = assessRemoteFeatureDocument(document);
const hadObservedVersion = this._highestObservedVersion > 0;
let newFeaturesAdded = false;
if (recordObservation) {
let changed = false;
if (assessment.status === "invalid-control") {
changed = !this._invalidControlObserved;
this._invalidControlObserved = true;
} else {
const version = (document as EntryVersionInfo).version;
if (version > this._highestObservedVersion) {
this._highestObservedVersion = version;
changed = true;
}
if (assessment.status === "supported" || assessment.status === "unknown-features") {
for (const feature of assessment.status === "supported"
? assessment.usedFeatures
: ((document as EntryVersionInfo).used_features ?? [])) {
if (this._observedFeatures.has(feature)) continue;
this._observedFeatures.add(feature);
changed = true;
newFeaturesAdded = true;
}
}
}
if (changed) this.triggerTakeSnapshot();
}
if (assessment.status === "supported" || assessment.status === "older-generation") {
// A live writer must recheck the shared Tweak policy after another
// client starts using a newly declared representation.
if (
assessment.status === "supported" &&
hadObservedVersion &&
newFeaturesAdded &&
!this._assessingDatabase
) {
this.context.requestActiveReplicatorRetirement();
}
return;
}
if (this._compatibilityBlocked) return;
this._compatibilityBlocked = true;
this.updateProcessingActivity();
this.log(describeRemoteFeatureRejection(assessment), LOG_LEVEL_NOTICE);
this.context.requestActiveReplicatorRetirement();
}
/**
* Whether the application accepts replicated documents being applied.
*
@@ -135,6 +232,8 @@ export class ReplicateResultProcessor {
public get isSuspended() {
return (
this._suspended ||
this._compatibilityBlocked ||
this._assessingDatabase ||
!this.acceptsResultApplication ||
this.context.currentSettings().suspendParseReplicationResult ||
this.services.appLifecycle.isSuspended()
@@ -145,17 +244,38 @@ export class ReplicateResultProcessor {
* Take a snapshot of the current processing state.
* This snapshot is stored in the KV database for recovery on restart.
*/
protected async _takeSnapshot() {
const snapshot = {
queued: this._queuedChanges.slice(),
processing: this._processingChanges.slice(),
} satisfies ReplicateResultProcessorState;
await this.context.getKeyValueDB().set(KV_KEY_REPLICATION_RESULT_PROCESSOR_SNAPSHOT, snapshot);
this.log(
`Snapshot taken. Queued: ${snapshot.queued.length}, Processing: ${snapshot.processing.length}`,
LOG_LEVEL_DEBUG
);
this.reportStatus();
private _snapshotWriter: Promise<void> = Promise.resolve();
protected _takeSnapshot(): Promise<void> {
// A blocked-batch flush must follow any earlier throttled write, or an
// older snapshot could replace the queue after the replication callback.
const write = this._snapshotWriter
.catch((): void => undefined)
.then(async () => {
const physicalDatabase = this.localDatabase.localDatabase;
const databaseId = await getPhysicalDatabaseId(physicalDatabase);
if (physicalDatabase !== this.localDatabase.localDatabase) return;
const snapshot = {
...(databaseId ? { databaseId } : {}),
observedFeatures: [...this._observedFeatures],
highestObservedVersion: this._highestObservedVersion,
invalidControlObserved: this._invalidControlObserved,
queued: this._queuedChanges.slice(),
processing: this._processingChanges.slice(),
} satisfies ReplicateResultProcessorState;
await this.context.getKeyValueDB().set(KV_KEY_REPLICATION_RESULT_PROCESSOR_SNAPSHOT, snapshot);
this.log(
`Snapshot taken. Queued: ${snapshot.queued.length}, Processing: ${snapshot.processing.length}`,
LOG_LEVEL_DEBUG
);
this.reportStatus();
});
this._snapshotWriter = write;
return write;
}
public async persistBlockedSnapshot(): Promise<void> {
if (this._compatibilityBlocked) await this._takeSnapshot();
}
/**
* Trigger taking a snapshot.
@@ -172,10 +292,42 @@ export class ReplicateResultProcessor {
* Restore from snapshot.
*/
public async restoreFromSnapshot() {
const physicalDatabase = this.localDatabase.localDatabase;
// Replication may have checkpointed a version document before its accompanying
// file changes reached the Vault. Assess the persisted requirement first.
let versionInfo: unknown;
try {
versionInfo = await this.localDatabase.getRaw(VERSIONING_DOCID);
} catch (error) {
if (!isNotFoundError(error)) throw error;
}
if (physicalDatabase !== this.localDatabase.localDatabase) return;
const snapshot = await this.context
.getKeyValueDB()
.get<ReplicateResultProcessorState>(KV_KEY_REPLICATION_RESULT_PROCESSOR_SNAPSHOT);
if (snapshot) {
if (physicalDatabase !== this.localDatabase.localDatabase) return;
const databaseId = await getPhysicalDatabaseId(physicalDatabase);
if (snapshot && (!snapshot.databaseId || !databaseId || snapshot.databaseId === databaseId)) {
for (const feature of snapshot.observedFeatures ?? []) this._observedFeatures.add(feature);
this._highestObservedVersion = Math.max(this._highestObservedVersion, snapshot.highestObservedVersion ?? 0);
this._invalidControlObserved ||= snapshot.invalidControlObserved === true;
}
if (versionInfo !== undefined) this.blockForIncompatibleVersion(versionInfo);
if (this._invalidControlObserved) this.blockForIncompatibleVersion(null, false);
if (this._highestObservedVersion > 0) {
this.blockForIncompatibleVersion(
{
_id: VERSIONING_DOCID,
type: "versioninfo",
version: this._highestObservedVersion,
...(this._highestObservedVersion >= REMOTE_FEATURE_GENERATION
? { used_features: [...this._observedFeatures] }
: {}),
},
false
);
}
if (snapshot && (!snapshot.databaseId || !databaseId || snapshot.databaseId === databaseId)) {
// Restoring the snapshot re-runs processing for both queued and processing items.
const newQueue = [...snapshot.processing, ...snapshot.queued, ...this._queuedChanges];
this._queuedChanges = [];
@@ -186,6 +338,9 @@ export class ReplicateResultProcessor {
);
// await this._takeSnapshot();
}
this._assessingDatabase = false;
this.updateProcessingActivity();
this.triggerProcessQueue();
}
private _restoreFromSnapshot: Promise<void> | undefined = undefined;
@@ -195,7 +350,9 @@ export class ReplicateResultProcessor {
* @returns Promise that resolves when restoration is complete.
*/
public restoreFromSnapshotOnce() {
this.refreshPhysicalDatabase();
if (!this._restoreFromSnapshot) {
this._assessingDatabase = true;
this._restoreFromSnapshot = this.restoreFromSnapshot();
}
return this._restoreFromSnapshot;
@@ -229,7 +386,17 @@ export class ReplicateResultProcessor {
* @param changes Changes to enqueue
*/
public enqueueAll(changes: PouchDB.Core.ExistingDocument<EntryDoc>[]) {
public enqueueAll(changes: PouchDB.Core.ExistingDocument<EntryDoc>[], sourceDatabase?: PouchDB.Database<EntryDoc>) {
if (sourceDatabase && sourceDatabase !== this.localDatabase.localDatabase) return;
const previousPhysicalDatabase = this._physicalDatabase;
this.refreshPhysicalDatabase();
if (previousPhysicalDatabase && previousPhysicalDatabase !== this._physicalDatabase) {
fireAndForget(() => this.restoreFromSnapshotOnce());
}
// Inspect every control document before a note in this batch can start applying.
for (const change of changes) {
if (change?._id === VERSIONING_DOCID) this.blockForIncompatibleVersion(change);
}
for (const change of changes) {
// Check if the change is not a document change (e.g., chunk, versioninfo, syncinfo), and processed it directly.
const isProcessed = this.processIfNonDocumentChange(change);
@@ -274,16 +441,8 @@ export class ReplicateResultProcessor {
this.log(`Processed chunk: ${shortenId(change._id)}`, LOG_LEVEL_DEBUG);
return true;
}
if (change.type == "versioninfo") {
if (change._id === VERSIONING_DOCID) {
this.log(`Version info document received: ${change._id}`, LOG_LEVEL_VERBOSE);
if (change.version > VER) {
// Fence and retire the active publication through its owner.
this.context.requestActiveReplicatorRetirement();
this.log(
`Remote database updated to incompatible version. update your Self-hosted LiveSync plugin.`,
LOG_LEVEL_NOTICE
);
}
return true;
}
if (
@@ -409,11 +568,12 @@ export class ReplicateResultProcessor {
// (per-document serialisation caps concurrency).
const releaser = await this._semaphore.acquire();
releaser();
if (this.isSuspended) break;
// Dequeue the next change
const doc = this._queuedChanges.shift();
if (doc) {
this._processingChanges.push(doc);
void this.parseDocumentChange(doc);
void this.parseDocumentChange(doc, this.localDatabase.localDatabase);
}
// Take snapshot (to be restored on next startup if needed)
this.triggerTakeSnapshot();
@@ -429,8 +589,12 @@ export class ReplicateResultProcessor {
* @param change
* @returns
*/
async parseDocumentChange(change: PouchDB.Core.ExistingDocument<EntryDoc>) {
async parseDocumentChange(
change: PouchDB.Core.ExistingDocument<EntryDoc>,
sourceDatabase: PouchDB.Database<EntryDoc> = this.localDatabase.localDatabase
) {
try {
if (this.shouldStopApplication(sourceDatabase)) return;
if (isAnyNote(change)) {
const docMtime = change.mtime ?? 0;
const maxMTime = this.context.currentSettings().maxMTimeForReflectEvents;
@@ -447,6 +611,7 @@ export class ReplicateResultProcessor {
}
// If the document is a virtual document, process it in the virtual document processor.
if (await this.services.replication.processVirtualDocument(change)) return;
if (this.shouldStopApplication(sourceDatabase)) return;
// If the document is version info, check compatibility and return.
if (isAnyNote(change)) {
const docPath = this.getPath(change);
@@ -454,6 +619,7 @@ export class ReplicateResultProcessor {
this.log(`Skipped: ${docPath}`, LOG_LEVEL_VERBOSE);
return;
}
if (this.shouldStopApplication(sourceDatabase)) return;
const size = change.size;
// Note that this size check depends size that in metadata, not the actual content size.
if (this.services.vault.isFileSizeTooLarge(size)) {
@@ -463,11 +629,20 @@ export class ReplicateResultProcessor {
);
return;
}
return await this.applyToDatabase(change);
return await this.applyToDatabase(change, sourceDatabase);
}
this.log(`Skipped unexpected non-note document: ${change._id}`, LOG_LEVEL_INFO);
return;
} finally {
// An in-flight parse may have started before the control document arrived.
// Retain it even if a later asynchronous boundary stopped application.
if (
this._compatibilityBlocked &&
sourceDatabase === this.localDatabase.localDatabase &&
!this._queuedChanges.includes(change)
) {
this._queuedChanges.push(change);
}
// Remove from processing queue
this._processingChanges = this._processingChanges.filter((e) => e !== change);
try {
@@ -487,12 +662,16 @@ export class ReplicateResultProcessor {
}
// Phase 2: apply the document to database
protected applyToDatabase(doc: PouchDB.Core.ExistingDocument<AnyEntry>) {
protected applyToDatabase(
doc: PouchDB.Core.ExistingDocument<AnyEntry>,
sourceDatabase: PouchDB.Database<EntryDoc> = this.localDatabase.localDatabase
) {
return this.withCounting(async () => {
let releaser: Awaited<ReturnType<typeof this._semaphore.acquire>> | undefined = undefined;
try {
releaser = await this._semaphore.acquire();
await this._applyToDatabase(doc);
if (this.shouldStopApplication(sourceDatabase)) return;
await this._applyToDatabase(doc, sourceDatabase);
} catch (e) {
this.log(`Error while processing replication result`, LOG_LEVEL_NOTICE);
this.logError(e);
@@ -506,12 +685,16 @@ export class ReplicateResultProcessor {
}
// Phase 2.1: process the document and apply to storage
// This function is serialized per document to avoid race-condition for the same document.
private _applyToDatabase(doc_: PouchDB.Core.ExistingDocument<AnyEntry>) {
private _applyToDatabase(
doc_: PouchDB.Core.ExistingDocument<AnyEntry>,
sourceDatabase: PouchDB.Database<EntryDoc>
) {
const dbDoc = doc_ as LoadedEntry; // It has no `data`
const path = this.getPath(dbDoc);
return serialized(`replication-process:${dbDoc._id}`, async () => {
const docNote = `${path} (${shortenId(dbDoc._id)}, ${shortenRev(dbDoc._rev)})`;
const isRequired = await this.checkIsChangeRequiredForDatabaseProcessing(dbDoc);
if (this.shouldStopApplication(sourceDatabase)) return;
if (!isRequired) {
this.log(`Skipped (Not latest): ${docNote}`, LOG_LEVEL_VERBOSE);
return;
@@ -525,6 +708,7 @@ export class ReplicateResultProcessor {
const doc = isDeleted
? { ...dbDoc, data: "" }
: await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, false, true);
if (this.shouldStopApplication(sourceDatabase)) return;
if (!doc) {
// Failed to gather content
this.log(`Failed to gather content of ${docNote}`, LOG_LEVEL_NOTICE);
@@ -535,9 +719,10 @@ export class ReplicateResultProcessor {
// Already processed
this.log(`Processed by other processor: ${docNote}`, LOG_LEVEL_DEBUG);
} else if (this.services.vault.isValidPath(this.getPath(doc))) {
if (this.shouldStopApplication(sourceDatabase)) return;
// Apply to storage if the path is valid
try {
const reflected = await this.applyToStorage(doc as MetaEntry);
const reflected = await this.applyToStorage(doc as MetaEntry, sourceDatabase);
if (!reflected) {
this.reportVaultReflectionFailure(doc as MetaEntry);
return;
@@ -558,9 +743,15 @@ export class ReplicateResultProcessor {
* @param entry
* @returns
*/
protected applyToStorage(entry: MetaEntry) {
protected applyToStorage(
entry: MetaEntry,
sourceDatabase: PouchDB.Database<EntryDoc> = this.localDatabase.localDatabase
) {
return this.withCounting(
() => this.services.replication.processSynchroniseResult(entry),
() =>
this.shouldStopApplication(sourceDatabase)
? Promise.resolve(false)
: this.services.replication.processSynchroniseResult(entry),
this.services.replication.storageApplyingCount
);
}
@@ -1,7 +1,12 @@
import { promiseWithResolvers } from "octagonal-wheels/promises";
import { reactiveSource } from "octagonal-wheels/dataobject/reactive";
import { describe, expect, it, vi } from "vitest";
import { VER, type EntryDoc, type FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
import {
VERSIONING_DOCID,
type EntryDoc,
type FilePathWithPrefix,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
import {
isValidFilenameInAndroid,
isValidFilenameInWidows,
@@ -37,6 +42,10 @@ type SetupOptions = {
isValidPath?: (path: string) => boolean;
processSynchroniseResult?: (entry: unknown) => Promise<boolean>;
setSnapshot?: (key: string, value: unknown) => Promise<unknown>;
getSnapshot?: (key: string) => Promise<unknown>;
localVersionInfo?: unknown;
isTargetFile?: (path: string) => Promise<boolean>;
databaseId?: string;
};
function setup(options: SetupOptions = {}) {
@@ -47,6 +56,9 @@ function setup(options: SetupOptions = {}) {
const isReady = vi.fn(() => options.applicationReady ?? true);
const isValidPath = vi.fn(options.isValidPath ?? (() => true));
const getDBEntryFromMeta = vi.fn(async (entry: object) => ({ ...entry, data: "x" }));
const localPhysicalDatabase = {
...(options.databaseId ? { id: vi.fn(async () => options.databaseId) } : {}),
} as PouchDB.Database<EntryDoc>;
const core = {
services: {
appLifecycle: { isReady, isSuspended: () => false },
@@ -62,14 +74,21 @@ function setup(options: SetupOptions = {}) {
},
replicator: { onCloseActiveReplication, runBoundedLocalApplicationActivity },
vault: {
isTargetFile: vi.fn(async () => true),
isTargetFile: vi.fn(options.isTargetFile ?? (async () => true)),
isFileSizeTooLarge: vi.fn(() => false),
isValidPath,
},
},
kvDB: { set: setSnapshot },
kvDB: { set: setSnapshot, get: vi.fn(options.getSnapshot ?? (async () => undefined)) },
localDatabase: {
getRaw: vi.fn(async (id: string) => ({ _id: id, _rev: "1-test" })),
localDatabase: localPhysicalDatabase,
getRaw: vi.fn(async (id: string) => {
if (id === VERSIONING_DOCID) {
if (options.localVersionInfo === undefined) throw { status: 404 };
return options.localVersionInfo;
}
return { _id: id, _rev: "1-test" };
}),
getDBEntryFromMeta,
},
};
@@ -88,6 +107,9 @@ function setup(options: SetupOptions = {}) {
} as never);
return {
getDBEntryFromMeta,
isTargetFile: core.services.vault.isTargetFile,
localPhysicalDatabase,
localDatabase: core.localDatabase,
isReady,
isValidPath,
onCloseActiveReplication,
@@ -145,9 +167,9 @@ describe("ReplicateResultProcessor", () => {
});
}
expect(processSynchroniseResult).toHaveBeenCalledTimes(11);
expect(processSynchroniseResult.mock.calls.some(([entry]) =>
(entry as { _id: string })._id === "unrelated-queue"
)).toBe(true);
expect(
processSynchroniseResult.mock.calls.some(([entry]) => (entry as { _id: string })._id === "unrelated-queue")
).toBe(true);
});
it("suspends result application while the application is not ready", () => {
@@ -199,10 +221,10 @@ describe("ReplicateResultProcessor", () => {
it("retires active ownership when a newer remote version is observed", async () => {
const { onCloseActiveReplication, processor } = setup();
const versionInfo = {
_id: "versioninfo",
_id: VERSIONING_DOCID,
_rev: "1-test",
type: "versioninfo",
version: VER + 1,
version: REMOTE_FEATURE_GENERATION + 1,
} as unknown as PouchDB.Core.ExistingDocument<EntryDoc>;
processor.enqueueAll([versionInfo]);
@@ -210,6 +232,251 @@ describe("ReplicateResultProcessor", () => {
await vi.waitFor(() => expect(onCloseActiveReplication).toHaveBeenCalledOnce());
});
it("continues applying documents after restoring a legacy local version document", async () => {
const { onCloseActiveReplication, processor, processSynchroniseResult } = setup({
localVersionInfo: {
_id: VERSIONING_DOCID,
type: "versioninfo",
version: 11,
},
});
await processor.restoreFromSnapshotOnce();
processor.enqueueAll([note("legacy-database-note")]);
await vi.waitFor(() => expect(processSynchroniseResult).toHaveBeenCalledOnce());
expect(processor.isSuspended).toBe(false);
expect(onCloseActiveReplication).not.toHaveBeenCalled();
});
it.each(["first", "last"] as const)(
"holds an entire received batch when an unknown feature is %s",
async (position) => {
const { onCloseActiveReplication, processor, processSynchroniseResult } = setup();
const versionInfo = {
_id: VERSIONING_DOCID,
_rev: "2-unknown",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: ["future-format-v7"],
} as PouchDB.Core.ExistingDocument<EntryDoc>;
const document = note("pending-after-compatibility-change");
processor.enqueueAll(position === "first" ? [versionInfo, document] : [document, versionInfo]);
await vi.waitFor(() => expect(onCloseActiveReplication).toHaveBeenCalledOnce());
await Promise.resolve();
expect(processor.isSuspended).toBe(true);
expect(processSynchroniseResult).not.toHaveBeenCalled();
processor.resume();
await Promise.resolve();
expect(processSynchroniseResult).not.toHaveBeenCalled();
}
);
it("continues when a newly received feature is supported", async () => {
const { onCloseActiveReplication, processor, processSynchroniseResult } = setup();
const versionInfo = {
_id: VERSIONING_DOCID,
_rev: "2-supported",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: [ENCRYPTED_INTERNAL_METADATA_FEATURE],
} as PouchDB.Core.ExistingDocument<EntryDoc>;
processor.enqueueAll([versionInfo, note("supported-update")]);
await vi.waitFor(() => expect(processSynchroniseResult).toHaveBeenCalledOnce());
expect(onCloseActiveReplication).not.toHaveBeenCalled();
});
it("rechecks shared writer settings when a supported feature is added to an active database", async () => {
const { onCloseActiveReplication, processor } = setup({
localVersionInfo: {
_id: VERSIONING_DOCID,
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: [],
},
});
await processor.restoreFromSnapshotOnce();
const versionInfo = {
_id: VERSIONING_DOCID,
_rev: "2-supported",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: [ENCRYPTED_INTERNAL_METADATA_FEATURE],
} as PouchDB.Core.ExistingDocument<EntryDoc>;
processor.enqueueAll([versionInfo]);
processor.enqueueAll([{ ...versionInfo, _rev: "3-same-features" }]);
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
expect(processor.isSuspended).toBe(false);
});
it("retains an in-flight note when a feature change arrives during an asynchronous check", async () => {
const targetCheck = promiseWithResolvers<boolean>();
const { isTargetFile, onCloseActiveReplication, processor, processSynchroniseResult } = setup({
isTargetFile: async () => targetCheck.promise,
});
const pending = note("in-flight");
processor.enqueueAll([pending]);
await vi.waitFor(() => expect(isTargetFile).toHaveBeenCalledOnce());
processor.enqueueAll([
{
_id: VERSIONING_DOCID,
_rev: "2-unknown",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: ["future-format-v7"],
} as PouchDB.Core.ExistingDocument<EntryDoc>,
]);
targetCheck.resolve(true);
await vi.waitFor(() => expect(processor["_processingChanges"]).toHaveLength(0));
expect(processor["_queuedChanges"]).toContain(pending);
expect(processSynchroniseResult).not.toHaveBeenCalled();
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
});
it("restores a checkpointed note only after assessing the persisted version document", async () => {
const pending = note("checkpointed");
const { onCloseActiveReplication, processor, processSynchroniseResult } = setup({
localVersionInfo: {
_id: VERSIONING_DOCID,
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: ["future-format-v7"],
},
getSnapshot: async () => ({ processing: [pending], queued: [] }),
});
await processor.restoreFromSnapshotOnce();
expect(processor.isSuspended).toBe(true);
expect(processor["_queuedChanges"]).toContain(pending);
expect(processSynchroniseResult).not.toHaveBeenCalled();
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
});
it("retains an observed unknown feature when the local version list is later shortened", async () => {
let snapshot: unknown;
const first = setup({
databaseId: "same-database",
setSnapshot: async (_key, value) => {
snapshot = value;
},
});
const pending = note("checkpointed-after-list-change");
first.processor.enqueueAll([
{
_id: VERSIONING_DOCID,
_rev: "2-unknown",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: ["future-format-v7"],
} as PouchDB.Core.ExistingDocument<EntryDoc>,
pending,
]);
await vi.waitFor(() => expect(snapshot).toBeDefined());
const resumed = setup({
databaseId: "same-database",
localVersionInfo: {
_id: VERSIONING_DOCID,
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: [],
},
getSnapshot: async () => snapshot,
});
await resumed.processor.restoreFromSnapshotOnce();
expect(resumed.processor.isSuspended).toBe(true);
expect(resumed.processor["_queuedChanges"]).toContain(pending);
expect(resumed.processSynchroniseResult).not.toHaveBeenCalled();
expect(resumed.onCloseActiveReplication).toHaveBeenCalledOnce();
});
it("does not restore pending work belonging to a replaced physical database", async () => {
const pending = note("old-database");
const { processor, processSynchroniseResult } = setup({
databaseId: "replacement-database",
getSnapshot: async () => ({ databaseId: "retired-database", processing: [pending], queued: [] }),
});
await processor.restoreFromSnapshotOnce();
expect(processor["_queuedChanges"]).toHaveLength(0);
expect(processSynchroniseResult).not.toHaveBeenCalled();
expect(processor.isSuspended).toBe(false);
});
it("reports unknown identifiers and retires only once for repeated notifications", () => {
const log = vi.fn((_message: unknown, _level?: number) => undefined);
setGlobalLogFunction(log);
try {
const { onCloseActiveReplication, processor } = setup();
const versionInfo = {
_id: VERSIONING_DOCID,
_rev: "2-unknown",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: ["future-format-v7"],
} as PouchDB.Core.ExistingDocument<EntryDoc>;
processor.enqueueAll([versionInfo]);
processor.enqueueAll([versionInfo]);
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
expect(log).toHaveBeenCalledWith(
"[ReplicateResultProcessor] Unknown features are in use: future-format-v7",
LOG_LEVEL_NOTICE,
undefined
);
} finally {
setGlobalLogFunction(defaultLogger);
}
});
it("ignores a late feature notification from a replaced physical database", () => {
const { onCloseActiveReplication, processor } = setup();
const oldPhysicalDatabase = {} as PouchDB.Database<EntryDoc>;
const versionInfo = {
_id: VERSIONING_DOCID,
_rev: "2-unknown",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: ["future-format-v7"],
} as PouchDB.Core.ExistingDocument<EntryDoc>;
processor.enqueueAll([versionInfo], oldPhysicalDatabase);
expect(processor.isSuspended).toBe(false);
expect(onCloseActiveReplication).not.toHaveBeenCalled();
});
it("reassesses compatibility for a replacement local database", async () => {
const { localDatabase, onCloseActiveReplication, processor } = setup();
processor.enqueueAll([
{
_id: VERSIONING_DOCID,
_rev: "2-unknown",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: ["future-format-v7"],
} as PouchDB.Core.ExistingDocument<EntryDoc>,
]);
expect(processor.isSuspended).toBe(true);
localDatabase.localDatabase = {} as PouchDB.Database<EntryDoc>;
await processor.restoreFromSnapshotOnce();
expect(processor.isSuspended).toBe(false);
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
});
it("scans normal-file metadata without loading chunk documents and requeues it", async () => {
const documents = [
{ _id: "first", _rev: "1-a", type: "plain", path: "first.md" },
@@ -1,4 +1,8 @@
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import {
VERSIONING_DOCID,
type EntryDoc,
type ObsidianLiveSyncSettings,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { assessTweakCompatibility } from "@vrtmrz/livesync-commonlib/settings";
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger";
import { skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
@@ -7,12 +11,27 @@ import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/rep
import {
CENTRAL_COMPATIBILITY_REJECTION_REASONS,
REPLICATION_PROGRESS_PRESENTATIONS,
assessRemoteFeatureDocument,
describeRemoteFeatureRejection,
type ReplicatorInstance,
type ReplicationFailureRequest,
} from "@vrtmrz/livesync-commonlib/replication";
import { $msg } from "@/common/translation";
import { usesLegacyIndexedDBAdapter } from "@/common/compatibilitySettings";
import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
import type PouchDB from "pouchdb-core";
async function canInterpretCleanupDatabase(db: PouchDB.Database<EntryDoc>): Promise<boolean> {
try {
const assessment = assessRemoteFeatureDocument(await db.get(VERSIONING_DOCID));
if (assessment.status === "supported" || assessment.status === "older-generation") return true;
Logger(`Database cleanup cancelled: ${describeRemoteFeatureRejection(assessment)}`, LOG_LEVEL_NOTICE);
} catch (error) {
Logger("Database cleanup cancelled: feature compatibility could not be checked.", LOG_LEVEL_NOTICE);
Logger(error, LOG_LEVEL_INFO);
}
return false;
}
type CentralCompatibilityRecoveryServices = Pick<
LiveSyncBaseCore["services"],
@@ -60,6 +79,7 @@ export function createCentralCompatibilityRecovery(context: CentralCompatibility
) {
Logger("The remote database has been cleaned.", showProgress ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
await skipIfDuplicated("cleanup", async () => {
if (!(await canInterpretCleanupDatabase(context.getLocalDatabase().localDatabase))) return;
const count = await purgeUnreferencedChunks(context.getLocalDatabase().localDatabase, true);
const message = `The remote database has been cleaned up.
To synchronize, this device must be also cleaned up. ${count} chunk(s) will be erased from this device.
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { VERSIONING_DOCID, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { assessTweakCompatibility } from "@vrtmrz/livesync-commonlib/settings";
import { defaultLogger, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, setGlobalLogFunction } from "octagonal-wheels/common/logger";
import {
@@ -24,6 +24,49 @@ import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/rep
import { createCentralCompatibilityRecovery } from "./centralCompatibilityRecovery";
describe("central compatibility recovery", () => {
it("does not count chunks for cleanup when local feature requirements are unknown", async () => {
chunkMocks.purgeUnreferencedChunks.mockClear();
const confirmWithMessage = vi.fn(async () => "Dismiss");
const recovery = createCentralCompatibilityRecovery({
confirm: { confirmWithMessage },
getLocalDatabase: () => ({
localDatabase: {
get: vi.fn(async (id: string) => ({
_id: id,
type: "versioninfo",
version: 13,
used_features: ["future-format-v7"],
})),
},
}),
services: { replicator: {} },
} as never);
await recovery.reconcileCleanedRemote(true, {} as ObsidianLiveSyncSettings, {} as never);
expect(chunkMocks.purgeUnreferencedChunks).not.toHaveBeenCalled();
expect(confirmWithMessage).not.toHaveBeenCalled();
});
it("allows cleanup counting for a legacy local version document", async () => {
chunkMocks.purgeUnreferencedChunks.mockClear();
const confirmWithMessage = vi.fn(async () => "Dismiss");
const recovery = createCentralCompatibilityRecovery({
confirm: { confirmWithMessage },
getLocalDatabase: () => ({
localDatabase: {
get: vi.fn(async (id: string) => ({ _id: id, type: "versioninfo", version: 11 })),
},
}),
services: { replicator: {} },
} as never);
await recovery.reconcileCleanedRemote(true, {} as ObsidianLiveSyncSettings, {} as never);
expect(chunkMocks.purgeUnreferencedChunks).toHaveBeenCalledWith(expect.anything(), true);
expect(confirmWithMessage).toHaveBeenCalledOnce();
});
it("passes the failed attempt's exact tweak assessment to mismatch resolution", async () => {
const setting = { customChunkSize: 0 };
const preferredTweakValue = { customChunkSize: 60 };
@@ -292,7 +335,9 @@ describe("central compatibility recovery", () => {
});
const runFiniteReplicationActivity = vi.fn(async (task: () => unknown) => await task());
const openOneShotReplication = vi.fn(async () => true);
const remoteDatabase = { close: vi.fn(async () => undefined) };
const remoteDatabase = {
close: vi.fn(async () => undefined),
};
const close = vi.fn(async () => undefined);
const activeReplicator = Object.assign(new LiveSyncCouchDBReplicator({} as never), {
connectRemoteCouchDBWithSetting: vi.fn(async () => ({ db: remoteDatabase, close })),
@@ -303,7 +348,12 @@ describe("central compatibility recovery", () => {
const runWithActiveReplicatorContext = vi.fn(async (task: (context: unknown) => unknown) =>
task(expectedContext)
);
const localDatabase = { localDatabase: {}, clearCaches: vi.fn() };
const localDatabase = {
localDatabase: {
get: vi.fn(async () => ({ _id: VERSIONING_DOCID, type: "versioninfo", version: 12 })),
},
clearCaches: vi.fn(),
};
const getLocalDatabase = vi.fn(() => localDatabase);
const recovery = createCentralCompatibilityRecovery({
confirm: { confirmWithMessage: vi.fn(async () => "Cleanup") },
@@ -335,7 +385,7 @@ describe("central compatibility recovery", () => {
activityFinished.mock.invocationCallOrder[0]
);
expect(chunkMocks.balanceChunkPurgedDBs).toHaveBeenCalledOnce();
expect(getLocalDatabase).toHaveBeenCalledTimes(2);
expect(getLocalDatabase).toHaveBeenCalled();
expect(close).toHaveBeenCalledOnce();
expect(close.mock.invocationCallOrder[0]).toBeLessThan(activityFinished.mock.invocationCallOrder[0]);
});
+8 -6
View File
@@ -98,19 +98,21 @@ export function useReplicationFeature<TContext extends ServiceContext, TCommands
clearHandlers();
return Promise.resolve(true);
});
services.databaseEvents.onDatabaseInitialised.addHandler(() => {
fireAndForget(() => resultProcessor.restoreFromSnapshotOnce());
return Promise.resolve(true);
services.databaseEvents.onDatabaseInitialised.addHandler(async () => {
await resultProcessor.restoreFromSnapshotOnce();
return true;
});
services.appLifecycle.onSettingLoaded.addHandler(initialiseAutomaticReplicationTriggers);
services.replication.parseSynchroniseResult.addHandler((documents) => {
resultProcessor.enqueueAll(documents);
return Promise.resolve(true);
services.replication.parseSynchroniseResult.addHandler(async (documents, sourceDatabase) => {
resultProcessor.enqueueAll(documents, sourceDatabase);
await resultProcessor.persistBlockedSnapshot();
return true;
});
services.replication.onBeforeReplicate.addHandler(onlinePreflight, 10);
services.replication.onPrepareCentralRemoteReplication.addHandler(securitySeedPreflight);
services.replication.onBeforeReplicate.addHandler(async () => {
await resultProcessor.restoreFromSnapshotOnce();
if (resultProcessor.isCompatibilityBlocked) return false;
unresolvedErrorManager.clearErrors();
return true;
}, 100);
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { createServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { VER, type EntryDoc } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { VERSIONING_DOCID, type EntryDoc } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
import { promiseWithResolvers } from "octagonal-wheels/promises";
import { useReplicationFeature } from "./index";
@@ -19,8 +20,12 @@ type SetupOptions = {
};
function setup(options: SetupOptions = {}) {
const defaultLocalDatabase = {
localDatabase: {},
getRaw: vi.fn(async () => { throw { status: 404 }; }),
};
const {
getLocalDatabase = () => ({}),
getLocalDatabase = () => defaultLocalDatabase,
keyValueDB = {
kvDB: {
get: vi.fn(async () => undefined),
@@ -39,7 +44,7 @@ function setup(options: SetupOptions = {}) {
API: { isMobile: vi.fn(() => false), isOnline: true },
appLifecycle: {
getUnresolvedMessages: { addHandler: vi.fn() },
isReady: true,
isReady: vi.fn(() => true),
isSuspended: vi.fn(() => false),
onSettingLoaded: { addHandler: vi.fn() },
},
@@ -48,6 +53,7 @@ function setup(options: SetupOptions = {}) {
keyValueDB,
path: { getPath: vi.fn((entry: { path: string }) => entry.path) },
replication: {
replicationResultCount: { value: 0 },
onBeforeReplicate: {
addHandler: vi.fn((handler: BooleanHandler, priority = 0) => {
beforeReplicateHandlers.set(priority, handler);
@@ -165,10 +171,10 @@ describe("replication serviceFeature composition", () => {
const onCloseActiveReplication = vi.fn(() => retirement.promise);
const harness = setup({ onCloseActiveReplication });
const versionInfo = {
_id: "versioninfo",
_id: VERSIONING_DOCID,
_rev: "1-test",
type: "versioninfo",
version: VER + 1,
version: REMOTE_FEATURE_GENERATION + 1,
} as unknown as PouchDB.Core.ExistingDocument<EntryDoc>;
expect(harness.parseHandler).toBeDefined();
@@ -177,4 +183,59 @@ describe("replication serviceFeature composition", () => {
retirement.resolve(true);
});
it("persists a blocked batch before its replication callback settles", async () => {
const writeFinished = promiseWithResolvers<void>();
const writes: Array<{ queued: Array<{ _id: string }> }> = [];
const { parseHandler } = setup({
keyValueDB: {
kvDB: {
get: vi.fn(async () => undefined),
set: vi.fn(async (_key, value) => {
writes.push(value as { queued: Array<{ _id: string }> });
await writeFinished.promise;
}),
},
},
});
const pending = {
_id: "checkpointed-note",
_rev: "1-test",
type: "plain",
path: "checkpointed-note.md",
} as PouchDB.Core.ExistingDocument<EntryDoc>;
const unknownVersion = {
_id: VERSIONING_DOCID,
_rev: "2-unknown",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: ["future-format-v7"],
} as PouchDB.Core.ExistingDocument<EntryDoc>;
let callbackSettled = false;
const callback = parseHandler!([pending, unknownVersion]).then((result) => {
callbackSettled = true;
return result;
});
await vi.waitFor(() => expect(writes.length).toBeGreaterThan(0));
expect(callbackSettled).toBe(false);
writeFinished.resolve();
await expect(callback).resolves.toBe(true);
expect(writes[writes.length - 1]?.queued.map((entry) => entry._id)).toContain(pending._id);
});
it("refuses another replication after observing an unknown feature locally", async () => {
const { beforeReplicateHandlers, parseHandler } = setup();
const unknownVersion = {
_id: VERSIONING_DOCID,
_rev: "2-unknown",
type: "versioninfo",
version: REMOTE_FEATURE_GENERATION,
used_features: ["future-format-v7"],
} as PouchDB.Core.ExistingDocument<EntryDoc>;
await parseHandler!([unknownVersion]);
await expect(beforeReplicateHandlers.get(100)!(false)).resolves.toBe(false);
});
});