mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-10-08 18:32:30 +00:00
Integrate encrypted internal metadata in LiveSync
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { usesEncryptedInternalMetadata } from "@vrtmrz/livesync-commonlib/replication";
|
||||
|
||||
type EndpointProjection = readonly [kind: "url" | "invalid-url", value: string];
|
||||
|
||||
@@ -77,6 +78,7 @@ export function getCouchDBReplicatorConfigurationIdentity(settings: RemoteDBSett
|
||||
settings.useRequestAPI,
|
||||
settings.disableRequestURI,
|
||||
projectRemoteSecurity(settings),
|
||||
usesEncryptedInternalMetadata(settings),
|
||||
settings.enableCompression,
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -67,6 +67,22 @@ describe("active Replicator configuration identity", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("recreates the CouchDB connection when internal Metadata encryption becomes effective", () => {
|
||||
const active = configuredSettings({ usePathObfuscation: true, encryptInternalMetadata: false });
|
||||
const enabled = { ...active, encryptInternalMetadata: true };
|
||||
|
||||
expect(getCouchDBReplicatorConfigurationIdentity(enabled)).not.toBe(
|
||||
getCouchDBReplicatorConfigurationIdentity(active)
|
||||
);
|
||||
const inactive = { ...active, usePathObfuscation: false };
|
||||
expect(getCouchDBReplicatorConfigurationIdentity({ ...inactive, encryptInternalMetadata: true })).toBe(
|
||||
getCouchDBReplicatorConfigurationIdentity(inactive)
|
||||
);
|
||||
expect(getObjectStorageReplicatorConfigurationIdentity(enabled)).toBe(
|
||||
getObjectStorageReplicatorConfigurationIdentity(active)
|
||||
);
|
||||
});
|
||||
|
||||
it("projects only the active CouchDB authentication mode", () => {
|
||||
const basic = configuredSettings({ useJWT: false, jwtKey: "inactive-a" });
|
||||
expect(getCouchDBReplicatorConfigurationIdentity({ ...basic, jwtKey: "inactive-b" })).toBe(
|
||||
|
||||
@@ -569,6 +569,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
|
||||
}
|
||||
}
|
||||
|
||||
// Internal Metadata encryption affects future Metadata writes and is not a rebuild requirement.
|
||||
isNeedRebuildLocal() {
|
||||
return this.isSomeDirty([
|
||||
"useIndexedDBAdapter",
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
DEFAULT_SETTINGS,
|
||||
LOG_LEVEL_NOTICE,
|
||||
type ObsidianLiveSyncSettings,
|
||||
type EncryptionSettings,
|
||||
LOG_LEVEL_VERBOSE,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { Menu, type ButtonComponent } from "@/deps.ts";
|
||||
@@ -31,11 +32,13 @@ import {
|
||||
import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
|
||||
import type { RemoteConfigurationResult } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
|
||||
import SetupRemote from "@/modules/features/SetupWizard/dialogs/SetupRemote.svelte";
|
||||
import SetupRemoteE2EE from "@/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte";
|
||||
import SetupRemoteCouchDB from "@/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte";
|
||||
import SetupRemoteBucket from "@/modules/features/SetupWizard/dialogs/SetupRemoteBucket.svelte";
|
||||
import type {
|
||||
SetupRemoteCouchDBInitialData,
|
||||
SetupRemoteCouchDBResultType,
|
||||
SetupRemoteE2EEResultType,
|
||||
} from "@/modules/features/SetupWizard/dialogs/setupDialogTypes.ts";
|
||||
import { syncActivatedRemoteSettings } from "./remoteConfigBuffer.ts";
|
||||
|
||||
@@ -116,7 +119,35 @@ export function paneRemoteConfig(
|
||||
.onClick(async () => {
|
||||
const setupManager = this.core.getModule(SetupManager);
|
||||
const originalSettings = getSettingsFromEditingSettings(this.editingSettings);
|
||||
await setupManager.onlyE2EEConfiguration(UserMode.Update, originalSettings);
|
||||
const e2eeConf = await setupManager.dialogManager.openWithExplicitCancel<
|
||||
SetupRemoteE2EEResultType,
|
||||
EncryptionSettings
|
||||
>(SetupRemoteE2EE, originalSettings);
|
||||
if (e2eeConf === "cancelled") {
|
||||
return;
|
||||
}
|
||||
const onlyInternalMetadataPreferenceChanged =
|
||||
originalSettings.encryptInternalMetadata !== e2eeConf.encryptInternalMetadata &&
|
||||
originalSettings.encrypt === e2eeConf.encrypt &&
|
||||
originalSettings.passphrase === e2eeConf.passphrase &&
|
||||
originalSettings.E2EEAlgorithm === e2eeConf.E2EEAlgorithm &&
|
||||
originalSettings.usePathObfuscation === e2eeConf.usePathObfuscation;
|
||||
if (onlyInternalMetadataPreferenceChanged) {
|
||||
await this.services.setting.applyPartial(
|
||||
{ encryptInternalMetadata: e2eeConf.encryptInternalMetadata },
|
||||
true
|
||||
);
|
||||
this.editingSettings.encryptInternalMetadata = e2eeConf.encryptInternalMetadata;
|
||||
if (this.initialSettings) {
|
||||
this.initialSettings.encryptInternalMetadata = e2eeConf.encryptInternalMetadata;
|
||||
}
|
||||
this.requestUpdate();
|
||||
} else {
|
||||
await setupManager.onConfirmApplySettingsFromWizard(
|
||||
{ ...originalSettings, ...e2eeConf },
|
||||
UserMode.Update
|
||||
);
|
||||
}
|
||||
updateE2EESummary();
|
||||
})
|
||||
.setButtonText("Configure")
|
||||
@@ -243,6 +274,7 @@ export function paneRemoteConfig(
|
||||
...DEFAULT_SETTINGS,
|
||||
encrypt: this.editingSettings.encrypt,
|
||||
usePathObfuscation: this.editingSettings.usePathObfuscation,
|
||||
encryptInternalMetadata: this.editingSettings.encryptInternalMetadata,
|
||||
passphrase: this.editingSettings.passphrase,
|
||||
configPassphraseStore: this.editingSettings.configPassphraseStore,
|
||||
});
|
||||
|
||||
@@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const runtime = vi.hoisted(() => ({
|
||||
buttonClasses: [] as string[],
|
||||
clickHandlers: [] as Array<() => Promise<void> | void>,
|
||||
panels: [] as Array<{ destroy: ReturnType<typeof vi.fn> }>,
|
||||
settingClasses: [] as string[],
|
||||
}));
|
||||
@@ -51,7 +52,8 @@ vi.mock("./LiveSyncSetting.ts", () => ({
|
||||
setDestructive() {
|
||||
return this;
|
||||
},
|
||||
onClick() {
|
||||
onClick(callback: () => Promise<void> | void) {
|
||||
runtime.clickHandlers.push(callback);
|
||||
return this;
|
||||
},
|
||||
setButtonText() {
|
||||
@@ -97,6 +99,7 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/common/ConnectionString", () => ({
|
||||
},
|
||||
}));
|
||||
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemote.svelte", () => ({ default: {} }));
|
||||
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteE2EE.svelte", () => ({ default: {} }));
|
||||
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte", () => ({ default: {} }));
|
||||
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteBucket.svelte", () => ({ default: {} }));
|
||||
vi.mock("@/modules/features/SetupWizard/dialogs/SetupRemoteP2P.svelte", () => ({ default: {} }));
|
||||
@@ -114,6 +117,7 @@ function createPanelElement(): HTMLElement {
|
||||
|
||||
afterEach(() => {
|
||||
runtime.buttonClasses.length = 0;
|
||||
runtime.clickHandlers.length = 0;
|
||||
runtime.panels.length = 0;
|
||||
runtime.settingClasses.length = 0;
|
||||
vi.clearAllMocks();
|
||||
@@ -148,4 +152,56 @@ describe("paneRemoteConfig", () => {
|
||||
|
||||
expect(runtime.panels[0].destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("applies an internal Metadata preference change without scheduling setup initialisation", async () => {
|
||||
const originalSettings = {
|
||||
encrypt: true,
|
||||
passphrase: "passphrase",
|
||||
E2EEAlgorithm: "v2",
|
||||
usePathObfuscation: true,
|
||||
encryptInternalMetadata: false,
|
||||
remoteConfigurations: {},
|
||||
};
|
||||
const applyPartial = vi.fn(async () => {});
|
||||
const onConfirmApplySettingsFromWizard = vi.fn(async () => {});
|
||||
const setupManager = {
|
||||
dialogManager: {
|
||||
openWithExplicitCancel: vi.fn(async () => ({
|
||||
encrypt: true,
|
||||
passphrase: "passphrase",
|
||||
E2EEAlgorithm: "v2",
|
||||
usePathObfuscation: true,
|
||||
encryptInternalMetadata: true,
|
||||
})),
|
||||
},
|
||||
onConfirmApplySettingsFromWizard,
|
||||
};
|
||||
const host = {
|
||||
editingSettings: { ...originalSettings },
|
||||
initialSettings: { ...originalSettings },
|
||||
services: { setting: { applyPartial } },
|
||||
core: {
|
||||
settings: { ...originalSettings },
|
||||
getModule: vi.fn(() => setupManager),
|
||||
},
|
||||
lifetimeComponent: { register: vi.fn() },
|
||||
requestUpdate: vi.fn(),
|
||||
};
|
||||
const addPanel = vi.fn((_parent: HTMLElement, heading: string) => ({
|
||||
then(callback: (paneEl: HTMLElement) => void) {
|
||||
if (heading === "E2EE Configuration") {
|
||||
callback(createPanelElement());
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
paneRemoteConfig.call(host as never, {} as HTMLElement, { addPanel } as never);
|
||||
await runtime.clickHandlers[0]();
|
||||
|
||||
expect(applyPartial).toHaveBeenCalledWith({ encryptInternalMetadata: true }, true);
|
||||
expect(onConfirmApplySettingsFromWizard).not.toHaveBeenCalled();
|
||||
expect(host.editingSettings.encryptInternalMetadata).toBe(true);
|
||||
expect(host.initialSettings.encryptInternalMetadata).toBe(true);
|
||||
expect(host.requestUpdate).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -26,7 +26,8 @@
|
||||
passphrase: "",
|
||||
E2EEAlgorithm: DEFAULT_SETTINGS.E2EEAlgorithm,
|
||||
usePathObfuscation: true,
|
||||
} as EncryptionSettings;
|
||||
encryptInternalMetadata: true,
|
||||
};
|
||||
|
||||
let encryptionSettings = $state<EncryptionSettings>({ ...default_encryption });
|
||||
|
||||
@@ -42,6 +43,11 @@
|
||||
if (!encryptionSettings.encrypt) return true;
|
||||
return encryptionSettings.passphrase.trim().length >= 1;
|
||||
});
|
||||
let canEncryptInternalMetadata = $derived(
|
||||
encryptionSettings.encrypt &&
|
||||
encryptionSettings.E2EEAlgorithm === E2EEAlgorithms.V2 &&
|
||||
encryptionSettings.usePathObfuscation
|
||||
);
|
||||
|
||||
function commit() {
|
||||
setResult(pickEncryptionSettings(encryptionSettings));
|
||||
@@ -87,6 +93,21 @@
|
||||
</InfoNote>
|
||||
{/if}
|
||||
|
||||
<InputRow label="Encrypt internal file Metadata">
|
||||
<input
|
||||
type="checkbox"
|
||||
bind:checked={encryptionSettings.encryptInternalMetadata}
|
||||
disabled={!canEncryptInternalMetadata}
|
||||
/>
|
||||
</InputRow>
|
||||
<InfoNote>
|
||||
This option applies only to CouchDB and requires End-to-End Encryption, the V2 algorithm, and Property Encryption
|
||||
(Obfuscate Properties). The remote type is selected later in this setup wizard.
|
||||
<br />
|
||||
It protects Metadata written after the option is enabled; existing Metadata is not rewritten. A manual remote
|
||||
Rebuild is strongly recommended to protect existing Metadata.
|
||||
</InfoNote>
|
||||
|
||||
<ExtraItems title={translateMessage("Advanced")}>
|
||||
<InputRow label={translateMessage("Encryption Algorithm")}>
|
||||
<select bind:value={encryptionSettings.E2EEAlgorithm} disabled={!encryptionSettings.encrypt}>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
SYNCINFO_ID,
|
||||
VER,
|
||||
VERSIONING_DOCID,
|
||||
type EntryVersionInfo,
|
||||
type AnyEntry,
|
||||
type EntryDoc,
|
||||
type EntryLeaf,
|
||||
@@ -8,6 +9,11 @@ import {
|
||||
type MetaEntry,
|
||||
type ObsidianLiveSyncSettings,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import {
|
||||
assessRemoteFeatureDocument,
|
||||
describeRemoteFeatureRejection,
|
||||
REMOTE_FEATURE_GENERATION,
|
||||
} from "@vrtmrz/livesync-commonlib/replication";
|
||||
import { isChunk } from "@vrtmrz/livesync-commonlib/compat/common/typeUtils";
|
||||
import {
|
||||
LOG_LEVEL_DEBUG,
|
||||
@@ -63,6 +69,10 @@ interface ReplicateResultProcessorContext {
|
||||
readonly services: ReplicateResultProcessorServices;
|
||||
}
|
||||
type ReplicateResultProcessorState = {
|
||||
databaseId?: string;
|
||||
observedFeatures?: string[];
|
||||
highestObservedVersion?: number;
|
||||
invalidControlObserved?: boolean;
|
||||
queued: PouchDB.Core.ExistingDocument<EntryDoc>[];
|
||||
processing: PouchDB.Core.ExistingDocument<EntryDoc>[];
|
||||
};
|
||||
@@ -73,6 +83,10 @@ function shortenRev(rev: string | undefined): string {
|
||||
if (!rev) return "undefined";
|
||||
return rev.length > 10 ? rev.substring(0, 10) : rev;
|
||||
}
|
||||
function getPhysicalDatabaseId(database: PouchDB.Database<EntryDoc>): Promise<string | undefined> {
|
||||
const identified = database as PouchDB.Database<EntryDoc> & { id?: () => Promise<string> };
|
||||
return typeof identified.id === "function" ? identified.id() : Promise.resolve(undefined);
|
||||
}
|
||||
export class ReplicateResultProcessor {
|
||||
private log(message: string, level: LOG_LEVEL = LOG_LEVEL_INFO) {
|
||||
Logger(`[ReplicateResultProcessor] ${message}`, level);
|
||||
@@ -115,6 +129,89 @@ export class ReplicateResultProcessor {
|
||||
// If true, the processing queue processor bails the loop.
|
||||
private _suspended: boolean = false;
|
||||
|
||||
// A temporary lifecycle resume cannot make an unknown remote format safe to apply.
|
||||
private _compatibilityBlocked = false;
|
||||
private _assessingDatabase = false;
|
||||
private _physicalDatabase?: PouchDB.Database<EntryDoc>;
|
||||
private _observedFeatures = new Set<string>();
|
||||
private _highestObservedVersion = 0;
|
||||
private _invalidControlObserved = false;
|
||||
|
||||
public get isCompatibilityBlocked() {
|
||||
return this._compatibilityBlocked;
|
||||
}
|
||||
|
||||
private refreshPhysicalDatabase() {
|
||||
const current = this.localDatabase.localDatabase;
|
||||
if (current === this._physicalDatabase) return;
|
||||
if (this._physicalDatabase) {
|
||||
this._compatibilityBlocked = false;
|
||||
this._assessingDatabase = true;
|
||||
this._observedFeatures.clear();
|
||||
this._highestObservedVersion = 0;
|
||||
this._invalidControlObserved = false;
|
||||
this._queuedChanges = [];
|
||||
this._processingChanges = [];
|
||||
this._restoreFromSnapshot = undefined;
|
||||
this.updateProcessingActivity();
|
||||
}
|
||||
this._physicalDatabase = current;
|
||||
}
|
||||
|
||||
private shouldStopApplication(sourceDatabase: PouchDB.Database<EntryDoc>) {
|
||||
return (
|
||||
this._compatibilityBlocked || this._assessingDatabase || sourceDatabase !== this.localDatabase.localDatabase
|
||||
);
|
||||
}
|
||||
|
||||
private blockForIncompatibleVersion(document: unknown, recordObservation = true) {
|
||||
const assessment = assessRemoteFeatureDocument(document);
|
||||
const hadObservedVersion = this._highestObservedVersion > 0;
|
||||
let newFeaturesAdded = false;
|
||||
if (recordObservation) {
|
||||
let changed = false;
|
||||
if (assessment.status === "invalid-control") {
|
||||
changed = !this._invalidControlObserved;
|
||||
this._invalidControlObserved = true;
|
||||
} else {
|
||||
const version = (document as EntryVersionInfo).version;
|
||||
if (version > this._highestObservedVersion) {
|
||||
this._highestObservedVersion = version;
|
||||
changed = true;
|
||||
}
|
||||
if (assessment.status === "supported" || assessment.status === "unknown-features") {
|
||||
for (const feature of assessment.status === "supported"
|
||||
? assessment.usedFeatures
|
||||
: ((document as EntryVersionInfo).used_features ?? [])) {
|
||||
if (this._observedFeatures.has(feature)) continue;
|
||||
this._observedFeatures.add(feature);
|
||||
changed = true;
|
||||
newFeaturesAdded = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (changed) this.triggerTakeSnapshot();
|
||||
}
|
||||
if (assessment.status === "supported" || assessment.status === "older-generation") {
|
||||
// A live writer must recheck the shared Tweak policy after another
|
||||
// client starts using a newly declared representation.
|
||||
if (
|
||||
assessment.status === "supported" &&
|
||||
hadObservedVersion &&
|
||||
newFeaturesAdded &&
|
||||
!this._assessingDatabase
|
||||
) {
|
||||
this.context.requestActiveReplicatorRetirement();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (this._compatibilityBlocked) return;
|
||||
this._compatibilityBlocked = true;
|
||||
this.updateProcessingActivity();
|
||||
this.log(describeRemoteFeatureRejection(assessment), LOG_LEVEL_NOTICE);
|
||||
this.context.requestActiveReplicatorRetirement();
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the application accepts replicated documents being applied.
|
||||
*
|
||||
@@ -135,6 +232,8 @@ export class ReplicateResultProcessor {
|
||||
public get isSuspended() {
|
||||
return (
|
||||
this._suspended ||
|
||||
this._compatibilityBlocked ||
|
||||
this._assessingDatabase ||
|
||||
!this.acceptsResultApplication ||
|
||||
this.context.currentSettings().suspendParseReplicationResult ||
|
||||
this.services.appLifecycle.isSuspended()
|
||||
@@ -145,17 +244,38 @@ export class ReplicateResultProcessor {
|
||||
* Take a snapshot of the current processing state.
|
||||
* This snapshot is stored in the KV database for recovery on restart.
|
||||
*/
|
||||
protected async _takeSnapshot() {
|
||||
const snapshot = {
|
||||
queued: this._queuedChanges.slice(),
|
||||
processing: this._processingChanges.slice(),
|
||||
} satisfies ReplicateResultProcessorState;
|
||||
await this.context.getKeyValueDB().set(KV_KEY_REPLICATION_RESULT_PROCESSOR_SNAPSHOT, snapshot);
|
||||
this.log(
|
||||
`Snapshot taken. Queued: ${snapshot.queued.length}, Processing: ${snapshot.processing.length}`,
|
||||
LOG_LEVEL_DEBUG
|
||||
);
|
||||
this.reportStatus();
|
||||
private _snapshotWriter: Promise<void> = Promise.resolve();
|
||||
|
||||
protected _takeSnapshot(): Promise<void> {
|
||||
// A blocked-batch flush must follow any earlier throttled write, or an
|
||||
// older snapshot could replace the queue after the replication callback.
|
||||
const write = this._snapshotWriter
|
||||
.catch((): void => undefined)
|
||||
.then(async () => {
|
||||
const physicalDatabase = this.localDatabase.localDatabase;
|
||||
const databaseId = await getPhysicalDatabaseId(physicalDatabase);
|
||||
if (physicalDatabase !== this.localDatabase.localDatabase) return;
|
||||
const snapshot = {
|
||||
...(databaseId ? { databaseId } : {}),
|
||||
observedFeatures: [...this._observedFeatures],
|
||||
highestObservedVersion: this._highestObservedVersion,
|
||||
invalidControlObserved: this._invalidControlObserved,
|
||||
queued: this._queuedChanges.slice(),
|
||||
processing: this._processingChanges.slice(),
|
||||
} satisfies ReplicateResultProcessorState;
|
||||
await this.context.getKeyValueDB().set(KV_KEY_REPLICATION_RESULT_PROCESSOR_SNAPSHOT, snapshot);
|
||||
this.log(
|
||||
`Snapshot taken. Queued: ${snapshot.queued.length}, Processing: ${snapshot.processing.length}`,
|
||||
LOG_LEVEL_DEBUG
|
||||
);
|
||||
this.reportStatus();
|
||||
});
|
||||
this._snapshotWriter = write;
|
||||
return write;
|
||||
}
|
||||
|
||||
public async persistBlockedSnapshot(): Promise<void> {
|
||||
if (this._compatibilityBlocked) await this._takeSnapshot();
|
||||
}
|
||||
/**
|
||||
* Trigger taking a snapshot.
|
||||
@@ -172,10 +292,42 @@ export class ReplicateResultProcessor {
|
||||
* Restore from snapshot.
|
||||
*/
|
||||
public async restoreFromSnapshot() {
|
||||
const physicalDatabase = this.localDatabase.localDatabase;
|
||||
// Replication may have checkpointed a version document before its accompanying
|
||||
// file changes reached the Vault. Assess the persisted requirement first.
|
||||
let versionInfo: unknown;
|
||||
try {
|
||||
versionInfo = await this.localDatabase.getRaw(VERSIONING_DOCID);
|
||||
} catch (error) {
|
||||
if (!isNotFoundError(error)) throw error;
|
||||
}
|
||||
if (physicalDatabase !== this.localDatabase.localDatabase) return;
|
||||
const snapshot = await this.context
|
||||
.getKeyValueDB()
|
||||
.get<ReplicateResultProcessorState>(KV_KEY_REPLICATION_RESULT_PROCESSOR_SNAPSHOT);
|
||||
if (snapshot) {
|
||||
if (physicalDatabase !== this.localDatabase.localDatabase) return;
|
||||
const databaseId = await getPhysicalDatabaseId(physicalDatabase);
|
||||
if (snapshot && (!snapshot.databaseId || !databaseId || snapshot.databaseId === databaseId)) {
|
||||
for (const feature of snapshot.observedFeatures ?? []) this._observedFeatures.add(feature);
|
||||
this._highestObservedVersion = Math.max(this._highestObservedVersion, snapshot.highestObservedVersion ?? 0);
|
||||
this._invalidControlObserved ||= snapshot.invalidControlObserved === true;
|
||||
}
|
||||
if (versionInfo !== undefined) this.blockForIncompatibleVersion(versionInfo);
|
||||
if (this._invalidControlObserved) this.blockForIncompatibleVersion(null, false);
|
||||
if (this._highestObservedVersion > 0) {
|
||||
this.blockForIncompatibleVersion(
|
||||
{
|
||||
_id: VERSIONING_DOCID,
|
||||
type: "versioninfo",
|
||||
version: this._highestObservedVersion,
|
||||
...(this._highestObservedVersion >= REMOTE_FEATURE_GENERATION
|
||||
? { used_features: [...this._observedFeatures] }
|
||||
: {}),
|
||||
},
|
||||
false
|
||||
);
|
||||
}
|
||||
if (snapshot && (!snapshot.databaseId || !databaseId || snapshot.databaseId === databaseId)) {
|
||||
// Restoring the snapshot re-runs processing for both queued and processing items.
|
||||
const newQueue = [...snapshot.processing, ...snapshot.queued, ...this._queuedChanges];
|
||||
this._queuedChanges = [];
|
||||
@@ -186,6 +338,9 @@ export class ReplicateResultProcessor {
|
||||
);
|
||||
// await this._takeSnapshot();
|
||||
}
|
||||
this._assessingDatabase = false;
|
||||
this.updateProcessingActivity();
|
||||
this.triggerProcessQueue();
|
||||
}
|
||||
|
||||
private _restoreFromSnapshot: Promise<void> | undefined = undefined;
|
||||
@@ -195,7 +350,9 @@ export class ReplicateResultProcessor {
|
||||
* @returns Promise that resolves when restoration is complete.
|
||||
*/
|
||||
public restoreFromSnapshotOnce() {
|
||||
this.refreshPhysicalDatabase();
|
||||
if (!this._restoreFromSnapshot) {
|
||||
this._assessingDatabase = true;
|
||||
this._restoreFromSnapshot = this.restoreFromSnapshot();
|
||||
}
|
||||
return this._restoreFromSnapshot;
|
||||
@@ -229,7 +386,17 @@ export class ReplicateResultProcessor {
|
||||
* @param changes Changes to enqueue
|
||||
*/
|
||||
|
||||
public enqueueAll(changes: PouchDB.Core.ExistingDocument<EntryDoc>[]) {
|
||||
public enqueueAll(changes: PouchDB.Core.ExistingDocument<EntryDoc>[], sourceDatabase?: PouchDB.Database<EntryDoc>) {
|
||||
if (sourceDatabase && sourceDatabase !== this.localDatabase.localDatabase) return;
|
||||
const previousPhysicalDatabase = this._physicalDatabase;
|
||||
this.refreshPhysicalDatabase();
|
||||
if (previousPhysicalDatabase && previousPhysicalDatabase !== this._physicalDatabase) {
|
||||
fireAndForget(() => this.restoreFromSnapshotOnce());
|
||||
}
|
||||
// Inspect every control document before a note in this batch can start applying.
|
||||
for (const change of changes) {
|
||||
if (change?._id === VERSIONING_DOCID) this.blockForIncompatibleVersion(change);
|
||||
}
|
||||
for (const change of changes) {
|
||||
// Check if the change is not a document change (e.g., chunk, versioninfo, syncinfo), and processed it directly.
|
||||
const isProcessed = this.processIfNonDocumentChange(change);
|
||||
@@ -274,16 +441,8 @@ export class ReplicateResultProcessor {
|
||||
this.log(`Processed chunk: ${shortenId(change._id)}`, LOG_LEVEL_DEBUG);
|
||||
return true;
|
||||
}
|
||||
if (change.type == "versioninfo") {
|
||||
if (change._id === VERSIONING_DOCID) {
|
||||
this.log(`Version info document received: ${change._id}`, LOG_LEVEL_VERBOSE);
|
||||
if (change.version > VER) {
|
||||
// Fence and retire the active publication through its owner.
|
||||
this.context.requestActiveReplicatorRetirement();
|
||||
this.log(
|
||||
`Remote database updated to incompatible version. update your Self-hosted LiveSync plugin.`,
|
||||
LOG_LEVEL_NOTICE
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
@@ -409,11 +568,12 @@ export class ReplicateResultProcessor {
|
||||
// (per-document serialisation caps concurrency).
|
||||
const releaser = await this._semaphore.acquire();
|
||||
releaser();
|
||||
if (this.isSuspended) break;
|
||||
// Dequeue the next change
|
||||
const doc = this._queuedChanges.shift();
|
||||
if (doc) {
|
||||
this._processingChanges.push(doc);
|
||||
void this.parseDocumentChange(doc);
|
||||
void this.parseDocumentChange(doc, this.localDatabase.localDatabase);
|
||||
}
|
||||
// Take snapshot (to be restored on next startup if needed)
|
||||
this.triggerTakeSnapshot();
|
||||
@@ -429,8 +589,12 @@ export class ReplicateResultProcessor {
|
||||
* @param change
|
||||
* @returns
|
||||
*/
|
||||
async parseDocumentChange(change: PouchDB.Core.ExistingDocument<EntryDoc>) {
|
||||
async parseDocumentChange(
|
||||
change: PouchDB.Core.ExistingDocument<EntryDoc>,
|
||||
sourceDatabase: PouchDB.Database<EntryDoc> = this.localDatabase.localDatabase
|
||||
) {
|
||||
try {
|
||||
if (this.shouldStopApplication(sourceDatabase)) return;
|
||||
if (isAnyNote(change)) {
|
||||
const docMtime = change.mtime ?? 0;
|
||||
const maxMTime = this.context.currentSettings().maxMTimeForReflectEvents;
|
||||
@@ -447,6 +611,7 @@ export class ReplicateResultProcessor {
|
||||
}
|
||||
// If the document is a virtual document, process it in the virtual document processor.
|
||||
if (await this.services.replication.processVirtualDocument(change)) return;
|
||||
if (this.shouldStopApplication(sourceDatabase)) return;
|
||||
// If the document is version info, check compatibility and return.
|
||||
if (isAnyNote(change)) {
|
||||
const docPath = this.getPath(change);
|
||||
@@ -454,6 +619,7 @@ export class ReplicateResultProcessor {
|
||||
this.log(`Skipped: ${docPath}`, LOG_LEVEL_VERBOSE);
|
||||
return;
|
||||
}
|
||||
if (this.shouldStopApplication(sourceDatabase)) return;
|
||||
const size = change.size;
|
||||
// Note that this size check depends size that in metadata, not the actual content size.
|
||||
if (this.services.vault.isFileSizeTooLarge(size)) {
|
||||
@@ -463,11 +629,20 @@ export class ReplicateResultProcessor {
|
||||
);
|
||||
return;
|
||||
}
|
||||
return await this.applyToDatabase(change);
|
||||
return await this.applyToDatabase(change, sourceDatabase);
|
||||
}
|
||||
this.log(`Skipped unexpected non-note document: ${change._id}`, LOG_LEVEL_INFO);
|
||||
return;
|
||||
} finally {
|
||||
// An in-flight parse may have started before the control document arrived.
|
||||
// Retain it even if a later asynchronous boundary stopped application.
|
||||
if (
|
||||
this._compatibilityBlocked &&
|
||||
sourceDatabase === this.localDatabase.localDatabase &&
|
||||
!this._queuedChanges.includes(change)
|
||||
) {
|
||||
this._queuedChanges.push(change);
|
||||
}
|
||||
// Remove from processing queue
|
||||
this._processingChanges = this._processingChanges.filter((e) => e !== change);
|
||||
try {
|
||||
@@ -487,12 +662,16 @@ export class ReplicateResultProcessor {
|
||||
}
|
||||
|
||||
// Phase 2: apply the document to database
|
||||
protected applyToDatabase(doc: PouchDB.Core.ExistingDocument<AnyEntry>) {
|
||||
protected applyToDatabase(
|
||||
doc: PouchDB.Core.ExistingDocument<AnyEntry>,
|
||||
sourceDatabase: PouchDB.Database<EntryDoc> = this.localDatabase.localDatabase
|
||||
) {
|
||||
return this.withCounting(async () => {
|
||||
let releaser: Awaited<ReturnType<typeof this._semaphore.acquire>> | undefined = undefined;
|
||||
try {
|
||||
releaser = await this._semaphore.acquire();
|
||||
await this._applyToDatabase(doc);
|
||||
if (this.shouldStopApplication(sourceDatabase)) return;
|
||||
await this._applyToDatabase(doc, sourceDatabase);
|
||||
} catch (e) {
|
||||
this.log(`Error while processing replication result`, LOG_LEVEL_NOTICE);
|
||||
this.logError(e);
|
||||
@@ -506,12 +685,16 @@ export class ReplicateResultProcessor {
|
||||
}
|
||||
// Phase 2.1: process the document and apply to storage
|
||||
// This function is serialized per document to avoid race-condition for the same document.
|
||||
private _applyToDatabase(doc_: PouchDB.Core.ExistingDocument<AnyEntry>) {
|
||||
private _applyToDatabase(
|
||||
doc_: PouchDB.Core.ExistingDocument<AnyEntry>,
|
||||
sourceDatabase: PouchDB.Database<EntryDoc>
|
||||
) {
|
||||
const dbDoc = doc_ as LoadedEntry; // It has no `data`
|
||||
const path = this.getPath(dbDoc);
|
||||
return serialized(`replication-process:${dbDoc._id}`, async () => {
|
||||
const docNote = `${path} (${shortenId(dbDoc._id)}, ${shortenRev(dbDoc._rev)})`;
|
||||
const isRequired = await this.checkIsChangeRequiredForDatabaseProcessing(dbDoc);
|
||||
if (this.shouldStopApplication(sourceDatabase)) return;
|
||||
if (!isRequired) {
|
||||
this.log(`Skipped (Not latest): ${docNote}`, LOG_LEVEL_VERBOSE);
|
||||
return;
|
||||
@@ -525,6 +708,7 @@ export class ReplicateResultProcessor {
|
||||
const doc = isDeleted
|
||||
? { ...dbDoc, data: "" }
|
||||
: await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, false, true);
|
||||
if (this.shouldStopApplication(sourceDatabase)) return;
|
||||
if (!doc) {
|
||||
// Failed to gather content
|
||||
this.log(`Failed to gather content of ${docNote}`, LOG_LEVEL_NOTICE);
|
||||
@@ -535,9 +719,10 @@ export class ReplicateResultProcessor {
|
||||
// Already processed
|
||||
this.log(`Processed by other processor: ${docNote}`, LOG_LEVEL_DEBUG);
|
||||
} else if (this.services.vault.isValidPath(this.getPath(doc))) {
|
||||
if (this.shouldStopApplication(sourceDatabase)) return;
|
||||
// Apply to storage if the path is valid
|
||||
try {
|
||||
const reflected = await this.applyToStorage(doc as MetaEntry);
|
||||
const reflected = await this.applyToStorage(doc as MetaEntry, sourceDatabase);
|
||||
if (!reflected) {
|
||||
this.reportVaultReflectionFailure(doc as MetaEntry);
|
||||
return;
|
||||
@@ -558,9 +743,15 @@ export class ReplicateResultProcessor {
|
||||
* @param entry
|
||||
* @returns
|
||||
*/
|
||||
protected applyToStorage(entry: MetaEntry) {
|
||||
protected applyToStorage(
|
||||
entry: MetaEntry,
|
||||
sourceDatabase: PouchDB.Database<EntryDoc> = this.localDatabase.localDatabase
|
||||
) {
|
||||
return this.withCounting(
|
||||
() => this.services.replication.processSynchroniseResult(entry),
|
||||
() =>
|
||||
this.shouldStopApplication(sourceDatabase)
|
||||
? Promise.resolve(false)
|
||||
: this.services.replication.processSynchroniseResult(entry),
|
||||
this.services.replication.storageApplyingCount
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import { promiseWithResolvers } from "octagonal-wheels/promises";
|
||||
import { reactiveSource } from "octagonal-wheels/dataobject/reactive";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { VER, type EntryDoc, type FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import {
|
||||
VERSIONING_DOCID,
|
||||
type EntryDoc,
|
||||
type FilePathWithPrefix,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
|
||||
import {
|
||||
isValidFilenameInAndroid,
|
||||
isValidFilenameInWidows,
|
||||
@@ -37,6 +42,10 @@ type SetupOptions = {
|
||||
isValidPath?: (path: string) => boolean;
|
||||
processSynchroniseResult?: (entry: unknown) => Promise<boolean>;
|
||||
setSnapshot?: (key: string, value: unknown) => Promise<unknown>;
|
||||
getSnapshot?: (key: string) => Promise<unknown>;
|
||||
localVersionInfo?: unknown;
|
||||
isTargetFile?: (path: string) => Promise<boolean>;
|
||||
databaseId?: string;
|
||||
};
|
||||
|
||||
function setup(options: SetupOptions = {}) {
|
||||
@@ -47,6 +56,9 @@ function setup(options: SetupOptions = {}) {
|
||||
const isReady = vi.fn(() => options.applicationReady ?? true);
|
||||
const isValidPath = vi.fn(options.isValidPath ?? (() => true));
|
||||
const getDBEntryFromMeta = vi.fn(async (entry: object) => ({ ...entry, data: "x" }));
|
||||
const localPhysicalDatabase = {
|
||||
...(options.databaseId ? { id: vi.fn(async () => options.databaseId) } : {}),
|
||||
} as PouchDB.Database<EntryDoc>;
|
||||
const core = {
|
||||
services: {
|
||||
appLifecycle: { isReady, isSuspended: () => false },
|
||||
@@ -62,14 +74,21 @@ function setup(options: SetupOptions = {}) {
|
||||
},
|
||||
replicator: { onCloseActiveReplication, runBoundedLocalApplicationActivity },
|
||||
vault: {
|
||||
isTargetFile: vi.fn(async () => true),
|
||||
isTargetFile: vi.fn(options.isTargetFile ?? (async () => true)),
|
||||
isFileSizeTooLarge: vi.fn(() => false),
|
||||
isValidPath,
|
||||
},
|
||||
},
|
||||
kvDB: { set: setSnapshot },
|
||||
kvDB: { set: setSnapshot, get: vi.fn(options.getSnapshot ?? (async () => undefined)) },
|
||||
localDatabase: {
|
||||
getRaw: vi.fn(async (id: string) => ({ _id: id, _rev: "1-test" })),
|
||||
localDatabase: localPhysicalDatabase,
|
||||
getRaw: vi.fn(async (id: string) => {
|
||||
if (id === VERSIONING_DOCID) {
|
||||
if (options.localVersionInfo === undefined) throw { status: 404 };
|
||||
return options.localVersionInfo;
|
||||
}
|
||||
return { _id: id, _rev: "1-test" };
|
||||
}),
|
||||
getDBEntryFromMeta,
|
||||
},
|
||||
};
|
||||
@@ -88,6 +107,9 @@ function setup(options: SetupOptions = {}) {
|
||||
} as never);
|
||||
return {
|
||||
getDBEntryFromMeta,
|
||||
isTargetFile: core.services.vault.isTargetFile,
|
||||
localPhysicalDatabase,
|
||||
localDatabase: core.localDatabase,
|
||||
isReady,
|
||||
isValidPath,
|
||||
onCloseActiveReplication,
|
||||
@@ -145,9 +167,9 @@ describe("ReplicateResultProcessor", () => {
|
||||
});
|
||||
}
|
||||
expect(processSynchroniseResult).toHaveBeenCalledTimes(11);
|
||||
expect(processSynchroniseResult.mock.calls.some(([entry]) =>
|
||||
(entry as { _id: string })._id === "unrelated-queue"
|
||||
)).toBe(true);
|
||||
expect(
|
||||
processSynchroniseResult.mock.calls.some(([entry]) => (entry as { _id: string })._id === "unrelated-queue")
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("suspends result application while the application is not ready", () => {
|
||||
@@ -199,10 +221,10 @@ describe("ReplicateResultProcessor", () => {
|
||||
it("retires active ownership when a newer remote version is observed", async () => {
|
||||
const { onCloseActiveReplication, processor } = setup();
|
||||
const versionInfo = {
|
||||
_id: "versioninfo",
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "1-test",
|
||||
type: "versioninfo",
|
||||
version: VER + 1,
|
||||
version: REMOTE_FEATURE_GENERATION + 1,
|
||||
} as unknown as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
|
||||
processor.enqueueAll([versionInfo]);
|
||||
@@ -210,6 +232,251 @@ describe("ReplicateResultProcessor", () => {
|
||||
await vi.waitFor(() => expect(onCloseActiveReplication).toHaveBeenCalledOnce());
|
||||
});
|
||||
|
||||
it("continues applying documents after restoring a legacy local version document", async () => {
|
||||
const { onCloseActiveReplication, processor, processSynchroniseResult } = setup({
|
||||
localVersionInfo: {
|
||||
_id: VERSIONING_DOCID,
|
||||
type: "versioninfo",
|
||||
version: 11,
|
||||
},
|
||||
});
|
||||
|
||||
await processor.restoreFromSnapshotOnce();
|
||||
processor.enqueueAll([note("legacy-database-note")]);
|
||||
|
||||
await vi.waitFor(() => expect(processSynchroniseResult).toHaveBeenCalledOnce());
|
||||
expect(processor.isSuspended).toBe(false);
|
||||
expect(onCloseActiveReplication).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["first", "last"] as const)(
|
||||
"holds an entire received batch when an unknown feature is %s",
|
||||
async (position) => {
|
||||
const { onCloseActiveReplication, processor, processSynchroniseResult } = setup();
|
||||
const versionInfo = {
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-unknown",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: ["future-format-v7"],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
const document = note("pending-after-compatibility-change");
|
||||
processor.enqueueAll(position === "first" ? [versionInfo, document] : [document, versionInfo]);
|
||||
|
||||
await vi.waitFor(() => expect(onCloseActiveReplication).toHaveBeenCalledOnce());
|
||||
await Promise.resolve();
|
||||
expect(processor.isSuspended).toBe(true);
|
||||
expect(processSynchroniseResult).not.toHaveBeenCalled();
|
||||
processor.resume();
|
||||
await Promise.resolve();
|
||||
expect(processSynchroniseResult).not.toHaveBeenCalled();
|
||||
}
|
||||
);
|
||||
|
||||
it("continues when a newly received feature is supported", async () => {
|
||||
const { onCloseActiveReplication, processor, processSynchroniseResult } = setup();
|
||||
const versionInfo = {
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-supported",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: [ENCRYPTED_INTERNAL_METADATA_FEATURE],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
|
||||
processor.enqueueAll([versionInfo, note("supported-update")]);
|
||||
|
||||
await vi.waitFor(() => expect(processSynchroniseResult).toHaveBeenCalledOnce());
|
||||
expect(onCloseActiveReplication).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rechecks shared writer settings when a supported feature is added to an active database", async () => {
|
||||
const { onCloseActiveReplication, processor } = setup({
|
||||
localVersionInfo: {
|
||||
_id: VERSIONING_DOCID,
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: [],
|
||||
},
|
||||
});
|
||||
await processor.restoreFromSnapshotOnce();
|
||||
const versionInfo = {
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-supported",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: [ENCRYPTED_INTERNAL_METADATA_FEATURE],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
|
||||
processor.enqueueAll([versionInfo]);
|
||||
processor.enqueueAll([{ ...versionInfo, _rev: "3-same-features" }]);
|
||||
|
||||
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
|
||||
expect(processor.isSuspended).toBe(false);
|
||||
});
|
||||
|
||||
it("retains an in-flight note when a feature change arrives during an asynchronous check", async () => {
|
||||
const targetCheck = promiseWithResolvers<boolean>();
|
||||
const { isTargetFile, onCloseActiveReplication, processor, processSynchroniseResult } = setup({
|
||||
isTargetFile: async () => targetCheck.promise,
|
||||
});
|
||||
const pending = note("in-flight");
|
||||
processor.enqueueAll([pending]);
|
||||
await vi.waitFor(() => expect(isTargetFile).toHaveBeenCalledOnce());
|
||||
|
||||
processor.enqueueAll([
|
||||
{
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-unknown",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: ["future-format-v7"],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>,
|
||||
]);
|
||||
targetCheck.resolve(true);
|
||||
|
||||
await vi.waitFor(() => expect(processor["_processingChanges"]).toHaveLength(0));
|
||||
expect(processor["_queuedChanges"]).toContain(pending);
|
||||
expect(processSynchroniseResult).not.toHaveBeenCalled();
|
||||
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("restores a checkpointed note only after assessing the persisted version document", async () => {
|
||||
const pending = note("checkpointed");
|
||||
const { onCloseActiveReplication, processor, processSynchroniseResult } = setup({
|
||||
localVersionInfo: {
|
||||
_id: VERSIONING_DOCID,
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: ["future-format-v7"],
|
||||
},
|
||||
getSnapshot: async () => ({ processing: [pending], queued: [] }),
|
||||
});
|
||||
|
||||
await processor.restoreFromSnapshotOnce();
|
||||
|
||||
expect(processor.isSuspended).toBe(true);
|
||||
expect(processor["_queuedChanges"]).toContain(pending);
|
||||
expect(processSynchroniseResult).not.toHaveBeenCalled();
|
||||
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("retains an observed unknown feature when the local version list is later shortened", async () => {
|
||||
let snapshot: unknown;
|
||||
const first = setup({
|
||||
databaseId: "same-database",
|
||||
setSnapshot: async (_key, value) => {
|
||||
snapshot = value;
|
||||
},
|
||||
});
|
||||
const pending = note("checkpointed-after-list-change");
|
||||
first.processor.enqueueAll([
|
||||
{
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-unknown",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: ["future-format-v7"],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>,
|
||||
pending,
|
||||
]);
|
||||
await vi.waitFor(() => expect(snapshot).toBeDefined());
|
||||
|
||||
const resumed = setup({
|
||||
databaseId: "same-database",
|
||||
localVersionInfo: {
|
||||
_id: VERSIONING_DOCID,
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: [],
|
||||
},
|
||||
getSnapshot: async () => snapshot,
|
||||
});
|
||||
await resumed.processor.restoreFromSnapshotOnce();
|
||||
|
||||
expect(resumed.processor.isSuspended).toBe(true);
|
||||
expect(resumed.processor["_queuedChanges"]).toContain(pending);
|
||||
expect(resumed.processSynchroniseResult).not.toHaveBeenCalled();
|
||||
expect(resumed.onCloseActiveReplication).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("does not restore pending work belonging to a replaced physical database", async () => {
|
||||
const pending = note("old-database");
|
||||
const { processor, processSynchroniseResult } = setup({
|
||||
databaseId: "replacement-database",
|
||||
getSnapshot: async () => ({ databaseId: "retired-database", processing: [pending], queued: [] }),
|
||||
});
|
||||
|
||||
await processor.restoreFromSnapshotOnce();
|
||||
|
||||
expect(processor["_queuedChanges"]).toHaveLength(0);
|
||||
expect(processSynchroniseResult).not.toHaveBeenCalled();
|
||||
expect(processor.isSuspended).toBe(false);
|
||||
});
|
||||
|
||||
it("reports unknown identifiers and retires only once for repeated notifications", () => {
|
||||
const log = vi.fn((_message: unknown, _level?: number) => undefined);
|
||||
setGlobalLogFunction(log);
|
||||
try {
|
||||
const { onCloseActiveReplication, processor } = setup();
|
||||
const versionInfo = {
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-unknown",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: ["future-format-v7"],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
|
||||
processor.enqueueAll([versionInfo]);
|
||||
processor.enqueueAll([versionInfo]);
|
||||
|
||||
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
|
||||
expect(log).toHaveBeenCalledWith(
|
||||
"[ReplicateResultProcessor] Unknown features are in use: future-format-v7",
|
||||
LOG_LEVEL_NOTICE,
|
||||
undefined
|
||||
);
|
||||
} finally {
|
||||
setGlobalLogFunction(defaultLogger);
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores a late feature notification from a replaced physical database", () => {
|
||||
const { onCloseActiveReplication, processor } = setup();
|
||||
const oldPhysicalDatabase = {} as PouchDB.Database<EntryDoc>;
|
||||
const versionInfo = {
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-unknown",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: ["future-format-v7"],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
|
||||
processor.enqueueAll([versionInfo], oldPhysicalDatabase);
|
||||
|
||||
expect(processor.isSuspended).toBe(false);
|
||||
expect(onCloseActiveReplication).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reassesses compatibility for a replacement local database", async () => {
|
||||
const { localDatabase, onCloseActiveReplication, processor } = setup();
|
||||
processor.enqueueAll([
|
||||
{
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-unknown",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: ["future-format-v7"],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>,
|
||||
]);
|
||||
expect(processor.isSuspended).toBe(true);
|
||||
|
||||
localDatabase.localDatabase = {} as PouchDB.Database<EntryDoc>;
|
||||
await processor.restoreFromSnapshotOnce();
|
||||
|
||||
expect(processor.isSuspended).toBe(false);
|
||||
expect(onCloseActiveReplication).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("scans normal-file metadata without loading chunk documents and requeues it", async () => {
|
||||
const documents = [
|
||||
{ _id: "first", _rev: "1-a", type: "plain", path: "first.md" },
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import {
|
||||
VERSIONING_DOCID,
|
||||
type EntryDoc,
|
||||
type ObsidianLiveSyncSettings,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { assessTweakCompatibility } from "@vrtmrz/livesync-commonlib/settings";
|
||||
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger";
|
||||
import { skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
|
||||
@@ -7,12 +11,27 @@ import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/rep
|
||||
import {
|
||||
CENTRAL_COMPATIBILITY_REJECTION_REASONS,
|
||||
REPLICATION_PROGRESS_PRESENTATIONS,
|
||||
assessRemoteFeatureDocument,
|
||||
describeRemoteFeatureRejection,
|
||||
type ReplicatorInstance,
|
||||
type ReplicationFailureRequest,
|
||||
} from "@vrtmrz/livesync-commonlib/replication";
|
||||
import { $msg } from "@/common/translation";
|
||||
import { usesLegacyIndexedDBAdapter } from "@/common/compatibilitySettings";
|
||||
import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
|
||||
import type PouchDB from "pouchdb-core";
|
||||
|
||||
async function canInterpretCleanupDatabase(db: PouchDB.Database<EntryDoc>): Promise<boolean> {
|
||||
try {
|
||||
const assessment = assessRemoteFeatureDocument(await db.get(VERSIONING_DOCID));
|
||||
if (assessment.status === "supported" || assessment.status === "older-generation") return true;
|
||||
Logger(`Database cleanup cancelled: ${describeRemoteFeatureRejection(assessment)}`, LOG_LEVEL_NOTICE);
|
||||
} catch (error) {
|
||||
Logger("Database cleanup cancelled: feature compatibility could not be checked.", LOG_LEVEL_NOTICE);
|
||||
Logger(error, LOG_LEVEL_INFO);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
type CentralCompatibilityRecoveryServices = Pick<
|
||||
LiveSyncBaseCore["services"],
|
||||
@@ -60,6 +79,7 @@ export function createCentralCompatibilityRecovery(context: CentralCompatibility
|
||||
) {
|
||||
Logger("The remote database has been cleaned.", showProgress ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
|
||||
await skipIfDuplicated("cleanup", async () => {
|
||||
if (!(await canInterpretCleanupDatabase(context.getLocalDatabase().localDatabase))) return;
|
||||
const count = await purgeUnreferencedChunks(context.getLocalDatabase().localDatabase, true);
|
||||
const message = `The remote database has been cleaned up.
|
||||
To synchronize, this device must be also cleaned up. ${count} chunk(s) will be erased from this device.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { VERSIONING_DOCID, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { assessTweakCompatibility } from "@vrtmrz/livesync-commonlib/settings";
|
||||
import { defaultLogger, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, setGlobalLogFunction } from "octagonal-wheels/common/logger";
|
||||
import {
|
||||
@@ -24,6 +24,49 @@ import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/rep
|
||||
import { createCentralCompatibilityRecovery } from "./centralCompatibilityRecovery";
|
||||
|
||||
describe("central compatibility recovery", () => {
|
||||
it("does not count chunks for cleanup when local feature requirements are unknown", async () => {
|
||||
chunkMocks.purgeUnreferencedChunks.mockClear();
|
||||
const confirmWithMessage = vi.fn(async () => "Dismiss");
|
||||
const recovery = createCentralCompatibilityRecovery({
|
||||
confirm: { confirmWithMessage },
|
||||
getLocalDatabase: () => ({
|
||||
localDatabase: {
|
||||
get: vi.fn(async (id: string) => ({
|
||||
_id: id,
|
||||
type: "versioninfo",
|
||||
version: 13,
|
||||
used_features: ["future-format-v7"],
|
||||
})),
|
||||
},
|
||||
}),
|
||||
services: { replicator: {} },
|
||||
} as never);
|
||||
|
||||
await recovery.reconcileCleanedRemote(true, {} as ObsidianLiveSyncSettings, {} as never);
|
||||
|
||||
expect(chunkMocks.purgeUnreferencedChunks).not.toHaveBeenCalled();
|
||||
expect(confirmWithMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows cleanup counting for a legacy local version document", async () => {
|
||||
chunkMocks.purgeUnreferencedChunks.mockClear();
|
||||
const confirmWithMessage = vi.fn(async () => "Dismiss");
|
||||
const recovery = createCentralCompatibilityRecovery({
|
||||
confirm: { confirmWithMessage },
|
||||
getLocalDatabase: () => ({
|
||||
localDatabase: {
|
||||
get: vi.fn(async (id: string) => ({ _id: id, type: "versioninfo", version: 11 })),
|
||||
},
|
||||
}),
|
||||
services: { replicator: {} },
|
||||
} as never);
|
||||
|
||||
await recovery.reconcileCleanedRemote(true, {} as ObsidianLiveSyncSettings, {} as never);
|
||||
|
||||
expect(chunkMocks.purgeUnreferencedChunks).toHaveBeenCalledWith(expect.anything(), true);
|
||||
expect(confirmWithMessage).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("passes the failed attempt's exact tweak assessment to mismatch resolution", async () => {
|
||||
const setting = { customChunkSize: 0 };
|
||||
const preferredTweakValue = { customChunkSize: 60 };
|
||||
@@ -292,7 +335,9 @@ describe("central compatibility recovery", () => {
|
||||
});
|
||||
const runFiniteReplicationActivity = vi.fn(async (task: () => unknown) => await task());
|
||||
const openOneShotReplication = vi.fn(async () => true);
|
||||
const remoteDatabase = { close: vi.fn(async () => undefined) };
|
||||
const remoteDatabase = {
|
||||
close: vi.fn(async () => undefined),
|
||||
};
|
||||
const close = vi.fn(async () => undefined);
|
||||
const activeReplicator = Object.assign(new LiveSyncCouchDBReplicator({} as never), {
|
||||
connectRemoteCouchDBWithSetting: vi.fn(async () => ({ db: remoteDatabase, close })),
|
||||
@@ -303,7 +348,12 @@ describe("central compatibility recovery", () => {
|
||||
const runWithActiveReplicatorContext = vi.fn(async (task: (context: unknown) => unknown) =>
|
||||
task(expectedContext)
|
||||
);
|
||||
const localDatabase = { localDatabase: {}, clearCaches: vi.fn() };
|
||||
const localDatabase = {
|
||||
localDatabase: {
|
||||
get: vi.fn(async () => ({ _id: VERSIONING_DOCID, type: "versioninfo", version: 12 })),
|
||||
},
|
||||
clearCaches: vi.fn(),
|
||||
};
|
||||
const getLocalDatabase = vi.fn(() => localDatabase);
|
||||
const recovery = createCentralCompatibilityRecovery({
|
||||
confirm: { confirmWithMessage: vi.fn(async () => "Cleanup") },
|
||||
@@ -335,7 +385,7 @@ describe("central compatibility recovery", () => {
|
||||
activityFinished.mock.invocationCallOrder[0]
|
||||
);
|
||||
expect(chunkMocks.balanceChunkPurgedDBs).toHaveBeenCalledOnce();
|
||||
expect(getLocalDatabase).toHaveBeenCalledTimes(2);
|
||||
expect(getLocalDatabase).toHaveBeenCalled();
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
expect(close.mock.invocationCallOrder[0]).toBeLessThan(activityFinished.mock.invocationCallOrder[0]);
|
||||
});
|
||||
|
||||
@@ -98,19 +98,21 @@ export function useReplicationFeature<TContext extends ServiceContext, TCommands
|
||||
clearHandlers();
|
||||
return Promise.resolve(true);
|
||||
});
|
||||
services.databaseEvents.onDatabaseInitialised.addHandler(() => {
|
||||
fireAndForget(() => resultProcessor.restoreFromSnapshotOnce());
|
||||
return Promise.resolve(true);
|
||||
services.databaseEvents.onDatabaseInitialised.addHandler(async () => {
|
||||
await resultProcessor.restoreFromSnapshotOnce();
|
||||
return true;
|
||||
});
|
||||
services.appLifecycle.onSettingLoaded.addHandler(initialiseAutomaticReplicationTriggers);
|
||||
services.replication.parseSynchroniseResult.addHandler((documents) => {
|
||||
resultProcessor.enqueueAll(documents);
|
||||
return Promise.resolve(true);
|
||||
services.replication.parseSynchroniseResult.addHandler(async (documents, sourceDatabase) => {
|
||||
resultProcessor.enqueueAll(documents, sourceDatabase);
|
||||
await resultProcessor.persistBlockedSnapshot();
|
||||
return true;
|
||||
});
|
||||
services.replication.onBeforeReplicate.addHandler(onlinePreflight, 10);
|
||||
services.replication.onPrepareCentralRemoteReplication.addHandler(securitySeedPreflight);
|
||||
services.replication.onBeforeReplicate.addHandler(async () => {
|
||||
await resultProcessor.restoreFromSnapshotOnce();
|
||||
if (resultProcessor.isCompatibilityBlocked) return false;
|
||||
unresolvedErrorManager.clearErrors();
|
||||
return true;
|
||||
}, 100);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { VER, type EntryDoc } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { VERSIONING_DOCID, type EntryDoc } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication";
|
||||
import { promiseWithResolvers } from "octagonal-wheels/promises";
|
||||
import { useReplicationFeature } from "./index";
|
||||
|
||||
@@ -19,8 +20,12 @@ type SetupOptions = {
|
||||
};
|
||||
|
||||
function setup(options: SetupOptions = {}) {
|
||||
const defaultLocalDatabase = {
|
||||
localDatabase: {},
|
||||
getRaw: vi.fn(async () => { throw { status: 404 }; }),
|
||||
};
|
||||
const {
|
||||
getLocalDatabase = () => ({}),
|
||||
getLocalDatabase = () => defaultLocalDatabase,
|
||||
keyValueDB = {
|
||||
kvDB: {
|
||||
get: vi.fn(async () => undefined),
|
||||
@@ -39,7 +44,7 @@ function setup(options: SetupOptions = {}) {
|
||||
API: { isMobile: vi.fn(() => false), isOnline: true },
|
||||
appLifecycle: {
|
||||
getUnresolvedMessages: { addHandler: vi.fn() },
|
||||
isReady: true,
|
||||
isReady: vi.fn(() => true),
|
||||
isSuspended: vi.fn(() => false),
|
||||
onSettingLoaded: { addHandler: vi.fn() },
|
||||
},
|
||||
@@ -48,6 +53,7 @@ function setup(options: SetupOptions = {}) {
|
||||
keyValueDB,
|
||||
path: { getPath: vi.fn((entry: { path: string }) => entry.path) },
|
||||
replication: {
|
||||
replicationResultCount: { value: 0 },
|
||||
onBeforeReplicate: {
|
||||
addHandler: vi.fn((handler: BooleanHandler, priority = 0) => {
|
||||
beforeReplicateHandlers.set(priority, handler);
|
||||
@@ -165,10 +171,10 @@ describe("replication serviceFeature composition", () => {
|
||||
const onCloseActiveReplication = vi.fn(() => retirement.promise);
|
||||
const harness = setup({ onCloseActiveReplication });
|
||||
const versionInfo = {
|
||||
_id: "versioninfo",
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "1-test",
|
||||
type: "versioninfo",
|
||||
version: VER + 1,
|
||||
version: REMOTE_FEATURE_GENERATION + 1,
|
||||
} as unknown as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
|
||||
expect(harness.parseHandler).toBeDefined();
|
||||
@@ -177,4 +183,59 @@ describe("replication serviceFeature composition", () => {
|
||||
|
||||
retirement.resolve(true);
|
||||
});
|
||||
|
||||
it("persists a blocked batch before its replication callback settles", async () => {
|
||||
const writeFinished = promiseWithResolvers<void>();
|
||||
const writes: Array<{ queued: Array<{ _id: string }> }> = [];
|
||||
const { parseHandler } = setup({
|
||||
keyValueDB: {
|
||||
kvDB: {
|
||||
get: vi.fn(async () => undefined),
|
||||
set: vi.fn(async (_key, value) => {
|
||||
writes.push(value as { queued: Array<{ _id: string }> });
|
||||
await writeFinished.promise;
|
||||
}),
|
||||
},
|
||||
},
|
||||
});
|
||||
const pending = {
|
||||
_id: "checkpointed-note",
|
||||
_rev: "1-test",
|
||||
type: "plain",
|
||||
path: "checkpointed-note.md",
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
const unknownVersion = {
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-unknown",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: ["future-format-v7"],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
let callbackSettled = false;
|
||||
const callback = parseHandler!([pending, unknownVersion]).then((result) => {
|
||||
callbackSettled = true;
|
||||
return result;
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(writes.length).toBeGreaterThan(0));
|
||||
expect(callbackSettled).toBe(false);
|
||||
writeFinished.resolve();
|
||||
await expect(callback).resolves.toBe(true);
|
||||
expect(writes[writes.length - 1]?.queued.map((entry) => entry._id)).toContain(pending._id);
|
||||
});
|
||||
|
||||
it("refuses another replication after observing an unknown feature locally", async () => {
|
||||
const { beforeReplicateHandlers, parseHandler } = setup();
|
||||
const unknownVersion = {
|
||||
_id: VERSIONING_DOCID,
|
||||
_rev: "2-unknown",
|
||||
type: "versioninfo",
|
||||
version: REMOTE_FEATURE_GENERATION,
|
||||
used_features: ["future-format-v7"],
|
||||
} as PouchDB.Core.ExistingDocument<EntryDoc>;
|
||||
|
||||
await parseHandler!([unknownVersion]);
|
||||
|
||||
await expect(beforeReplicateHandlers.get(100)!(false)).resolves.toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user