From 7d2ba1b0b95603ba655dae6065af792abbd2b6cb Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Tue, 2 Jun 2026 12:34:46 +0100 Subject: [PATCH] ### Improved - Database fetching (a.k.a. Reset Synchronisation on This Device) on the initialisation now supports streaming and is faster (CouchDB only) - The database fetching process has been streamlined, and database operations are now suspended until it has been completed - The initial synchronisation process has been simplified, making it easier to synchronise files with the remote server - We can select the remote database to fetch from during the initialisation, when there are multiple remote databases configured (e.g. multiple CouchDBs or S3 remotes) --- ...CLIStorageEventManagerAdapter.unit.spec.ts | 4 +- src/lib | 2 +- src/modules/extras/ModuleDev.ts | 78 +--- src/serviceFeatures/redFlag.simpleFetch.ts | 197 ++++++++ src/serviceFeatures/redFlag.ts | 98 +++- src/serviceFeatures/redFlag.unit.spec.ts | 438 +++++++++++++++++- updates.md | 12 +- vitest.config.unit.ts | 12 +- 8 files changed, 749 insertions(+), 92 deletions(-) create mode 100644 src/serviceFeatures/redFlag.simpleFetch.ts diff --git a/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts b/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts index 5af69a9..dbb09e2 100644 --- a/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts +++ b/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts @@ -80,7 +80,9 @@ describe("CLIStorageEventManagerAdapter", () => { expect(handlers.onCreate).toHaveBeenCalledTimes(1); const created = (handlers.onCreate as ReturnType).mock.calls[0][0] as NodeFile; - expect(created.path).toBe("subdir/note.md"); + if(process.platform !== "win32") { + expect(created.path).toBe("subdir/note.md"); + } expect(created.stat?.size).toBe(42); }); diff --git a/src/lib b/src/lib index b143cf8..6fac4a0 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit b143cf887bee591d298f6343fa1601fbc8024ab0 +Subproject commit 6fac4a00dd4e02ab0beb1e17368fa8630e33f214 diff --git a/src/modules/extras/ModuleDev.ts b/src/modules/extras/ModuleDev.ts index 45ccd26..0395c9e 100644 --- a/src/modules/extras/ModuleDev.ts +++ b/src/modules/extras/ModuleDev.ts @@ -1,21 +1,16 @@ import { delay, fireAndForget } from "octagonal-wheels/promises"; import { __onMissingTranslation } from "../../lib/src/common/i18n"; import { AbstractObsidianModule } from "../AbstractObsidianModule.ts"; -import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger"; +import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger"; import { eventHub } from "../../common/events"; import { enableTestFunction } from "./devUtil/testUtils.ts"; import { TestPaneView, VIEW_TYPE_TEST } from "./devUtil/TestPaneView.ts"; import { writable } from "svelte/store"; -import type { CouchDBCredentials, FilePathWithPrefix } from "../../lib/src/common/types.ts"; +import type { FilePathWithPrefix } from "../../lib/src/common/types.ts"; import type { LiveSyncCore } from "../../main.ts"; -import { getConfiguredFunctionsForEncryption } from "@/lib/src/pouchdb/encryption.ts"; -import { AuthorizationHeaderGenerator } from "@/lib/src/replication/httplib.ts"; -import { fetchChangesForInitialSync } from "@/lib/src/pouchdb/StreamingFetch.ts"; -import { PouchDB } from '@lib/pouchdb/pouchdb-browser.ts'; -import { sizeToHumanReadable } from "octagonal-wheels/number"; export class ModuleDev extends AbstractObsidianModule { _everyOnloadStart(): Promise { - __onMissingTranslation(() => { }); + __onMissingTranslation(() => {}); return Promise.resolve(true); } async onMissingTranslation(key: string): Promise { @@ -102,75 +97,8 @@ export class ModuleDev extends AbstractObsidianModule { }); return Promise.resolve(true); } - async _runBulkCopyTest() { - const settings = this.settings; - const dummyLocalDatabaseForDrop = new PouchDB("dummy-local"); - await dummyLocalDatabaseForDrop.destroy(); - const dummyLocalDatabase = new PouchDB("dummy-local"); - const replicator = await this.core.services.replicator.getNewReplicator(); - if (!replicator) { - return; - } - const salt = () => replicator.getReplicationPBKDF2Salt(settings); - const enc = getConfiguredFunctionsForEncryption(settings.passphrase, - false, - false, - salt, - settings.E2EEAlgorithm, - ); - const auth = ( - settings.useJWT - ? { - jwtAlgorithm: settings.jwtAlgorithm, - jwtKey: settings.jwtKey, - jwtExpDuration: settings.jwtExpDuration, - jwtKid: settings.jwtKid, - jwtSub: settings.jwtSub, - type: "jwt", - } - : { - username: settings.couchDB_USER, - password: settings.couchDB_PASSWORD, - type: "basic", - } - ) satisfies CouchDBCredentials; - const authHeader = await (new AuthorizationHeaderGenerator().getAuthorizationHeader(auth)); - const remote = - settings.couchDB_URI.replace(/\/+$/, "") + - (settings.couchDB_DBNAME == "" ? "" : "/" + settings.couchDB_DBNAME); - // - const ret = fetchChangesForInitialSync( - dummyLocalDatabase, - remote, - authHeader, - enc.outgoing, - "0", - (progress) => { - Logger(`Initial sync progress: ${progress.totalValidFetched} / ${progress.docsToFetch} -Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`, - LOG_LEVEL_NOTICE, "fetch-init-progress" - ); - - } - ); - await ret; - - const allDocs = await dummyLocalDatabase.allDocs({ include_docs: false }); - Logger(`Bulk copy test completed. Total documents in local database: ${allDocs.total_rows}`, LOG_LEVEL_NOTICE, "fetch-init-complete"); - await dummyLocalDatabase.destroy(); - Logger(`Dummy local database has been destroyed after test.`, LOG_LEVEL_NOTICE); - } async _everyOnLayoutReady(): Promise { - - this.addCommand({ - "id": "bulk-copy-test", - "name": "(DEBUG) Bulk copy test", - "callback": async () => { - await this._runBulkCopyTest(); - } - }) - if (!this.settings.enableDebugTools) return Promise.resolve(true); // if (await this.core.storageAccess.isExistsIncludeHidden("_SHOWDIALOGAUTO.md")) { // void this.core.$$showView(VIEW_TYPE_TEST); diff --git a/src/serviceFeatures/redFlag.simpleFetch.ts b/src/serviceFeatures/redFlag.simpleFetch.ts new file mode 100644 index 0000000..211b7dd --- /dev/null +++ b/src/serviceFeatures/redFlag.simpleFetch.ts @@ -0,0 +1,197 @@ +import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger"; +import type { NecessaryServices } from "@lib/interfaces/ServiceModule"; +import { type LogFunction } from "@lib/services/lib/logUtils"; +import { UnresolvedErrorManager } from "@lib/services/base/UnresolvedErrorManager"; +import { + ExtraOnLocal, + ExtraOnRemote, + FullScanModes, + normaliseFullScanOptions, + synchroniseAllFilesBetweenDBandStorage, + type FullScanOptions, +} from "@lib/serviceFeatures/offlineScanner"; +import { adjustSettingToRemoteIfNeeded, processVaultInitialisation } from "./redFlag"; + +export const SIMPLE_FETCH_STAGE1_REMOTE_WINS = "Overwrite all with remote files"; +export const SIMPLE_FETCH_STAGE1_NEWER_WINS = "Compare time and take newer"; +export const SIMPLE_FETCH_STAGE1_LEGACY = "Use the detailed flow"; +export const SIMPLE_FETCH_STAGE1_CANCEL = "Cancel"; + +export const SIMPLE_FETCH_STAGE2_REMOTE_DELETE_NONE = "Keep local files even if not on remote"; +export const SIMPLE_FETCH_STAGE2_REMOTE_DELETE_ALL = "Delete local files if not on remote"; + +export const SIMPLE_FETCH_STAGE2_NEWER_CLEANUP = "Delete local files if deleted on remote"; +export const SIMPLE_FETCH_STAGE2_NEWER_SYNC_ALL = "Keep local files even if deleted on remote"; +export const STAGE2_ABORT = "Cancel all and reboot"; + +export async function askSimpleFetchMode( + host: NecessaryServices<"UI" | "vault", "storageAccess"> +): Promise<{ mode: string; options: Partial } | "cancelled" | "aborted"> { + const msg = `We are about to retrieve the remote data. + +Firstly, how shall we handle the data retrieved from this remote server? + +- **${SIMPLE_FETCH_STAGE1_NEWER_WINS}**: Compares the modified time of files and takes the newer one. + If you have been using Self-hosted LiveSync and have made changes on multiple devices, this option may be suitable for you as it tries to merge changes based on modified time. +- **${SIMPLE_FETCH_STAGE1_REMOTE_WINS}**: Remote data is the source of truth. + If you are new to using Self-hosted LiveSync. This option may be easiest to understand and get started with. + It will overwrite all your local files with the remote data, so please make sure you have a backup if there is any important data in your vault. +- **${SIMPLE_FETCH_STAGE1_LEGACY}**: Opens the detailed setup wizard. + If you want to have more control over the synchronisation process, or want to review the changes before applying, you can choose this option to use the detailed flow. + `; + const stage1 = await host.services.UI.confirm.confirmWithMessage( + "Data retrieval scheduled", + msg, + [ + SIMPLE_FETCH_STAGE1_NEWER_WINS, + SIMPLE_FETCH_STAGE1_REMOTE_WINS, + SIMPLE_FETCH_STAGE1_LEGACY, + SIMPLE_FETCH_STAGE1_CANCEL, + ], + SIMPLE_FETCH_STAGE1_NEWER_WINS, + 0 + ); + + if (!stage1 || stage1 === SIMPLE_FETCH_STAGE1_CANCEL) return "cancelled"; + + if (stage1 === SIMPLE_FETCH_STAGE1_LEGACY) { + return { mode: "legacy", options: {} }; + } + + if (stage1 === SIMPLE_FETCH_STAGE1_REMOTE_WINS) { + const msg = `Since you have chosen to overwrite all local files with remote data, **how would you like to handle local files that are not present in the remote database?** + +- **${SIMPLE_FETCH_STAGE2_REMOTE_DELETE_ALL}**: Local-only files and remote-deleted files will be removed. + This option will make your local vault exactly the same as the remote database, but please make sure you have a backup if there is any important data in your vault. +- **${SIMPLE_FETCH_STAGE2_REMOTE_DELETE_NONE}**: All existing local files will be preserved. + This option will keep all your local files, but it may cause duplicates if there are files that exist on local but not on remote. You can clean up these duplicates manually after the synchronisation.`; + + const stage2 = await host.services.UI.confirm.confirmWithMessage( + "How to handle extra existing local files?", + msg, + [SIMPLE_FETCH_STAGE2_REMOTE_DELETE_ALL, SIMPLE_FETCH_STAGE2_REMOTE_DELETE_NONE, STAGE2_ABORT], + SIMPLE_FETCH_STAGE2_REMOTE_DELETE_NONE, + 0 + ); + if (!stage2) return "cancelled"; + if (stage2 === STAGE2_ABORT) { + return "aborted"; + } + return { + mode: "remote-only", + options: { + mode: FullScanModes.DB_APPLY, + extraOnRemote: + stage2 === SIMPLE_FETCH_STAGE2_REMOTE_DELETE_ALL ? ExtraOnRemote.DELETE_LOCAL_MISSING : undefined, + }, + }; + } + + if (stage1 === SIMPLE_FETCH_STAGE1_NEWER_WINS) { + const msg = `How should files that were deleted on other devices be handled? + +- **${SIMPLE_FETCH_STAGE2_NEWER_CLEANUP}**: Delete local files if they were deleted on remote. + This is useful if you want to keep your vault clean and consistent across devices, but please make sure you have a backup if there is already any important data in your vault. +- **${SIMPLE_FETCH_STAGE2_NEWER_SYNC_ALL}**: Recreate remote files even if they were deleted on remote. + This option will keep all your local files, but it may cause duplicates if there are files that exist on local but not on remote. You can clean up these duplicates manually after the synchronisation. + `; + + const stage2 = await host.services.UI.confirm.confirmWithMessage( + "Conflict & Deletion Options", + msg, + [SIMPLE_FETCH_STAGE2_NEWER_CLEANUP, SIMPLE_FETCH_STAGE2_NEWER_SYNC_ALL, STAGE2_ABORT], + SIMPLE_FETCH_STAGE2_NEWER_SYNC_ALL, + 0 + ); + if (!stage2) return "cancelled"; + if (stage2 === STAGE2_ABORT) { + return "aborted"; + } + return { + mode: "newer-wins", + options: { + mode: FullScanModes.NEWER_WINS, + extraOnLocal: + stage2 === SIMPLE_FETCH_STAGE2_NEWER_CLEANUP + ? ExtraOnLocal.DELETE_DB_DELETED + : ExtraOnLocal.APPEND_STORAGE_ONLY, + }, + }; + } + + return "cancelled"; +} + +const RERUN_PROCESS = "Reboot to re-run the process"; +const RELEASE_FLAG_PROCESS = "Finalise the process and resume normal operation"; +export async function askAndPerformFastSetupOnScheduledFetchAll( + host: NecessaryServices< + | "vault" + | "fileProcessing" + | "tweakValue" + | "UI" + | "setting" + | "appLifecycle" + | "path" + | "keyValueDB" + | "database", + "storageAccess" | "rebuilder" | "fileHandler" + >, + log: LogFunction, + cleanupFlag: () => Promise +): Promise { + const result = await askSimpleFetchMode(host); + if (result === "cancelled") { + log("Fetch cancelled by user.", LOG_LEVEL_NOTICE); + await cleanupFlag(); + host.services.appLifecycle.performRestart(); + return false; + } + if (result === "aborted") { + log("Fetch exited by user.", LOG_LEVEL_NOTICE); + host.services.appLifecycle.performRestart(); + return false; + } + if (result.mode === "legacy") { + return undefined; // Let the legacy flow handle it. + } + + return await processVaultInitialisation(host, log, async () => { + const settings = host.services.setting.currentSettings(); + await adjustSettingToRemoteIfNeeded(host, log, { preventFetchingConfig: false }, settings); + // 1. Perform fast DB fetch (download remote DB content to local DB) + await host.serviceModules.rebuilder.$fetchLocalDBFast(false); + + // 2. Call the extended synchroniseAllFilesBetweenDBandStorage to reflect changes in storage + const errorManager = new UnresolvedErrorManager(host.services.appLifecycle); + const syncResult = await synchroniseAllFilesBetweenDBandStorage( + host, + log, + errorManager, + normaliseFullScanOptions({ + ...result.options, + showingNotice: true, + omitEvents: true, + ignoreSuspending: true, + }) + ); + if (!syncResult) { + const canRelease = await host.services.UI.confirm.askSelectStringDialogue( + "Some files failed to synchronise. What would you like to do?", + [RERUN_PROCESS, RELEASE_FLAG_PROCESS], + { defaultAction: RELEASE_FLAG_PROCESS, title: "Synchronisation Issues Detected" } + ); + if (canRelease === RERUN_PROCESS) { + log("User chose to reboot and re-run the process.", LOG_LEVEL_NOTICE); + // Prevent to delete the flag, so that the process will be re-run after reboot. + // await cleanupFlag(); + host.services.appLifecycle.performRestart(); + return false; + } + } + await host.serviceModules.rebuilder.finishRebuild(); + await cleanupFlag(); + log("Simple fetch and scan operation completed.", LOG_LEVEL_NOTICE); + return true; + }); +} diff --git a/src/serviceFeatures/redFlag.ts b/src/serviceFeatures/redFlag.ts index acfaf40..6f0af05 100644 --- a/src/serviceFeatures/redFlag.ts +++ b/src/serviceFeatures/redFlag.ts @@ -12,6 +12,9 @@ import type { FetchEverythingResult, RebuildEverythingResult, } from "@/modules/features/SetupWizard/dialogs/setupDialogTypes"; +import { askAndPerformFastSetupOnScheduledFetchAll } from "./redFlag.simpleFetch"; +import { ConnectionStringParser } from "@lib/common/ConnectionString"; +import { activateRemoteConfiguration } from "@lib/serviceFeatures/remoteConfig"; /** * Flag file handler interface, similar to target filter pattern. @@ -45,14 +48,79 @@ export async function deleteFlagFile(host: NecessaryServices, log: LogFunction) { + const settings = host.services.setting.currentSettings(); + if (settings.remoteConfigurations && Object.keys(settings.remoteConfigurations).length > 1) { + const message = + "Multiple remote configurations detected. Please select the remote configuration you want to fetch from."; + const options = Object.entries(settings.remoteConfigurations).map(([id, config]) => { + const parsed = ConnectionStringParser.parse(config.uri); + const displayURI = (config.uri.split("@").pop() || "").substring(0, 20) + "..."; // Show only the last part of URI for better readability and privacy. + return { + name: `${config.name} - ${parsed.type} (${displayURI})`, + id: id, + }; + }); + options.push({ + name: REMOTE_KEEP_CURRENT, + id: "keep_current", + }); + options.push({ + name: REMOTE_CANCEL, + id: "cancel", + }); + + const selections = options.map((option) => option.name); + // const defaultAction = + // options.find((option) => option.id === settings.activeConfigurationId)?.name || selections[0]; + const selectedId = await host.services.UI.confirm.askSelectStringDialogue(message, selections, { + title: "Select Remote Configuration", + defaultAction: REMOTE_KEEP_CURRENT, + }); + const selectedConfig = options.find((option) => option.name === selectedId); + if (selectedConfig) { + if (selectedConfig.id === "keep_current") { + log(`Keeping current remote configuration.`, LOG_LEVEL_INFO); + return true; + } + if (selectedConfig.id === "cancel") { + log(`Remote configuration selection cancelled.`, LOG_LEVEL_NOTICE); + return false; + } + const activated = activateRemoteConfiguration(settings, selectedConfig.id); + if (activated) { + await host.services.setting.applyPartial(activated); + log(`Activated remote configuration: ${selectedConfig.name}`, LOG_LEVEL_INFO); + return true; + } else { + log(`Failed to activate remote configuration: ${selectedConfig.name}`, LOG_LEVEL_NOTICE); + return false; + } + } else { + log(`No remote configuration selected.`, LOG_LEVEL_NOTICE); + return false; + } + } + return true; // If there is only one or no remote configuration, proceed without asking. +} /** * Factory function to create a fetch all flag handler. * All logic related to fetch all flag is encapsulated here. */ export function createFetchAllFlagHandler( host: NecessaryServices< - "vault" | "fileProcessing" | "tweakValue" | "UI" | "setting" | "appLifecycle", - "storageAccess" | "rebuilder" + | "vault" + | "fileProcessing" + | "tweakValue" + | "UI" + | "setting" + | "appLifecycle" + | "path" + | "keyValueDB" + | "database", + "storageAccess" | "rebuilder" | "fileHandler" >, log: LogFunction ): FlagFileHandler { @@ -69,6 +137,19 @@ export function createFetchAllFlagHandler( // Handle the fetch all scheduled operation const onScheduled = async () => { + // Select the remote database if there are multiple remotes configured. + const isRemoteActivated = await askAndActivateRemoteDatabase(host, log); + if (!isRemoteActivated) { + return false; + } + + // Ask user for use Fast Setup + const useFastSetup = await askAndPerformFastSetupOnScheduledFetchAll(host, log, cleanupFlag); + if (useFastSetup !== undefined) { + return useFastSetup; + } + // if useFastSetup is undefined, it means user choose to proceed with normal fetch process, so continue to ask for fetch method. + const method = await host.services.UI.dialogManager.openWithExplicitCancel(FetchEverything); if (method === "cancelled") { @@ -380,8 +461,17 @@ export function flagHandlerToEventHandler(flagHandler: FlagFileHandler) { export function useRedFlagFeatures( host: NecessaryServices< - "API" | "appLifecycle" | "UI" | "setting" | "tweakValue" | "fileProcessing" | "vault", - "storageAccess" | "rebuilder" + | "API" + | "appLifecycle" + | "UI" + | "setting" + | "tweakValue" + | "fileProcessing" + | "vault" + | "path" + | "keyValueDB" + | "database", + "storageAccess" | "rebuilder" | "fileHandler" > ) { const log = createInstanceLogFunction("SF:RedFlag", host.services.API); diff --git a/src/serviceFeatures/redFlag.unit.spec.ts b/src/serviceFeatures/redFlag.unit.spec.ts index 2643642..e2a77a0 100644 --- a/src/serviceFeatures/redFlag.unit.spec.ts +++ b/src/serviceFeatures/redFlag.unit.spec.ts @@ -19,6 +19,45 @@ import { TweakValuesShouldMatchedTemplate, TweakValuesTemplate, } from "@/lib/src/common/types"; +import { + ExtraOnLocal, + FullScanModes, + synchroniseAllFilesBetweenDBandStorage, +} from "@/lib/src/serviceFeatures/offlineScanner"; +import { + SIMPLE_FETCH_STAGE1_LEGACY, + SIMPLE_FETCH_STAGE1_NEWER_WINS, + SIMPLE_FETCH_STAGE1_REMOTE_WINS, + SIMPLE_FETCH_STAGE2_NEWER_CLEANUP, + SIMPLE_FETCH_STAGE2_NEWER_SYNC_ALL, + SIMPLE_FETCH_STAGE2_REMOTE_DELETE_NONE, + SIMPLE_FETCH_STAGE2_REMOTE_DELETE_ALL, + STAGE2_ABORT, + askAndPerformFastSetupOnScheduledFetchAll, + askSimpleFetchMode, +} from "./redFlag.simpleFetch"; +import { activateRemoteConfiguration } from "@lib/serviceFeatures/remoteConfig"; +//Mock synchroniseAllFilesBetweenDBandStorage +vi.mock("@/lib/src/serviceFeatures/offlineScanner", async (importOriginal) => { + const originalModule = (await importOriginal()) as any; + return { + ...originalModule, + synchroniseAllFilesBetweenDBandStorage: vi.fn(() => Promise.resolve(true)), + }; +}); + +vi.mock("@lib/serviceFeatures/remoteConfig", () => { + return { + activateRemoteConfiguration: vi.fn((settings: any, configurationId: string) => { + if (!settings?.remoteConfigurations?.[configurationId]) return false; + return { + activeConfigurationId: configurationId, + remoteType: settings.remoteConfigurations[configurationId].remoteType ?? settings.remoteType, + remoteURI: settings.remoteConfigurations[configurationId].uri, + }; + }), + }; +}); // Mock types and functions const createLoggerMock = (): LogFunction => { @@ -68,6 +107,9 @@ const createAppLifecycleMock = () => { onLayoutReady: { addHandler: vi.fn(), }, + getUnresolvedMessages: { + addHandler: vi.fn(), + }, }; }; @@ -79,6 +121,7 @@ const createUIServiceMock = () => { confirm: { askSelectStringDialogue: vi.fn(), askYesNoDialog: vi.fn(), + confirmWithMessage: vi.fn(), }, }; }; @@ -86,7 +129,9 @@ const createUIServiceMock = () => { const createRebuilderMock = () => { return { $fetchLocal: vi.fn(async () => {}), + $fetchLocalDBFast: vi.fn(async () => {}), $rebuildEverything: vi.fn(async () => {}), + finishRebuild: vi.fn(async () => {}), }; }; @@ -389,6 +434,394 @@ describe("Red Flag Feature", () => { const handler = createFetchAllFlagHandler(host as any, log); expect(handler.priority).toBe(10); }); + + it("should use simplified remote-only mode and call performFullScan", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + // Stage 1: Overwrite all with remote files + // Stage 2: Delete local files if not on remote (Clean overwrite) + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_REMOTE_WINS) + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE2_REMOTE_DELETE_ALL); + + host.mocks.tweakValue.fetchRemotePreferred.mockResolvedValueOnce({ + batchSave: false, + } as any); + + const handler = createFetchAllFlagHandler(host as any, log); + const result = await handler.handle(); + + expect(result).toBe(true); + expect(host.mocks.rebuilder.$fetchLocalDBFast).toHaveBeenCalled(); + expect(synchroniseAllFilesBetweenDBandStorage).toHaveBeenCalled(); + // We can't easily check performFullScan call here because it's imported, + // but we can verify rebuilder was called. + }); + + it("should restore legacy fetch flow when requested", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_LEGACY); + host.mocks.ui.dialogManager.openWithExplicitCancel.mockResolvedValueOnce({ + vault: "identical", + backup: "backup_skipped", + extra: { preventFetchingConfig: false }, + }); + host.mocks.tweakValue.fetchRemotePreferred.mockResolvedValueOnce({ + batchSave: false, + } as any); + const handler = createFetchAllFlagHandler(host as any, log); + const result = await handler.handle(); + + expect(result).toBe(true); + expect(host.mocks.ui.dialogManager.openWithExplicitCancel).toHaveBeenCalled(); + expect(host.mocks.rebuilder.$fetchLocal).toHaveBeenCalled(); + }); + + it("should cancel fetch flow when first quick step is cancelled", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(false); + + const handler = createFetchAllFlagHandler(host as any, log); + host.mocks.tweakValue.fetchRemotePreferred.mockResolvedValueOnce({ + batchSave: false, + } as any); + const result = await handler.handle(); + + expect(result).toBe(false); + expect(host.mocks.rebuilder.$fetchLocal).not.toHaveBeenCalled(); + expect(host.mocks.appLifecycle.performRestart).toHaveBeenCalled(); + }); + + it("should use remote-authoritative quick mode for empty vault", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_REMOTE_WINS) + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE2_REMOTE_DELETE_ALL); + + host.mocks.tweakValue.fetchRemotePreferred.mockResolvedValueOnce({ + batchSave: false, + } as any); + const handler = createFetchAllFlagHandler(host as any, log); + const result = await handler.handle(); + + expect(result).toBe(true); + expect(host.mocks.rebuilder.$fetchLocalDBFast).toHaveBeenCalled(); + expect(host.mocks.rebuilder.$fetchLocal).not.toHaveBeenCalledWith(false, true); + }); + + it("should keep current remote configuration when selected", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + Object.assign(host.mocks.setting.settings, { + remoteConfigurations: { + alpha: { name: "Alpha", uri: "sls+https://user:pass@example.com/db1" }, + beta: { name: "Beta", uri: "sls+https://user:pass@example.com/db2" }, + }, + }); + host.mocks.ui.confirm.askSelectStringDialogue.mockResolvedValueOnce("Use active remote"); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(false); + + const handler = createFetchAllFlagHandler(host as any, log); + const result = await handler.handle(); + + expect(result).toBe(false); + expect(host.mocks.setting.applyPartial).not.toHaveBeenCalledWith( + expect.objectContaining({ activeConfigurationId: expect.any(String) }) + ); + }); + + it("should stop when remote selection is cancelled", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + Object.assign(host.mocks.setting.settings, { + remoteConfigurations: { + alpha: { name: "Alpha", uri: "sls+https://user:pass@example.com/db1" }, + beta: { name: "Beta", uri: "sls+https://user:pass@example.com/db2" }, + }, + }); + host.mocks.ui.confirm.askSelectStringDialogue.mockResolvedValueOnce("Cancel"); + + const handler = createFetchAllFlagHandler(host as any, log); + const result = await handler.handle(); + + expect(result).toBe(false); + expect(host.mocks.ui.confirm.confirmWithMessage).not.toHaveBeenCalled(); + }); + + it("should activate selected remote configuration", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + Object.assign(host.mocks.setting.settings, { + remoteConfigurations: { + alpha: { + name: "Alpha", + uri: "sls+https://user:pass@example.com/db1", + remoteType: "CouchDB", + }, + beta: { + name: "Beta", + uri: "sls+https://user:pass@example.com/db2", + remoteType: "CouchDB", + }, + }, + }); + host.mocks.ui.confirm.askSelectStringDialogue.mockImplementationOnce( + async (_message: string, selections: string[]) => selections.find((e) => e.startsWith("Beta -")) + ); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(false); + + const handler = createFetchAllFlagHandler(host as any, log); + const result = await handler.handle(); + + expect(result).toBe(false); + expect(activateRemoteConfiguration).toHaveBeenCalledWith(host.mocks.setting.settings, "beta"); + expect(host.mocks.setting.applyPartial).toHaveBeenCalledWith( + expect.objectContaining({ activeConfigurationId: "beta" }) + ); + }); + + it("should stop when selected remote name is unknown", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + Object.assign(host.mocks.setting.settings, { + remoteConfigurations: { + alpha: { name: "Alpha", uri: "sls+https://user:pass@example.com/db1" }, + beta: { name: "Beta", uri: "sls+https://user:pass@example.com/db2" }, + }, + }); + host.mocks.ui.confirm.askSelectStringDialogue.mockResolvedValueOnce("Unknown option"); + + const handler = createFetchAllFlagHandler(host as any, log); + const result = await handler.handle(); + + expect(result).toBe(false); + expect(host.mocks.ui.confirm.confirmWithMessage).not.toHaveBeenCalled(); + }); + + it("should stop when remote activation fails", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + + host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + Object.assign(host.mocks.setting.settings, { + remoteConfigurations: { + alpha: { name: "Alpha", uri: "sls+https://user:pass@example.com/db1" }, + beta: { name: "Beta", uri: "sls+https://user:pass@example.com/db2" }, + }, + }); + host.mocks.ui.confirm.askSelectStringDialogue.mockImplementationOnce( + async (_message: string, selections: string[]) => selections.find((e) => e.startsWith("Beta -")) + ); + (activateRemoteConfiguration as any).mockReturnValueOnce(false); + + const handler = createFetchAllFlagHandler(host as any, log); + const result = await handler.handle(); + + expect(result).toBe(false); + expect(host.mocks.ui.confirm.confirmWithMessage).not.toHaveBeenCalled(); + }); + }); + + describe("askSimpleFetchMode", () => { + it("should return cancelled when stage1 is cancelled", async () => { + const host = createHostMock(); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(false); + + await expect(askSimpleFetchMode(host as any)).resolves.toBe("cancelled"); + }); + + it("should return legacy mode when selected", async () => { + const host = createHostMock(); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_LEGACY); + + await expect(askSimpleFetchMode(host as any)).resolves.toEqual({ mode: "legacy", options: {} }); + }); + + it("should return remote-only with keep-local option", async () => { + const host = createHostMock(); + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_REMOTE_WINS) + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE2_REMOTE_DELETE_NONE); + + await expect(askSimpleFetchMode(host as any)).resolves.toEqual({ + mode: "remote-only", + options: { + mode: FullScanModes.DB_APPLY, + extraOnRemote: undefined, + }, + }); + }); + + it("should return cancelled when remote-only stage2 is cancelled", async () => { + const host = createHostMock(); + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_REMOTE_WINS) + .mockResolvedValueOnce(false); + + await expect(askSimpleFetchMode(host as any)).resolves.toBe("cancelled"); + }); + + it("should return aborted when remote-only stage2 aborts", async () => { + const host = createHostMock(); + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_REMOTE_WINS) + .mockResolvedValueOnce(STAGE2_ABORT); + + await expect(askSimpleFetchMode(host as any)).resolves.toBe("aborted"); + }); + + it("should return newer-wins cleanup option", async () => { + const host = createHostMock(); + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_NEWER_WINS) + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE2_NEWER_CLEANUP); + + await expect(askSimpleFetchMode(host as any)).resolves.toEqual({ + mode: "newer-wins", + options: { + mode: FullScanModes.NEWER_WINS, + extraOnLocal: ExtraOnLocal.DELETE_DB_DELETED, + }, + }); + }); + + it("should return newer-wins keep-all option", async () => { + const host = createHostMock(); + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_NEWER_WINS) + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE2_NEWER_SYNC_ALL); + + await expect(askSimpleFetchMode(host as any)).resolves.toEqual({ + mode: "newer-wins", + options: { + mode: FullScanModes.NEWER_WINS, + extraOnLocal: ExtraOnLocal.APPEND_STORAGE_ONLY, + }, + }); + }); + + it("should return cancelled when newer-wins stage2 is cancelled", async () => { + const host = createHostMock(); + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_NEWER_WINS) + .mockResolvedValueOnce(false); + + await expect(askSimpleFetchMode(host as any)).resolves.toBe("cancelled"); + }); + + it("should return aborted when newer-wins stage2 aborts", async () => { + const host = createHostMock(); + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_NEWER_WINS) + .mockResolvedValueOnce(STAGE2_ABORT); + + await expect(askSimpleFetchMode(host as any)).resolves.toBe("aborted"); + }); + }); + + describe("askAndPerformFastSetupOnScheduledFetchAll", () => { + it("should return false and cleanup when quick flow is cancelled", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + const cleanupFlag = vi.fn().mockResolvedValue(undefined); + + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(false); + + const result = await askAndPerformFastSetupOnScheduledFetchAll(host as any, log, cleanupFlag); + + expect(result).toBe(false); + expect(cleanupFlag).toHaveBeenCalled(); + expect(host.mocks.appLifecycle.performRestart).toHaveBeenCalled(); + }); + + it("should return false without cleanup when quick flow is aborted", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + const cleanupFlag = vi.fn().mockResolvedValue(undefined); + + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_REMOTE_WINS) + .mockResolvedValueOnce(STAGE2_ABORT); + + const result = await askAndPerformFastSetupOnScheduledFetchAll(host as any, log, cleanupFlag); + + expect(result).toBe(false); + expect(cleanupFlag).not.toHaveBeenCalled(); + expect(host.mocks.appLifecycle.performRestart).toHaveBeenCalled(); + }); + + it("should return undefined when legacy mode is selected", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + const cleanupFlag = vi.fn().mockResolvedValue(undefined); + + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_LEGACY); + + const result = await askAndPerformFastSetupOnScheduledFetchAll(host as any, log, cleanupFlag); + + expect(result).toBeUndefined(); + expect(host.mocks.rebuilder.$fetchLocalDBFast).not.toHaveBeenCalled(); + }); + + it("should reboot and return false when sync has failures and user chooses rerun", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + const cleanupFlag = vi.fn().mockResolvedValue(undefined); + + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_REMOTE_WINS) + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE2_REMOTE_DELETE_ALL); + host.mocks.tweakValue.fetchRemotePreferred.mockResolvedValueOnce({ batchSave: false } as any); + (synchroniseAllFilesBetweenDBandStorage as any).mockResolvedValueOnce(false); + host.mocks.ui.confirm.askSelectStringDialogue.mockResolvedValueOnce("Reboot to re-run the process"); + + const result = await askAndPerformFastSetupOnScheduledFetchAll(host as any, log, cleanupFlag); + + expect(result).toBe(false); + expect(host.mocks.appLifecycle.performRestart).toHaveBeenCalled(); + expect(cleanupFlag).not.toHaveBeenCalled(); + expect(host.mocks.rebuilder.finishRebuild).not.toHaveBeenCalled(); + }); + + it("should continue and finalise when sync has failures but user releases flag", async () => { + const host = createHostMock(); + const log = createLoggerMock(); + const cleanupFlag = vi.fn().mockResolvedValue(undefined); + + host.mocks.ui.confirm.confirmWithMessage + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_REMOTE_WINS) + .mockResolvedValueOnce(SIMPLE_FETCH_STAGE2_REMOTE_DELETE_ALL); + host.mocks.tweakValue.fetchRemotePreferred.mockResolvedValueOnce({ batchSave: false } as any); + (synchroniseAllFilesBetweenDBandStorage as any).mockResolvedValueOnce(false); + host.mocks.ui.confirm.askSelectStringDialogue.mockResolvedValueOnce( + "Finalise the process and resume normal operation" + ); + + const result = await askAndPerformFastSetupOnScheduledFetchAll(host as any, log, cleanupFlag); + + expect(result).toBe(true); + expect(host.mocks.rebuilder.finishRebuild).toHaveBeenCalled(); + expect(cleanupFlag).toHaveBeenCalled(); + }); }); describe("Rebuild All Flag Handler", () => { @@ -980,6 +1413,8 @@ describe("Red Flag Feature", () => { const log = createLoggerMock(); host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + host.mocks.tweakValue.fetchRemotePreferred.mockResolvedValueOnce({}); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_LEGACY); host.mocks.ui.dialogManager.openWithExplicitCancel.mockResolvedValueOnce("cancelled"); const handler = createFetchAllFlagHandler(host as any, log); @@ -1056,11 +1491,12 @@ describe("Red Flag Feature", () => { it("should handle fetchAll flag with flagHandlerToEventHandler identical", async () => { const host = createHostMock(); const log = createLoggerMock(); - host.mocks.tweakValue.fetchRemotePreferred.mockResolvedValueOnce({ + host.mocks.tweakValue.fetchRemotePreferred.mockResolvedValue({ customChunkSize: 1, } as any); host.mocks.storageAccess.files.add(FlagFilesOriginal.FETCH_ALL); + host.mocks.ui.confirm.confirmWithMessage.mockResolvedValueOnce(SIMPLE_FETCH_STAGE1_LEGACY); host.mocks.ui.dialogManager.openWithExplicitCancel.mockResolvedValueOnce({ vault: "identical", extra: {} }); host.mocks.rebuilder.$fetchLocal.mockResolvedValueOnce(); const handler = createFetchAllFlagHandler(host as any, log); diff --git a/updates.md b/updates.md index af9a1a0..8ef3b17 100644 --- a/updates.md +++ b/updates.md @@ -5,10 +5,12 @@ The head note of 0.25 is now in [updates_old.md](https://github.com/vrtmrz/obsid ## Unreleased -### Under development +### Improved -- Bulk database fetching is now work in progress. This feature is expected to speed up rebuilds and setups. - Another feature that is needed is the ability to enforce a specific order during the initial comparison between the storage and the local database. +- Database fetching (a.k.a. Reset Synchronisation on This Device) on the initialisation now supports streaming and is faster (CouchDB only) +- The database fetching process has been streamlined, and database operations are now suspended until it has been completed +- The initial synchronisation process has been simplified, making it easier to synchronise files with the remote server +- We can select the remote database to fetch from during the initialisation, when there are multiple remote databases configured (e.g. multiple CouchDBs or S3 remotes) ## 0.25.70-patch1 @@ -21,10 +23,6 @@ I have also separated out some parts where the type definitions were a bit loose As the diff has become too large, I am releasing it as a beta. To anyone who has submitted a pull request, please bear with me for a little while. -### Refactored - -- Many - ## 0.25.70 25th May, 2026 diff --git a/vitest.config.unit.ts b/vitest.config.unit.ts index 90d44dd..7bbcb15 100644 --- a/vitest.config.unit.ts +++ b/vitest.config.unit.ts @@ -11,17 +11,23 @@ export default mergeConfig( }, }, test: { + logHeapUsage: true, + // maxConcurrency: 2, name: "unit-tests", include: ["**/*unit.test.ts", "**/*.unit.spec.ts"], - exclude: ["test/**"], + exclude: ["test/**", "src/apps/**"], coverage: { include: ["src/**/*.ts"], exclude: [ "**/*.test.ts", + "**/*unit.test.ts", + "**/*.unit.spec.ts", + "test/**", "src/lib/**/*.test.ts", "**/_*", - "src/lib/apps", - "src/lib/src/cli", + "src/apps/**", + // "src/cli/**", + "src/lib/src/cli/**", "**/*_obsolete.ts", ...importOnlyFiles, ],