From 5a280c791974083956072b5d68a9db5718ffa0d8 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Mon, 1 Jun 2026 10:41:48 +0100 Subject: [PATCH] (feat): Bulk database fetching is now work in progress. This feature is expected to speed up rebuilds and setups. (WIP) --- src/lib | 2 +- src/modules/extras/ModuleDev.ts | 80 +++++++++++++++++++++++++++++++-- updates.md | 7 +++ 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/lib b/src/lib index 6f97753..b143cf8 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 6f977537f42bafc41f231a61379b613b22f1773b +Subproject commit b143cf887bee591d298f6343fa1601fbc8024ab0 diff --git a/src/modules/extras/ModuleDev.ts b/src/modules/extras/ModuleDev.ts index ab48c31..45ccd26 100644 --- a/src/modules/extras/ModuleDev.ts +++ b/src/modules/extras/ModuleDev.ts @@ -1,17 +1,21 @@ import { delay, fireAndForget } from "octagonal-wheels/promises"; import { __onMissingTranslation } from "../../lib/src/common/i18n"; import { AbstractObsidianModule } from "../AbstractObsidianModule.ts"; -import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger"; +import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger"; import { eventHub } from "../../common/events"; import { enableTestFunction } from "./devUtil/testUtils.ts"; import { TestPaneView, VIEW_TYPE_TEST } from "./devUtil/TestPaneView.ts"; import { writable } from "svelte/store"; -import type { FilePathWithPrefix } from "../../lib/src/common/types.ts"; +import type { CouchDBCredentials, FilePathWithPrefix } from "../../lib/src/common/types.ts"; import type { LiveSyncCore } from "../../main.ts"; - +import { getConfiguredFunctionsForEncryption } from "@/lib/src/pouchdb/encryption.ts"; +import { AuthorizationHeaderGenerator } from "@/lib/src/replication/httplib.ts"; +import { fetchChangesForInitialSync } from "@/lib/src/pouchdb/StreamingFetch.ts"; +import { PouchDB } from '@lib/pouchdb/pouchdb-browser.ts'; +import { sizeToHumanReadable } from "octagonal-wheels/number"; export class ModuleDev extends AbstractObsidianModule { _everyOnloadStart(): Promise { - __onMissingTranslation(() => {}); + __onMissingTranslation(() => { }); return Promise.resolve(true); } async onMissingTranslation(key: string): Promise { @@ -98,7 +102,75 @@ export class ModuleDev extends AbstractObsidianModule { }); return Promise.resolve(true); } + async _runBulkCopyTest() { + const settings = this.settings; + const dummyLocalDatabaseForDrop = new PouchDB("dummy-local"); + await dummyLocalDatabaseForDrop.destroy(); + const dummyLocalDatabase = new PouchDB("dummy-local"); + const replicator = await this.core.services.replicator.getNewReplicator(); + if (!replicator) { + return; + } + const salt = () => replicator.getReplicationPBKDF2Salt(settings); + const enc = getConfiguredFunctionsForEncryption(settings.passphrase, + false, + false, + salt, + settings.E2EEAlgorithm, + ); + + const auth = ( + settings.useJWT + ? { + jwtAlgorithm: settings.jwtAlgorithm, + jwtKey: settings.jwtKey, + jwtExpDuration: settings.jwtExpDuration, + jwtKid: settings.jwtKid, + jwtSub: settings.jwtSub, + type: "jwt", + } + : { + username: settings.couchDB_USER, + password: settings.couchDB_PASSWORD, + type: "basic", + } + ) satisfies CouchDBCredentials; + const authHeader = await (new AuthorizationHeaderGenerator().getAuthorizationHeader(auth)); + const remote = + settings.couchDB_URI.replace(/\/+$/, "") + + (settings.couchDB_DBNAME == "" ? "" : "/" + settings.couchDB_DBNAME); + // + const ret = fetchChangesForInitialSync( + dummyLocalDatabase, + remote, + authHeader, + enc.outgoing, + "0", + (progress) => { + Logger(`Initial sync progress: ${progress.totalValidFetched} / ${progress.docsToFetch} +Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`, + LOG_LEVEL_NOTICE, "fetch-init-progress" + ); + + } + ); + await ret; + + const allDocs = await dummyLocalDatabase.allDocs({ include_docs: false }); + Logger(`Bulk copy test completed. Total documents in local database: ${allDocs.total_rows}`, LOG_LEVEL_NOTICE, "fetch-init-complete"); + await dummyLocalDatabase.destroy(); + Logger(`Dummy local database has been destroyed after test.`, LOG_LEVEL_NOTICE); + } async _everyOnLayoutReady(): Promise { + + this.addCommand({ + "id": "bulk-copy-test", + "name": "(DEBUG) Bulk copy test", + "callback": async () => { + await this._runBulkCopyTest(); + } + }) + if (!this.settings.enableDebugTools) return Promise.resolve(true); // if (await this.core.storageAccess.isExistsIncludeHidden("_SHOWDIALOGAUTO.md")) { // void this.core.$$showView(VIEW_TYPE_TEST); diff --git a/updates.md b/updates.md index a81c82a..cd54957 100644 --- a/updates.md +++ b/updates.md @@ -3,6 +3,13 @@ Since 19th July, 2025 (beta1 in 0.25.0-beta1, 13th July, 2025) The head note of 0.25 is now in [updates_old.md](https://github.com/vrtmrz/obsidian-livesync/blob/main/updates_old.md). Because 0.25 got a lot of updates, thankfully, compatibility is kept and we do not need breaking changes! In other words, when get enough stabled. The next version will be v1.0.0. Even though it my hope. +## Unreleased + +### Under development + +- Bulk database fetching is now work in progress. This feature is expected to speed up rebuilds and setups. + Another feature that is needed is the ability to enforce a specific order during the initial comparison between the storage and the local database. + ## 0.25.70 25th May, 2026