(feat): Bulk database fetching is now work in progress. This feature is expected to speed up rebuilds and setups. (WIP)

src/modules/extras/ModuleDev.ts

@@ -1,17 +1,21 @@
 import { delay, fireAndForget } from "octagonal-wheels/promises";
 import { __onMissingTranslation } from "../../lib/src/common/i18n";
 import { AbstractObsidianModule } from "../AbstractObsidianModule.ts";
-import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
+import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
 import { eventHub } from "../../common/events";
 import { enableTestFunction } from "./devUtil/testUtils.ts";
 import { TestPaneView, VIEW_TYPE_TEST } from "./devUtil/TestPaneView.ts";
 import { writable } from "svelte/store";
-import type { FilePathWithPrefix } from "../../lib/src/common/types.ts";
+import type { CouchDBCredentials, FilePathWithPrefix } from "../../lib/src/common/types.ts";
 import type { LiveSyncCore } from "../../main.ts";
-
+import { getConfiguredFunctionsForEncryption } from "@/lib/src/pouchdb/encryption.ts";
+import { AuthorizationHeaderGenerator } from "@/lib/src/replication/httplib.ts";
+import { fetchChangesForInitialSync } from "@/lib/src/pouchdb/StreamingFetch.ts";
+import { PouchDB } from '@lib/pouchdb/pouchdb-browser.ts';
+import { sizeToHumanReadable } from "octagonal-wheels/number";
 export class ModuleDev extends AbstractObsidianModule {
     _everyOnloadStart(): Promise<boolean> {
-        __onMissingTranslation(() => {});
+        __onMissingTranslation(() => { });
         return Promise.resolve(true);
     }
     async onMissingTranslation(key: string): Promise<void> {
@@ -98,7 +102,75 @@ export class ModuleDev extends AbstractObsidianModule {
         });
         return Promise.resolve(true);
     }
+    async _runBulkCopyTest() {
+        const settings = this.settings;
+        const dummyLocalDatabaseForDrop = new PouchDB("dummy-local");
+        await dummyLocalDatabaseForDrop.destroy();
+        const dummyLocalDatabase = new PouchDB("dummy-local");
+        const replicator = await this.core.services.replicator.getNewReplicator();
+        if (!replicator) {
+            return;
+        }
+        const salt = () => replicator.getReplicationPBKDF2Salt(settings);
+        const enc = getConfiguredFunctionsForEncryption(settings.passphrase,
+            false,
+            false,
+            salt,
+            settings.E2EEAlgorithm,
+        );
+
+        const auth = (
+            settings.useJWT
+                ? {
+                    jwtAlgorithm: settings.jwtAlgorithm,
+                    jwtKey: settings.jwtKey,
+                    jwtExpDuration: settings.jwtExpDuration,
+                    jwtKid: settings.jwtKid,
+                    jwtSub: settings.jwtSub,
+                    type: "jwt",
+                }
+                : {
+                    username: settings.couchDB_USER,
+                    password: settings.couchDB_PASSWORD,
+                    type: "basic",
+                }
+        ) satisfies CouchDBCredentials;
+        const authHeader = await (new AuthorizationHeaderGenerator().getAuthorizationHeader(auth));
+        const remote =
+            settings.couchDB_URI.replace(/\/+$/, "") +
+            (settings.couchDB_DBNAME == "" ? "" : "/" + settings.couchDB_DBNAME);
+        //
+        const ret = fetchChangesForInitialSync(
+            dummyLocalDatabase,
+            remote,
+            authHeader,
+            enc.outgoing,
+            "0",
+            (progress) => {
+                Logger(`Initial sync progress: ${progress.totalValidFetched} / ${progress.docsToFetch}
+Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`,
+                    LOG_LEVEL_NOTICE, "fetch-init-progress"
+                );
+
+            }
+        );
+        await ret;
+
+        const allDocs = await dummyLocalDatabase.allDocs({ include_docs: false });
+        Logger(`Bulk copy test completed. Total documents in local database: ${allDocs.total_rows}`, LOG_LEVEL_NOTICE, "fetch-init-complete");
+        await dummyLocalDatabase.destroy();
+        Logger(`Dummy local database has been destroyed after test.`, LOG_LEVEL_NOTICE);
+    }
     async _everyOnLayoutReady(): Promise<boolean> {
+
+        this.addCommand({
+            "id": "bulk-copy-test",
+            "name": "(DEBUG) Bulk copy test",
+            "callback": async () => {
+                await this._runBulkCopyTest();
+            }
+        })
+
         if (!this.settings.enableDebugTools) return Promise.resolve(true);
         // if (await this.core.storageAccess.isExistsIncludeHidden("_SHOWDIALOGAUTO.md")) {
         //     void this.core.$$showView(VIEW_TYPE_TEST);

updates.md

@@ -3,6 +3,13 @@ Since 19th July, 2025 (beta1 in 0.25.0-beta1, 13th July, 2025)
 
 The head note of 0.25 is now in [updates_old.md](https://github.com/vrtmrz/obsidian-livesync/blob/main/updates_old.md). Because 0.25 got a lot of updates, thankfully, compatibility is kept and we do not need breaking changes! In other words, when get enough stabled. The next version will be v1.0.0. Even though it my hope.
 
+## Unreleased
+
+### Under development
+
+- Bulk database fetching is now work in progress. This feature is expected to speed up rebuilds and setups.
+  Another feature that is needed is the ability to enforce a specific order during the initial comparison between the storage and the local database.
+
 ## 0.25.70
 
 25th May, 2026