(feat): Bulk database fetching is now work in progress. This feature is expected to speed up rebuilds and setups. (WIP)

src/modules/extras/ModuleDev.ts

@@ -1,17 +1,21 @@
 import { delay, fireAndForget } from "octagonal-wheels/promises";
 import { __onMissingTranslation } from "../../lib/src/common/i18n";
 import { AbstractObsidianModule } from "../AbstractObsidianModule.ts";
-import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
+import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
 import { eventHub } from "../../common/events";
 import { enableTestFunction } from "./devUtil/testUtils.ts";
 import { TestPaneView, VIEW_TYPE_TEST } from "./devUtil/TestPaneView.ts";
 import { writable } from "svelte/store";
-import type { FilePathWithPrefix } from "../../lib/src/common/types.ts";
+import type { CouchDBCredentials, FilePathWithPrefix } from "../../lib/src/common/types.ts";
 import type { LiveSyncCore } from "../../main.ts";
-
+import { getConfiguredFunctionsForEncryption } from "@/lib/src/pouchdb/encryption.ts";
+import { AuthorizationHeaderGenerator } from "@/lib/src/replication/httplib.ts";
+import { fetchChangesForInitialSync } from "@/lib/src/pouchdb/StreamingFetch.ts";
+import { PouchDB } from '@lib/pouchdb/pouchdb-browser.ts';
+import { sizeToHumanReadable } from "octagonal-wheels/number";
 export class ModuleDev extends AbstractObsidianModule {
     _everyOnloadStart(): Promise<boolean> {
-        __onMissingTranslation(() => {});
+        __onMissingTranslation(() => { });
         return Promise.resolve(true);
     }
     async onMissingTranslation(key: string): Promise<void> {
@@ -98,7 +102,75 @@ export class ModuleDev extends AbstractObsidianModule {
         });
         return Promise.resolve(true);
     }
+    async _runBulkCopyTest() {
+        const settings = this.settings;
+        const dummyLocalDatabaseForDrop = new PouchDB("dummy-local");
+        await dummyLocalDatabaseForDrop.destroy();
+        const dummyLocalDatabase = new PouchDB("dummy-local");
+        const replicator = await this.core.services.replicator.getNewReplicator();
+        if (!replicator) {
+            return;
+        }
+        const salt = () => replicator.getReplicationPBKDF2Salt(settings);
+        const enc = getConfiguredFunctionsForEncryption(settings.passphrase,
+            false,
+            false,
+            salt,
+            settings.E2EEAlgorithm,
+        );
+
+        const auth = (
+            settings.useJWT
+                ? {
+                    jwtAlgorithm: settings.jwtAlgorithm,
+                    jwtKey: settings.jwtKey,
+                    jwtExpDuration: settings.jwtExpDuration,
+                    jwtKid: settings.jwtKid,
+                    jwtSub: settings.jwtSub,
+                    type: "jwt",
+                }
+                : {
+                    username: settings.couchDB_USER,
+                    password: settings.couchDB_PASSWORD,
+                    type: "basic",
+                }
+        ) satisfies CouchDBCredentials;
+        const authHeader = await (new AuthorizationHeaderGenerator().getAuthorizationHeader(auth));
+        const remote =
+            settings.couchDB_URI.replace(/\/+$/, "") +
+            (settings.couchDB_DBNAME == "" ? "" : "/" + settings.couchDB_DBNAME);
+        //
+        const ret = fetchChangesForInitialSync(
+            dummyLocalDatabase,
+            remote,
+            authHeader,
+            enc.outgoing,
+            "0",
+            (progress) => {
+                Logger(`Initial sync progress: ${progress.totalValidFetched} / ${progress.docsToFetch}
+Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`,
+                    LOG_LEVEL_NOTICE, "fetch-init-progress"
+                );
+
+            }
+        );
+        await ret;
+
+        const allDocs = await dummyLocalDatabase.allDocs({ include_docs: false });
+        Logger(`Bulk copy test completed. Total documents in local database: ${allDocs.total_rows}`, LOG_LEVEL_NOTICE, "fetch-init-complete");
+        await dummyLocalDatabase.destroy();
+        Logger(`Dummy local database has been destroyed after test.`, LOG_LEVEL_NOTICE);
+    }
     async _everyOnLayoutReady(): Promise<boolean> {
+
+        this.addCommand({
+            "id": "bulk-copy-test",
+            "name": "(DEBUG) Bulk copy test",
+            "callback": async () => {
+                await this._runBulkCopyTest();
+            }
+        })
+
         if (!this.settings.enableDebugTools) return Promise.resolve(true);
         // if (await this.core.storageAccess.isExistsIncludeHidden("_SHOWDIALOGAUTO.md")) {
         //     void this.core.$$showView(VIEW_TYPE_TEST);