bump

- No longer data corrupting due to false BASE64 detections.

README.md

@@ -45,7 +45,7 @@ This plug-in might be useful for researchers, engineers, and developers with a n
 2. Configure plug-in in [Quick Setup](docs/quick_setup.md)
 
 > [!TIP]
-> We are still able to use IBM Cloudant. However, it is not recommended for several reasons nowadays. Here is [Setup IBM Cloudant](docs/setup_cloudant.md)
+> Now, fly.io has become not free. Fortunately, even though there are some issues, we are still able to use IBM Cloudant. Here is [Setup IBM Cloudant](docs/setup_cloudant.md). It will be updated soon!
 
 
 ## Information in StatusBar

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.22.17",
+    "version": "0.23.0",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.22.17",
+    "version": "0.23.0",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",
@@ -54,7 +54,12 @@
         "typescript": "^5.4.2"
     },
     "dependencies": {
+        "@aws-sdk/client-s3": "^3.556.0",
+        "@smithy/fetch-http-handler": "^2.5.0",
+        "@smithy/protocol-http": "^3.3.0",
+        "@smithy/querystring-builder": "^2.2.0",
         "diff-match-patch": "^1.0.5",
+        "fflate": "^0.8.2",
         "idb": "^8.0.0",
         "minimatch": "^9.0.3",
         "xxhash-wasm": "0.4.2",

src/CmdSetupLiveSync.ts

@@ -1,4 +1,4 @@
-import { type EntryDoc, type ObsidianLiveSyncSettings, DEFAULT_SETTINGS, LOG_LEVEL_NOTICE } from "./lib/src/types";
+import { type EntryDoc, type ObsidianLiveSyncSettings, DEFAULT_SETTINGS, LOG_LEVEL_NOTICE, REMOTE_COUCHDB, REMOTE_MINIO } from "./lib/src/types";
 import { configURIBase } from "./types";
 import { Logger } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";
@@ -9,6 +9,7 @@ import { delay, fireAndForget } from "./lib/src/utils";
 import { confirmWithMessage } from "./dialogs";
 import { Platform } from "./deps";
 import { fetchAllUsedChunks } from "./lib/src/utils_couchdb";
+import type { LiveSyncCouchDBReplicator } from "./lib/src/LiveSyncReplicator.js";
 
 export class SetupLiveSync extends LiveSyncCommands {
     onunload() { }
@@ -311,6 +312,7 @@ Of course, we are able to disable these features.`
     }
     async suspendReflectingDatabase() {
         if (this.plugin.settings.doNotSuspendOnFetching) return;
+        if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
         Logger(`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`, LOG_LEVEL_NOTICE);
         this.plugin.settings.suspendParseReplicationResult = true;
         this.plugin.settings.suspendFileWatching = true;
@@ -318,6 +320,7 @@ Of course, we are able to disable these features.`
     }
     async resumeReflectingDatabase() {
         if (this.plugin.settings.doNotSuspendOnFetching) return;
+        if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
         Logger(`Database and storage reflection has been resumed!`, LOG_LEVEL_NOTICE);
         this.plugin.settings.suspendParseReplicationResult = false;
         this.plugin.settings.suspendFileWatching = false;
@@ -348,9 +351,10 @@ Of course, we are able to disable these features.`
         await this.plugin.resetLocalDatabase();
     }
     async fetchRemoteChunks() {
-        if (!this.plugin.settings.doNotSuspendOnFetching && this.plugin.settings.readChunksOnline) {
+        if (!this.plugin.settings.doNotSuspendOnFetching && this.plugin.settings.readChunksOnline && this.plugin.settings.remoteType == REMOTE_COUCHDB) {
             Logger(`Fetching chunks`, LOG_LEVEL_NOTICE);
-            const remoteDB = await this.plugin.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.plugin.getIsMobile(), true);
+            const replicator = this.plugin.getReplicator() as LiveSyncCouchDBReplicator;
+            const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.plugin.getIsMobile(), true);
             if (typeof remoteDB == "string") {
                 Logger(remoteDB, LOG_LEVEL_NOTICE);
             } else {

src/ObsHttpHandler.ts

@@ -0,0 +1,133 @@
+// This file is based on a file that was published by the @remotely-save, under the Apache 2 License.
+// I would love to express my deepest gratitude to the original authors for their hard work and dedication. Without their contributions, this project would not have been possible.
+// 
+// Original Implementation is here: https://github.com/remotely-save/remotely-save/blob/28b99557a864ef59c19d2ad96101196e401718f0/src/remoteForS3.ts
+
+import {
+    FetchHttpHandler,
+    type FetchHttpHandlerOptions,
+} from "@smithy/fetch-http-handler";
+import { HttpRequest, HttpResponse, type HttpHandlerOptions } from "@smithy/protocol-http";
+//@ts-ignore
+import { requestTimeout } from "@smithy/fetch-http-handler/dist-es/request-timeout";
+import { buildQueryString } from "@smithy/querystring-builder";
+import { requestUrl, type RequestUrlParam } from "./deps";
+////////////////////////////////////////////////////////////////////////////////
+// special handler using Obsidian requestUrl
+////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * This is close to origin implementation of FetchHttpHandler
+ * https://github.com/aws/aws-sdk-js-v3/blob/main/packages/fetch-http-handler/src/fetch-http-handler.ts
+ * that is released under Apache 2 License.
+ * But this uses Obsidian requestUrl instead.
+ */
+export class ObsHttpHandler extends FetchHttpHandler {
+    requestTimeoutInMs: number | undefined;
+    reverseProxyNoSignUrl: string | undefined;
+    constructor(
+        options?: FetchHttpHandlerOptions,
+        reverseProxyNoSignUrl?: string
+    ) {
+        super(options);
+        this.requestTimeoutInMs =
+            options === undefined ? undefined : options.requestTimeout;
+        this.reverseProxyNoSignUrl = reverseProxyNoSignUrl;
+    }
+    async handle(
+        request: HttpRequest,
+        { abortSignal }: HttpHandlerOptions = {}
+    ): Promise<{ response: HttpResponse }> {
+        if (abortSignal?.aborted) {
+            const abortError = new Error("Request aborted");
+            abortError.name = "AbortError";
+            return Promise.reject(abortError);
+        }
+
+        let path = request.path;
+        if (request.query) {
+            const queryString = buildQueryString(request.query);
+            if (queryString) {
+                path += `?${queryString}`;
+            }
+        }
+
+        const { port, method } = request;
+        let url = `${request.protocol}//${request.hostname}${port ? `:${port}` : ""
+            }${path}`;
+        if (
+            this.reverseProxyNoSignUrl !== undefined &&
+            this.reverseProxyNoSignUrl !== ""
+        ) {
+            const urlObj = new URL(url);
+            urlObj.host = this.reverseProxyNoSignUrl;
+            url = urlObj.href;
+        }
+        const body =
+            method === "GET" || method === "HEAD" ? undefined : request.body;
+
+        const transformedHeaders: Record<string, string> = {};
+        for (const key of Object.keys(request.headers)) {
+            const keyLower = key.toLowerCase();
+            if (keyLower === "host" || keyLower === "content-length") {
+                continue;
+            }
+            transformedHeaders[keyLower] = request.headers[key];
+        }
+
+        let contentType: string | undefined = undefined;
+        if (transformedHeaders["content-type"] !== undefined) {
+            contentType = transformedHeaders["content-type"];
+        }
+
+        let transformedBody: any = body;
+        if (ArrayBuffer.isView(body)) {
+            transformedBody = new Uint8Array(body.buffer).buffer;
+        }
+
+        const param: RequestUrlParam = {
+            body: transformedBody,
+            headers: transformedHeaders,
+            method: method,
+            url: url,
+            contentType: contentType,
+        };
+
+        const raceOfPromises = [
+            requestUrl(param).then((rsp) => {
+                const headers = rsp.headers;
+                const headersLower: Record<string, string> = {};
+                for (const key of Object.keys(headers)) {
+                    headersLower[key.toLowerCase()] = headers[key];
+                }
+                const stream = new ReadableStream<Uint8Array>({
+                    start(controller) {
+                        controller.enqueue(new Uint8Array(rsp.arrayBuffer));
+                        controller.close();
+                    },
+                });
+                return {
+                    response: new HttpResponse({
+                        headers: headersLower,
+                        statusCode: rsp.status,
+                        body: stream,
+                    }),
+                };
+            }),
+            requestTimeout(this.requestTimeoutInMs),
+        ];
+
+        if (abortSignal) {
+            raceOfPromises.push(
+                new Promise<never>((resolve, reject) => {
+                    abortSignal.onabort = () => {
+                        const abortError = new Error("Request aborted");
+                        abortError.name = "AbortError";
+                        reject(abortError);
+                    };
+                })
+            );
+        }
+        return Promise.race(raceOfPromises);
+    }
+}

src/main.ts

@@ -2,7 +2,7 @@ const isDebug = false;
 
 import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch, stringifyYaml, parseYaml } from "./deps";
 import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
-import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, } from "./lib/src/types";
+import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, REMOTE_MINIO, REMOTE_COUCHDB, type BucketSyncSetting, } from "./lib/src/types";
 import { type InternalFileInfo, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";
 import { arrayToChunkedArray, createBlob, delay, determineTypeFromBlob, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue, throttle } from "./lib/src/utils";
 import { Logger, setGlobalLogFunction } from "./lib/src/logger";
@@ -12,7 +12,7 @@ import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab";
 import { DocumentHistoryModal } from "./DocumentHistoryModal";
 import { applyPatch, cancelAllPeriodicTask, cancelAllTasks, cancelTask, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, flattenObject, path2id, scheduleTask, tryParseJSON, isValidPath, isInternalMetadata, isPluginMetadata, stripInternalMetadataPrefix, isChunk, askSelectString, askYesNo, askString, PeriodicProcessor, getPath, getPathWithoutPrefix, getPathFromTFile, performRebuildDB, memoIfNotExist, memoObject, retrieveMemoObject, disposeMemoObject, isCustomisationSyncMetadata, compareFileFreshness, BASE_IS_NEW, TARGET_IS_NEW, EVEN, compareMTime, markChangesAreSame } from "./utils";
 import { encrypt, tryDecrypt } from "./lib/src/e2ee_v2";
-import { balanceChunkPurgedDBs, enableEncryption, isCloudantURI, isErrorOfMissingDoc, isValidRemoteCouchDBURI, purgeUnreferencedChunks } from "./lib/src/utils_couchdb";
+import { balanceChunkPurgedDBs, enableCompression, enableEncryption, isCloudantURI, isErrorOfMissingDoc, isValidRemoteCouchDBURI, purgeUnreferencedChunks } from "./lib/src/utils_couchdb";
 import { logStore, type LogEntry, collectingChunks, pluginScanningCount, hiddenFilesProcessingCount, hiddenFilesEventCount, logMessages } from "./lib/src/stores";
 import { setNoticeClass } from "./lib/src/wrapper";
 import { versionNumberString2Number, writeString, decodeBinary, readString } from "./lib/src/strbin";
@@ -20,7 +20,7 @@ import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixe
 import { isLockAcquired, serialized, shareRunningResult, skipIfDuplicated } from "./lib/src/lock";
 import { StorageEventManager, StorageEventManagerObsidian } from "./StorageEventManager";
 import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "./lib/src/LiveSyncLocalDB";
-import { LiveSyncDBReplicator, type LiveSyncReplicatorEnv } from "./lib/src/LiveSyncReplicator";
+import { LiveSyncAbstractReplicator, type LiveSyncReplicatorEnv } from "./lib/src/LiveSyncAbstractReplicator.js";
 import { type KeyValueDatabase, OpenKeyValueDatabase } from "./KeyValueDB";
 import { LiveSyncCommands } from "./LiveSyncCommands";
 import { HiddenFileSync } from "./CmdHiddenFileSync";
@@ -34,6 +34,11 @@ import { SerializedFileAccess } from "./SerializedFileAccess.js";
 import { QueueProcessor } from "./lib/src/processor.js";
 import { reactive, reactiveSource } from "./lib/src/reactive.js";
 import { initializeStores } from "./stores.js";
+import { JournalSyncMinio } from "./lib/src/JournalSyncMinio.js";
+import { LiveSyncJournalReplicator, type LiveSyncJournalReplicatorEnv } from "./lib/src/LiveSyncJournalReplicator.js";
+import { LiveSyncCouchDBReplicator, type LiveSyncCouchDBReplicatorEnv } from "./lib/src/LiveSyncReplicator.js";
+import type { CheckPointInfo, SimpleStore } from "./lib/src/JournalSyncTypes.js";
+import { ObsHttpHandler } from "./ObsHttpHandler.js";
 
 setNoticeClass(Notice);
 
@@ -69,11 +74,16 @@ const SETTING_HEADER = "````yaml:livesync-setting\n";
 const SETTING_FOOTER = "\n````";
 
 export default class ObsidianLiveSyncPlugin extends Plugin
-    implements LiveSyncLocalDBEnv, LiveSyncReplicatorEnv {
+    implements LiveSyncLocalDBEnv, LiveSyncReplicatorEnv, LiveSyncJournalReplicatorEnv, LiveSyncCouchDBReplicatorEnv {
+    _customHandler!: ObsHttpHandler;
+    customFetchHandler() {
+        if (!this._customHandler) this._customHandler = new ObsHttpHandler(undefined, undefined);
+        return this._customHandler;
+    }
 
     settings!: ObsidianLiveSyncSettings;
     localDatabase!: LiveSyncLocalDB;
-    replicator!: LiveSyncDBReplicator;
+    replicator!: LiveSyncAbstractReplicator;
 
     statusBar?: HTMLElement;
     _suspended = false;
@@ -119,7 +129,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
     requestCount = reactiveSource(0);
     responseCount = reactiveSource(0);
     processReplication = (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => this.parseReplicationResult(e);
-    async connectRemoteCouchDB(uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | false, useDynamicIterationCount: boolean, performSetup: boolean, skipInfo: boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> {
+    async connectRemoteCouchDB(uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | false, useDynamicIterationCount: boolean, performSetup: boolean, skipInfo: boolean, compression: boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> {
         if (!isValidRemoteCouchDBURI(uri)) return "Remote URI is not valid";
         if (uri.toLowerCase() != uri) return "Remote URI and database name could not contain capital letters.";
         if (uri.indexOf(" ") !== -1) return "Remote URI and database name could not contain spaces.";
@@ -237,6 +247,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         };
 
         const db: PouchDB.Database<EntryDoc> = new PouchDB<EntryDoc>(uri, conf);
+        enableCompression(db, compression);
         if (passphrase !== "false" && typeof passphrase === "string") {
             enableEncryption(db, passphrase, useDynamicIterationCount, false);
         }
@@ -307,9 +318,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin
     onClose(db: LiveSyncLocalDB): void {
         this.kvDB.close();
     }
+    getNewReplicator(settingOverride: Partial<ObsidianLiveSyncSettings> = {}): LiveSyncAbstractReplicator {
+        const settings = { ...this.settings, ...settingOverride };
+        if (settings.remoteType == REMOTE_MINIO) {
+            return new LiveSyncJournalReplicator(this);
+        }
+        return new LiveSyncCouchDBReplicator(this);
+    }
     async onInitializeDatabase(db: LiveSyncLocalDB): Promise<void> {
         this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv");
-        this.replicator = new LiveSyncDBReplicator(this);
+        this.replicator = this.getNewReplicator();
     }
     async onResetDatabase(db: LiveSyncLocalDB): Promise<void> {
         const kvDBKey = "queued-files"
@@ -317,7 +335,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         // localStorage.removeItem(lsKey);
         await this.kvDB.destroy();
         this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv");
-        this.replicator = new LiveSyncDBReplicator(this);
+        this.replicator = this.getNewReplicator()
     }
     getReplicator() {
         return this.replicator;
@@ -446,6 +464,52 @@ export default class ObsidianLiveSyncPlugin extends Plugin
         }
         Logger(`Checking expired file history done`);
     }
+
+    simpleStore: SimpleStore<CheckPointInfo> = {
+        get: async (key: string) => {
+            return await this.kvDB.get(`os-${key}`);
+        },
+        set: async (key: string, value: any) => {
+            await this.kvDB.set(`os-${key}`, value);
+        },
+        delete: async (key) => {
+            await this.kvDB.del(`os-${key}`);
+        },
+        keys: async (from: string | undefined, to: string | undefined, count?: number | undefined): Promise<string[]> => {
+            const ret = this.kvDB.keys(IDBKeyRange.bound(`os-${from || ""}`, `os-${to || ""}`), count);
+            return (await ret).map(e => e.toString());
+        }
+    }
+    getMinioJournalSyncClient() {
+        const id = this.settings.accessKey
+        const key = this.settings.secretKey
+        const bucket = this.settings.bucket
+        const region = this.settings.region
+        const endpoint = this.settings.endpoint
+        const useCustomRequestHandler = this.settings.useCustomRequestHandler;
+        return new JournalSyncMinio(id, key, endpoint, bucket, this.simpleStore, this, useCustomRequestHandler, region);
+    }
+    async resetRemoteBucket() {
+        const minioJournal = this.getMinioJournalSyncClient();
+        await minioJournal.resetBucket();
+    }
+    async resetJournalSync() {
+        const minioJournal = this.getMinioJournalSyncClient();
+        await minioJournal.resetCheckpointInfo();
+    }
+    async journalSendTest() {
+        const minioJournal = this.getMinioJournalSyncClient();
+        await minioJournal.sendLocalJournal();
+    }
+    async journalFetchTest() {
+        const minioJournal = this.getMinioJournalSyncClient();
+        await minioJournal.receiveRemoteJournal();
+    }
+
+    async journalSyncTest() {
+        const minioJournal = this.getMinioJournalSyncClient();
+        await minioJournal.sync();
+    }
     async onLayoutReady() {
         this.registerFileWatchEvents();
         if (!this.localDatabase.isReady) {
@@ -537,7 +601,7 @@ Click anywhere to stop counting down.
             await this.realizeSettingSyncMode();
             this.swapSaveCommand();
             if (!this.settings.liveSync && this.settings.syncOnStart) {
-                this.replicator.openReplication(this.settings, false, false);
+                this.replicator.openReplication(this.settings, false, false, false);
             }
             this.scanStat();
         } catch (ex) {
@@ -618,6 +682,63 @@ Note: We can always able to read V1 format. It will be progressively converted.
             this.addOnConfigSync.showPluginSyncModal();
         }).addClass("livesync-ribbon-showcustom");
 
+        this.addCommand({
+            id: "debug-x1",
+            name: "Journal send",
+            callback: () => {
+                this.journalSendTest();
+            }
+        });
+        this.addCommand({
+            id: "debug-x3",
+            name: "Journal receive",
+            callback: () => {
+                this.journalFetchTest();
+            }
+        });
+        this.addCommand({
+            id: "debug-x4",
+            name: "Sync By Journal",
+            callback: () => {
+                this.journalSyncTest();
+            }
+        });
+        this.addCommand({
+            id: "debug-x5",
+            name: "Reset journal sync",
+            callback: () => {
+                this.resetJournalSync();
+            }
+        });
+        this.addCommand({
+            id: "debug-x6",
+            name: "Reset journal sync and delete all items on the bucket",
+            callback: () => {
+                this.resetRemoteBucket();
+            }
+        })
+        this.addCommand({
+            id: "debug-x7",
+            name: "Perform Test",
+            callback: () => {
+                // const p = getMockedPouch();
+                // this.localDatabase.localDatabase.replicate.to(p, { since: 1000, checkpoint: "source" });
+            }
+        })
+        this.addCommand({
+            id: "debug-x8",
+            name: "Pack test",
+            callback: async () => {
+                const minioJournal = this.getMinioJournalSyncClient();
+                // const pack = await minioJournal.createJournalPack();
+                // console.warn();
+                console.warn(await minioJournal._createJournalPack());
+            }
+        })
+
+
+
+
         this.addCommand({
             id: "view-log",
             name: "Show log",
@@ -954,17 +1075,19 @@ Note: We can always able to read V1 format. It will be progressively converted.
             Logger("Could not determine passphrase for reading data.json! DO NOT synchronize with the remote before making sure your configuration is!", LOG_LEVEL_URGENT);
         } else {
             if (settings.encryptedCouchDBConnection) {
-                const keys = ["couchDB_URI", "couchDB_USER", "couchDB_PASSWORD", "couchDB_DBNAME"] as (keyof CouchDBConnection)[];
-                const decrypted = this.tryDecodeJson(await this.decryptConfigurationItem(settings.encryptedCouchDBConnection, passphrase)) as CouchDBConnection;
+                const keys = ["couchDB_URI", "couchDB_USER", "couchDB_PASSWORD", "couchDB_DBNAME", "accessKey", "bucket", "endpoint", "region", "secretKey"] as (keyof CouchDBConnection | keyof BucketSyncSetting)[];
+                const decrypted = this.tryDecodeJson(await this.decryptConfigurationItem(settings.encryptedCouchDBConnection, passphrase)) as (CouchDBConnection & BucketSyncSetting);
                 if (decrypted) {
                     for (const key of keys) {
                         if (key in decrypted) {
+                            //@ts-ignore
                             settings[key] = decrypted[key]
                         }
                     }
                 } else {
                     Logger("Could not decrypt passphrase for reading data.json! DO NOT synchronize with the remote before making sure your configuration is!", LOG_LEVEL_URGENT);
                     for (const key of keys) {
+                        //@ts-ignore
                         settings[key] = "";
                     }
                 }
@@ -1021,22 +1144,34 @@ Note: We can always able to read V1 format. It will be progressively converted.
             Logger("Could not determine passphrase for saving data.json! Our data.json have insecure items!", LOG_LEVEL_NOTICE);
         } else {
             if (settings.couchDB_PASSWORD != "" || settings.couchDB_URI != "" || settings.couchDB_USER != "" || settings.couchDB_DBNAME) {
-                const connectionSetting: CouchDBConnection = {
+                const connectionSetting: CouchDBConnection & BucketSyncSetting = {
                     couchDB_DBNAME: settings.couchDB_DBNAME,
                     couchDB_PASSWORD: settings.couchDB_PASSWORD,
                     couchDB_URI: settings.couchDB_URI,
                     couchDB_USER: settings.couchDB_USER,
+                    accessKey: settings.accessKey,
+                    bucket: settings.bucket,
+                    endpoint: settings.endpoint,
+                    region: settings.region,
+                    secretKey: settings.secretKey,
+                    useCustomRequestHandler: settings.useCustomRequestHandler
                 };
                 settings.encryptedCouchDBConnection = await this.encryptConfigurationItem(JSON.stringify(connectionSetting), settings);
                 settings.couchDB_PASSWORD = "";
                 settings.couchDB_DBNAME = "";
                 settings.couchDB_URI = "";
                 settings.couchDB_USER = "";
+                settings.accessKey = "";
+                settings.bucket = "";
+                settings.region = "";
+                settings.secretKey = "";
+                settings.endpoint = "";
             }
             if (settings.encrypt && settings.passphrase != "") {
                 settings.encryptedPassphrase = await this.encryptConfigurationItem(settings.passphrase, settings);
                 settings.passphrase = "";
             }
+
         }
         await this.saveData(settings);
         this.localDatabase.settings = this.settings;
@@ -1292,11 +1427,13 @@ We can perform a command in this file.
             // suspend all temporary.
             if (this.suspended) return;
             await Promise.all(this.addOns.map(e => e.onResume()));
-            if (this.settings.liveSync) {
-                this.replicator.openReplication(this.settings, true, false);
+            if (this.settings.remoteType == REMOTE_COUCHDB) {
+                if (this.settings.liveSync) {
+                    this.replicator.openReplication(this.settings, true, false, false);
+                }
             }
             if (this.settings.syncOnStart) {
-                this.replicator.openReplication(this.settings, false, false);
+                this.replicator.openReplication(this.settings, false, false, false);
             }
             this.periodicSyncProcessor.enable(this.settings.periodicReplication ? this.settings.periodicReplicationInterval * 1000 : 0);
         }
@@ -1789,8 +1926,10 @@ We can perform a command in this file.
         // disable all sync temporary.
         if (this.suspended) return;
         await Promise.all(this.addOns.map(e => e.onResume()));
-        if (this.settings.liveSync) {
-            this.replicator.openReplication(this.settings, true, false);
+        if (this.settings.remoteType == REMOTE_COUCHDB) {
+            if (this.settings.liveSync) {
+                this.replicator.openReplication(this.settings, true, false, false);
+            }
         }
 
         const q = activeDocument.querySelector(`.livesync-ribbon-showcustom`);
@@ -1863,6 +2002,11 @@ We can perform a command in this file.
             let pushLast = "";
             let pullLast = "";
             let w = "";
+            const labels: Partial<Record<DatabaseConnectingStatus, string>> = {
+                "CONNECTED": "⚡",
+                "JOURNAL_SEND": "📦↑",
+                "JOURNAL_RECEIVE": "📦↓",
+            }
             switch (e.syncStatus) {
                 case "CLOSED":
                 case "COMPLETED":
@@ -1876,7 +2020,9 @@ We can perform a command in this file.
                     w = "💤";
                     break;
                 case "CONNECTED":
-                    w = "⚡";
+                case "JOURNAL_SEND":
+                case "JOURNAL_RECEIVE":
+                    w = labels[e.syncStatus] || "⚡";
                     pushLast = ((lastSyncPushSeq == 0) ? "" : (lastSyncPushSeq >= maxPushSeq ? " (LIVE)" : ` (${maxPushSeq - lastSyncPushSeq})`));
                     pullLast = ((lastSyncPullSeq == 0) ? "" : (lastSyncPullSeq >= maxPullSeq ? " (LIVE)" : ` (${maxPullSeq - lastSyncPullSeq})`));
                     break;
@@ -1956,7 +2102,7 @@ We can perform a command in this file.
         await this.applyBatchChange();
         await Promise.all(this.addOns.map(e => e.beforeReplicate(showMessage)));
         await this.loadQueuedFiles();
-        const ret = await this.replicator.openReplication(this.settings, false, showMessage);
+        const ret = await this.replicator.openReplication(this.settings, false, showMessage, false);
         if (!ret) {
             if (this.replicator.remoteLockedAndDeviceNotAccepted) {
                 if (this.replicator.remoteCleaned && this.settings.useIndexedDBAdapter) {
@@ -1976,7 +2122,9 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
                             await performRebuildDB(this, "localOnly");
                         }
                         if (ret == CHOICE_CLEAN) {
-                            const remoteDB = await this.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.getIsMobile(), true);
+                            const replicator = this.getReplicator();
+                            if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
+                            const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.getIsMobile(), true);
                             if (typeof remoteDB == "string") {
                                 Logger(remoteDB, LOG_LEVEL_NOTICE);
                                 return false;
@@ -2210,7 +2358,7 @@ Or if you are sure know what had been happened, we can unlock the database from
         const step = 25;
         const remainLog = (remain: number) => {
             if (lastRemain - remain > step) {
-                const msg = ` CHECK AND SYNC: ${remain} / ${allSyncFiles}`;
+                const msg = ` CHECK AND SYNC: ${allSyncFiles - remain} / ${allSyncFiles}`;
                 updateLog("sync", msg);
                 lastRemain = remain;
             }
@@ -2934,7 +3082,9 @@ Or if you are sure know what had been happened, we can unlock the database from
     }
     async dryRunGC() {
         await skipIfDuplicated("cleanup", async () => {
-            const remoteDBConn = await this.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.isMobile)
+            const replicator = this.getReplicator();
+            if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
+            const remoteDBConn = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.isMobile)
             if (typeof (remoteDBConn) == "string") {
                 Logger(remoteDBConn);
                 return;
@@ -2948,8 +3098,10 @@ Or if you are sure know what had been happened, we can unlock the database from
     async dbGC() {
         // Lock the remote completely once.
         await skipIfDuplicated("cleanup", async () => {
+            const replicator = this.getReplicator();
+            if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
             this.getReplicator().markRemoteLocked(this.settings, true, true);
-            const remoteDBConn = await this.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.isMobile)
+            const remoteDBConn = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.isMobile)
             if (typeof (remoteDBConn) == "string") {
                 Logger(remoteDBConn);
                 return;

updates.md

@@ -1,46 +1,22 @@
-### 0.22.0
-A few years passed since Self-hosted LiveSync was born, and our codebase had been very complicated. This could be patient now, but it should be a tremendous hurt.
-Therefore at v0.22.0, for future maintainability, I refined task scheduling logic totally.
+### 0.23.0
+Incredibly new features!
 
-Of course, I think this would be our suffering in some cases. However, I would love to ask you for your cooperation and contribution.
+Now, we can use object storage (MinIO, S3, R2 or anything you like) for synchronising! Moreover, despite that, we can use all the features as if we were using CouchDB.
+Note: As this is a pretty experimental feature, hence we have some limitations.
+- This is built on the append-only architecture. It will not shrink used storage if we do not perform a rebuild.
+- A bit fragile. However, our version x.yy.0 is always so.
+- When the first synchronisation, the entire history to date is transferred. For this reason, it is preferable to do this under the WiFi network.
+- Do not worry, from the second synchronisation, we always transfer only differences.
 
-Sorry for being absent so much long. And thank you for your patience!
+I hope this feature empowers users to maintain independence and self-host their data, offering an alternative for those who prefer to manage their own storage solutions and avoid being stuck on the right side of a sudden change in business model.
 
-Note: we got a very performance improvement.
-Note at 0.22.2: **Now, to rescue mobile devices, Maximum file size is set to 50 by default**. Please configure the limit as you need. If you do not want to limit the sizes, set zero manually, please.
+Of course, I use Self-hosted MinIO for testing and recommend this. It is for the same reason as using CouchDB. -- open, controllable, auditable and indeed already audited by numerous eyes.
+
+Let me write one more acknowledgement.
+
+I have a lot of respect for that plugin, even though it is sometimes treated as if it is a competitor, remotely-save. I think it is a great architecture that embodies a different approach to my approach of recreating history. This time, with all due respect, I have used some of its code as a reference.
+Hooray for open source, and generous licences, and the sharing of knowledge by experts.
 
 #### Version history
-- 0.22.17:
-  - Fixed:
-    - Error handling on booting now works fine.
-    - Replication is now started automatically in LiveSync mode.
-    - Batch database update is now disabled in LiveSync mode.
-    - No longer automatically reconnection while off-focused.
-    - Status saves are thinned out.
-    - Now Self-hosted LiveSync waits for all files between the local database and storage to be surely checked.
-  - Improved:
-    - The job scheduler is now more robust and stable.
-    - The status indicator no longer flickers and keeps zero for a while.
-    - No longer meaningless frequent updates of status indicators.
-    - Now we can configure regular expression filters in handy UI. Thank you so much, @eth-p!
-    - `Fetch` or `Rebuild everything` is now more safely performed.
-  - Minor things
-    - Some utility function has been added.
-    - Customisation sync now less wrong messages.
-    - Digging the weeds for eradication of type errors.
-- 0.22.16:
-  - Fixed:
-    - Fixed the issue that binary files were sometimes corrupted.
-    - Fixed customisation sync data could be corrupted.
-  - Improved:
-    - Now the remote database costs lower memory.
-      - This release requires a brief wait on the first synchronisation, to track the latest changeset again.
-    - Description added for the `Device name`.
-  - Refactored:
-    - Many type-errors have been resolved.
-    - Obsolete file has been deleted.
-- 0.22.15:
-  - Improved:
-    - Faster start-up by removing too many logs which indicates normality
-    - By streamlined scanning of customised synchronisation extra phases have been deleted.
-... To continue on to `updates_old.md`.
+- New feature:
+  - Now we can use Object Storage.

updates_old.md

@@ -10,6 +10,51 @@ Note: we got a very performance improvement.
 Note at 0.22.2: **Now, to rescue mobile devices, Maximum file size is set to 50 by default**. Please configure the limit as you need. If you do not want to limit the sizes, set zero manually, please.
 
 #### Version history
+- 0.22.19
+  - Fixed:
+    - No longer data corrupting due to false BASE64 detections.
+  - Improved:
+    - A bit more efficient in Automatic data compression.
+- 0.22.18
+  - New feature (Very Experimental):
+    - Now we can use `Automatic data compression` to reduce amount of traffic and the usage of remote database.
+      - Please make sure all devices are updated to v0.22.18 before trying this feature.
+      - If you are using some other utilities which connected to your vault, please make sure that they have compatibilities. 
+      - Note: Setting `File Compression` on the remote database works for shrink the size of remote database. Please refer the [Doc](https://docs.couchdb.org/en/stable/config/couchdb.html#couchdb/file_compression).
+- 0.22.17:
+  - Fixed:
+    - Error handling on booting now works fine.
+    - Replication is now started automatically in LiveSync mode.
+    - Batch database update is now disabled in LiveSync mode.
+    - No longer automatically reconnection while off-focused.
+    - Status saves are thinned out.
+    - Now Self-hosted LiveSync waits for all files between the local database and storage to be surely checked.
+  - Improved:
+    - The job scheduler is now more robust and stable.
+    - The status indicator no longer flickers and keeps zero for a while.
+    - No longer meaningless frequent updates of status indicators.
+    - Now we can configure regular expression filters in handy UI. Thank you so much, @eth-p!
+    - `Fetch` or `Rebuild everything` is now more safely performed.
+  - Minor things
+    - Some utility function has been added.
+    - Customisation sync now less wrong messages.
+    - Digging the weeds for eradication of type errors.
+- 0.22.16:
+  - Fixed:
+    - Fixed the issue that binary files were sometimes corrupted.
+    - Fixed customisation sync data could be corrupted.
+  - Improved:
+    - Now the remote database costs lower memory.
+      - This release requires a brief wait on the first synchronisation, to track the latest changeset again.
+    - Description added for the `Device name`.
+  - Refactored:
+    - Many type-errors have been resolved.
+    - Obsolete file has been deleted.
+- 0.22.15:
+  - Improved:
+    - Faster start-up by removing too many logs which indicates normality
+    - By streamlined scanning of customised synchronisation extra phases have been deleted.
+... To continue on to `updates_old.md`.
 - 0.22.14:
   - New feature:
     - We can disable the status bar in the setting dialogue.