New feature: Object storage support

This commit is contained in:
vorotamoroz
2024-04-27 03:33:59 +09:00
parent a1ab1efd5d
commit 2c0fcf74d0
5 changed files with 905 additions and 456 deletions

View File

@@ -1,4 +1,4 @@
import { type EntryDoc, type ObsidianLiveSyncSettings, DEFAULT_SETTINGS, LOG_LEVEL_NOTICE } from "./lib/src/types"; import { type EntryDoc, type ObsidianLiveSyncSettings, DEFAULT_SETTINGS, LOG_LEVEL_NOTICE, REMOTE_COUCHDB, REMOTE_MINIO } from "./lib/src/types";
import { configURIBase } from "./types"; import { configURIBase } from "./types";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { PouchDB } from "./lib/src/pouchdb-browser.js"; import { PouchDB } from "./lib/src/pouchdb-browser.js";
@@ -9,6 +9,7 @@ import { delay, fireAndForget } from "./lib/src/utils";
import { confirmWithMessage } from "./dialogs"; import { confirmWithMessage } from "./dialogs";
import { Platform } from "./deps"; import { Platform } from "./deps";
import { fetchAllUsedChunks } from "./lib/src/utils_couchdb"; import { fetchAllUsedChunks } from "./lib/src/utils_couchdb";
import type { LiveSyncCouchDBReplicator } from "./lib/src/LiveSyncReplicator.js";
export class SetupLiveSync extends LiveSyncCommands { export class SetupLiveSync extends LiveSyncCommands {
onunload() { } onunload() { }
@@ -311,6 +312,7 @@ Of course, we are able to disable these features.`
} }
async suspendReflectingDatabase() { async suspendReflectingDatabase() {
if (this.plugin.settings.doNotSuspendOnFetching) return; if (this.plugin.settings.doNotSuspendOnFetching) return;
if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
Logger(`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`, LOG_LEVEL_NOTICE); Logger(`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`, LOG_LEVEL_NOTICE);
this.plugin.settings.suspendParseReplicationResult = true; this.plugin.settings.suspendParseReplicationResult = true;
this.plugin.settings.suspendFileWatching = true; this.plugin.settings.suspendFileWatching = true;
@@ -318,6 +320,7 @@ Of course, we are able to disable these features.`
} }
async resumeReflectingDatabase() { async resumeReflectingDatabase() {
if (this.plugin.settings.doNotSuspendOnFetching) return; if (this.plugin.settings.doNotSuspendOnFetching) return;
if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
Logger(`Database and storage reflection has been resumed!`, LOG_LEVEL_NOTICE); Logger(`Database and storage reflection has been resumed!`, LOG_LEVEL_NOTICE);
this.plugin.settings.suspendParseReplicationResult = false; this.plugin.settings.suspendParseReplicationResult = false;
this.plugin.settings.suspendFileWatching = false; this.plugin.settings.suspendFileWatching = false;
@@ -348,9 +351,10 @@ Of course, we are able to disable these features.`
await this.plugin.resetLocalDatabase(); await this.plugin.resetLocalDatabase();
} }
async fetchRemoteChunks() { async fetchRemoteChunks() {
if (!this.plugin.settings.doNotSuspendOnFetching && this.plugin.settings.readChunksOnline) { if (!this.plugin.settings.doNotSuspendOnFetching && this.plugin.settings.readChunksOnline && this.plugin.settings.remoteType == REMOTE_COUCHDB) {
Logger(`Fetching chunks`, LOG_LEVEL_NOTICE); Logger(`Fetching chunks`, LOG_LEVEL_NOTICE);
const remoteDB = await this.plugin.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.plugin.getIsMobile(), true); const replicator = this.plugin.getReplicator() as LiveSyncCouchDBReplicator;
const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.plugin.getIsMobile(), true);
if (typeof remoteDB == "string") { if (typeof remoteDB == "string") {
Logger(remoteDB, LOG_LEVEL_NOTICE); Logger(remoteDB, LOG_LEVEL_NOTICE);
} else { } else {

133
src/ObsHttpHandler.ts Normal file
View File

@@ -0,0 +1,133 @@
// This file is based on a file that was published by the @remotely-save, under the Apache 2 License.
// I would love to express my deepest gratitude to the original authors for their hard work and dedication. Without their contributions, this project would not have been possible.
//
// Original Implementation is here: https://github.com/remotely-save/remotely-save/blob/28b99557a864ef59c19d2ad96101196e401718f0/src/remoteForS3.ts
import {
FetchHttpHandler,
type FetchHttpHandlerOptions,
} from "@smithy/fetch-http-handler";
import { HttpRequest, HttpResponse, type HttpHandlerOptions } from "@smithy/protocol-http";
//@ts-ignore
import { requestTimeout } from "@smithy/fetch-http-handler/dist-es/request-timeout";
import { buildQueryString } from "@smithy/querystring-builder";
import { requestUrl, type RequestUrlParam } from "./deps";
////////////////////////////////////////////////////////////////////////////////
// special handler using Obsidian requestUrl
////////////////////////////////////////////////////////////////////////////////
/**
* This is close to origin implementation of FetchHttpHandler
* https://github.com/aws/aws-sdk-js-v3/blob/main/packages/fetch-http-handler/src/fetch-http-handler.ts
* that is released under Apache 2 License.
* But this uses Obsidian requestUrl instead.
*/
export class ObsHttpHandler extends FetchHttpHandler {
requestTimeoutInMs: number | undefined;
reverseProxyNoSignUrl: string | undefined;
constructor(
options?: FetchHttpHandlerOptions,
reverseProxyNoSignUrl?: string
) {
super(options);
this.requestTimeoutInMs =
options === undefined ? undefined : options.requestTimeout;
this.reverseProxyNoSignUrl = reverseProxyNoSignUrl;
}
async handle(
request: HttpRequest,
{ abortSignal }: HttpHandlerOptions = {}
): Promise<{ response: HttpResponse }> {
if (abortSignal?.aborted) {
const abortError = new Error("Request aborted");
abortError.name = "AbortError";
return Promise.reject(abortError);
}
let path = request.path;
if (request.query) {
const queryString = buildQueryString(request.query);
if (queryString) {
path += `?${queryString}`;
}
}
const { port, method } = request;
let url = `${request.protocol}//${request.hostname}${port ? `:${port}` : ""
}${path}`;
if (
this.reverseProxyNoSignUrl !== undefined &&
this.reverseProxyNoSignUrl !== ""
) {
const urlObj = new URL(url);
urlObj.host = this.reverseProxyNoSignUrl;
url = urlObj.href;
}
const body =
method === "GET" || method === "HEAD" ? undefined : request.body;
const transformedHeaders: Record<string, string> = {};
for (const key of Object.keys(request.headers)) {
const keyLower = key.toLowerCase();
if (keyLower === "host" || keyLower === "content-length") {
continue;
}
transformedHeaders[keyLower] = request.headers[key];
}
let contentType: string | undefined = undefined;
if (transformedHeaders["content-type"] !== undefined) {
contentType = transformedHeaders["content-type"];
}
let transformedBody: any = body;
if (ArrayBuffer.isView(body)) {
transformedBody = new Uint8Array(body.buffer).buffer;
}
const param: RequestUrlParam = {
body: transformedBody,
headers: transformedHeaders,
method: method,
url: url,
contentType: contentType,
};
const raceOfPromises = [
requestUrl(param).then((rsp) => {
const headers = rsp.headers;
const headersLower: Record<string, string> = {};
for (const key of Object.keys(headers)) {
headersLower[key.toLowerCase()] = headers[key];
}
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(rsp.arrayBuffer));
controller.close();
},
});
return {
response: new HttpResponse({
headers: headersLower,
statusCode: rsp.status,
body: stream,
}),
};
}),
requestTimeout(this.requestTimeoutInMs),
];
if (abortSignal) {
raceOfPromises.push(
new Promise<never>((resolve, reject) => {
abortSignal.onabort = () => {
const abortError = new Error("Request aborted");
abortError.name = "AbortError";
reject(abortError);
};
})
);
}
return Promise.race(raceOfPromises);
}
}

View File

@@ -1,6 +1,6 @@
import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, MarkdownRenderer, stringifyYaml } from "./deps"; import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, MarkdownRenderer, stringifyYaml } from "./deps";
import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO, type LoadedEntry, PREFERRED_SETTING_CLOUDANT, PREFERRED_SETTING_SELF_HOSTED, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR } from "./lib/src/types"; import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO, type LoadedEntry, PREFERRED_SETTING_CLOUDANT, PREFERRED_SETTING_SELF_HOSTED, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, REMOTE_COUCHDB, REMOTE_MINIO, type BucketSyncSetting, type RemoteType, PREFERRED_JOURNAL_SYNC } from "./lib/src/types";
import { createBlob, delay, isDocContentSame, readAsBlob } from "./lib/src/utils"; import { createBlob, delay, extractObject, isDocContentSame, readAsBlob } from "./lib/src/utils";
import { versionNumberString2Number } from "./lib/src/strbin"; import { versionNumberString2Number } from "./lib/src/strbin";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb"; import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb";
@@ -10,6 +10,7 @@ import { askYesNo, performRebuildDB, requestToCouchDB, scheduleTask } from "./ut
import { request, type ButtonComponent, TFile } from "obsidian"; import { request, type ButtonComponent, TFile } from "obsidian";
import { shouldBeIgnored } from "./lib/src/path"; import { shouldBeIgnored } from "./lib/src/path";
import MultipleRegExpControl from './MultipleRegExpControl.svelte'; import MultipleRegExpControl from './MultipleRegExpControl.svelte';
import { LiveSyncCouchDBReplicator } from "./lib/src/LiveSyncReplicator";
export class ObsidianLiveSyncSettingTab extends PluginSettingTab { export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
@@ -20,17 +21,15 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
super(app, plugin); super(app, plugin);
this.plugin = plugin; this.plugin = plugin;
} }
async testConnection(): Promise<void> { async testConnection(settingOverride: Partial<ObsidianLiveSyncSettings> = {}): Promise<void> {
const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(this.plugin.settings, this.plugin.isMobile, true); const trialSetting = { ...this.plugin.settings, ...settingOverride };
if (typeof db === "string") { const replicator = this.plugin.getNewReplicator(trialSetting);
this.plugin.addLog(`could not connect to ${this.plugin.settings.couchDB_URI} : ${this.plugin.settings.couchDB_DBNAME} \n(${db})`, LOG_LEVEL_NOTICE);
return; await replicator.tryConnectRemote(trialSetting);
} }
this.plugin.addLog(`Connected to ${db.info.db_name}`, LOG_LEVEL_NOTICE); askReload(message?: string) {
}
askReload() {
scheduleTask("configReload", 250, async () => { scheduleTask("configReload", 250, async () => {
if (await askYesNo(this.app, "Do you want to restart and reload Obsidian now?") == "yes") { if (await askYesNo(this.app, message || "Do you want to restart and reload Obsidian now?") == "yes") {
// @ts-ignore // @ts-ignore
this.app.commands.executeCommandById("app:reload") this.app.commands.executeCommandById("app:reload")
} }
@@ -129,6 +128,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
addScreenElement("100", containerInformationEl); addScreenElement("100", containerInformationEl);
const isAnySyncEnabled = (): boolean => { const isAnySyncEnabled = (): boolean => {
if (!this.plugin.settings.isConfigured) return false;
if (this.plugin.settings.liveSync) return true; if (this.plugin.settings.liveSync) return true;
if (this.plugin.settings.periodicReplication) return true; if (this.plugin.settings.periodicReplication) return true;
if (this.plugin.settings.syncOnFileOpen) return true; if (this.plugin.settings.syncOnFileOpen) return true;
@@ -141,6 +141,9 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
return false; return false;
}; };
let inWizard = false; let inWizard = false;
if (containerEl.hasClass("inWizard")) {
inWizard = true;
}
const setupWizardEl = containerEl.createDiv(); const setupWizardEl = containerEl.createDiv();
setupWizardEl.createEl("h3", { text: "Setup wizard" }); setupWizardEl.createEl("h3", { text: "Setup wizard" });
@@ -261,8 +264,129 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
addScreenElement("110", setupWizardEl); addScreenElement("110", setupWizardEl);
const containerRemoteDatabaseEl = containerEl.createDiv(); const containerRemoteDatabaseEl = containerEl.createDiv();
containerRemoteDatabaseEl.createEl("h3", { text: "Remote Database configuration" }); containerRemoteDatabaseEl.createEl("h3", { text: "Remote configuration" });
const syncWarn = containerRemoteDatabaseEl.createEl("div", { text: `These settings are kept locked while any synchronization options are enabled. Disable these options in the "Sync Settings" tab to unlock.` }); new Setting(containerRemoteDatabaseEl)
.setName("Remote Type")
.setDesc("Remote server type")
.addDropdown((dropdown) => {
dropdown
.addOptions({ [REMOTE_COUCHDB]: "CouchDB", [REMOTE_MINIO]: "Minio,S3,R2" })
.setValue(this.plugin.settings.remoteType)
.onChange(async (value) => {
if (this.plugin.settings.remoteType != value) {
if (value != REMOTE_COUCHDB && this.plugin.settings.liveSync) {
this.plugin.settings.liveSync = false;
}
this.plugin.settings.remoteType = value as RemoteType;
await this.plugin.saveSettings();
this.selectedScreen = "";
this.closeSetting();
Logger(`Please reopen the wizard if you have changed the remote type.`, LOG_LEVEL_NOTICE);
}
})
})
let applyDisplayEnabled = () => { }
const editing = extractObject<BucketSyncSetting>({
accessKey: "",
bucket: "",
endpoint: "",
region: "",
secretKey: "",
useCustomRequestHandler: false,
}, this.plugin.settings);
if (this.plugin.settings.remoteType == REMOTE_MINIO) {
const syncWarnMinio = containerRemoteDatabaseEl.createEl("div", {
text: ""
});
const ObjectStorageMessage = `Kindly notice: this is a pretty experimental feature, hence we have some limitations.
- Append only architecture. It will not shrink used storage if we do not perform a rebuild.
- A bit fragile.
- During the first synchronization, the entire history to date will be transferred. For this reason, it is preferable to do this under the WiFi network.
- From the second, we always transfer only differences.
However, your report is needed to stabilise this. I appreciate you for your great dedication.
`;
MarkdownRenderer.render(this.plugin.app, ObjectStorageMessage, syncWarnMinio, "/", this.plugin);
syncWarnMinio.addClass("op-warn-info");
new Setting(containerRemoteDatabaseEl).setName("Endpoint URL").addText((text) =>
text
.setPlaceholder("https://........")
.setValue(editing.endpoint)
.onChange(async (value) => {
editing.endpoint = value;
})
)
new Setting(containerRemoteDatabaseEl).setName("Access Key").addText((text) =>
text
.setPlaceholder("")
.setValue(editing.accessKey)
.onChange(async (value) => {
editing.accessKey = value;
})
)
new Setting(containerRemoteDatabaseEl).setName("Secret Key").addText((text) =>
text
.setPlaceholder("")
.setValue(editing.secretKey)
.onChange(async (value) => {
editing.secretKey = value;
})
.inputEl.setAttribute("type", "password")
)
new Setting(containerRemoteDatabaseEl).setName("Region").addText((text) =>
text
.setPlaceholder("auto")
.setValue(editing.region)
.onChange(async (value) => {
editing.region = value;
})
)
new Setting(containerRemoteDatabaseEl).setName("Bucket Name").addText((text) =>
text
.setPlaceholder("")
.setValue(editing.bucket)
.onChange(async (value) => {
editing.bucket = value;
})
)
new Setting(containerRemoteDatabaseEl).setName("Use Custom HTTP Handler")
.setDesc("If your Object Storage could not configured accepting CORS, enable this.")
.addToggle((toggle) => {
toggle.setValue(editing.useCustomRequestHandler).onChange(async (value) => {
editing.useCustomRequestHandler = value;
})
})
new Setting(containerRemoteDatabaseEl)
.setName("Test Connection")
.addButton((button) =>
button
.setButtonText("Test")
.setDisabled(false)
.onClick(async () => {
await this.testConnection(editing);
})
);
new Setting(containerRemoteDatabaseEl)
.setName("Apply Setting")
.setClass("wizardHidden")
.addButton((button) =>
button
.setButtonText("Apply")
.setDisabled(false)
.onClick(async () => {
this.plugin.settings = { ...this.plugin.settings, ...editing };
await this.plugin.saveSettings();
this.display();
// await this.testConnection();
})
);
} else {
if (this.plugin.settings.couchDB_URI.startsWith("http://")) { if (this.plugin.settings.couchDB_URI.startsWith("http://")) {
if (this.plugin.isMobile) { if (this.plugin.isMobile) {
containerRemoteDatabaseEl.createEl("div", { text: `Configured as using plain HTTP. We cannot connect to the remote. Please set up the credentials and use HTTPS for the remote URI.` }) containerRemoteDatabaseEl.createEl("div", { text: `Configured as using plain HTTP. We cannot connect to the remote. Please set up the credentials and use HTTPS for the remote URI.` })
@@ -272,12 +396,11 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.addClass("op-warn-info"); .addClass("op-warn-info");
} }
} }
const syncWarn = containerRemoteDatabaseEl.createEl("div", { text: `These settings are kept locked while any synchronization options are enabled. Disable these options in the "Sync Settings" tab to unlock.` });
syncWarn.addClass("op-warn-info");
syncWarn.addClass("sls-hidden"); syncWarn.addClass("sls-hidden");
const applyDisplayEnabled = () => { applyDisplayEnabled = () => {
if (isAnySyncEnabled()) { if (isAnySyncEnabled()) {
dbSettings.forEach((e) => { dbSettings.forEach((e) => {
e.setDisabled(true).setTooltip("Could not change this while any synchronization options are enabled."); e.setDisabled(true).setTooltip("Could not change this while any synchronization options are enabled.");
@@ -544,7 +667,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
text: "", text: "",
}); });
containerRemoteDatabaseEl.createEl("h4", { text: "Effective Storage Using" }); containerRemoteDatabaseEl.createEl("h4", { text: "Effective Storage Using" });
new Setting(containerRemoteDatabaseEl) new Setting(containerRemoteDatabaseEl)
.setName("Data Compression (Experimental)") .setName("Data Compression (Experimental)")
@@ -556,6 +678,9 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
this.display(); this.display();
}) })
); );
}
containerRemoteDatabaseEl.createEl("h4", { text: "Confidentiality" }); containerRemoteDatabaseEl.createEl("h4", { text: "Confidentiality" });
@@ -616,6 +741,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.onChange(async (value) => { .onChange(async (value) => {
if (inWizard) { if (inWizard) {
this.plugin.settings.passphrase = value; this.plugin.settings.passphrase = value;
passphrase = value;
await this.plugin.saveSettings(); await this.plugin.saveSettings();
} else { } else {
passphrase = value; passphrase = value;
@@ -633,6 +759,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
toggle.setValue(usePathObfuscation).onChange(async (value) => { toggle.setValue(usePathObfuscation).onChange(async (value) => {
if (inWizard) { if (inWizard) {
this.plugin.settings.usePathObfuscation = value; this.plugin.settings.usePathObfuscation = value;
usePathObfuscation = value;
await this.plugin.saveSettings(); await this.plugin.saveSettings();
} else { } else {
usePathObfuscation = value; usePathObfuscation = value;
@@ -650,6 +777,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.onChange(async (value) => { .onChange(async (value) => {
if (inWizard) { if (inWizard) {
this.plugin.settings.useDynamicIterationCount = value; this.plugin.settings.useDynamicIterationCount = value;
useDynamicIterationCount = value;
await this.plugin.saveSettings(); await this.plugin.saveSettings();
} else { } else {
useDynamicIterationCount = value; useDynamicIterationCount = value;
@@ -695,14 +823,17 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
updateE2EControls(); updateE2EControls();
const checkWorkingPassphrase = async (): Promise<boolean> => { const checkWorkingPassphrase = async (): Promise<boolean> => {
if (this.plugin.settings.remoteType == REMOTE_MINIO) return true;
const settingForCheck: RemoteDBSettings = { const settingForCheck: RemoteDBSettings = {
...this.plugin.settings, ...this.plugin.settings,
encrypt: encrypt, encrypt: encrypt,
passphrase: passphrase, passphrase: passphrase,
useDynamicIterationCount: useDynamicIterationCount, useDynamicIterationCount: useDynamicIterationCount,
}; };
console.dir(settingForCheck); const replicator = this.plugin.getReplicator();
const db = await this.plugin.replicator.connectRemoteCouchDBWithSetting(settingForCheck, this.plugin.isMobile, true); if (!(replicator instanceof LiveSyncCouchDBReplicator)) return true;
const db = await replicator.connectRemoteCouchDBWithSetting(settingForCheck, this.plugin.isMobile, true);
if (typeof db === "string") { if (typeof db === "string") {
Logger("Could not connect to the database.", LOG_LEVEL_NOTICE); Logger("Could not connect to the database.", LOG_LEVEL_NOTICE);
return false; return false;
@@ -788,9 +919,12 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
if (!this.plugin.settings.encrypt) { if (!this.plugin.settings.encrypt) {
this.plugin.settings.passphrase = ""; this.plugin.settings.passphrase = "";
} }
this.plugin.settings = { ...this.plugin.settings, ...editing };
if (isCloudantURI(this.plugin.settings.couchDB_URI)) { if (isCloudantURI(this.plugin.settings.couchDB_URI)) {
// this.plugin.settings.customChunkSize = 0; // this.plugin.settings.customChunkSize = 0;
this.plugin.settings = { ...this.plugin.settings, ...PREFERRED_SETTING_CLOUDANT }; this.plugin.settings = { ...this.plugin.settings, ...PREFERRED_SETTING_CLOUDANT };
} else if (this.plugin.settings.remoteType == REMOTE_MINIO) {
this.plugin.settings = { ...this.plugin.settings, ...PREFERRED_JOURNAL_SYNC };
} else { } else {
this.plugin.settings = { ...this.plugin.settings, ...PREFERRED_SETTING_SELF_HOSTED }; this.plugin.settings = { ...this.plugin.settings, ...PREFERRED_SETTING_SELF_HOSTED };
} }
@@ -1008,12 +1142,13 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
containerSyncSettingEl.createEl("div", containerSyncSettingEl.createEl("div",
{ text: `Please select any preset to complete wizard.` } { text: `Please select any preset to complete wizard.` }
).addClasses(["op-warn-info", "wizardOnly"]); ).addClasses(["op-warn-info", "wizardOnly"]);
const options: Record<string, string> = this.plugin.settings.remoteType == REMOTE_COUCHDB ? { NONE: "", LIVESYNC: "LiveSync", PERIODIC: "Periodic w/ batch", DISABLE: "Disable all automatic" } : { NONE: "", PERIODIC: "Periodic w/ batch", DISABLE: "Disable all automatic" };
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Presets") .setName("Presets")
.setDesc("Apply preset configuration") .setDesc("Apply preset configuration")
.addDropdown((dropdown) => .addDropdown((dropdown) =>
dropdown dropdown
.addOptions({ NONE: "", LIVESYNC: "LiveSync", PERIODIC: "Periodic w/ batch", DISABLE: "Disable all automatic" }) .addOptions(options)
.setValue(currentPreset) .setValue(currentPreset)
.onChange((value) => (currentPreset = value)) .onChange((value) => (currentPreset = value))
) )
@@ -1098,12 +1233,14 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
} else if (this.plugin.settings.periodicReplication) { } else if (this.plugin.settings.periodicReplication) {
syncMode = "PERIODIC"; syncMode = "PERIODIC";
} }
const optionsSyncMode = this.plugin.settings.remoteType == REMOTE_COUCHDB ? { "": "On events", PERIODIC: "Periodic and On events", "LIVESYNC": "LiveSync" } : { "": "On events", PERIODIC: "Periodic and On events" }
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Sync Mode") .setName("Sync Mode")
.setClass("wizardHidden") .setClass("wizardHidden")
.addDropdown((dropdown) => .addDropdown((dropdown) =>
dropdown dropdown
.addOptions({ "": "On events", PERIODIC: "Periodic and On events", "LIVESYNC": "LiveSync" }) .addOptions(optionsSyncMode as Record<string, string>)
.setValue(syncMode) .setValue(syncMode)
.onChange(async (value) => { .onChange(async (value) => {
this.plugin.settings.liveSync = false; this.plugin.settings.liveSync = false;
@@ -1357,6 +1494,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
const pat = this.plugin.settings.syncInternalFilesIgnorePatterns.split(",").map(x => x.trim()).filter(x => x != ""); const pat = this.plugin.settings.syncInternalFilesIgnorePatterns.split(",").map(x => x.trim()).filter(x => x != "");
const patSetting = new Setting(containerSyncSettingEl) const patSetting = new Setting(containerSyncSettingEl)
.setName("Hidden files ignore patterns") .setName("Hidden files ignore patterns")
.setClass("wizardHidden")
.setDesc(""); .setDesc("");
new MultipleRegExpControl( new MultipleRegExpControl(
@@ -1427,6 +1565,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
text.inputEl.setAttribute("type", "number"); text.inputEl.setAttribute("type", "number");
}); });
if (this.plugin.settings.remoteType == REMOTE_COUCHDB) {
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Fetch chunks on demand") .setName("Fetch chunks on demand")
.setDesc("(ex. Read chunks online) If this option is enabled, LiveSync reads chunks online directly instead of replicating them locally. Increasing Custom chunk size is recommended.") .setDesc("(ex. Read chunks online) If this option is enabled, LiveSync reads chunks online directly instead of replicating them locally. Increasing Custom chunk size is recommended.")
@@ -1440,6 +1579,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}) })
return toggle; return toggle;
}); });
}
containerSyncSettingEl.createEl("h4", { containerSyncSettingEl.createEl("h4", {
text: sanitizeHTMLToDom(`Targets`), text: sanitizeHTMLToDom(`Targets`),
@@ -1528,6 +1668,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
} }
); );
} }
if (this.plugin.settings.remoteType == REMOTE_COUCHDB) {
containerSyncSettingEl.createEl("h4", { containerSyncSettingEl.createEl("h4", {
text: sanitizeHTMLToDom(`Advanced settings`), text: sanitizeHTMLToDom(`Advanced settings`),
}).addClass("wizardHidden"); }).addClass("wizardHidden");
@@ -1619,7 +1761,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}); });
text.inputEl.setAttribute("type", "number"); text.inputEl.setAttribute("type", "number");
}); });
}
addScreenElement("30", containerSyncSettingEl); addScreenElement("30", containerSyncSettingEl);
const containerHatchEl = containerEl.createDiv(); const containerHatchEl = containerEl.createDiv();
@@ -1635,6 +1777,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.onClick(async () => { .onClick(async () => {
let responseConfig: any = {}; let responseConfig: any = {};
const REDACTED = "𝑅𝐸𝐷𝐴𝐶𝑇𝐸𝐷"; const REDACTED = "𝑅𝐸𝐷𝐴𝐶𝑇𝐸𝐷";
if (this.plugin.settings.remoteType == REMOTE_COUCHDB) {
try { try {
const r = await requestToCouchDB(this.plugin.settings.couchDB_URI, this.plugin.settings.couchDB_USER, this.plugin.settings.couchDB_PASSWORD, window.origin); const r = await requestToCouchDB(this.plugin.settings.couchDB_URI, this.plugin.settings.couchDB_USER, this.plugin.settings.couchDB_PASSWORD, window.origin);
@@ -1650,6 +1793,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
} catch (ex) { } catch (ex) {
responseConfig = "Requesting information to the remote CouchDB has been failed. If you are using IBM Cloudant, it is the normal behaviour." responseConfig = "Requesting information to the remote CouchDB has been failed. If you are using IBM Cloudant, it is the normal behaviour."
} }
} else if (this.plugin.settings.remoteType == REMOTE_MINIO) {
responseConfig = "Object Storage Synchronisation";
//
}
const pluginConfig = JSON.parse(JSON.stringify(this.plugin.settings)) as ObsidianLiveSyncSettings; const pluginConfig = JSON.parse(JSON.stringify(this.plugin.settings)) as ObsidianLiveSyncSettings;
pluginConfig.couchDB_DBNAME = REDACTED; pluginConfig.couchDB_DBNAME = REDACTED;
pluginConfig.couchDB_PASSWORD = REDACTED; pluginConfig.couchDB_PASSWORD = REDACTED;
@@ -1659,7 +1806,18 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
pluginConfig.passphrase = REDACTED; pluginConfig.passphrase = REDACTED;
pluginConfig.encryptedPassphrase = REDACTED; pluginConfig.encryptedPassphrase = REDACTED;
pluginConfig.encryptedCouchDBConnection = REDACTED; pluginConfig.encryptedCouchDBConnection = REDACTED;
pluginConfig.accessKey = REDACTED;
pluginConfig.secretKey = REDACTED;
pluginConfig.region = `${REDACTED}(${pluginConfig.region.length} letters)`;
pluginConfig.bucket = `${REDACTED}(${pluginConfig.bucket.length} letters)`;
pluginConfig.pluginSyncExtendedSetting = {}; pluginConfig.pluginSyncExtendedSetting = {};
const endpoint = pluginConfig.endpoint;
if (endpoint == "") {
pluginConfig.endpoint = "Not configured or AWS";
} else {
const endpointScheme = pluginConfig.endpoint.startsWith("http:") ? "(HTTP)" : (pluginConfig.endpoint.startsWith("https:")) ? "(HTTPS)" : "";
pluginConfig.endpoint = `${endpoint.indexOf(".r2.cloudflarestorage.") !== -1 ? "R2" : "self-hosted?"}(${endpointScheme})`;
}
const obsidianInfo = navigator.userAgent; const obsidianInfo = navigator.userAgent;
const msgConfig = `---- Obsidian info ---- const msgConfig = `---- Obsidian info ----
${obsidianInfo} ${obsidianInfo}
@@ -2133,13 +2291,13 @@ ${stringifyYaml(pluginConfig)}`;
const containerMaintenanceEl = containerEl.createDiv(); const containerMaintenanceEl = containerEl.createDiv();
containerMaintenanceEl.createEl("h3", { text: "Maintain databases" }); containerMaintenanceEl.createEl("h3", { text: "Maintenance" });
containerMaintenanceEl.createEl("h4", { text: "The remote database" }); containerMaintenanceEl.createEl("h4", { text: "Remote" });
new Setting(containerMaintenanceEl) new Setting(containerMaintenanceEl)
.setName("Lock remote database") .setName("Lock remote")
.setDesc("Lock remote database to prevent synchronization with other devices.") .setDesc("Lock remote to prevent synchronization with other devices.")
.addButton((button) => .addButton((button) =>
button button
.setButtonText("Lock") .setButtonText("Lock")
@@ -2151,8 +2309,8 @@ ${stringifyYaml(pluginConfig)}`;
); );
new Setting(containerMaintenanceEl) new Setting(containerMaintenanceEl)
.setName("Overwrite remote database") .setName("Overwrite remote")
.setDesc("Overwrite remote database with local DB and passphrase.") .setDesc("Overwrite remote with local DB and passphrase.")
.addButton((button) => .addButton((button) =>
button button
.setButtonText("Send") .setButtonText("Send")
@@ -2164,11 +2322,11 @@ ${stringifyYaml(pluginConfig)}`;
) )
containerMaintenanceEl.createEl("h4", { text: "The local database" }); containerMaintenanceEl.createEl("h4", { text: "Local database" });
new Setting(containerMaintenanceEl) new Setting(containerMaintenanceEl)
.setName("Fetch rebuilt DB") .setName("Fetch from remote")
.setDesc("Restore or reconstruct local database from remote database.") .setDesc("Restore or reconstruct local database from remote.")
.addButton((button) => .addButton((button) =>
button button
.setButtonText("Fetch") .setButtonText("Fetch")
@@ -2188,6 +2346,7 @@ ${stringifyYaml(pluginConfig)}`;
}) })
) )
if (this.plugin.settings.remoteType == REMOTE_COUCHDB) {
new Setting(containerMaintenanceEl) new Setting(containerMaintenanceEl)
.setName("Fetch rebuilt DB (Save local documents before)") .setName("Fetch rebuilt DB (Save local documents before)")
.setDesc("Restore or reconstruct local database from remote database but use local chunks.") .setDesc("Restore or reconstruct local database from remote database but use local chunks.")
@@ -2200,7 +2359,7 @@ ${stringifyYaml(pluginConfig)}`;
await rebuildDB("localOnlyWithChunks"); await rebuildDB("localOnlyWithChunks");
}) })
) )
}
new Setting(containerMaintenanceEl) new Setting(containerMaintenanceEl)
.setName("Discard local database to reset or uninstall Self-hosted LiveSync") .setName("Discard local database to reset or uninstall Self-hosted LiveSync")
.addButton((button) => .addButton((button) =>
@@ -2216,6 +2375,7 @@ ${stringifyYaml(pluginConfig)}`;
containerMaintenanceEl.createEl("h4", { text: "Both databases" }); containerMaintenanceEl.createEl("h4", { text: "Both databases" });
if (this.plugin.settings.remoteType == REMOTE_COUCHDB) {
new Setting(containerMaintenanceEl) new Setting(containerMaintenanceEl)
.setName("(Beta2) Clean up databases") .setName("(Beta2) Clean up databases")
.setDesc("Delete unused chunks to shrink the database. This feature requires disabling 'Use an old adapter for compatibility'") .setDesc("Delete unused chunks to shrink the database. This feature requires disabling 'Use an old adapter for compatibility'")
@@ -2234,6 +2394,7 @@ ${stringifyYaml(pluginConfig)}`;
await this.plugin.dbGC(); await this.plugin.dbGC();
}) })
); );
}
new Setting(containerMaintenanceEl) new Setting(containerMaintenanceEl)
.setName("Rebuild everything") .setName("Rebuild everything")
.setDesc("Rebuild local and remote database with local files.") .setDesc("Rebuild local and remote database with local files.")

Submodule src/lib updated: b05e493258...5da1dbc7fc

View File

@@ -2,7 +2,7 @@ const isDebug = false;
import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch, stringifyYaml, parseYaml } from "./deps"; import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch, stringifyYaml, parseYaml } from "./deps";
import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps"; import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, } from "./lib/src/types"; import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, REMOTE_MINIO, REMOTE_COUCHDB, type BucketSyncSetting, } from "./lib/src/types";
import { type InternalFileInfo, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types"; import { type InternalFileInfo, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";
import { arrayToChunkedArray, createBlob, delay, determineTypeFromBlob, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue, throttle } from "./lib/src/utils"; import { arrayToChunkedArray, createBlob, delay, determineTypeFromBlob, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue, throttle } from "./lib/src/utils";
import { Logger, setGlobalLogFunction } from "./lib/src/logger"; import { Logger, setGlobalLogFunction } from "./lib/src/logger";
@@ -20,7 +20,7 @@ import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixe
import { isLockAcquired, serialized, shareRunningResult, skipIfDuplicated } from "./lib/src/lock"; import { isLockAcquired, serialized, shareRunningResult, skipIfDuplicated } from "./lib/src/lock";
import { StorageEventManager, StorageEventManagerObsidian } from "./StorageEventManager"; import { StorageEventManager, StorageEventManagerObsidian } from "./StorageEventManager";
import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "./lib/src/LiveSyncLocalDB"; import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "./lib/src/LiveSyncLocalDB";
import { LiveSyncDBReplicator, type LiveSyncReplicatorEnv } from "./lib/src/LiveSyncReplicator"; import { LiveSyncAbstractReplicator, type LiveSyncReplicatorEnv } from "./lib/src/LiveSyncAbstractReplicator.js";
import { type KeyValueDatabase, OpenKeyValueDatabase } from "./KeyValueDB"; import { type KeyValueDatabase, OpenKeyValueDatabase } from "./KeyValueDB";
import { LiveSyncCommands } from "./LiveSyncCommands"; import { LiveSyncCommands } from "./LiveSyncCommands";
import { HiddenFileSync } from "./CmdHiddenFileSync"; import { HiddenFileSync } from "./CmdHiddenFileSync";
@@ -34,6 +34,11 @@ import { SerializedFileAccess } from "./SerializedFileAccess.js";
import { QueueProcessor } from "./lib/src/processor.js"; import { QueueProcessor } from "./lib/src/processor.js";
import { reactive, reactiveSource } from "./lib/src/reactive.js"; import { reactive, reactiveSource } from "./lib/src/reactive.js";
import { initializeStores } from "./stores.js"; import { initializeStores } from "./stores.js";
import { JournalSyncMinio } from "./lib/src/JournalSyncMinio.js";
import { LiveSyncJournalReplicator, type LiveSyncJournalReplicatorEnv } from "./lib/src/LiveSyncJournalReplicator.js";
import { LiveSyncCouchDBReplicator, type LiveSyncCouchDBReplicatorEnv } from "./lib/src/LiveSyncReplicator.js";
import type { CheckPointInfo, SimpleStore } from "./lib/src/JournalSyncTypes.js";
import { ObsHttpHandler } from "./ObsHttpHandler.js";
setNoticeClass(Notice); setNoticeClass(Notice);
@@ -69,11 +74,16 @@ const SETTING_HEADER = "````yaml:livesync-setting\n";
const SETTING_FOOTER = "\n````"; const SETTING_FOOTER = "\n````";
export default class ObsidianLiveSyncPlugin extends Plugin export default class ObsidianLiveSyncPlugin extends Plugin
implements LiveSyncLocalDBEnv, LiveSyncReplicatorEnv { implements LiveSyncLocalDBEnv, LiveSyncReplicatorEnv, LiveSyncJournalReplicatorEnv, LiveSyncCouchDBReplicatorEnv {
_customHandler!: ObsHttpHandler;
customFetchHandler() {
if (!this._customHandler) this._customHandler = new ObsHttpHandler(undefined, undefined);
return this._customHandler;
}
settings!: ObsidianLiveSyncSettings; settings!: ObsidianLiveSyncSettings;
localDatabase!: LiveSyncLocalDB; localDatabase!: LiveSyncLocalDB;
replicator!: LiveSyncDBReplicator; replicator!: LiveSyncAbstractReplicator;
statusBar?: HTMLElement; statusBar?: HTMLElement;
_suspended = false; _suspended = false;
@@ -308,9 +318,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin
onClose(db: LiveSyncLocalDB): void { onClose(db: LiveSyncLocalDB): void {
this.kvDB.close(); this.kvDB.close();
} }
getNewReplicator(settingOverride: Partial<ObsidianLiveSyncSettings> = {}): LiveSyncAbstractReplicator {
const settings = { ...this.settings, ...settingOverride };
if (settings.remoteType == REMOTE_MINIO) {
return new LiveSyncJournalReplicator(this);
}
return new LiveSyncCouchDBReplicator(this);
}
async onInitializeDatabase(db: LiveSyncLocalDB): Promise<void> { async onInitializeDatabase(db: LiveSyncLocalDB): Promise<void> {
this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv"); this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv");
this.replicator = new LiveSyncDBReplicator(this); this.replicator = this.getNewReplicator();
} }
async onResetDatabase(db: LiveSyncLocalDB): Promise<void> { async onResetDatabase(db: LiveSyncLocalDB): Promise<void> {
const kvDBKey = "queued-files" const kvDBKey = "queued-files"
@@ -318,7 +335,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin
// localStorage.removeItem(lsKey); // localStorage.removeItem(lsKey);
await this.kvDB.destroy(); await this.kvDB.destroy();
this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv"); this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv");
this.replicator = new LiveSyncDBReplicator(this); this.replicator = this.getNewReplicator()
} }
getReplicator() { getReplicator() {
return this.replicator; return this.replicator;
@@ -447,6 +464,52 @@ export default class ObsidianLiveSyncPlugin extends Plugin
} }
Logger(`Checking expired file history done`); Logger(`Checking expired file history done`);
} }
simpleStore: SimpleStore<CheckPointInfo> = {
get: async (key: string) => {
return await this.kvDB.get(`os-${key}`);
},
set: async (key: string, value: any) => {
await this.kvDB.set(`os-${key}`, value);
},
delete: async (key) => {
await this.kvDB.del(`os-${key}`);
},
keys: async (from: string | undefined, to: string | undefined, count?: number | undefined): Promise<string[]> => {
const ret = this.kvDB.keys(IDBKeyRange.bound(`os-${from || ""}`, `os-${to || ""}`), count);
return (await ret).map(e => e.toString());
}
}
getMinioJournalSyncClient() {
const id = this.settings.accessKey
const key = this.settings.secretKey
const bucket = this.settings.bucket
const region = this.settings.region
const endpoint = this.settings.endpoint
const useCustomRequestHandler = this.settings.useCustomRequestHandler;
return new JournalSyncMinio(id, key, endpoint, bucket, this.simpleStore, this, useCustomRequestHandler, region);
}
async resetRemoteBucket() {
const minioJournal = this.getMinioJournalSyncClient();
await minioJournal.resetBucket();
}
async resetJournalSync() {
const minioJournal = this.getMinioJournalSyncClient();
await minioJournal.resetCheckpointInfo();
}
async journalSendTest() {
const minioJournal = this.getMinioJournalSyncClient();
await minioJournal.sendLocalJournal();
}
async journalFetchTest() {
const minioJournal = this.getMinioJournalSyncClient();
await minioJournal.receiveRemoteJournal();
}
async journalSyncTest() {
const minioJournal = this.getMinioJournalSyncClient();
await minioJournal.sync();
}
async onLayoutReady() { async onLayoutReady() {
this.registerFileWatchEvents(); this.registerFileWatchEvents();
if (!this.localDatabase.isReady) { if (!this.localDatabase.isReady) {
@@ -538,7 +601,7 @@ Click anywhere to stop counting down.
await this.realizeSettingSyncMode(); await this.realizeSettingSyncMode();
this.swapSaveCommand(); this.swapSaveCommand();
if (!this.settings.liveSync && this.settings.syncOnStart) { if (!this.settings.liveSync && this.settings.syncOnStart) {
this.replicator.openReplication(this.settings, false, false); this.replicator.openReplication(this.settings, false, false, false);
} }
this.scanStat(); this.scanStat();
} catch (ex) { } catch (ex) {
@@ -619,6 +682,63 @@ Note: We can always able to read V1 format. It will be progressively converted.
this.addOnConfigSync.showPluginSyncModal(); this.addOnConfigSync.showPluginSyncModal();
}).addClass("livesync-ribbon-showcustom"); }).addClass("livesync-ribbon-showcustom");
this.addCommand({
id: "debug-x1",
name: "Journal send",
callback: () => {
this.journalSendTest();
}
});
this.addCommand({
id: "debug-x3",
name: "Journal receive",
callback: () => {
this.journalFetchTest();
}
});
this.addCommand({
id: "debug-x4",
name: "Sync By Journal",
callback: () => {
this.journalSyncTest();
}
});
this.addCommand({
id: "debug-x5",
name: "Reset journal sync",
callback: () => {
this.resetJournalSync();
}
});
this.addCommand({
id: "debug-x6",
name: "Reset journal sync and delete all items on the bucket",
callback: () => {
this.resetRemoteBucket();
}
})
this.addCommand({
id: "debug-x7",
name: "Perform Test",
callback: () => {
// const p = getMockedPouch();
// this.localDatabase.localDatabase.replicate.to(p, { since: 1000, checkpoint: "source" });
}
})
this.addCommand({
id: "debug-x8",
name: "Pack test",
callback: async () => {
const minioJournal = this.getMinioJournalSyncClient();
// const pack = await minioJournal.createJournalPack();
// console.warn();
console.warn(await minioJournal._createJournalPack());
}
})
this.addCommand({ this.addCommand({
id: "view-log", id: "view-log",
name: "Show log", name: "Show log",
@@ -955,17 +1075,19 @@ Note: We can always able to read V1 format. It will be progressively converted.
Logger("Could not determine passphrase for reading data.json! DO NOT synchronize with the remote before making sure your configuration is!", LOG_LEVEL_URGENT); Logger("Could not determine passphrase for reading data.json! DO NOT synchronize with the remote before making sure your configuration is!", LOG_LEVEL_URGENT);
} else { } else {
if (settings.encryptedCouchDBConnection) { if (settings.encryptedCouchDBConnection) {
const keys = ["couchDB_URI", "couchDB_USER", "couchDB_PASSWORD", "couchDB_DBNAME"] as (keyof CouchDBConnection)[]; const keys = ["couchDB_URI", "couchDB_USER", "couchDB_PASSWORD", "couchDB_DBNAME", "accessKey", "bucket", "endpoint", "region", "secretKey"] as (keyof CouchDBConnection | keyof BucketSyncSetting)[];
const decrypted = this.tryDecodeJson(await this.decryptConfigurationItem(settings.encryptedCouchDBConnection, passphrase)) as CouchDBConnection; const decrypted = this.tryDecodeJson(await this.decryptConfigurationItem(settings.encryptedCouchDBConnection, passphrase)) as (CouchDBConnection & BucketSyncSetting);
if (decrypted) { if (decrypted) {
for (const key of keys) { for (const key of keys) {
if (key in decrypted) { if (key in decrypted) {
//@ts-ignore
settings[key] = decrypted[key] settings[key] = decrypted[key]
} }
} }
} else { } else {
Logger("Could not decrypt passphrase for reading data.json! DO NOT synchronize with the remote before making sure your configuration is!", LOG_LEVEL_URGENT); Logger("Could not decrypt passphrase for reading data.json! DO NOT synchronize with the remote before making sure your configuration is!", LOG_LEVEL_URGENT);
for (const key of keys) { for (const key of keys) {
//@ts-ignore
settings[key] = ""; settings[key] = "";
} }
} }
@@ -1022,22 +1144,34 @@ Note: We can always able to read V1 format. It will be progressively converted.
Logger("Could not determine passphrase for saving data.json! Our data.json have insecure items!", LOG_LEVEL_NOTICE); Logger("Could not determine passphrase for saving data.json! Our data.json have insecure items!", LOG_LEVEL_NOTICE);
} else { } else {
if (settings.couchDB_PASSWORD != "" || settings.couchDB_URI != "" || settings.couchDB_USER != "" || settings.couchDB_DBNAME) { if (settings.couchDB_PASSWORD != "" || settings.couchDB_URI != "" || settings.couchDB_USER != "" || settings.couchDB_DBNAME) {
const connectionSetting: CouchDBConnection = { const connectionSetting: CouchDBConnection & BucketSyncSetting = {
couchDB_DBNAME: settings.couchDB_DBNAME, couchDB_DBNAME: settings.couchDB_DBNAME,
couchDB_PASSWORD: settings.couchDB_PASSWORD, couchDB_PASSWORD: settings.couchDB_PASSWORD,
couchDB_URI: settings.couchDB_URI, couchDB_URI: settings.couchDB_URI,
couchDB_USER: settings.couchDB_USER, couchDB_USER: settings.couchDB_USER,
accessKey: settings.accessKey,
bucket: settings.bucket,
endpoint: settings.endpoint,
region: settings.region,
secretKey: settings.secretKey,
useCustomRequestHandler: settings.useCustomRequestHandler
}; };
settings.encryptedCouchDBConnection = await this.encryptConfigurationItem(JSON.stringify(connectionSetting), settings); settings.encryptedCouchDBConnection = await this.encryptConfigurationItem(JSON.stringify(connectionSetting), settings);
settings.couchDB_PASSWORD = ""; settings.couchDB_PASSWORD = "";
settings.couchDB_DBNAME = ""; settings.couchDB_DBNAME = "";
settings.couchDB_URI = ""; settings.couchDB_URI = "";
settings.couchDB_USER = ""; settings.couchDB_USER = "";
settings.accessKey = "";
settings.bucket = "";
settings.region = "";
settings.secretKey = "";
settings.endpoint = "";
} }
if (settings.encrypt && settings.passphrase != "") { if (settings.encrypt && settings.passphrase != "") {
settings.encryptedPassphrase = await this.encryptConfigurationItem(settings.passphrase, settings); settings.encryptedPassphrase = await this.encryptConfigurationItem(settings.passphrase, settings);
settings.passphrase = ""; settings.passphrase = "";
} }
} }
await this.saveData(settings); await this.saveData(settings);
this.localDatabase.settings = this.settings; this.localDatabase.settings = this.settings;
@@ -1293,11 +1427,13 @@ We can perform a command in this file.
// suspend all temporary. // suspend all temporary.
if (this.suspended) return; if (this.suspended) return;
await Promise.all(this.addOns.map(e => e.onResume())); await Promise.all(this.addOns.map(e => e.onResume()));
if (this.settings.remoteType == REMOTE_COUCHDB) {
if (this.settings.liveSync) { if (this.settings.liveSync) {
this.replicator.openReplication(this.settings, true, false); this.replicator.openReplication(this.settings, true, false, false);
}
} }
if (this.settings.syncOnStart) { if (this.settings.syncOnStart) {
this.replicator.openReplication(this.settings, false, false); this.replicator.openReplication(this.settings, false, false, false);
} }
this.periodicSyncProcessor.enable(this.settings.periodicReplication ? this.settings.periodicReplicationInterval * 1000 : 0); this.periodicSyncProcessor.enable(this.settings.periodicReplication ? this.settings.periodicReplicationInterval * 1000 : 0);
} }
@@ -1790,8 +1926,10 @@ We can perform a command in this file.
// disable all sync temporary. // disable all sync temporary.
if (this.suspended) return; if (this.suspended) return;
await Promise.all(this.addOns.map(e => e.onResume())); await Promise.all(this.addOns.map(e => e.onResume()));
if (this.settings.remoteType == REMOTE_COUCHDB) {
if (this.settings.liveSync) { if (this.settings.liveSync) {
this.replicator.openReplication(this.settings, true, false); this.replicator.openReplication(this.settings, true, false, false);
}
} }
const q = activeDocument.querySelector(`.livesync-ribbon-showcustom`); const q = activeDocument.querySelector(`.livesync-ribbon-showcustom`);
@@ -1864,6 +2002,11 @@ We can perform a command in this file.
let pushLast = ""; let pushLast = "";
let pullLast = ""; let pullLast = "";
let w = ""; let w = "";
const labels: Partial<Record<DatabaseConnectingStatus, string>> = {
"CONNECTED": "⚡",
"JOURNAL_SEND": "📦↑",
"JOURNAL_RECEIVE": "📦↓",
}
switch (e.syncStatus) { switch (e.syncStatus) {
case "CLOSED": case "CLOSED":
case "COMPLETED": case "COMPLETED":
@@ -1877,7 +2020,9 @@ We can perform a command in this file.
w = "💤"; w = "💤";
break; break;
case "CONNECTED": case "CONNECTED":
w = "⚡"; case "JOURNAL_SEND":
case "JOURNAL_RECEIVE":
w = labels[e.syncStatus] || "⚡";
pushLast = ((lastSyncPushSeq == 0) ? "" : (lastSyncPushSeq >= maxPushSeq ? " (LIVE)" : ` (${maxPushSeq - lastSyncPushSeq})`)); pushLast = ((lastSyncPushSeq == 0) ? "" : (lastSyncPushSeq >= maxPushSeq ? " (LIVE)" : ` (${maxPushSeq - lastSyncPushSeq})`));
pullLast = ((lastSyncPullSeq == 0) ? "" : (lastSyncPullSeq >= maxPullSeq ? " (LIVE)" : ` (${maxPullSeq - lastSyncPullSeq})`)); pullLast = ((lastSyncPullSeq == 0) ? "" : (lastSyncPullSeq >= maxPullSeq ? " (LIVE)" : ` (${maxPullSeq - lastSyncPullSeq})`));
break; break;
@@ -1957,7 +2102,7 @@ We can perform a command in this file.
await this.applyBatchChange(); await this.applyBatchChange();
await Promise.all(this.addOns.map(e => e.beforeReplicate(showMessage))); await Promise.all(this.addOns.map(e => e.beforeReplicate(showMessage)));
await this.loadQueuedFiles(); await this.loadQueuedFiles();
const ret = await this.replicator.openReplication(this.settings, false, showMessage); const ret = await this.replicator.openReplication(this.settings, false, showMessage, false);
if (!ret) { if (!ret) {
if (this.replicator.remoteLockedAndDeviceNotAccepted) { if (this.replicator.remoteLockedAndDeviceNotAccepted) {
if (this.replicator.remoteCleaned && this.settings.useIndexedDBAdapter) { if (this.replicator.remoteCleaned && this.settings.useIndexedDBAdapter) {
@@ -1977,7 +2122,9 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
await performRebuildDB(this, "localOnly"); await performRebuildDB(this, "localOnly");
} }
if (ret == CHOICE_CLEAN) { if (ret == CHOICE_CLEAN) {
const remoteDB = await this.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.getIsMobile(), true); const replicator = this.getReplicator();
if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.getIsMobile(), true);
if (typeof remoteDB == "string") { if (typeof remoteDB == "string") {
Logger(remoteDB, LOG_LEVEL_NOTICE); Logger(remoteDB, LOG_LEVEL_NOTICE);
return false; return false;
@@ -2211,7 +2358,7 @@ Or if you are sure know what had been happened, we can unlock the database from
const step = 25; const step = 25;
const remainLog = (remain: number) => { const remainLog = (remain: number) => {
if (lastRemain - remain > step) { if (lastRemain - remain > step) {
const msg = ` CHECK AND SYNC: ${remain} / ${allSyncFiles}`; const msg = ` CHECK AND SYNC: ${allSyncFiles - remain} / ${allSyncFiles}`;
updateLog("sync", msg); updateLog("sync", msg);
lastRemain = remain; lastRemain = remain;
} }
@@ -2935,7 +3082,9 @@ Or if you are sure know what had been happened, we can unlock the database from
} }
async dryRunGC() { async dryRunGC() {
await skipIfDuplicated("cleanup", async () => { await skipIfDuplicated("cleanup", async () => {
const remoteDBConn = await this.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.isMobile) const replicator = this.getReplicator();
if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
const remoteDBConn = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.isMobile)
if (typeof (remoteDBConn) == "string") { if (typeof (remoteDBConn) == "string") {
Logger(remoteDBConn); Logger(remoteDBConn);
return; return;
@@ -2949,8 +3098,10 @@ Or if you are sure know what had been happened, we can unlock the database from
async dbGC() { async dbGC() {
// Lock the remote completely once. // Lock the remote completely once.
await skipIfDuplicated("cleanup", async () => { await skipIfDuplicated("cleanup", async () => {
const replicator = this.getReplicator();
if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
this.getReplicator().markRemoteLocked(this.settings, true, true); this.getReplicator().markRemoteLocked(this.settings, true, true);
const remoteDBConn = await this.getReplicator().connectRemoteCouchDBWithSetting(this.settings, this.isMobile) const remoteDBConn = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.isMobile)
if (typeof (remoteDBConn) == "string") { if (typeof (remoteDBConn) == "string") {
Logger(remoteDBConn); Logger(remoteDBConn);
return; return;