From d7d4f1e6f2e6aaf66bcc212fe025ee8070bb6923 Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Sun, 5 May 2024 23:45:45 +0900 Subject: [PATCH] New feature: - We can now use `Incubate Chunks in Document` to reduce non-well-formed chunks. Fixed: - No longer experimental configuration is shown on the Minimal Setup. --- src/lib | 2 +- src/main.ts | 6 + src/ui/ObsidianLiveSyncSettingTab.ts | 164 ++++++++++++++++++++++++--- 3 files changed, 154 insertions(+), 18 deletions(-) diff --git a/src/lib b/src/lib index 4dfdcbe..1417452 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 4dfdcbe2ea6004f74a36058bd4533c3eaaf524bc +Subproject commit 1417452fec9badd89506810eb47fa7bd02468fe8 diff --git a/src/main.ts b/src/main.ts index 9dbc7ee..bc4d539 100644 --- a/src/main.ts +++ b/src/main.ts @@ -39,6 +39,7 @@ import { LiveSyncJournalReplicator, type LiveSyncJournalReplicatorEnv } from "./ import { LiveSyncCouchDBReplicator, type LiveSyncCouchDBReplicatorEnv } from "./lib/src/replication/couchdb/LiveSyncReplicator.js"; import type { CheckPointInfo } from "./lib/src/replication/journal/JournalSyncTypes.js"; import { ObsHttpHandler } from "./common/ObsHttpHandler.js"; +// import { Trench } from "./lib/src/memory/memutil.js"; setNoticeClass(Notice); @@ -327,6 +328,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin } async onInitializeDatabase(db: LiveSyncLocalDB): Promise { this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv"); + // this.trench = new Trench(this.simpleStore); this.replicator = this.getNewReplicator(); } async onResetDatabase(db: LiveSyncLocalDB): Promise { @@ -335,6 +337,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin // localStorage.removeItem(lsKey); await this.kvDB.destroy(); this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv"); + // this.trench = new Trench(this.simpleStore); this.replicator = this.getNewReplicator() } getReplicator() { @@ -480,6 +483,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin return (await ret).map(e => e.toString()).filter(e => e.startsWith("os-")).map(e => e.substring(3)); } } + // trench!: Trench; + getMinioJournalSyncClient() { const id = this.settings.accessKey const key = this.settings.secretKey @@ -2873,6 +2878,7 @@ Or if you are sure know what had been happened, we can unlock the database from children: [], datatype: datatype, type: datatype, + eden: {}, }; //upsert should locked const msg = `STORAGE -> DB (${datatype}) `; diff --git a/src/ui/ObsidianLiveSyncSettingTab.ts b/src/ui/ObsidianLiveSyncSettingTab.ts index 008160c..99b7028 100644 --- a/src/ui/ObsidianLiveSyncSettingTab.ts +++ b/src/ui/ObsidianLiveSyncSettingTab.ts @@ -1,5 +1,26 @@ import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, MarkdownRenderer, stringifyYaml } from "../deps.ts"; -import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO, type LoadedEntry, PREFERRED_SETTING_CLOUDANT, PREFERRED_SETTING_SELF_HOSTED, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, REMOTE_COUCHDB, REMOTE_MINIO, type BucketSyncSetting, type RemoteType, PREFERRED_JOURNAL_SYNC } from "../lib/src/common/types.ts"; +import { + DEFAULT_SETTINGS, + type ObsidianLiveSyncSettings, + type ConfigPassphraseStore, + type RemoteDBSettings, + type FilePathWithPrefix, + type HashAlgorithm, + type DocumentID, + LOG_LEVEL_NOTICE, + LOG_LEVEL_VERBOSE, + LOG_LEVEL_INFO, + type LoadedEntry, + PREFERRED_SETTING_CLOUDANT, + PREFERRED_SETTING_SELF_HOSTED, + FLAGMD_REDFLAG2_HR, + FLAGMD_REDFLAG3_HR, + REMOTE_COUCHDB, + REMOTE_MINIO, + type BucketSyncSetting, + type RemoteType, + PREFERRED_JOURNAL_SYNC +} from "../lib/src/common/types.ts"; import { createBlob, delay, extractObject, isDocContentSame, readAsBlob } from "../lib/src/common/utils.ts"; import { versionNumberString2Number } from "../lib/src/string_and_binary/strbin.ts"; import { Logger } from "../lib/src/common/logger.ts"; @@ -21,12 +42,14 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { super(app, plugin); this.plugin = plugin; } + async testConnection(settingOverride: Partial = {}): Promise { const trialSetting = { ...this.plugin.settings, ...settingOverride }; const replicator = this.plugin.getNewReplicator(trialSetting); await replicator.tryConnectRemote(trialSetting); } + askReload(message?: string) { scheduleTask("configReload", 250, async () => { if (await askYesNo(this.app, message || "Do you want to restart and reload Obsidian now?") == "yes") { @@ -35,10 +58,12 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { } }) } + closeSetting() { // @ts-ignore this.plugin.app.setting.close() } + display(): void { const { containerEl } = this; let encrypt = this.plugin.settings.encrypt; @@ -236,7 +261,11 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { // Render markdown await MarkdownRenderer.render(this.plugin.app, ` [Tips and Troubleshooting](${topPath}) [PageTop](${filename})\n\n${remoteTroubleShootMD}`, troubleShootEl, `${rawRepoURI}`, this.plugin); // Menu - troubleShootEl.querySelector(".sls-troubleshoot-anchor")?.parentElement?.setCssStyles({ position: "sticky", top: "-1em", backgroundColor: "var(--modal-background)" }); + troubleShootEl.querySelector(".sls-troubleshoot-anchor")?.parentElement?.setCssStyles({ + position: "sticky", + top: "-1em", + backgroundColor: "var(--modal-background)" + }); // Trap internal links. troubleShootEl.querySelectorAll("a.internal-link").forEach((anchorEl) => { anchorEl.addEventListener("click", async (evt) => { @@ -286,7 +315,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { }) }) - let applyDisplayEnabled = () => { } + let applyDisplayEnabled = () => { + } const editing = extractObject({ accessKey: "", bucket: "", @@ -480,7 +510,6 @@ However, your report is needed to stabilise this. I appreciate you for your grea await this.plugin.saveSettings(); }) ) - ); new Setting(containerRemoteDatabaseEl) @@ -667,7 +696,71 @@ However, your report is needed to stabilise this. I appreciate you for your grea text: "", }); - containerRemoteDatabaseEl.createEl("h4", { text: "Effective Storage Using" }); + containerRemoteDatabaseEl.createEl("h4", { text: "Effective Storage Using" }).addClass("wizardHidden") + new Setting(containerRemoteDatabaseEl) + .setName("Incubate Chunks in Document") + .setDesc("If enabled, newly created chunks are temporarily kept within the document, and graduated to become independent chunks once stabilised.") + .addToggle((toggle) => + toggle.setValue(this.plugin.settings.useEden).onChange(async (value) => { + this.plugin.settings.useEden = value; + await this.plugin.saveSettings(); + this.display(); + }) + ) + .setClass("wizardHidden"); + if (this.plugin.settings.useEden) { + new Setting(containerRemoteDatabaseEl) + .setName("Maximum Incubating Chunks") + .setDesc("The maximum number of chunks that can be incubated within the document. Chunks exceeding this number will immediately graduate to independent chunks.") + .addText((text) => { + text.setPlaceholder("") + .setValue(this.plugin.settings.maxChunksInEden + "") + .onChange(async (value) => { + let v = Number(value); + if (isNaN(v) || v < 3) { + v = 3; + } + this.plugin.settings.maxChunksInEden = v; + await this.plugin.saveSettings(); + }); + text.inputEl.setAttribute("type", "number"); + }) + .setClass("wizardHidden"); + new Setting(containerRemoteDatabaseEl) + .setName("Maximum Incubating Chunk Size") + .setDesc("The maximum total size of chunks that can be incubated within the document. Chunks exceeding this size will immediately graduate to independent chunks.") + .addText((text) => { + text.setPlaceholder("") + .setValue(this.plugin.settings.maxTotalLengthInEden + "") + .onChange(async (value) => { + let v = Number(value); + if (isNaN(v) || v < 100) { + v = 100; + } + this.plugin.settings.maxTotalLengthInEden = v; + await this.plugin.saveSettings(); + }); + text.inputEl.setAttribute("type", "number"); + }) + .setClass("wizardHidden"); + new Setting(containerRemoteDatabaseEl) + .setName("Maximum Incubation Period") + .setDesc("The maximum duration for which chunks can be incubated within the document. Chunks exceeding this period will graduate to independent chunks.") + .addText((text) => { + text.setPlaceholder("") + .setValue(this.plugin.settings.maxAgeInEden + "") + .onChange(async (value) => { + let v = Number(value); + if (isNaN(v) || v < 3) { + v = 3; + } + this.plugin.settings.maxAgeInEden = v; + await this.plugin.saveSettings(); + }); + text.inputEl.setAttribute("type", "number"); + }) + .setClass("wizardHidden"); + } new Setting(containerRemoteDatabaseEl) .setName("Data Compression (Experimental)") .setDesc("Compresses data during transfer, saving space in the remote database. Note: Please ensure that all devices have v0.22.18 and connected tools are also supported compression.") @@ -677,11 +770,11 @@ However, your report is needed to stabilise this. I appreciate you for your grea await this.plugin.saveSettings(); this.display(); }) - ); + ) + .setClass("wizardHidden"); } - containerRemoteDatabaseEl.createEl("h4", { text: "Confidentiality" }); const e2e = new Setting(containerRemoteDatabaseEl) @@ -1142,7 +1235,12 @@ However, your report is needed to stabilise this. I appreciate you for your grea containerSyncSettingEl.createEl("div", { text: `Please select any preset to complete wizard.` } ).addClasses(["op-warn-info", "wizardOnly"]); - const options: Record = this.plugin.settings.remoteType == REMOTE_COUCHDB ? { NONE: "", LIVESYNC: "LiveSync", PERIODIC: "Periodic w/ batch", DISABLE: "Disable all automatic" } : { NONE: "", PERIODIC: "Periodic w/ batch", DISABLE: "Disable all automatic" }; + const options: Record = this.plugin.settings.remoteType == REMOTE_COUCHDB ? { + NONE: "", + LIVESYNC: "LiveSync", + PERIODIC: "Periodic w/ batch", + DISABLE: "Disable all automatic" + } : { NONE: "", PERIODIC: "Periodic w/ batch", DISABLE: "Disable all automatic" }; new Setting(containerSyncSettingEl) .setName("Presets") .setDesc("Apply preset configuration") @@ -1234,7 +1332,11 @@ However, your report is needed to stabilise this. I appreciate you for your grea syncMode = "PERIODIC"; } - const optionsSyncMode = this.plugin.settings.remoteType == REMOTE_COUCHDB ? { "": "On events", PERIODIC: "Periodic and On events", "LIVESYNC": "LiveSync" } : { "": "On events", PERIODIC: "Periodic and On events" } + const optionsSyncMode = this.plugin.settings.remoteType == REMOTE_COUCHDB ? { + "": "On events", + PERIODIC: "Periodic and On events", + "LIVESYNC": "LiveSync" + } : { "": "On events", PERIODIC: "Periodic and On events" } new Setting(containerSyncSettingEl) .setName("Sync Mode") .setClass("wizardHidden") @@ -1593,7 +1695,9 @@ However, your report is needed to stabilise this. I appreciate you for your grea { target: syncFilesSetting.controlEl, props: { - patterns: this.plugin.settings.syncOnlyRegEx.split("|[]|"), originals: [...this.plugin.settings.syncOnlyRegEx.split("|[]|")], apply: async (newPatterns) => { + patterns: this.plugin.settings.syncOnlyRegEx.split("|[]|"), + originals: [...this.plugin.settings.syncOnlyRegEx.split("|[]|")], + apply: async (newPatterns) => { this.plugin.settings.syncOnlyRegEx = newPatterns.map(e => e.trim()).filter(e => e != "").join("|[]|"); await this.plugin.saveSettings(); this.display(); @@ -1611,7 +1715,9 @@ However, your report is needed to stabilise this. I appreciate you for your grea { target: nonSyncFilesSetting.controlEl, props: { - patterns: this.plugin.settings.syncIgnoreRegEx.split("|[]|"), originals: [...this.plugin.settings.syncIgnoreRegEx.split("|[]|")], apply: async (newPatterns) => { + patterns: this.plugin.settings.syncIgnoreRegEx.split("|[]|"), + originals: [...this.plugin.settings.syncIgnoreRegEx.split("|[]|")], + apply: async (newPatterns) => { this.plugin.settings.syncIgnoreRegEx = newPatterns.map(e => e.trim()).filter(e => e != "").join("|[]|"); await this.plugin.saveSettings(); this.display(); @@ -1875,7 +1981,6 @@ ${stringifyYaml(pluginConfig)}`; hatchWarn.addClass("op-warn-info"); - const addResult = (path: string, file: TFile | false, fileOnDB: LoadedEntry | false) => { resultArea.appendChild(resultArea.createEl("div", {}, el => { el.appendChild(el.createEl("h6", { text: path })); @@ -2044,7 +2149,11 @@ ${stringifyYaml(pluginConfig)}`; .setWarning() .onClick(async () => { Logger(`Deleting customization sync data`, LOG_LEVEL_NOTICE); - const entriesToDelete = (await this.plugin.localDatabase.allDocsRaw({ startkey: "ix:", endkey: "ix:\u{10ffff}", include_docs: true })); + const entriesToDelete = (await this.plugin.localDatabase.allDocsRaw({ + startkey: "ix:", + endkey: "ix:\u{10ffff}", + include_docs: true + })); const newData = entriesToDelete.rows.map(e => ({ ...e.doc, _deleted: true })); const r = await this.plugin.localDatabase.bulkDocsRaw(newData as any[]); // Do not care about the result. @@ -2180,7 +2289,12 @@ ${stringifyYaml(pluginConfig)}`; .setClass("wizardHidden") .addDropdown((dropdown) => dropdown - .addOptions({ "": "Old Algorithm", "xxhash32": "xxhash32 (Fast)", "xxhash64": "xxhash64 (Fastest)", "sha1": "Fallback (Without WebAssembly)" } as Record) + .addOptions({ + "": "Old Algorithm", + "xxhash32": "xxhash32 (Fast)", + "xxhash64": "xxhash64 (Fastest)", + "sha1": "Fallback (Without WebAssembly)" + } as Record) .setValue(this.plugin.settings.hashAlg) .onChange(async (value) => { this.plugin.settings.hashAlg = value as HashAlgorithm; @@ -2331,7 +2445,11 @@ ${stringifyYaml(pluginConfig)}`; .setWarning() .setDisabled(false) .onClick(async () => { - await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, receivedFiles: new Set(), knownIDs: new Set() })); + await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ + ...info, + receivedFiles: new Set(), + knownIDs: new Set() + })); Logger(`Journal received history has been cleared.`, LOG_LEVEL_NOTICE); }) ) @@ -2344,7 +2462,12 @@ ${stringifyYaml(pluginConfig)}`; .setWarning() .setDisabled(false) .onClick(async () => { - await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, lastLocalSeq: 0, sentIDs: new Set(), sentFiles: new Set() })); + await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ + ...info, + lastLocalSeq: 0, + sentIDs: new Set(), + sentFiles: new Set() + })); Logger(`Journal sent history has been cleared.`, LOG_LEVEL_NOTICE); }) ) @@ -2384,7 +2507,14 @@ ${stringifyYaml(pluginConfig)}`; .setWarning() .setDisabled(false) .onClick(async () => { - await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ ...info, receivedFiles: new Set(), knownIDs: new Set(), lastLocalSeq: 0, sentIDs: new Set(), sentFiles: new Set() })); + await this.plugin.getMinioJournalSyncClient().updateCheckPointInfo((info) => ({ + ...info, + receivedFiles: new Set(), + knownIDs: new Set(), + lastLocalSeq: 0, + sentIDs: new Set(), + sentFiles: new Set() + })); await this.plugin.resetRemoteBucket(); Logger(`the bucket has been cleared.`, LOG_LEVEL_NOTICE); })