diff --git a/src/CmdConfigSync.ts b/src/CmdConfigSync.ts index 6f3d4a2..eb7d1b5 100644 --- a/src/CmdConfigSync.ts +++ b/src/CmdConfigSync.ts @@ -4,7 +4,7 @@ import { Notice, type PluginManifest, parseYaml, normalizePath, type ListedFiles import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry, SavingEntry } from "./lib/src/types"; import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "./lib/src/types"; import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types"; -import { createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocData, isDocContentSame } from "./lib/src/utils"; +import { createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocData, isDocContentSame, throttle } from "./lib/src/utils"; import { Logger } from "./lib/src/logger"; import { readString, decodeBinary, arrayBufferToBase64, digestHash } from "./lib/src/strbin"; import { serialized } from "./lib/src/lock"; @@ -305,7 +305,8 @@ export class ConfigSync extends LiveSyncCommands { } return false; } - createMissingConfigurationEntry() { + createMissingConfigurationEntry = throttle(() => this._createMissingConfigurationEntry(), 1000); + _createMissingConfigurationEntry() { let saveRequired = false; for (const v of this.pluginList) { const key = `${v.category}/${v.name}`; @@ -349,8 +350,7 @@ export class ConfigSync extends LiveSyncCommands { Logger(ex, LOG_LEVEL_VERBOSE); } return []; - }, { suspended: false, batchSize: 1, concurrentLimit: 10, delay: 100, yieldThreshold: 10, maintainDelay: false, totalRemainingReactiveSource: pluginScanningCount }).startPipeline().root.onIdle(() => { - // Logger(`All files enumerated`, LOG_LEVEL_INFO, "get-plugins"); + }, { suspended: false, batchSize: 1, concurrentLimit: 10, delay: 100, yieldThreshold: 10, maintainDelay: false, totalRemainingReactiveSource: pluginScanningCount }).startPipeline().root.onUpdateProgress(() => { this.createMissingConfigurationEntry(); }); diff --git a/src/CmdHiddenFileSync.ts b/src/CmdHiddenFileSync.ts index a4a9059..0f05519 100644 --- a/src/CmdHiddenFileSync.ts +++ b/src/CmdHiddenFileSync.ts @@ -9,7 +9,7 @@ import { serialized } from "./lib/src/lock"; import { JsonResolveModal } from "./JsonResolveModal"; import { LiveSyncCommands } from "./LiveSyncCommands"; import { addPrefix, stripAllPrefixes } from "./lib/src/path"; -import { KeyedQueueProcessor, QueueProcessor } from "./lib/src/processor"; +import { QueueProcessor } from "./lib/src/processor"; import { hiddenFilesEventCount, hiddenFilesProcessingCount } from "./lib/src/stores"; export class HiddenFileSync extends LiveSyncCommands { @@ -73,15 +73,15 @@ export class HiddenFileSync extends LiveSyncCommands { } procInternalFile(filename: string) { - this.internalFileProcessor.enqueueWithKey(filename, filename); + this.internalFileProcessor.enqueue(filename); } - internalFileProcessor = new KeyedQueueProcessor( + internalFileProcessor = new QueueProcessor( async (filenames) => { Logger(`START :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE); await this.syncInternalFilesAndDatabase("pull", false, false, filenames); Logger(`DONE :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE); return; - }, { batchSize: 100, concurrentLimit: 1, delay: 10, yieldThreshold: 10, suspended: false, totalRemainingReactiveSource: hiddenFilesEventCount } + }, { batchSize: 100, concurrentLimit: 1, delay: 10, yieldThreshold: 100, suspended: false, totalRemainingReactiveSource: hiddenFilesEventCount } ); recentProcessedInternalFiles = [] as string[]; diff --git a/src/CmdSetupLiveSync.ts b/src/CmdSetupLiveSync.ts index c8964cd..5516fae 100644 --- a/src/CmdSetupLiveSync.ts +++ b/src/CmdSetupLiveSync.ts @@ -50,7 +50,7 @@ export class SetupLiveSync extends LiveSyncCommands { const encryptingPassphrase = await askString(this.app, "Encrypt your settings", "The passphrase to encrypt the setup URI", "", true); if (encryptingPassphrase === false) return; - const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" }; + const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" } as Partial; if (stripExtra) { delete setting.pluginSyncExtendedSetting; } @@ -377,9 +377,6 @@ Of course, we are able to disable these features.` await this.plugin.replicateAllFromServer(true); await delay(1000); await this.plugin.replicateAllFromServer(true); - // if (!tryLessFetching) { - // await this.fetchRemoteChunks(); - // } await this.resumeReflectingDatabase(); await this.askHiddenFileConfiguration({ enableFetch: true }); } diff --git a/src/KeyValueDB.ts b/src/KeyValueDB.ts index d380169..3945688 100644 --- a/src/KeyValueDB.ts +++ b/src/KeyValueDB.ts @@ -6,7 +6,7 @@ export interface KeyValueDatabase { clear(): Promise; keys(query?: IDBValidKey | IDBKeyRange, count?: number): Promise; close(): void; - destroy(): void; + destroy(): Promise; } const databaseCache: { [key: string]: IDBPDatabase } = {}; export const OpenKeyValueDatabase = async (dbKey: string): Promise => { @@ -20,8 +20,7 @@ export const OpenKeyValueDatabase = async (dbKey: string): Promise = null; - db = await dbPromise; + const db = await dbPromise; databaseCache[dbKey] = db; return { get(key: string): Promise { diff --git a/src/MultipleRegExpControl.svelte b/src/MultipleRegExpControl.svelte new file mode 100644 index 0000000..e8b1f7b --- /dev/null +++ b/src/MultipleRegExpControl.svelte @@ -0,0 +1,83 @@ + + +
    + {#each patterns as pattern, idx} +
    • + {/each} +
  • + +
    • +
  • + + +
    • +
+ + diff --git a/src/ObsidianLiveSyncSettingTab.ts b/src/ObsidianLiveSyncSettingTab.ts index 1691ea8..c0f820e 100644 --- a/src/ObsidianLiveSyncSettingTab.ts +++ b/src/ObsidianLiveSyncSettingTab.ts @@ -1,5 +1,5 @@ -import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "./deps"; -import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO, type LoadedEntry, PREFERRED_SETTING_CLOUDANT, PREFERRED_SETTING_SELF_HOSTED } from "./lib/src/types"; +import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, MarkdownRenderer, stringifyYaml } from "./deps"; +import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO, type LoadedEntry, PREFERRED_SETTING_CLOUDANT, PREFERRED_SETTING_SELF_HOSTED, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR } from "./lib/src/types"; import { createBlob, delay, isDocContentSame, readAsBlob } from "./lib/src/utils"; import { versionNumberString2Number } from "./lib/src/strbin"; import { Logger } from "./lib/src/logger"; @@ -9,6 +9,7 @@ import ObsidianLiveSyncPlugin from "./main"; import { askYesNo, performRebuildDB, requestToCouchDB, scheduleTask } from "./utils"; import { request, type ButtonComponent, TFile } from "obsidian"; import { shouldBeIgnored } from "./lib/src/path"; +import MultipleRegExpControl from './MultipleRegExpControl.svelte'; export class ObsidianLiveSyncSettingTab extends PluginSettingTab { @@ -46,11 +47,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { let useDynamicIterationCount = this.plugin.settings.useDynamicIterationCount; containerEl.empty(); - - // const preferred_setting = isCloudantURI(this.plugin.settings.couchDB_URI) ? PREFERRED_SETTING_CLOUDANT : PREFERRED_SETTING_SELF_HOSTED; - // const default_setting = { ...DEFAULT_SETTINGS }; - - containerEl.createEl("h2", { text: "Settings for Self-hosted LiveSync." }); containerEl.addClass("sls-setting"); containerEl.removeClass("isWizard"); @@ -1342,43 +1338,48 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { }); text.inputEl.setAttribute("type", "number"); }); - let skipPatternTextArea: TextAreaComponent; - const defaultSkipPattern = "\\/node_modules\\/, \\/\\.git\\/, \\/obsidian-livesync\\/"; + const defaultSkipPattern = "\\/node_modules\\/, \\/\\.git\\/, ^\\.git\\/, \\/obsidian-livesync\\/"; const defaultSkipPatternXPlat = defaultSkipPattern + ",\\/workspace$ ,\\/workspace.json$,\\/workspace-mobile.json$"; - new Setting(containerSyncSettingEl) - .setName("Folders and files to ignore") - .setDesc( - "Regular expression, If you use hidden file sync between desktop and mobile, adding `workspace$` is recommended." - ) - .setClass("wizardHidden") - .addTextArea((text) => { - text - .setValue(this.plugin.settings.syncInternalFilesIgnorePatterns) - .setPlaceholder("\\/node_modules\\/, \\/\\.git\\/") - .onChange(async (value) => { - this.plugin.settings.syncInternalFilesIgnorePatterns = value; + + const pat = this.plugin.settings.syncInternalFilesIgnorePatterns.split(",").map(x => x.trim()).filter(x => x != ""); + const patSetting = new Setting(containerSyncSettingEl) + .setName("Hidden files ignore patterns") + .setDesc(""); + + new MultipleRegExpControl( + { + target: patSetting.controlEl, + props: { + patterns: pat, originals: [...pat], apply: async (newPatterns) => { + this.plugin.settings.syncInternalFilesIgnorePatterns = newPatterns.map(e => e.trim()).filter(e => e != "").join(", "); await this.plugin.saveSettings(); - }) - skipPatternTextArea = text; - return text; + this.display(); + } + } } - ); + ) + + const addDefaultPatterns = async (patterns: string) => { + const oldList = this.plugin.settings.syncInternalFilesIgnorePatterns.split(",").map(x => x.trim()).filter(x => x != ""); + const newList = patterns.split(",").map(x => x.trim()).filter(x => x != ""); + const allSet = new Set([...oldList, ...newList]); + this.plugin.settings.syncInternalFilesIgnorePatterns = [...allSet].join(", "); + await this.plugin.saveSettings(); + this.display(); + } + new Setting(containerSyncSettingEl) - .setName("Restore the skip pattern to default") + .setName("Add default patterns") .setClass("wizardHidden") .addButton((button) => { button.setButtonText("Default") .onClick(async () => { - skipPatternTextArea.setValue(defaultSkipPattern); - this.plugin.settings.syncInternalFilesIgnorePatterns = defaultSkipPattern; - await this.plugin.saveSettings(); + await addDefaultPatterns(defaultSkipPattern); }) }).addButton((button) => { button.setButtonText("Cross-platform") .onClick(async () => { - skipPatternTextArea.setValue(defaultSkipPatternXPlat); - this.plugin.settings.syncInternalFilesIgnorePatterns = defaultSkipPatternXPlat; - await this.plugin.saveSettings(); + await addDefaultPatterns(defaultSkipPatternXPlat); }) }) @@ -1430,54 +1431,41 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab { containerSyncSettingEl.createEl("h4", { text: sanitizeHTMLToDom(`Targets`), }).addClass("wizardHidden"); - new Setting(containerSyncSettingEl) + + const syncFilesSetting = new Setting(containerSyncSettingEl) .setName("Synchronising files") .setDesc("(RegExp) Empty to sync all files. set filter as a regular expression to limit synchronising files.") .setClass("wizardHidden") - .addTextArea((text) => { - text - .setValue(this.plugin.settings.syncOnlyRegEx) - .setPlaceholder("\\.md$|\\.txt") - .onChange(async (value) => { - let isValidRegExp = false; - try { - new RegExp(value); - isValidRegExp = true; - } catch (_) { - // NO OP. - } - if (isValidRegExp || value.trim() == "") { - this.plugin.settings.syncOnlyRegEx = value; - await this.plugin.saveSettings(); - } - }) - return text; + new MultipleRegExpControl( + { + target: syncFilesSetting.controlEl, + props: { + patterns: this.plugin.settings.syncOnlyRegEx.split("|[]|"), originals: [...this.plugin.settings.syncOnlyRegEx.split("|[]|")], apply: async (newPatterns) => { + this.plugin.settings.syncOnlyRegEx = newPatterns.map(e => e.trim()).filter(e => e != "").join("|[]|"); + await this.plugin.saveSettings(); + this.display(); + } + } } - ); - new Setting(containerSyncSettingEl) + ) + + const nonSyncFilesSetting = new Setting(containerSyncSettingEl) .setName("Non-Synchronising files") .setDesc("(RegExp) If this is set, any changes to local and remote files that match this will be skipped.") - .setClass("wizardHidden") - .addTextArea((text) => { - text - .setValue(this.plugin.settings.syncIgnoreRegEx) - .setPlaceholder("\\.pdf$") - .onChange(async (value) => { - let isValidRegExp = false; - try { - new RegExp(value); - isValidRegExp = true; - } catch (_) { - // NO OP. - } - if (isValidRegExp || value.trim() == "") { - this.plugin.settings.syncIgnoreRegEx = value; - await this.plugin.saveSettings(); - } - }) - return text; + .setClass("wizardHidden"); + + new MultipleRegExpControl( + { + target: nonSyncFilesSetting.controlEl, + props: { + patterns: this.plugin.settings.syncIgnoreRegEx.split("|[]|"), originals: [...this.plugin.settings.syncIgnoreRegEx.split("|[]|")], apply: async (newPatterns) => { + this.plugin.settings.syncIgnoreRegEx = newPatterns.map(e => e.trim()).filter(e => e != "").join("|[]|"); + await this.plugin.saveSettings(); + this.display(); + } + } } - ); + ) new Setting(containerSyncSettingEl) .setName("Maximum file size") .setDesc("(MB) If this is set, changes to local and remote files that are larger than this will be skipped. If the file becomes smaller again, a newer one will be used.") @@ -2173,6 +2161,15 @@ ${stringifyYaml(pluginConfig)}`; .setButtonText("Fetch") .setWarning() .setDisabled(false) + .onClick(async () => { + await this.plugin.vaultAccess.vaultCreate(FLAGMD_REDFLAG3_HR, ""); + this.plugin.performAppReload(); + }) + ).addButton((button) => + button + .setButtonText("Fetch w/o restarting") + .setWarning() + .setDisabled(false) .onClick(async () => { await rebuildDB("localOnly"); }) @@ -2232,10 +2229,21 @@ ${stringifyYaml(pluginConfig)}`; .setButtonText("Rebuild") .setWarning() .setDisabled(false) + .onClick(async () => { + await this.plugin.vaultAccess.vaultCreate(FLAGMD_REDFLAG2_HR, ""); + this.plugin.performAppReload(); + }) + ) + .addButton((button) => + button + .setButtonText("Rebuild w/o restarting") + .setWarning() + .setDisabled(false) .onClick(async () => { await rebuildDB("rebuildBothByThisDevice"); }) ) + applyDisplayEnabled(); addScreenElement("70", containerMaintenanceEl); diff --git a/src/StorageEventManager.ts b/src/StorageEventManager.ts index ca7ec98..00899dc 100644 --- a/src/StorageEventManager.ts +++ b/src/StorageEventManager.ts @@ -2,7 +2,7 @@ import type { SerializedFileAccess } from "./SerializedFileAccess"; import { Plugin, TAbstractFile, TFile, TFolder } from "./deps"; import { Logger } from "./lib/src/logger"; import { shouldBeIgnored } from "./lib/src/path"; -import type { KeyedQueueProcessor } from "./lib/src/processor"; +import type { QueueProcessor } from "./lib/src/processor"; import { LOG_LEVEL_NOTICE, type FilePath, type ObsidianLiveSyncSettings } from "./lib/src/types"; import { delay } from "./lib/src/utils"; import { type FileEventItem, type FileEventType, type FileInfo, type InternalFileInfo } from "./types"; @@ -19,7 +19,7 @@ type LiveSyncForStorageEventManager = Plugin & vaultAccess: SerializedFileAccess } & { isTargetFile: (file: string | TAbstractFile) => Promise, - fileEventQueue: KeyedQueueProcessor, + fileEventQueue: QueueProcessor, isFileSizeExceeded: (size: number) => boolean; }; @@ -133,8 +133,7 @@ export class StorageEventManagerObsidian extends StorageEventManager { path: file.path, size: file.stat.size } as FileInfo : file as InternalFileInfo; - - this.plugin.fileEventQueue.enqueueWithKey(`file-${fileInfo.path}`, { + this.plugin.fileEventQueue.enqueue({ type, args: { file: fileInfo, diff --git a/src/lib b/src/lib index 98809f3..0d21724 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 98809f37df086250702e21033872321ea0ece2ff +Subproject commit 0d217242a8dbc2d052c687c6a5031b5cd638676f diff --git a/src/main.ts b/src/main.ts index d8c9517..3ee8033 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,7 +4,7 @@ import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch, stri import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps"; import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, } from "./lib/src/types"; import { type InternalFileInfo, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types"; -import { arrayToChunkedArray, createBlob, determineTypeFromBlob, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue } from "./lib/src/utils"; +import { arrayToChunkedArray, createBlob, delay, determineTypeFromBlob, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue, throttle } from "./lib/src/utils"; import { Logger, setGlobalLogFunction } from "./lib/src/logger"; import { PouchDB } from "./lib/src/pouchdb-browser.js"; import { ConflictResolveModal } from "./ConflictResolveModal"; @@ -31,7 +31,7 @@ import { GlobalHistoryView, VIEW_TYPE_GLOBAL_HISTORY } from "./GlobalHistoryView import { LogPaneView, VIEW_TYPE_LOG } from "./LogPaneView"; import { LRUCache } from "./lib/src/LRUCache"; import { SerializedFileAccess } from "./SerializedFileAccess.js"; -import { KeyedQueueProcessor, QueueProcessor, type QueueItemWithKey } from "./lib/src/processor.js"; +import { QueueProcessor } from "./lib/src/processor.js"; import { reactive, reactiveSource } from "./lib/src/reactive.js"; import { initializeStores } from "./stores.js"; @@ -312,8 +312,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin this.replicator = new LiveSyncDBReplicator(this); } async onResetDatabase(db: LiveSyncLocalDB): Promise { - const lsKey = "obsidian-livesync-queuefiles-" + this.getVaultName(); - localStorage.removeItem(lsKey); + const kvDBKey = "queued-files" + this.kvDB.del(kvDBKey); + // localStorage.removeItem(lsKey); await this.kvDB.destroy(); this.kvDB = await OpenKeyValueDatabase(db.dbname + "-livesync-kv"); this.replicator = new LiveSyncDBReplicator(this); @@ -535,7 +536,7 @@ Click anywhere to stop counting down. this.registerWatchEvents(); await this.realizeSettingSyncMode(); this.swapSaveCommand(); - if (this.settings.syncOnStart) { + if (!this.settings.liveSync && this.settings.syncOnStart) { this.replicator.openReplication(this.settings, false, false); } this.scanStat(); @@ -1007,7 +1008,7 @@ Note: We can always able to read V1 format. It will be progressively converted. } this.deviceAndVaultName = localStorage.getItem(lsKey) || ""; this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim()); - this.fileEventQueue.delay = this.settings.batchSave ? 5000 : 100; + this.fileEventQueue.delay = (!this.settings.liveSync && this.settings.batchSave) ? 5000 : 100; } async saveSettingData() { @@ -1039,7 +1040,7 @@ Note: We can always able to read V1 format. It will be progressively converted. } await this.saveData(settings); this.localDatabase.settings = this.settings; - this.fileEventQueue.delay = this.settings.batchSave ? 5000 : 100; + this.fileEventQueue.delay = (!this.settings.liveSync && this.settings.batchSave) ? 5000 : 100; this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim()); if (this.settings.settingSyncFile != "") { fireAndForget(() => this.saveSettingToMarkdown(this.settings.settingSyncFile)); @@ -1237,9 +1238,13 @@ We can perform a command in this file. _this.performCommand('editor:save-file'); }; } + hasFocus = true; + isLastHidden = false; registerWatchEvents() { this.registerEvent(this.app.workspace.on("file-open", this.watchWorkspaceOpen)); this.registerDomEvent(document, "visibilitychange", this.watchWindowVisibility); + this.registerDomEvent(window, "focus", () => this.setHasFocus(true)); + this.registerDomEvent(window, "blur", () => this.setHasFocus(false)); this.registerDomEvent(window, "online", this.watchOnline); this.registerDomEvent(window, "offline", this.watchOnline); } @@ -1255,15 +1260,30 @@ We can perform a command in this file. await this.syncAllFiles(); } } + setHasFocus(hasFocus: boolean) { + this.hasFocus = hasFocus; + this.watchWindowVisibility(); + } watchWindowVisibility() { - scheduleTask("watch-window-visibility", 500, () => fireAndForget(() => this.watchWindowVisibilityAsync())); + scheduleTask("watch-window-visibility", 100, () => fireAndForget(() => this.watchWindowVisibilityAsync())); } async watchWindowVisibilityAsync() { if (this.settings.suspendFileWatching) return; if (!this.settings.isConfigured) return; if (!this.isReady) return; + + if (this.isLastHidden && !this.hasFocus) { + // NO OP while non-focused after made hidden; + return; + } + const isHidden = document.hidden; + if (this.isLastHidden === isHidden) { + return; + } + this.isLastHidden = isHidden; + await this.applyBatchChange(); if (isHidden) { this.replicator.closeReplication(); @@ -1283,12 +1303,12 @@ We can perform a command in this file. } cancelRelativeEvent(item: FileEventItem) { - this.fileEventQueue.modifyQueue((items) => [...items.filter(e => e.entity.key != item.key)]) + this.fileEventQueue.modifyQueue((items) => [...items.filter(e => e.key != item.key)]) } - queueNextFileEvent(items: QueueItemWithKey[], newItem: QueueItemWithKey): QueueItemWithKey[] { + queueNextFileEvent(items: FileEventItem[], newItem: FileEventItem): FileEventItem[] { if (this.settings.batchSave && !this.settings.liveSync) { - const file = newItem.entity.args.file; + const file = newItem.args.file; // if the latest event is the same type, omit that // a.md MODIFY <- this should be cancelled when a.md MODIFIED // b.md MODIFY <- this should be cancelled when b.md MODIFIED @@ -1300,16 +1320,16 @@ We can perform a command in this file. while (i >= 0) { i--; if (i < 0) break L1; - if (items[i].entity.args.file.path != file.path) { + if (items[i].args.file.path != file.path) { continue L1; } - if (items[i].entity.type != newItem.entity.type) break L1; + if (items[i].type != newItem.type) break L1; items.remove(items[i]); } } items.push(newItem); // When deleting or renaming, the queue must be flushed once before processing subsequent processes to prevent unexpected race condition. - if (newItem.entity.type == "DELETE" || newItem.entity.type == "RENAME") { + if (newItem.type == "DELETE" || newItem.type == "RENAME") { this.fileEventQueue.requestNextFlush(); } return items; @@ -1363,7 +1383,7 @@ We can perform a command in this file. pendingFileEventCount = reactiveSource(0); processingFileEventCount = reactiveSource(0); fileEventQueue = - new KeyedQueueProcessor( + new QueueProcessor( (items: FileEventItem[]) => this.handleFileEvent(items[0]), { suspended: true, batchSize: 1, concurrentLimit: 5, delay: 100, yieldThreshold: FileWatchEventQueueMax, totalRemainingReactiveSource: this.pendingFileEventCount, processingEntitiesReactiveSource: this.processingFileEventCount } ).replaceEnqueueProcessor((items, newItem) => this.queueNextFileEvent(items, newItem)); @@ -1622,21 +1642,32 @@ We can perform a command in this file. this.conflictCheckQueue.enqueue(path); } + _saveQueuedFiles = throttle(() => { + const saveData = this.replicationResultProcessor._queue.filter(e => e !== undefined && e !== null).map((e) => e?._id ?? "" as string) as string[]; + const kvDBKey = "queued-files" + // localStorage.setItem(lsKey, saveData); + fireAndForget(() => this.kvDB.set(kvDBKey, saveData)); + }, 100); saveQueuedFiles() { - const saveData = JSON.stringify(this.replicationResultProcessor._queue.map((e) => e._id)); - const lsKey = "obsidian-livesync-queuefiles-" + this.getVaultName(); - localStorage.setItem(lsKey, saveData); + this._saveQueuedFiles(); } async loadQueuedFiles() { if (this.settings.suspendParseReplicationResult) return; if (!this.settings.isConfigured) return; - const lsKey = "obsidian-livesync-queuefiles-" + this.getVaultName(); - const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[]; + const kvDBKey = "queued-files" + // const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[]; + const ids = [...new Set(await this.kvDB.get(kvDBKey) ?? [])]; const batchSize = 100; const chunkedIds = arrayToChunkedArray(ids, batchSize); for await (const idsBatch of chunkedIds) { const ret = await this.localDatabase.allDocsRaw({ keys: idsBatch, include_docs: true, limit: 100 }); - this.replicationResultProcessor.enqueueAll(ret.rows.map(doc => doc.doc!)); + const docs = ret.rows.filter(e => e.doc).map(e => e.doc) as PouchDB.Core.ExistingDocument[]; + const errors = ret.rows.filter(e => !e.doc && !e.value.deleted); + if (errors.length > 0) { + Logger("Some queued processes were not resurrected"); + Logger(JSON.stringify(errors), LOG_LEVEL_VERBOSE); + } + this.replicationResultProcessor.enqueueAll(docs); await this.replicationResultProcessor.waitForPipeline(); } @@ -1658,34 +1689,43 @@ We can perform a command in this file. const filename = this.getPathWithoutPrefix(doc); this.isTargetFile(filename).then((ret) => ret ? this.addOnHiddenFileSync.procInternalFile(filename) : Logger(`Skipped (Not target:${filename})`, LOG_LEVEL_VERBOSE)); } else if (isValidPath(this.getPath(doc))) { - this.storageApplyingProcessor.enqueueWithKey(doc.path, doc); + this.storageApplyingProcessor.enqueue(doc); } else { Logger(`Skipped: ${doc._id.substring(0, 8)}`, LOG_LEVEL_VERBOSE); } return; - }, { suspended: true, batchSize: 1, concurrentLimit: 10, yieldThreshold: 1, delay: 0, totalRemainingReactiveSource: this.databaseQueueCount }).startPipeline(); + }, { suspended: true, batchSize: 1, concurrentLimit: 10, yieldThreshold: 1, delay: 0, totalRemainingReactiveSource: this.databaseQueueCount }).replaceEnqueueProcessor((queue, newItem) => { + const q = queue.filter(e => e._id != newItem._id); + return [...q, newItem]; + }).startPipeline(); storageApplyingCount = reactiveSource(0); - storageApplyingProcessor = new KeyedQueueProcessor(async (docs: LoadedEntry[]) => { + storageApplyingProcessor = new QueueProcessor(async (docs: LoadedEntry[]) => { const entry = docs[0]; - const path = this.getPath(entry); - Logger(`Processing ${path} (${entry._id.substring(0, 8)}: ${entry._rev?.substring(0, 5)}) :Started...`, LOG_LEVEL_VERBOSE); - const targetFile = this.vaultAccess.getAbstractFileByPath(this.getPathWithoutPrefix(entry)); - if (targetFile instanceof TFolder) { - Logger(`${this.getPath(entry)} is already exist as the folder`); - } else { - await this.processEntryDoc(entry, targetFile instanceof TFile ? targetFile : undefined); - Logger(`Processing ${path} (${entry._id.substring(0, 8)} :${entry._rev?.substring(0, 5)}) : Done`); - } + await serialized(entry.path, async () => { + const path = this.getPath(entry); + Logger(`Processing ${path} (${entry._id.substring(0, 8)}: ${entry._rev?.substring(0, 5)}) :Started...`, LOG_LEVEL_VERBOSE); + const targetFile = this.vaultAccess.getAbstractFileByPath(this.getPathWithoutPrefix(entry)); + if (targetFile instanceof TFolder) { + Logger(`${this.getPath(entry)} is already exist as the folder`); + } else { + await this.processEntryDoc(entry, targetFile instanceof TFile ? targetFile : undefined); + Logger(`Processing ${path} (${entry._id.substring(0, 8)} :${entry._rev?.substring(0, 5)}) : Done`); + } + }); return; - }, { suspended: true, batchSize: 1, concurrentLimit: 2, yieldThreshold: 1, delay: 0, totalRemainingReactiveSource: this.storageApplyingCount }).startPipeline() + }, { suspended: true, batchSize: 1, concurrentLimit: 6, yieldThreshold: 1, delay: 0, totalRemainingReactiveSource: this.storageApplyingCount }).replaceEnqueueProcessor((queue, newItem) => { + const q = queue.filter(e => e._id != newItem._id); + return [...q, newItem]; + }).startPipeline() replicationResultCount = reactiveSource(0); replicationResultProcessor = new QueueProcessor(async (docs: PouchDB.Core.ExistingDocument[]) => { if (this.settings.suspendParseReplicationResult) return; const change = docs[0]; + if (!change) return; if (isChunk(change._id)) { // SendSignal? // this.parseIncomingChunk(change); @@ -1722,16 +1762,19 @@ We can perform a command in this file. this.databaseQueuedProcessor.enqueue(change); } return; - }, { batchSize: 1, suspended: true, concurrentLimit: 100, delay: 0, totalRemainingReactiveSource: this.replicationResultCount }).startPipeline().onUpdateProgress(() => { + }, { batchSize: 1, suspended: true, concurrentLimit: 100, delay: 0, totalRemainingReactiveSource: this.replicationResultCount }).replaceEnqueueProcessor((queue, newItem) => { + const q = queue.filter(e => e._id != newItem._id); + return [...q, newItem]; + }).startPipeline().onUpdateProgress(() => { this.saveQueuedFiles(); }); //---> Sync parseReplicationResult(docs: Array>) { - if (this.settings.suspendParseReplicationResult) { + if (this.settings.suspendParseReplicationResult && !this.replicationResultProcessor.isSuspended) { this.replicationResultProcessor.suspend() } this.replicationResultProcessor.enqueueAll(docs); - if (!this.settings.suspendParseReplicationResult) { + if (!this.settings.suspendParseReplicationResult && this.replicationResultProcessor.isSuspended) { this.replicationResultProcessor.resume() } } @@ -1761,8 +1804,33 @@ We can perform a command in this file. lastMessage = ""; observeForLogs() { + const padSpaces = `\u{2007}`.repeat(10); + // const emptyMark = `\u{2003}`; + const rerenderTimer = new Map, number]>; + const tick = reactiveSource(0); + function padLeftSp(num: number, mark: string) { + const numLen = `${num}`.length + 1; + const [timer, len] = rerenderTimer.get(mark) ?? [undefined, numLen]; + if (num || timer) { + if (num) { + if (timer) clearTimeout(timer); + rerenderTimer.set(mark, [setTimeout(async () => { + rerenderTimer.delete(mark); + await delay(100); + tick.value = tick.value + 1; + }, 3000), Math.max(len, numLen)]); + } + return ` ${mark}${`${padSpaces}${num}`.slice(-(len))}`; + } else { + return ""; + } + } // const logStore const queueCountLabel = reactive(() => { + // For invalidating + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const _ = tick.value; const dbCount = this.databaseQueueCount.value; const replicationCount = this.replicationResultCount.value; const storageApplyingCount = this.storageApplyingCount.value; @@ -1770,13 +1838,13 @@ We can perform a command in this file. const pluginScanCount = pluginScanningCount.value; const hiddenFilesCount = hiddenFilesEventCount.value + hiddenFilesProcessingCount.value; const conflictProcessCount = this.conflictProcessQueueCount.value; - const labelReplication = replicationCount ? `📥 ${replicationCount} ` : ""; - const labelDBCount = dbCount ? `📄 ${dbCount} ` : ""; - const labelStorageCount = storageApplyingCount ? `💾 ${storageApplyingCount}` : ""; - const labelChunkCount = chunkCount ? `🧩${chunkCount} ` : ""; - const labelPluginScanCount = pluginScanCount ? `🔌${pluginScanCount} ` : ""; - const labelHiddenFilesCount = hiddenFilesCount ? `⚙️${hiddenFilesCount} ` : ""; - const labelConflictProcessCount = conflictProcessCount ? `🔩${conflictProcessCount} ` : ""; + const labelReplication = padLeftSp(replicationCount, `📥`); + const labelDBCount = padLeftSp(dbCount, `📄`); + const labelStorageCount = padLeftSp(storageApplyingCount, `💾`); + const labelChunkCount = padLeftSp(chunkCount, `🧩`); + const labelPluginScanCount = padLeftSp(pluginScanCount, `🔌`); + const labelHiddenFilesCount = padLeftSp(hiddenFilesCount, `⚙️`) + const labelConflictProcessCount = padLeftSp(conflictProcessCount, `🔩`); return `${labelReplication}${labelDBCount}${labelStorageCount}${labelChunkCount}${labelPluginScanCount}${labelHiddenFilesCount}${labelConflictProcessCount}`; }) const requestingStatLabel = reactive(() => { @@ -1821,11 +1889,15 @@ We can perform a command in this file. return { w, sent, pushLast, arrived, pullLast }; }) const waitingLabel = reactive(() => { + // For invalidating + // @ts-ignore + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const _ = tick.value; const e = this.pendingFileEventCount.value; const proc = this.processingFileEventCount.value; const pend = e - proc; - const labelProc = proc != 0 ? `⏳${proc} ` : ""; - const labelPend = pend != 0 ? ` 🛫${pend}` : ""; + const labelProc = padLeftSp(proc, `⏳`); + const labelPend = padLeftSp(pend, `🛫`); return `${labelProc}${labelPend}`; }) const statusLineLabel = reactive(() => { @@ -1834,7 +1906,7 @@ We can perform a command in this file. const waiting = waitingLabel.value; const networkActivity = requestingStatLabel.value; return { - message: `${networkActivity}Sync: ${w} ↑${sent}${pushLast} ↓${arrived}${pullLast}${waiting} ${queued}`, + message: `${networkActivity}Sync: ${w} ↑ ${sent}${pushLast} ↓ ${arrived}${pullLast}${waiting}${queued}`, }; }) const statusBarLabels = reactive(() => { @@ -1845,31 +1917,20 @@ We can perform a command in this file. message, status } }) - let last = 0; - const applyToDisplay = () => { + + const applyToDisplay = throttle(() => { const v = statusBarLabels.value; - const now = Date.now(); - if (now - last < 10) { - scheduleTask("applyToDisplay", 20, () => applyToDisplay()); - return; - } this.applyStatusBarText(v.message, v.status); - last = now; - } + + }, 20); statusBarLabels.onChanged(applyToDisplay); } applyStatusBarText(message: string, log: string) { - const newMsg = message; - const newLog = log; - // scheduleTask("update-display", 50, () => { + const newMsg = message.replace(/\n/g, "\\A "); + const newLog = log.replace(/\n/g, "\\A "); + this.statusBar?.setText(newMsg.split("\n")[0]); - // const selector = `.CodeMirror-wrap,` + - // `.markdown-preview-view.cm-s-obsidian,` + - // `.markdown-source-view.cm-s-obsidian,` + - // `.canvas-wrapper,` + - // `.empty-state` - // ; if (this.settings.showStatusOnEditor) { const root = activeDocument.documentElement; root.style.setProperty("--sls-log-text", "'" + (newMsg + "\\A " + newLog) + "'"); @@ -1877,7 +1938,6 @@ We can perform a command in this file. // const root = activeDocument.documentElement; // root.style.setProperty("--log-text", "'" + (newMsg + "\\A " + newLog) + "'"); } - // }, true); scheduleTask("log-hide", 3000, () => { this.statusLog.value = "" }); @@ -2053,9 +2113,15 @@ Or if you are sure know what had been happened, we can unlock the database from const syncFiles = filesStorage.filter((e) => onlyInStorageNames.indexOf(e.path) == -1); Logger("Updating database by new files"); + const processStatus = {} as Record; + const logLevel = showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; + const updateLog = throttle((key: string, msg: string) => { + processStatus[key] = msg; + const log = Object.values(processStatus).join("\n"); + Logger(log, logLevel, "syncAll"); + }, 25); const initProcess = []; - const logLevel = showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO; const runAll = async(procedureName: string, objects: T[], callback: (arg: T) => Promise) => { if (objects.length == 0) { Logger(`${procedureName}: Nothing to do`); @@ -2077,12 +2143,14 @@ Or if you are sure know what had been happened, we can unlock the database from failed++; } if ((success + failed) % step == 0) { - Logger(`${procedureName}: DONE:${success}, FAILED:${failed}, LAST:${processor._queue.length}`, logLevel, `log-${procedureName}`); + const msg = `${procedureName}: DONE:${success}, FAILED:${failed}, LAST:${processor._queue.length}`; + updateLog(procedureName, msg); } return; }, { batchSize: 1, concurrentLimit: 10, delay: 0, suspended: true }, objects) await processor.waitForPipeline(); - Logger(`${procedureName} All done: DONE:${success}, FAILED:${failed}`, logLevel, `log-${procedureName}`); + const msg = `${procedureName} All done: DONE:${success}, FAILED:${failed}`; + updateLog(procedureName, msg) } initProcess.push(runAll("UPDATE DATABASE", onlyInStorage, async (e) => { if (!this.isFileSizeExceeded(e.stat.size)) { @@ -2116,7 +2184,6 @@ Or if you are sure know what had been happened, we can unlock the database from const id = await this.path2id(getPathFromTFile(file)); const pair: FileDocPair = { file, id }; return [pair]; - // processSyncFile.enqueue(pair); } , { batchSize: 1, concurrentLimit: 10, delay: 0, suspended: true }, syncFiles); processPrepareSyncFile @@ -2138,10 +2205,18 @@ Or if you are sure know what had been happened, we can unlock the database from }, { batchSize: 1, concurrentLimit: 5, delay: 10, suspended: false } )) - processPrepareSyncFile.startPipeline(); - initProcess.push(async () => { - await processPrepareSyncFile.waitForPipeline(); - }) + const allSyncFiles = syncFiles.length; + let lastRemain = allSyncFiles; + const step = 25; + const remainLog = (remain: number) => { + if (lastRemain - remain > step) { + const msg = ` CHECK AND SYNC: ${remain} / ${allSyncFiles}`; + updateLog("sync", msg); + lastRemain = remain; + } + } + processPrepareSyncFile.startPipeline().onUpdateProgress(() => remainLog(processPrepareSyncFile.totalRemaining + processPrepareSyncFile.nowProcessing)) + initProcess.push(processPrepareSyncFile.waitForPipeline()); await Promise.all(initProcess); // this.setStatusBarText(`NOW TRACKING!`); @@ -2501,38 +2576,39 @@ Or if you are sure know what had been happened, we can unlock the database from conflictProcessQueueCount = reactiveSource(0); conflictResolveQueue = - new KeyedQueueProcessor(async (entries: { filename: FilePathWithPrefix }[]) => { - const entry = entries[0]; - const filename = entry.filename; - const conflictCheckResult = await this.checkConflictAndPerformAutoMerge(filename); - if (conflictCheckResult === MISSING_OR_ERROR || conflictCheckResult === NOT_CONFLICTED || conflictCheckResult === CANCELLED) { - // nothing to do. - return; - } - if (conflictCheckResult === AUTO_MERGED) { - //auto resolved, but need check again; - if (this.settings.syncAfterMerge && !this.suspended) { - //Wait for the running replication, if not running replication, run it once. - await shareRunningResult(`replication`, () => this.replicate()); - } - Logger("conflict:Automatically merged, but we have to check it again"); - this.conflictCheckQueue.enqueue(filename); - return; - } - if (this.settings.showMergeDialogOnlyOnActive) { - const af = this.getActiveFile(); - if (af && af.path != filename) { - Logger(`${filename} is conflicted. Merging process has been postponed to the file have got opened.`, LOG_LEVEL_NOTICE); + new QueueProcessor(async (filenames: FilePathWithPrefix[]) => { + const filename = filenames[0]; + await serialized(`conflict-resolve:${filename}`, async () => { + const conflictCheckResult = await this.checkConflictAndPerformAutoMerge(filename); + if (conflictCheckResult === MISSING_OR_ERROR || conflictCheckResult === NOT_CONFLICTED || conflictCheckResult === CANCELLED) { + // nothing to do. return; } - } - Logger("conflict:Manual merge required!"); - await this.resolveConflictByUI(filename, conflictCheckResult); + if (conflictCheckResult === AUTO_MERGED) { + //auto resolved, but need check again; + if (this.settings.syncAfterMerge && !this.suspended) { + //Wait for the running replication, if not running replication, run it once. + await shareRunningResult(`replication`, () => this.replicate()); + } + Logger("conflict:Automatically merged, but we have to check it again"); + this.conflictCheckQueue.enqueue(filename); + return; + } + if (this.settings.showMergeDialogOnlyOnActive) { + const af = this.getActiveFile(); + if (af && af.path != filename) { + Logger(`${filename} is conflicted. Merging process has been postponed to the file have got opened.`, LOG_LEVEL_NOTICE); + return; + } + } + Logger("conflict:Manual merge required!"); + await this.resolveConflictByUI(filename, conflictCheckResult); + }); }, { suspended: false, batchSize: 1, concurrentLimit: 1, delay: 10, keepResultUntilDownstreamConnected: false }).replaceEnqueueProcessor( (queue, newEntity) => { - const filename = newEntity.entity.filename; + const filename = newEntity; sendValue("cancel-resolve-conflict:" + filename, true); - const newQueue = [...queue].filter(e => e.key != newEntity.key); + const newQueue = [...queue].filter(e => e != newEntity); return [...newQueue, newEntity]; }); @@ -2544,10 +2620,9 @@ Or if you are sure know what had been happened, we can unlock the database from const file = this.vaultAccess.getAbstractFileByPath(filename); // if (!file) return; // if (!(file instanceof TFile)) return; - if ((file instanceof TFolder)) return; + if ((file instanceof TFolder)) return []; // Check again? - - return [{ key: filename, entity: { filename } }]; + return [filename]; // this.conflictResolveQueue.enqueueWithKey(filename, { filename, file }); }, { suspended: false, batchSize: 1, concurrentLimit: 5, delay: 10, keepResultUntilDownstreamConnected: true, pipeTo: this.conflictResolveQueue, totalRemainingReactiveSource: this.conflictProcessQueueCount diff --git a/styles.css b/styles.css index f163bd5..96fe576 100644 --- a/styles.css +++ b/styles.css @@ -103,6 +103,9 @@ .canvas-wrapper::before, .empty-state::before { content: var(--sls-log-text, ""); + font-variant-numeric: tabular-nums; + font-variant-emoji: emoji; + tab-size: 4; text-align: right; white-space: pre-wrap; position: absolute;