- No longer batch-saving ignores editor inputs.
- The file-watching and serialisation processes have been changed to the one which is similar to previous implementations.
- We can configure the settings (Especially about text-boxes) even if we have configured the device name.
Improved:
- We can configure the delay of batch-saving.
  - Default: 5 seconds, the same as the previous hard-coded value. (Note: also, the previous behaviour was not correct).
- Also, we can configure the limit of delaying batch-saving.
- The performance of showing status indicators has been improved.
This commit is contained in:
vorotamoroz
2024-06-14 12:35:56 +01:00
parent 16c6c52128
commit 29ed4d2b95
7 changed files with 308 additions and 136 deletions

View File

@@ -68,6 +68,7 @@ Synchronization status is shown in the status bar with the following icons.
- 💾 Working write storage processes - 💾 Working write storage processes
- ⏳ Working read storage processes - ⏳ Working read storage processes
- 🛫 Pending read storage processes - 🛫 Pending read storage processes
- 📬 Batched read storage processes
- ⚙️ Working or pending storage processes of hidden files - ⚙️ Working or pending storage processes of hidden files
- 🧩 Waiting chunks - 🧩 Waiting chunks
- 🔌 Working Customisation items (Configuration, snippets, and plug-ins) - 🔌 Working Customisation items (Configuration, snippets, and plug-ins)

View File

@@ -60,6 +60,9 @@ export type FileEventItem = {
type: FileEventType, type: FileEventType,
args: FileEventArgs, args: FileEventArgs,
key: string, key: string,
skipBatchWait?: boolean,
cancelled?: boolean,
batched?: boolean
} }
// Hidden items (Now means `chunk`) // Hidden items (Now means `chunk`)

Submodule src/lib updated: eb39d533e4...acad314d1a

View File

@@ -3,8 +3,8 @@ const isDebug = false;
import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch, stringifyYaml, parseYaml } from "./deps"; import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch, stringifyYaml, parseYaml } from "./deps";
import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps"; import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, REMOTE_MINIO, REMOTE_COUCHDB, type BucketSyncSetting, TweakValuesShouldMatchedTemplate, confName, type TweakValues, } from "./lib/src/common/types.ts"; import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, REMOTE_MINIO, REMOTE_COUCHDB, type BucketSyncSetting, TweakValuesShouldMatchedTemplate, confName, type TweakValues, } from "./lib/src/common/types.ts";
import { type InternalFileInfo, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./common/types.ts"; import { type InternalFileInfo, type CacheData, type FileEventItem } from "./common/types.ts";
import { arrayToChunkedArray, createBlob, delay, determineTypeFromBlob, escapeMarkdownValue, extractObject, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue, throttle, type SimpleStore } from "./lib/src/common/utils.ts"; import { arrayToChunkedArray, createBlob, determineTypeFromBlob, escapeMarkdownValue, extractObject, fireAndForget, getDocData, isAnyNote, isDocContentSame, isObjectDifferent, readContent, sendValue, throttle, type SimpleStore } from "./lib/src/common/utils.ts";
import { Logger, setGlobalLogFunction } from "./lib/src/common/logger.ts"; import { Logger, setGlobalLogFunction } from "./lib/src/common/logger.ts";
import { PouchDB } from "./lib/src/pouchdb/pouchdb-browser.js"; import { PouchDB } from "./lib/src/pouchdb/pouchdb-browser.js";
import { ConflictResolveModal } from "./ui/ConflictResolveModal.ts"; import { ConflictResolveModal } from "./ui/ConflictResolveModal.ts";
@@ -18,7 +18,7 @@ import { setNoticeClass } from "./lib/src/mock_and_interop/wrapper.ts";
import { versionNumberString2Number, writeString, decodeBinary, readString } from "./lib/src/string_and_binary/convert.ts"; import { versionNumberString2Number, writeString, decodeBinary, readString } from "./lib/src/string_and_binary/convert.ts";
import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/string_and_binary/path.ts"; import { addPrefix, isAcceptedAll, isPlainText, shouldBeIgnored, stripAllPrefixes } from "./lib/src/string_and_binary/path.ts";
import { isLockAcquired, serialized, shareRunningResult, skipIfDuplicated } from "./lib/src/concurrency/lock.ts"; import { isLockAcquired, serialized, shareRunningResult, skipIfDuplicated } from "./lib/src/concurrency/lock.ts";
import { StorageEventManager, StorageEventManagerObsidian } from "./storages/StorageEventManager.ts"; import { StorageEventManager, StorageEventManagerObsidian, type FileEvent } from "./storages/StorageEventManager.ts";
import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "./lib/src/pouchdb/LiveSyncLocalDB.ts"; import { LiveSyncLocalDB, type LiveSyncLocalDBEnv } from "./lib/src/pouchdb/LiveSyncLocalDB.ts";
import { LiveSyncAbstractReplicator, type LiveSyncReplicatorEnv } from "./lib/src/replication/LiveSyncAbstractReplicator.js"; import { LiveSyncAbstractReplicator, type LiveSyncReplicatorEnv } from "./lib/src/replication/LiveSyncAbstractReplicator.js";
import { type KeyValueDatabase, OpenKeyValueDatabase } from "./common/KeyValueDB.ts"; import { type KeyValueDatabase, OpenKeyValueDatabase } from "./common/KeyValueDB.ts";
@@ -32,7 +32,7 @@ import { LogPaneView, VIEW_TYPE_LOG } from "./ui/LogPaneView.ts";
import { LRUCache } from "./lib/src/memory/LRUCache.ts"; import { LRUCache } from "./lib/src/memory/LRUCache.ts";
import { SerializedFileAccess } from "./storages/SerializedFileAccess.js"; import { SerializedFileAccess } from "./storages/SerializedFileAccess.js";
import { QueueProcessor, stopAllRunningProcessors } from "./lib/src/concurrency/processor.js"; import { QueueProcessor, stopAllRunningProcessors } from "./lib/src/concurrency/processor.js";
import { reactive, reactiveSource, type ReactiveValue } from "./lib/src/dataobject/reactive.js"; import { computed, reactive, reactiveSource, type ReactiveValue } from "./lib/src/dataobject/reactive.js";
import { initializeStores } from "./common/stores.js"; import { initializeStores } from "./common/stores.js";
import { JournalSyncMinio } from "./lib/src/replication/journal/objectstore/JournalSyncMinio.js"; import { JournalSyncMinio } from "./lib/src/replication/journal/objectstore/JournalSyncMinio.js";
import { LiveSyncJournalReplicator, type LiveSyncJournalReplicatorEnv } from "./lib/src/replication/journal/LiveSyncJournalReplicator.js"; import { LiveSyncJournalReplicator, type LiveSyncJournalReplicatorEnv } from "./lib/src/replication/journal/LiveSyncJournalReplicator.js";
@@ -99,6 +99,15 @@ export default class ObsidianLiveSyncPlugin extends Plugin
set suspended(value: boolean) { set suspended(value: boolean) {
this._suspended = value; this._suspended = value;
} }
get shouldBatchSave() {
return this.settings?.batchSave && this.settings?.liveSync != true;
}
get batchSaveMinimumDelay(): number {
return this.settings?.batchSaveMinimumDelay ?? DEFAULT_SETTINGS.batchSaveMinimumDelay
}
get batchSaveMaximumDelay(): number {
return this.settings?.batchSaveMaximumDelay ?? DEFAULT_SETTINGS.batchSaveMaximumDelay
}
deviceAndVaultName = ""; deviceAndVaultName = "";
isReady = false; isReady = false;
packageVersion = ""; packageVersion = "";
@@ -895,6 +904,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
const writePiece = piece.substring(1, piece.length - 1) + ","; const writePiece = piece.substring(1, piece.length - 1) + ",";
fireAndForget(async () => { fireAndForget(async () => {
try { try {
await this.vaultAccess.ensureDirectory(this.app.vault.configDir + "/ls-debug/");
await this.vaultAccess.adapterAppend(this.app.vault.configDir + "/ls-debug/" + outFile, writePiece + "\n") await this.vaultAccess.adapterAppend(this.app.vault.configDir + "/ls-debug/" + outFile, writePiece + "\n")
} catch (ex) { } catch (ex) {
Logger(`Could not write ${outFile}`, LOG_LEVEL_VERBOSE); Logger(`Could not write ${outFile}`, LOG_LEVEL_VERBOSE);
@@ -946,6 +956,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
localStorage.setItem(lsKey, `${VER}`); localStorage.setItem(lsKey, `${VER}`);
await this.openDatabase(); await this.openDatabase();
this.watchWorkspaceOpen = this.watchWorkspaceOpen.bind(this); this.watchWorkspaceOpen = this.watchWorkspaceOpen.bind(this);
this.watchEditorChange = this.watchEditorChange.bind(this);
this.watchWindowVisibility = this.watchWindowVisibility.bind(this) this.watchWindowVisibility = this.watchWindowVisibility.bind(this)
this.watchOnline = this.watchOnline.bind(this); this.watchOnline = this.watchOnline.bind(this);
this.realizeSettingSyncMode = this.realizeSettingSyncMode.bind(this); this.realizeSettingSyncMode = this.realizeSettingSyncMode.bind(this);
@@ -1136,7 +1147,6 @@ Note: We can always able to read V1 format. It will be progressively converted.
} }
this.deviceAndVaultName = localStorage.getItem(lsKey) || ""; this.deviceAndVaultName = localStorage.getItem(lsKey) || "";
this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim()); this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim());
this.fileEventQueue.delay = (!this.settings.liveSync && this.settings.batchSave) ? 5000 : 100;
this.settingTab.requestReload() this.settingTab.requestReload()
} }
@@ -1185,7 +1195,6 @@ Note: We can always able to read V1 format. It will be progressively converted.
this.localDatabase.settings = this.settings; this.localDatabase.settings = this.settings;
setLang(this.settings.displayLanguage); setLang(this.settings.displayLanguage);
this.settingTab.requestReload(); this.settingTab.requestReload();
this.fileEventQueue.delay = (!this.settings.liveSync && this.settings.batchSave) ? 5000 : 100;
this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim()); this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim());
if (this.settings.settingSyncFile != "") { if (this.settings.settingSyncFile != "") {
fireAndForget(() => this.saveSettingToMarkdown(this.settings.settingSyncFile)); fireAndForget(() => this.saveSettingToMarkdown(this.settings.settingSyncFile));
@@ -1351,6 +1360,7 @@ We can perform a command in this file.
vaultManager: StorageEventManager = new StorageEventManagerObsidian(this); vaultManager: StorageEventManager = new StorageEventManagerObsidian(this);
registerFileWatchEvents() { registerFileWatchEvents() {
this.vaultManager.beginWatch(); this.vaultManager.beginWatch();
this.registerEvent(this.app.workspace.on("editor-change", this.watchEditorChange));
} }
_initialCallback: any; _initialCallback: any;
swapSaveCommand() { swapSaveCommand() {
@@ -1368,8 +1378,8 @@ We can perform a command in this file.
saveCommandDefinition.callback = this._initialCallback; saveCommandDefinition.callback = this._initialCallback;
this._initialCallback = undefined; this._initialCallback = undefined;
} else { } else {
Logger("Sync on Editor Save.", LOG_LEVEL_VERBOSE);
if (this.settings.syncOnEditorSave) { if (this.settings.syncOnEditorSave) {
Logger("Sync on Editor Save.", LOG_LEVEL_VERBOSE);
this.replicate(); this.replicate();
} }
} }
@@ -1452,37 +1462,10 @@ We can perform a command in this file.
} }
cancelRelativeEvent(item: FileEventItem) { cancelRelativeEvent(item: FileEventItem) {
this.fileEventQueue.modifyQueue((items) => [...items.filter(e => e.key != item.key)]) this.vaultManager.cancelQueue(item.key);
} }
queueNextFileEvent(items: FileEventItem[], newItem: FileEventItem): FileEventItem[] {
if (this.settings.batchSave && !this.settings.liveSync) {
const file = newItem.args.file;
// if the latest event is the same type, omit that
// a.md MODIFY <- this should be cancelled when a.md MODIFIED
// b.md MODIFY <- this should be cancelled when b.md MODIFIED
// a.md MODIFY
// a.md CREATE
// :
let i = items.length;
L1:
while (i >= 0) {
i--;
if (i < 0) break L1;
if (items[i].args.file.path != file.path) {
continue L1;
}
if (items[i].type != newItem.type) break L1;
items.remove(items[i]);
}
}
items.push(newItem);
// When deleting or renaming, the queue must be flushed once before processing subsequent processes to prevent unexpected race condition.
if (newItem.type == "DELETE" || newItem.type == "RENAME") {
this.fileEventQueue.requestNextFlush();
}
return items;
}
async handleFileEvent(queue: FileEventItem): Promise<any> { async handleFileEvent(queue: FileEventItem): Promise<any> {
const file = queue.args.file; const file = queue.args.file;
const lockKey = `handleFile:${file.path}`; const lockKey = `handleFile:${file.path}`;
@@ -1536,21 +1519,8 @@ We can perform a command in this file.
pendingFileEventCount = reactiveSource(0); pendingFileEventCount = reactiveSource(0);
processingFileEventCount = reactiveSource(0); processingFileEventCount = reactiveSource(0);
fileEventQueue =
new QueueProcessor(
async (items: FileEventItem[]) => {
await this.handleFileEvent(items[0]);
return []
}
,
{ suspended: true, batchSize: 1, concurrentLimit: 5, delay: 100, yieldThreshold: FileWatchEventQueueMax, totalRemainingReactiveSource: this.pendingFileEventCount, processingEntitiesReactiveSource: this.processingFileEventCount }
).replaceEnqueueProcessor((items, newItem) => this.queueNextFileEvent(items, newItem));
flushFileEventQueue() {
return this.fileEventQueue.flush();
}
watchWorkspaceOpen(file: TFile | null) { watchWorkspaceOpen(file: TFile | null) {
if (this.settings.suspendFileWatching) return; if (this.settings.suspendFileWatching) return;
if (!this.settings.isConfigured) return; if (!this.settings.isConfigured) return;
@@ -1559,6 +1529,34 @@ We can perform a command in this file.
scheduleTask("watch-workspace-open", 500, () => fireAndForget(() => this.watchWorkspaceOpenAsync(file))); scheduleTask("watch-workspace-open", 500, () => fireAndForget(() => this.watchWorkspaceOpenAsync(file)));
} }
flushFileEventQueue() {
return this.vaultManager.flushQueue();
}
watchEditorChange(editor: Editor, info: any) {
if (!("path" in info)) {
return;
}
if (!this.shouldBatchSave) {
return;
}
const file = info?.file as TFile;
if (!file) return;
if (!this.vaultManager.isWaiting(file.path as FilePath)) {
return;
}
const data = info?.data as string;
const fi: FileEvent = {
type: "CHANGED",
file: file,
cachedData: data,
}
this.vaultManager.appendQueue([
fi
])
}
async watchWorkspaceOpenAsync(file: TFile) { async watchWorkspaceOpenAsync(file: TFile) {
if (this.settings.suspendFileWatching) return; if (this.settings.suspendFileWatching) return;
if (!this.settings.isConfigured) return; if (!this.settings.isConfigured) return;
@@ -1966,53 +1964,44 @@ We can perform a command in this file.
observeForLogs() { observeForLogs() {
const padSpaces = `\u{2007}`.repeat(10); const padSpaces = `\u{2007}`.repeat(10);
// const emptyMark = `\u{2003}`; // const emptyMark = `\u{2003}`;
const rerenderTimer = new Map<string, [ReturnType<typeof setTimeout>, number]>(); function padLeftSpComputed(numI: ReactiveValue<number>, mark: string) {
const tick = reactiveSource(0); const formatted = reactiveSource("");
function padLeftSp(num: number, mark: string) { let timer: ReturnType<typeof setTimeout> | undefined = undefined;
const numLen = `${num}`.length + 1; let maxLen = 1;
const [timer, len] = rerenderTimer.get(mark) ?? [undefined, numLen]; numI.onChanged(numX => {
if (num || timer) { const num = numX.value;
if (num) { const numLen = `${Math.abs(num)}`.length + 1;
if (timer) clearTimeout(timer); maxLen = maxLen < numLen ? numLen : maxLen;
rerenderTimer.set(mark, [setTimeout(async () => { if (timer) clearTimeout(timer);
rerenderTimer.delete(mark); if (num == 0) {
await delay(100); timer = setTimeout(() => {
tick.value = tick.value + 1; formatted.value = "";
}, 3000), Math.max(len, numLen)]); maxLen = 1;
}, 3000);
} }
return ` ${mark}${`${padSpaces}${num}`.slice(-(len))}`; formatted.value = ` ${mark}${`${padSpaces}${num}`.slice(-(maxLen))}`;
} else { })
return ""; return computed(() => formatted.value);
}
} }
// const logStore const labelReplication = padLeftSpComputed(this.replicationResultCount, `📥`);
const queueCountLabel = reactive(() => { const labelDBCount = padLeftSpComputed(this.databaseQueueCount, `📄`);
// For invalidating const labelStorageCount = padLeftSpComputed(this.storageApplyingCount, `💾`);
// @ts-ignore const labelChunkCount = padLeftSpComputed(collectingChunks, `🧩`);
// eslint-disable-next-line @typescript-eslint/no-unused-vars const labelPluginScanCount = padLeftSpComputed(pluginScanningCount, `🔌`);
const _ = tick.value; const labelConflictProcessCount = padLeftSpComputed(this.conflictProcessQueueCount, `🔩`);
const dbCount = this.databaseQueueCount.value; const hiddenFilesCount = reactive(() => hiddenFilesEventCount.value + hiddenFilesProcessingCount.value);
const replicationCount = this.replicationResultCount.value; const labelHiddenFilesCount = padLeftSpComputed(hiddenFilesCount, `⚙️`)
const storageApplyingCount = this.storageApplyingCount.value; const queueCountLabelX = reactive(() => {
const chunkCount = collectingChunks.value; return `${labelReplication()}${labelDBCount()}${labelStorageCount()}${labelChunkCount()}${labelPluginScanCount()}${labelHiddenFilesCount()}${labelConflictProcessCount()}`;
const pluginScanCount = pluginScanningCount.value;
const hiddenFilesCount = hiddenFilesEventCount.value + hiddenFilesProcessingCount.value;
const conflictProcessCount = this.conflictProcessQueueCount.value;
const labelReplication = padLeftSp(replicationCount, `📥`);
const labelDBCount = padLeftSp(dbCount, `📄`);
const labelStorageCount = padLeftSp(storageApplyingCount, `💾`);
const labelChunkCount = padLeftSp(chunkCount, `🧩`);
const labelPluginScanCount = padLeftSp(pluginScanCount, `🔌`);
const labelHiddenFilesCount = padLeftSp(hiddenFilesCount, `⚙️`)
const labelConflictProcessCount = padLeftSp(conflictProcessCount, `🔩`);
return `${labelReplication}${labelDBCount}${labelStorageCount}${labelChunkCount}${labelPluginScanCount}${labelHiddenFilesCount}${labelConflictProcessCount}`;
}) })
const requestingStatLabel = reactive(() => { const queueCountLabel = () => queueCountLabelX.value;
const requestingStatLabel = computed(() => {
const diff = this.requestCount.value - this.responseCount.value; const diff = this.requestCount.value - this.responseCount.value;
return diff != 0 ? "📲 " : ""; return diff != 0 ? "📲 " : "";
}) })
const replicationStatLabel = reactive(() => { const replicationStatLabel = computed(() => {
const e = this.replicationStat.value; const e = this.replicationStat.value;
const sent = e.sent; const sent = e.sent;
const arrived = e.arrived; const arrived = e.arrived;
@@ -2055,42 +2044,36 @@ We can perform a command in this file.
} }
return { w, sent, pushLast, arrived, pullLast }; return { w, sent, pushLast, arrived, pullLast };
}) })
const waitingLabel = reactive(() => { const labelProc = padLeftSpComputed(this.vaultManager.processing, ``);
// For invalidating const labelPend = padLeftSpComputed(this.vaultManager.totalQueued, `🛫`);
// @ts-ignore const labelInBatchDelay = padLeftSpComputed(this.vaultManager.batched, `📬`);
// eslint-disable-next-line @typescript-eslint/no-unused-vars const waitingLabel = computed(() => {
const _ = tick.value; return `${labelProc()}${labelPend()}${labelInBatchDelay()}`;
const e = this.pendingFileEventCount.value;
const proc = this.processingFileEventCount.value;
const pend = e - proc;
const labelProc = padLeftSp(proc, ``);
const labelPend = padLeftSp(pend, `🛫`);
return `${labelProc}${labelPend}`;
}) })
const statusLineLabel = reactive(() => { const statusLineLabel = computed(() => {
const { w, sent, pushLast, arrived, pullLast } = replicationStatLabel.value; const { w, sent, pushLast, arrived, pullLast } = replicationStatLabel();
const queued = queueCountLabel.value; const queued = queueCountLabel();
const waiting = waitingLabel.value; const waiting = waitingLabel();
const networkActivity = requestingStatLabel.value; const networkActivity = requestingStatLabel();
return { return {
message: `${networkActivity}Sync: ${w}${sent}${pushLast}${arrived}${pullLast}${waiting}${queued}`, message: `${networkActivity}Sync: ${w}${sent}${pushLast}${arrived}${pullLast}${waiting}${queued}`,
}; };
}) })
const statusBarLabels = reactive(() => { const statusBarLabels = reactive(() => {
const scheduleMessage = this.isReloadingScheduled ? `WARNING! RESTARTING OBSIDIAN IS SCHEDULED!\n` : ""; const scheduleMessage = this.isReloadingScheduled ? `WARNING! RESTARTING OBSIDIAN IS SCHEDULED!\n` : "";
const { message } = statusLineLabel.value; const { message } = statusLineLabel();
const status = scheduleMessage + this.statusLog.value; const status = scheduleMessage + this.statusLog.value;
return { return {
message, status message, status
} }
}) })
const applyToDisplay = throttle(() => { const applyToDisplay = throttle((label: typeof statusBarLabels.value) => {
const v = statusBarLabels.value; const v = label;
this.applyStatusBarText(v.message, v.status); this.applyStatusBarText(v.message, v.status);
}, 20); }, 20);
statusBarLabels.onChanged(applyToDisplay); statusBarLabels.onChanged(label => applyToDisplay(label.value))
} }
applyStatusBarText(message: string, log: string) { applyStatusBarText(message: string, log: string) {
@@ -2106,7 +2089,6 @@ We can perform a command in this file.
// root.style.setProperty("--log-text", "'" + (newMsg + "\\A " + newLog) + "'"); // root.style.setProperty("--log-text", "'" + (newMsg + "\\A " + newLog) + "'");
} }
scheduleTask("log-hide", 3000, () => { this.statusLog.value = "" }); scheduleTask("log-hide", 3000, () => { this.statusLog.value = "" });
} }
async askResolvingMismatchedTweaks(): Promise<"OK" | "CHECKAGAIN" | "IGNORE"> { async askResolvingMismatchedTweaks(): Promise<"OK" | "CHECKAGAIN" | "IGNORE"> {

View File

@@ -2,14 +2,34 @@ import type { SerializedFileAccess } from "./SerializedFileAccess.ts";
import { Plugin, TAbstractFile, TFile, TFolder } from "../deps.ts"; import { Plugin, TAbstractFile, TFile, TFolder } from "../deps.ts";
import { Logger } from "../lib/src/common/logger.ts"; import { Logger } from "../lib/src/common/logger.ts";
import { shouldBeIgnored } from "../lib/src/string_and_binary/path.ts"; import { shouldBeIgnored } from "../lib/src/string_and_binary/path.ts";
import type { QueueProcessor } from "../lib/src/concurrency/processor.ts"; import { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, type FilePath, type ObsidianLiveSyncSettings } from "../lib/src/common/types.ts";
import { LOG_LEVEL_NOTICE, type FilePath, type ObsidianLiveSyncSettings } from "../lib/src/common/types.ts"; import { delay, fireAndForget } from "../lib/src/common/utils.ts";
import { delay } from "../lib/src/common/utils.ts";
import { type FileEventItem, type FileEventType, type FileInfo, type InternalFileInfo } from "../common/types.ts"; import { type FileEventItem, type FileEventType, type FileInfo, type InternalFileInfo } from "../common/types.ts";
import { skipIfDuplicated } from "../lib/src/concurrency/lock.ts";
import { finishAllWaitingForTimeout, finishWaitingForTimeout, isWaitingForTimeout, waitForTimeout } from "../lib/src/concurrency/task.ts";
import { reactiveSource, type ReactiveSource } from "../lib/src/dataobject/reactive.ts";
import { Semaphore } from "../lib/src/concurrency/semaphore.ts";
export type FileEvent = {
type: FileEventType;
file: TAbstractFile | InternalFileInfo;
oldPath?: string;
cachedData?: string;
skipBatchWait?: boolean;
};
export abstract class StorageEventManager { export abstract class StorageEventManager {
abstract beginWatch(): void; abstract beginWatch(): void;
abstract flushQueue(): void;
abstract appendQueue(items: FileEvent[], ctx?: any): void;
abstract cancelQueue(key: string): void;
abstract isWaiting(filename: FilePath): boolean;
abstract totalQueued: ReactiveSource<number>;
abstract batched: ReactiveSource<number>;
abstract processing: ReactiveSource<number>;
} }
type LiveSyncForStorageEventManager = Plugin & type LiveSyncForStorageEventManager = Plugin &
@@ -17,15 +37,33 @@ type LiveSyncForStorageEventManager = Plugin &
settings: ObsidianLiveSyncSettings settings: ObsidianLiveSyncSettings
ignoreFiles: string[], ignoreFiles: string[],
vaultAccess: SerializedFileAccess vaultAccess: SerializedFileAccess
shouldBatchSave: boolean
batchSaveMinimumDelay: number;
batchSaveMaximumDelay: number;
} & { } & {
isTargetFile: (file: string | TAbstractFile) => Promise<boolean>, isTargetFile: (file: string | TAbstractFile) => Promise<boolean>,
fileEventQueue: QueueProcessor<FileEventItem, any>, // fileEventQueue: QueueProcessor<FileEventItem, any>,
handleFileEvent: (queue: FileEventItem) => Promise<any>,
isFileSizeExceeded: (size: number) => boolean; isFileSizeExceeded: (size: number) => boolean;
}; };
export class StorageEventManagerObsidian extends StorageEventManager { export class StorageEventManagerObsidian extends StorageEventManager {
totalQueued = reactiveSource(0);
batched = reactiveSource(0);
processing = reactiveSource(0);
plugin: LiveSyncForStorageEventManager; plugin: LiveSyncForStorageEventManager;
get shouldBatchSave() {
return this.plugin.shouldBatchSave;
}
get batchSaveMinimumDelay(): number {
return this.plugin.batchSaveMinimumDelay;
}
get batchSaveMaximumDelay(): number {
return this.plugin.batchSaveMaximumDelay
}
constructor(plugin: LiveSyncForStorageEventManager) { constructor(plugin: LiveSyncForStorageEventManager) {
super(); super();
this.plugin = plugin; this.plugin = plugin;
@@ -43,25 +81,25 @@ export class StorageEventManagerObsidian extends StorageEventManager {
plugin.registerEvent(plugin.app.vault.on("create", this.watchVaultCreate)); plugin.registerEvent(plugin.app.vault.on("create", this.watchVaultCreate));
//@ts-ignore : Internal API //@ts-ignore : Internal API
plugin.registerEvent(plugin.app.vault.on("raw", this.watchVaultRawEvents)); plugin.registerEvent(plugin.app.vault.on("raw", this.watchVaultRawEvents));
plugin.fileEventQueue.startPipeline(); // plugin.fileEventQueue.startPipeline();
} }
watchVaultCreate(file: TAbstractFile, ctx?: any) { watchVaultCreate(file: TAbstractFile, ctx?: any) {
this.appendWatchEvent([{ type: "CREATE", file }], ctx); this.appendQueue([{ type: "CREATE", file }], ctx);
} }
watchVaultChange(file: TAbstractFile, ctx?: any) { watchVaultChange(file: TAbstractFile, ctx?: any) {
this.appendWatchEvent([{ type: "CHANGED", file }], ctx); this.appendQueue([{ type: "CHANGED", file }], ctx);
} }
watchVaultDelete(file: TAbstractFile, ctx?: any) { watchVaultDelete(file: TAbstractFile, ctx?: any) {
this.appendWatchEvent([{ type: "DELETE", file }], ctx); this.appendQueue([{ type: "DELETE", file }], ctx);
} }
watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) { watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) {
if (file instanceof TFile) { if (file instanceof TFile) {
this.appendWatchEvent([ this.appendQueue([
{ type: "DELETE", file: { path: oldFile as FilePath, mtime: file.stat.mtime, ctime: file.stat.ctime, size: file.stat.size, deleted: true } }, { type: "DELETE", file: { path: oldFile as FilePath, mtime: file.stat.mtime, ctime: file.stat.ctime, size: file.stat.size, deleted: true }, skipBatchWait: true },
{ type: "CREATE", file }, { type: "CREATE", file, skipBatchWait: true },
], ctx); ], ctx);
} }
} }
@@ -83,16 +121,17 @@ export class StorageEventManagerObsidian extends StorageEventManager {
.replace(/\n| /g, "") .replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e, "i")); .split(",").filter(e => e).map(e => new RegExp(e, "i"));
if (ignorePatterns.some(e => path.match(e))) return; if (ignorePatterns.some(e => path.match(e))) return;
this.appendWatchEvent( this.appendQueue(
[{ [{
type: "INTERNAL", type: "INTERNAL",
file: { path, mtime: 0, ctime: 0, size: 0 } file: { path, mtime: 0, ctime: 0, size: 0 }
}], null); }], null);
} }
// Cache file and waiting to can be proceed. // Cache file and waiting to can be proceed.
async appendWatchEvent(params: { type: FileEventType, file: TAbstractFile | InternalFileInfo, oldPath?: string }[], ctx?: any) { async appendQueue(params: FileEvent[], ctx?: any) {
if (!this.plugin.settings.isConfigured) return; if (!this.plugin.settings.isConfigured) return;
if (this.plugin.settings.suspendFileWatching) return; if (this.plugin.settings.suspendFileWatching) return;
const processFiles = new Set<FilePath>();
for (const param of params) { for (const param of params) {
if (shouldBeIgnored(param.file.path)) { if (shouldBeIgnored(param.file.path)) {
continue; continue;
@@ -118,13 +157,6 @@ export class StorageEventManagerObsidian extends StorageEventManager {
if (this.plugin.vaultAccess.recentlyTouched(file)) { if (this.plugin.vaultAccess.recentlyTouched(file)) {
continue; continue;
} }
// cache = await this.plugin.vaultAccess.vaultReadAuto(file);
// if (!isPlainText(file.name)) {
// cache = await this.plugin.vaultAccess.vaultReadBinary(file);
// } else {
// cache = await this.plugin.vaultAccess.vaultCacheRead(file);
// if (!cache) cache = await this.plugin.vaultAccess.vaultRead(file);
// }
} }
const fileInfo = file instanceof TFile ? { const fileInfo = file instanceof TFile ? {
ctime: file.stat.ctime, ctime: file.stat.ctime,
@@ -133,16 +165,143 @@ export class StorageEventManagerObsidian extends StorageEventManager {
path: file.path, path: file.path,
size: file.stat.size size: file.stat.size
} as FileInfo : file as InternalFileInfo; } as FileInfo : file as InternalFileInfo;
this.plugin.fileEventQueue.enqueue({ let cache: string | undefined = undefined;
if (param.cachedData) {
cache = param.cachedData
}
this.enqueue({
type, type,
args: { args: {
file: fileInfo, file: fileInfo,
oldPath, oldPath,
// cache, cache,
ctx ctx,
}, },
skipBatchWait: param.skipBatchWait,
key: atomicKey key: atomicKey
}) })
processFiles.add(file.path as FilePath);
if (oldPath) {
processFiles.add(oldPath as FilePath);
}
}
for (const path of processFiles) {
fireAndForget(() => this.startStandingBy(path));
} }
} }
bufferedQueuedItems = [] as FileEventItem[];
enqueue(newItem: FileEventItem) {
const filename = newItem.args.file.path;
if (this.shouldBatchSave) {
Logger(`Request cancel for waiting of previous ${filename}`, LOG_LEVEL_DEBUG);
finishWaitingForTimeout(`storage-event-manager-batchsave-${filename}`);
}
this.bufferedQueuedItems.push(newItem);
// When deleting or renaming, the queue must be flushed once before processing subsequent processes to prevent unexpected race condition.
if (newItem.type == "DELETE" || newItem.type == "RENAME") {
return this.flushQueue();
}
}
concurrentProcessing = Semaphore(5);
waitedSince = new Map<FilePath, number>();
async startStandingBy(filename: FilePath) {
// If waited, cancel previous waiting.
await skipIfDuplicated(`storage-event-manager-${filename}`, async () => {
Logger(`Processing ${filename}: Starting`, LOG_LEVEL_DEBUG);
const release = await this.concurrentProcessing.acquire();
try {
Logger(`Processing ${filename}: Started`, LOG_LEVEL_DEBUG);
let noMoreFiles = false;
do {
const target = this.bufferedQueuedItems.find(e => e.args.file.path == filename);
if (target === undefined) {
noMoreFiles = true;
break;
}
const operationType = target.type;
// if (target.waitedFrom + this.batchSaveMaximumDelay > now) {
// this.requestProcessQueue(target);
// continue;
// }
const type = target.type;
if (target.cancelled) {
Logger(`Processing ${filename}: Cancelled (scheduled): ${operationType}`, LOG_LEVEL_DEBUG)
this.cancelStandingBy(target);
continue;
}
if (!target.skipBatchWait) {
if (this.shouldBatchSave && (type == "CREATE" || type == "CHANGED")) {
const waitedSince = this.waitedSince.get(filename);
let canWait = true;
const now = Date.now();
if (waitedSince !== undefined) {
if (waitedSince + (this.batchSaveMaximumDelay * 1000) < now) {
Logger(`Processing ${filename}: Could not wait no more: ${operationType}`, LOG_LEVEL_INFO)
canWait = false;
}
}
if (canWait) {
if (waitedSince === undefined) this.waitedSince.set(filename, now)
target.batched = true
Logger(`Processing ${filename}: Waiting for batch save delay: ${operationType}`, LOG_LEVEL_DEBUG)
this.updateStatus();
const result = await waitForTimeout(`storage-event-manager-batchsave-${filename}`, this.batchSaveMinimumDelay * 1000);
if (!result) {
Logger(`Processing ${filename}: Cancelled by new queue: ${operationType}`, LOG_LEVEL_DEBUG)
// If could not wait for the timeout, possibly we got a new queue. therefore, currently processing one should be cancelled
this.cancelStandingBy(target);
continue;
}
}
}
} else {
Logger(`Processing ${filename}:Requested to perform immediately ${filename}: ${operationType}`, LOG_LEVEL_DEBUG)
}
Logger(`Processing ${filename}: Request main to process: ${operationType}`, LOG_LEVEL_DEBUG)
this.requestProcessQueue(target);
} while (!noMoreFiles)
} finally {
release()
}
Logger(`Processing ${filename}: Finished`, LOG_LEVEL_DEBUG);
})
}
cancelStandingBy(fei: FileEventItem) {
this.bufferedQueuedItems.remove(fei);
this.updateStatus();
}
processingCount = 0;
async requestProcessQueue(fei: FileEventItem) {
try {
this.processingCount++;
this.bufferedQueuedItems.remove(fei);
this.updateStatus()
this.waitedSince.delete(fei.args.file.path);
await this.plugin.handleFileEvent(fei);
} finally {
this.processingCount--;
this.updateStatus()
}
}
isWaiting(filename: FilePath) {
return isWaitingForTimeout(`storage-event-manager-batchsave-${filename}`);
}
flushQueue() {
this.bufferedQueuedItems.forEach(e => e.skipBatchWait = true)
finishAllWaitingForTimeout("storage-event-manager-batchsave-", true);
}
cancelQueue(key: string) {
this.bufferedQueuedItems.forEach(e => {
if (e.key === key) e.skipBatchWait = true
})
}
updateStatus() {
const allItems = this.bufferedQueuedItems.filter(e => !e.cancelled)
this.batched.value = allItems.filter(e => e.batched && !e.skipBatchWait).length;
this.processing.value = this.processingCount;
this.totalQueued.value = allItems.length - this.batched.value;
}
} }

View File

@@ -644,6 +644,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
const keys = Object.keys(newConf) as (keyof ObsidianLiveSyncSettings)[]; const keys = Object.keys(newConf) as (keyof ObsidianLiveSyncSettings)[];
let hasLoaded = false; let hasLoaded = false;
for (const k of keys) { for (const k of keys) {
if (k === "deviceAndVaultName") continue;
if (isObjectDifferent(newConf[k], this.initialSettings?.[k])) { if (isObjectDifferent(newConf[k], this.initialSettings?.[k])) {
// Something has changed // Something has changed
if (this.isDirty(k as AllSettingItemKey)) { if (this.isDirty(k as AllSettingItemKey)) {
@@ -1739,6 +1740,24 @@ However, your report is needed to stabilise this. I appreciate you for your grea
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setClass("wizardHidden") .setClass("wizardHidden")
.autoWireToggle("batchSave") .autoWireToggle("batchSave")
new Setting(containerSyncSettingEl)
.setClass("wizardHidden")
.autoWireNumeric("batchSaveMinimumDelay",
{
acceptZero: true,
onUpdate: visibleOnly(() => this.isConfiguredAs("batchSave", true))
}
)
new Setting(containerSyncSettingEl)
.setClass("wizardHidden")
.autoWireNumeric("batchSaveMaximumDelay",
{
acceptZero: true,
onUpdate: visibleOnly(() => this.isConfiguredAs("batchSave", true))
}
)
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setClass("wizardHidden") .setClass("wizardHidden")

View File

@@ -67,11 +67,11 @@ export const SettingInformation: Partial<Record<keyof AllSettings, Configuration
}, },
"lessInformationInLog": { "lessInformationInLog": {
"name": "Show only notifications", "name": "Show only notifications",
"desc": "Prevent logging and show only notification" "desc": "Prevent logging and show only notification. Please disable when you report the logs"
}, },
"showVerboseLog": { "showVerboseLog": {
"name": "Verbose Log", "name": "Verbose Log",
"desc": "Show verbose log" "desc": "Show verbose log. Please enable when you report the logs"
}, },
"hashCacheMaxCount": { "hashCacheMaxCount": {
"name": "Memory cache size (by total items)" "name": "Memory cache size (by total items)"
@@ -312,6 +312,14 @@ export const SettingInformation: Partial<Record<keyof AllSettings, Configuration
processSmallFilesInUIThread: { processSmallFilesInUIThread: {
name: "Process small files in the foreground", name: "Process small files in the foreground",
desc: "If enabled, the file under 1kb will be processed in the UI thread." desc: "If enabled, the file under 1kb will be processed in the UI thread."
},
batchSaveMinimumDelay: {
name: "Minimum delay for batch database updating",
desc: "Seconds. Saving to the local database will be delayed until this value after we stop typing or saving."
},
batchSaveMaximumDelay: {
name: "Maximum delay for batch database updating",
desc: "Saving will be performed forcefully after this number of seconds."
} }
} }
function translateInfo(infoSrc: ConfigurationItem | undefined | false) { function translateInfo(infoSrc: ConfigurationItem | undefined | false) {