diff --git a/manifest.json b/manifest.json index 9a15043..c803f98 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "0.25.43-patched-5", + "version": "0.25.43-patched-6", "minAppVersion": "0.9.12", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index ae871f3..8274cf0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "0.25.43-patched-5", + "version": "0.25.43-patched-6", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "0.25.43-patched-5", + "version": "0.25.43-patched-6", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.808.0", diff --git a/package.json b/package.json index f2b6e94..28fdbea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "0.25.43-patched-5", + "version": "0.25.43-patched-6", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", diff --git a/src/deps.ts b/src/deps.ts index 08a78f7..cf8ed5b 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -40,6 +40,7 @@ export type { MarkdownFileInfo, ListedFiles, ValueComponent, + Stat, } from "obsidian"; import { normalizePath as normalizePath_ } from "obsidian"; const normalizePath = normalizePath_ as (from: T) => T; diff --git a/src/lib b/src/lib index 56fc24e..744a1b0 160000 --- a/src/lib +++ b/src/lib @@ -1 +1 @@ -Subproject commit 56fc24e001347948bafbc85c342f94fd0ee0a0b5 +Subproject commit 744a1b01ebb30acebd3d55a8a99ac3b61d228828 diff --git a/src/main.ts b/src/main.ts index e623df0..2769f8a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -60,12 +60,12 @@ import type { ServiceContext } from "./lib/src/services/base/ServiceBase.ts"; import { ServiceRebuilder } from "@lib/serviceModules/Rebuilder.ts"; import type { IFileHandler } from "@lib/interfaces/FileHandler.ts"; import { ServiceDatabaseFileAccess } from "@/serviceModules/DatabaseFileAccess.ts"; -import { ServiceFileAccessObsidian } from "@/serviceModules/ServiceFileAccessObsidian.ts"; -import { StorageEventManagerObsidian } from "@/modules/coreObsidian/storageLib/StorageEventManager.ts"; -import { ObsidianFileAccess } from "@/modules/coreObsidian/storageLib/SerializedFileAccess.ts"; +import { ServiceFileAccessObsidian } from "@/serviceModules/ServiceFileAccessImpl.ts"; import { StorageAccessManager } from "@lib/managers/StorageProcessingManager.ts"; import { __$checkInstanceBinding } from "./lib/src/dev/checks.ts"; import { ServiceFileHandler } from "./serviceModules/FileHandler.ts"; +import { FileAccessObsidian } from "./serviceModules/FileAccessObsidian.ts"; +import { StorageEventManagerObsidian } from "./managers/StorageEventManagerObsidian.ts"; export default class ObsidianLiveSyncPlugin extends Plugin @@ -302,8 +302,19 @@ export default class ObsidianLiveSyncPlugin private initialiseServiceModules() { const storageAccessManager = new StorageAccessManager(); // If we want to implement to the other platform, implement ObsidianXXXXXService. - const vaultAccess = new ObsidianFileAccess(this.app, this, storageAccessManager); - const storageEventManager = new StorageEventManagerObsidian(this, this, storageAccessManager); + const vaultAccess = new FileAccessObsidian(this.app, { + storageAccessManager: storageAccessManager, + vaultService: this.services.vault, + settingService: this.services.setting, + APIService: this.services.API, + }); + const storageEventManager = new StorageEventManagerObsidian(this, this, { + fileProcessing: this.services.fileProcessing, + setting: this.services.setting, + vaultService: this.services.vault, + storageAccessManager: storageAccessManager, + APIService: this.services.API, + }); const storageAccess = new ServiceFileAccessObsidian({ API: this.services.API, setting: this.services.setting, diff --git a/src/managers/StorageEventManagerObsidian.ts b/src/managers/StorageEventManagerObsidian.ts new file mode 100644 index 0000000..aaedacf --- /dev/null +++ b/src/managers/StorageEventManagerObsidian.ts @@ -0,0 +1,210 @@ +import type { FileEventItem } from "@/common/types"; +import { HiddenFileSync } from "@/features/HiddenFileSync/CmdHiddenFileSync"; +import type { FilePath, UXFileInfoStub, UXFolderInfo, UXInternalFileInfoStub } from "@lib/common/types"; +import type { FileEvent } from "@lib/interfaces/StorageEventManager"; +import { TFile, type TAbstractFile, TFolder } from "@/deps"; +import { LOG_LEVEL_DEBUG } from "octagonal-wheels/common/logger"; +import type ObsidianLiveSyncPlugin from "@/main"; +import type { LiveSyncCore } from "@/main"; +import { + StorageEventManagerBase, + type FileEventItemSentinel, + type StorageEventManagerBaseDependencies, +} from "@lib/managers/StorageEventManager"; +import { InternalFileToUXFileInfoStub, TFileToUXFileInfoStub } from "@/modules/coreObsidian/storageLib/utilObsidian"; + +export class StorageEventManagerObsidian extends StorageEventManagerBase { + plugin: ObsidianLiveSyncPlugin; + core: LiveSyncCore; + + // Necessary evil. + cmdHiddenFileSync: HiddenFileSync; + + override isFile(file: UXFileInfoStub | UXInternalFileInfoStub | UXFolderInfo | TFile): boolean { + if (file instanceof TFile) { + return true; + } + if (super.isFile(file)) { + return true; + } + return !file.isFolder; + } + override isFolder(file: UXFileInfoStub | UXInternalFileInfoStub | UXFolderInfo | TFolder): boolean { + if (file instanceof TFolder) { + return true; + } + if (super.isFolder(file)) { + return true; + } + return !!file.isFolder; + } + + constructor(plugin: ObsidianLiveSyncPlugin, core: LiveSyncCore, dependencies: StorageEventManagerBaseDependencies) { + super(dependencies); + this.plugin = plugin; + this.core = core; + this.cmdHiddenFileSync = this.plugin.getAddOn(HiddenFileSync.name) as HiddenFileSync; + } + + async beginWatch() { + await this.snapShotRestored; + const plugin = this.plugin; + this.watchVaultChange = this.watchVaultChange.bind(this); + this.watchVaultCreate = this.watchVaultCreate.bind(this); + this.watchVaultDelete = this.watchVaultDelete.bind(this); + this.watchVaultRename = this.watchVaultRename.bind(this); + this.watchVaultRawEvents = this.watchVaultRawEvents.bind(this); + this.watchEditorChange = this.watchEditorChange.bind(this); + plugin.registerEvent(plugin.app.vault.on("modify", this.watchVaultChange)); + plugin.registerEvent(plugin.app.vault.on("delete", this.watchVaultDelete)); + plugin.registerEvent(plugin.app.vault.on("rename", this.watchVaultRename)); + plugin.registerEvent(plugin.app.vault.on("create", this.watchVaultCreate)); + //@ts-ignore : Internal API + plugin.registerEvent(plugin.app.vault.on("raw", this.watchVaultRawEvents)); + plugin.registerEvent(plugin.app.workspace.on("editor-change", this.watchEditorChange)); + } + watchEditorChange(editor: any, info: any) { + if (!("path" in info)) { + return; + } + if (!this.shouldBatchSave) { + return; + } + const file = info?.file as TFile; + if (!file) return; + if (this.storageAccess.isFileProcessing(file.path as FilePath)) { + // this._log(`Editor change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE); + return; + } + if (!this.isWaiting(file.path as FilePath)) { + return; + } + const data = info?.data as string; + const fi: FileEvent = { + type: "CHANGED", + file: TFileToUXFileInfoStub(file), + cachedData: data, + }; + void this.appendQueue([fi]); + } + + watchVaultCreate(file: TAbstractFile, ctx?: any) { + if (file instanceof TFolder) return; + if (this.storageAccess.isFileProcessing(file.path as FilePath)) { + // this._log(`File create skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE); + return; + } + const fileInfo = TFileToUXFileInfoStub(file); + void this.appendQueue([{ type: "CREATE", file: fileInfo }], ctx); + } + + watchVaultChange(file: TAbstractFile, ctx?: any) { + if (file instanceof TFolder) return; + if (this.storageAccess.isFileProcessing(file.path as FilePath)) { + // this._log(`File change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE); + return; + } + const fileInfo = TFileToUXFileInfoStub(file); + void this.appendQueue([{ type: "CHANGED", file: fileInfo }], ctx); + } + + watchVaultDelete(file: TAbstractFile, ctx?: any) { + if (file instanceof TFolder) return; + if (this.storageAccess.isFileProcessing(file.path as FilePath)) { + // this._log(`File delete skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE); + return; + } + const fileInfo = TFileToUXFileInfoStub(file, true); + void this.appendQueue([{ type: "DELETE", file: fileInfo }], ctx); + } + watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) { + // vault Rename will not be raised for self-events (Self-hosted LiveSync will not handle 'rename'). + if (file instanceof TFile) { + const fileInfo = TFileToUXFileInfoStub(file); + void this.appendQueue( + [ + { + type: "DELETE", + file: { + path: oldFile as FilePath, + name: file.name, + stat: { + mtime: file.stat.mtime, + ctime: file.stat.ctime, + size: file.stat.size, + type: "file", + }, + deleted: true, + }, + skipBatchWait: true, + }, + { type: "CREATE", file: fileInfo, skipBatchWait: true }, + ], + ctx + ); + } + } + // Watch raw events (Internal API) + watchVaultRawEvents(path: FilePath) { + if (this.storageAccess.isFileProcessing(path)) { + // this._log(`Raw file event skipped because the file is being processed: ${path}`, LOG_LEVEL_VERBOSE); + return; + } + // Only for internal files. + if (!this.settings) return; + // if (this.plugin.settings.useIgnoreFiles && this.plugin.ignoreFiles.some(e => path.endsWith(e.trim()))) { + if (this.settings.useIgnoreFiles) { + // If it is one of ignore files, refresh the cached one. + // (Calling$$isTargetFile will refresh the cache) + void this.vaultService.isTargetFile(path).then(() => this._watchVaultRawEvents(path)); + } else { + void this._watchVaultRawEvents(path); + } + } + + async _watchVaultRawEvents(path: FilePath) { + if (!this.settings.syncInternalFiles && !this.settings.usePluginSync) return; + if (!this.settings.watchInternalFileChanges) return; + if (!path.startsWith(this.plugin.app.vault.configDir)) return; + if (path.endsWith("/")) { + // Folder + return; + } + const isTargetFile = await this.cmdHiddenFileSync.isTargetFile(path); + if (!isTargetFile) return; + + void this.appendQueue( + [ + { + type: "INTERNAL", + file: InternalFileToUXFileInfoStub(path), + skipBatchWait: true, // Internal files should be processed immediately. + }, + ], + null + ); + } + + async _saveSnapshot(snapshot: (FileEventItem | FileEventItemSentinel)[]) { + await this.core.kvDB.set("storage-event-manager-snapshot", snapshot); + this._log(`Storage operation snapshot saved: ${snapshot.length} items`, LOG_LEVEL_DEBUG); + } + + async _loadSnapshot() { + const snapShot = await this.core.kvDB.get<(FileEventItem | FileEventItemSentinel)[]>( + "storage-event-manager-snapshot" + ); + return snapShot; + } + + updateStatus() { + const allFileEventItems = this.bufferedQueuedItems.filter((e): e is FileEventItem => "args" in e); + const allItems = allFileEventItems.filter((e) => !e.cancelled); + const totalItems = allItems.length + this.concurrentProcessing.waiting; + const processing = this.processingCount; + const batchedCount = this._waitingMap.size; + this.core.batched.value = batchedCount; + this.core.processing.value = processing; + this.core.totalQueued.value = totalItems + batchedCount + processing; + } +} diff --git a/src/modules/core/ModuleTargetFilter.ts b/src/modules/core/ModuleTargetFilter.ts index 4014976..956f657 100644 --- a/src/modules/core/ModuleTargetFilter.ts +++ b/src/modules/core/ModuleTargetFilter.ts @@ -141,10 +141,10 @@ export class ModuleTargetFilter extends AbstractModule { services.vault.markFileListPossiblyChanged.setHandler(this._markFileListPossiblyChanged.bind(this)); services.appLifecycle.onLoaded.addHandler(this._everyOnload.bind(this)); services.vault.isIgnoredByIgnoreFile.setHandler(this._isTargetIgnoredByIgnoreFiles.bind(this)); - services.vault.isTargetFile.addHandler(this._isTargetFileByFileNameDuplication.bind(this)); - services.vault.isTargetFile.addHandler(this._isTargetIgnoredByIgnoreFiles.bind(this)); - services.vault.isTargetFile.addHandler(this._isTargetFileByLocalDB.bind(this)); - services.vault.isTargetFile.addHandler(this._isTargetFileFinal.bind(this)); + services.vault.isTargetFile.addHandler(this._isTargetFileByFileNameDuplication.bind(this), 10); + services.vault.isTargetFile.addHandler(this._isTargetIgnoredByIgnoreFiles.bind(this), 20); + services.vault.isTargetFile.addHandler(this._isTargetFileByLocalDB.bind(this), 30); + services.vault.isTargetFile.addHandler(this._isTargetFileFinal.bind(this), 100); services.setting.onSettingRealised.addHandler(this.refreshSettings.bind(this)); } } diff --git a/src/modules/coreObsidian/storageLib/SerializedFileAccess.ts b/src/modules/coreObsidian/storageLib/SerializedFileAccess.ts deleted file mode 100644 index ac4b2d4..0000000 --- a/src/modules/coreObsidian/storageLib/SerializedFileAccess.ts +++ /dev/null @@ -1,264 +0,0 @@ -import { type App, TFile, type DataWriteOptions, TFolder, TAbstractFile } from "../../../deps.ts"; -import { Logger } from "../../../lib/src/common/logger.ts"; -import { isPlainText } from "../../../lib/src/string_and_binary/path.ts"; -import type { FilePath, UXFileInfoStub } from "../../../lib/src/common/types.ts"; -import { createBinaryBlob, isDocContentSame } from "../../../lib/src/common/utils.ts"; -import type { InternalFileInfo } from "../../../common/types.ts"; -import { markChangesAreSame } from "../../../common/utils.ts"; -import type { IStorageAccessManager } from "@lib/interfaces/StorageAccess.ts"; -import type { LiveSyncCore } from "@/main.ts"; -function toArrayBuffer(arr: Uint8Array | ArrayBuffer | DataView): ArrayBuffer { - if (arr instanceof Uint8Array) { - return arr.buffer; - } - if (arr instanceof DataView) { - return arr.buffer; - } - return arr; -} -// TODO: add abstraction for the file access (as wrapping TFile or something similar) -export abstract class FileAccessBase { - storageAccessManager: IStorageAccessManager; - constructor(storageAccessManager: IStorageAccessManager) { - this.storageAccessManager = storageAccessManager; - } - abstract getPath(file: TNativeFile | string): FilePath; -} - -export class ObsidianFileAccess extends FileAccessBase { - app: App; - plugin: LiveSyncCore; - - getPath(file: string | TFile): FilePath { - return (typeof file === "string" ? file : file.path) as FilePath; - } - - constructor(app: App, plugin: LiveSyncCore, storageAccessManager: IStorageAccessManager) { - super(storageAccessManager); - this.app = app; - this.plugin = plugin; - } - - async tryAdapterStat(file: TFile | string) { - const path = file instanceof TFile ? file.path : file; - return await this.storageAccessManager.processReadFile(path as FilePath, async () => { - if (!(await this.app.vault.adapter.exists(path))) return null; - return this.app.vault.adapter.stat(path); - }); - } - async adapterStat(file: TFile | string) { - const path = file instanceof TFile ? file.path : file; - return await this.storageAccessManager.processReadFile(path as FilePath, () => - this.app.vault.adapter.stat(path) - ); - } - async adapterExists(file: TFile | string) { - const path = file instanceof TFile ? file.path : file; - return await this.storageAccessManager.processReadFile(path as FilePath, () => - this.app.vault.adapter.exists(path) - ); - } - async adapterRemove(file: TFile | string) { - const path = file instanceof TFile ? file.path : file; - return await this.storageAccessManager.processWriteFile(path as FilePath, () => - this.app.vault.adapter.remove(path) - ); - } - - async adapterRead(file: TFile | string) { - const path = file instanceof TFile ? file.path : file; - return await this.storageAccessManager.processReadFile(path as FilePath, () => - this.app.vault.adapter.read(path) - ); - } - async adapterReadBinary(file: TFile | string) { - const path = file instanceof TFile ? file.path : file; - return await this.storageAccessManager.processReadFile(path as FilePath, () => - this.app.vault.adapter.readBinary(path) - ); - } - - async adapterReadAuto(file: TFile | string) { - const path = file instanceof TFile ? file.path : file; - if (isPlainText(path)) { - return await this.storageAccessManager.processReadFile(path as FilePath, () => - this.app.vault.adapter.read(path) - ); - } - return await this.storageAccessManager.processReadFile(path as FilePath, () => - this.app.vault.adapter.readBinary(path) - ); - } - - async adapterWrite( - file: TFile | string, - data: string | ArrayBuffer | Uint8Array, - options?: DataWriteOptions - ) { - const path = file instanceof TFile ? file.path : file; - if (typeof data === "string") { - return await this.storageAccessManager.processWriteFile(path as FilePath, () => - this.app.vault.adapter.write(path, data, options) - ); - } else { - return await this.storageAccessManager.processWriteFile(path as FilePath, () => - this.app.vault.adapter.writeBinary(path, toArrayBuffer(data), options) - ); - } - } - - adapterList(basePath: string): Promise<{ files: string[]; folders: string[] }> { - return Promise.resolve(this.app.vault.adapter.list(basePath)); - } - - async vaultCacheRead(file: TFile) { - return await this.storageAccessManager.processReadFile(file.path as FilePath, () => - this.app.vault.cachedRead(file) - ); - } - - async vaultRead(file: TFile) { - return await this.storageAccessManager.processReadFile(file.path as FilePath, () => this.app.vault.read(file)); - } - - async vaultReadBinary(file: TFile) { - return await this.storageAccessManager.processReadFile(file.path as FilePath, () => - this.app.vault.readBinary(file) - ); - } - - async vaultReadAuto(file: TFile) { - const path = file.path; - if (isPlainText(path)) { - return await this.storageAccessManager.processReadFile(path as FilePath, () => this.app.vault.read(file)); - } - return await this.storageAccessManager.processReadFile(path as FilePath, () => this.app.vault.readBinary(file)); - } - - async vaultModify(file: TFile, data: string | ArrayBuffer | Uint8Array, options?: DataWriteOptions) { - if (typeof data === "string") { - return await this.storageAccessManager.processWriteFile(file.path as FilePath, async () => { - const oldData = await this.app.vault.read(file); - if (data === oldData) { - if (options && options.mtime) markChangesAreSame(file.path, file.stat.mtime, options.mtime); - return true; - } - await this.app.vault.modify(file, data, options); - return true; - }); - } else { - return await this.storageAccessManager.processWriteFile(file.path as FilePath, async () => { - const oldData = await this.app.vault.readBinary(file); - if (await isDocContentSame(createBinaryBlob(oldData), createBinaryBlob(data))) { - if (options && options.mtime) markChangesAreSame(file.path, file.stat.mtime, options.mtime); - return true; - } - await this.app.vault.modifyBinary(file, toArrayBuffer(data), options); - return true; - }); - } - } - async vaultCreate( - path: string, - data: string | ArrayBuffer | Uint8Array, - options?: DataWriteOptions - ): Promise { - if (typeof data === "string") { - return await this.storageAccessManager.processWriteFile(path as FilePath, () => - this.app.vault.create(path, data, options) - ); - } else { - return await this.storageAccessManager.processWriteFile(path as FilePath, () => - this.app.vault.createBinary(path, toArrayBuffer(data), options) - ); - } - } - - trigger(name: string, ...data: any[]) { - return this.app.vault.trigger(name, ...data); - } - async reconcileInternalFile(path: string) { - await (this.app.vault.adapter as any)?.reconcileInternalFile(path); - } - - async adapterAppend(normalizedPath: string, data: string, options?: DataWriteOptions) { - return await this.app.vault.adapter.append(normalizedPath, data, options); - } - - async delete(file: TFile | TFolder, force = false) { - return await this.storageAccessManager.processWriteFile(file.path as FilePath, () => - this.app.vault.delete(file, force) - ); - } - async trash(file: TFile | TFolder, force = false) { - return await this.storageAccessManager.processWriteFile(file.path as FilePath, () => - this.app.vault.trash(file, force) - ); - } - - isStorageInsensitive(): boolean { - return this.plugin.services.vault.isStorageInsensitive(); - } - - getAbstractFileByPathInsensitive(path: FilePath | string): TAbstractFile | null { - //@ts-ignore - return this.app.vault.getAbstractFileByPathInsensitive(path); - } - - getAbstractFileByPath(path: FilePath | string): TAbstractFile | null { - if (!this.plugin.settings.handleFilenameCaseSensitive || this.isStorageInsensitive()) { - return this.getAbstractFileByPathInsensitive(path); - } - return this.app.vault.getAbstractFileByPath(path); - } - - getFiles() { - return this.app.vault.getFiles(); - } - - async ensureDirectory(fullPath: string) { - const pathElements = fullPath.split("/"); - pathElements.pop(); - let c = ""; - for (const v of pathElements) { - c += v; - try { - await this.app.vault.adapter.mkdir(c); - } catch (ex: any) { - if (ex?.message == "Folder already exists.") { - // Skip if already exists. - } else { - Logger("Folder Create Error"); - Logger(ex); - } - } - c += "/"; - } - } - - touchedFiles: string[] = []; - - _statInternal(file: FilePath) { - return this.app.vault.adapter.stat(file); - } - - async touch(file: TFile | FilePath) { - const path = file instanceof TFile ? (file.path as FilePath) : file; - const statOrg = file instanceof TFile ? file.stat : await this._statInternal(path); - const stat = statOrg || { mtime: 0, size: 0 }; - const key = `${path}-${stat.mtime}-${stat.size}`; - this.touchedFiles.unshift(key); - this.touchedFiles = this.touchedFiles.slice(0, 100); - } - recentlyTouched(file: TFile | InternalFileInfo | UXFileInfoStub) { - const key = - "stat" in file - ? `${file.path}-${file.stat.mtime}-${file.stat.size}` - : `${file.path}-${file.mtime}-${file.size}`; - if (this.touchedFiles.indexOf(key) == -1) return false; - return true; - } - clearTouched() { - this.touchedFiles = []; - } -} diff --git a/src/modules/coreObsidian/storageLib/StorageEventManager.ts b/src/modules/coreObsidian/storageLib/StorageEventManager.ts deleted file mode 100644 index 90b1dc6..0000000 --- a/src/modules/coreObsidian/storageLib/StorageEventManager.ts +++ /dev/null @@ -1,631 +0,0 @@ -import { TAbstractFile, TFile, TFolder } from "../../../deps.ts"; -import { Logger } from "../../../lib/src/common/logger.ts"; -import { shouldBeIgnored } from "../../../lib/src/string_and_binary/path.ts"; -import { - DEFAULT_SETTINGS, - LOG_LEVEL_DEBUG, - LOG_LEVEL_INFO, - LOG_LEVEL_NOTICE, - LOG_LEVEL_VERBOSE, - type FileEventType, - type FilePath, - type UXFileInfoStub, -} from "../../../lib/src/common/types.ts"; -import { delay, fireAndForget, throttle } from "../../../lib/src/common/utils.ts"; -import { type FileEventItem } from "../../../common/types.ts"; -import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock"; -import { isWaitingForTimeout } from "octagonal-wheels/concurrency/task"; -import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; -import type { LiveSyncCore } from "../../../main.ts"; -import { InternalFileToUXFileInfoStub, TFileToUXFileInfoStub } from "./utilObsidian.ts"; -import ObsidianLiveSyncPlugin from "../../../main.ts"; -import type { IStorageAccessManager } from "@lib/interfaces/StorageAccess.ts"; -import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts"; -import { promiseWithResolvers, type PromiseWithResolvers } from "octagonal-wheels/promises"; -import { StorageEventManager, type FileEvent } from "@lib/interfaces/StorageEventManager.ts"; - -type WaitInfo = { - since: number; - type: FileEventType; - canProceed: PromiseWithResolvers; - timerHandler: ReturnType; - event: FileEventItem; -}; -const TYPE_SENTINEL_FLUSH = "SENTINEL_FLUSH"; -type FileEventItemSentinelFlush = { - type: typeof TYPE_SENTINEL_FLUSH; -}; -type FileEventItemSentinel = FileEventItemSentinelFlush; - -export class StorageEventManagerObsidian extends StorageEventManager { - plugin: ObsidianLiveSyncPlugin; - core: LiveSyncCore; - storageAccess: IStorageAccessManager; - get services() { - return this.core.services; - } - - get shouldBatchSave() { - return this.core.settings?.batchSave && this.core.settings?.liveSync != true; - } - get batchSaveMinimumDelay(): number { - return this.core.settings?.batchSaveMinimumDelay ?? DEFAULT_SETTINGS.batchSaveMinimumDelay; - } - get batchSaveMaximumDelay(): number { - return this.core.settings?.batchSaveMaximumDelay ?? DEFAULT_SETTINGS.batchSaveMaximumDelay; - } - // Necessary evil. - cmdHiddenFileSync: HiddenFileSync; - - /** - * Snapshot restoration promise. - * Snapshot will be restored before starting to watch vault changes. - * In designed time, this has been called from Initialisation process, which has been implemented on `ModuleInitializerFile.ts`. - */ - snapShotRestored: Promise | null = null; - - constructor(plugin: ObsidianLiveSyncPlugin, core: LiveSyncCore, storageAccessManager: IStorageAccessManager) { - super(); - this.storageAccess = storageAccessManager; - this.plugin = plugin; - this.core = core; - this.cmdHiddenFileSync = this.plugin.getAddOn(HiddenFileSync.name) as HiddenFileSync; - } - - /** - * Restore the previous snapshot if exists. - * @returns - */ - restoreState(): Promise { - this.snapShotRestored = this._restoreFromSnapshot(); - return this.snapShotRestored; - } - - async beginWatch() { - await this.snapShotRestored; - const plugin = this.plugin; - this.watchVaultChange = this.watchVaultChange.bind(this); - this.watchVaultCreate = this.watchVaultCreate.bind(this); - this.watchVaultDelete = this.watchVaultDelete.bind(this); - this.watchVaultRename = this.watchVaultRename.bind(this); - this.watchVaultRawEvents = this.watchVaultRawEvents.bind(this); - this.watchEditorChange = this.watchEditorChange.bind(this); - plugin.registerEvent(plugin.app.vault.on("modify", this.watchVaultChange)); - plugin.registerEvent(plugin.app.vault.on("delete", this.watchVaultDelete)); - plugin.registerEvent(plugin.app.vault.on("rename", this.watchVaultRename)); - plugin.registerEvent(plugin.app.vault.on("create", this.watchVaultCreate)); - //@ts-ignore : Internal API - plugin.registerEvent(plugin.app.vault.on("raw", this.watchVaultRawEvents)); - plugin.registerEvent(plugin.app.workspace.on("editor-change", this.watchEditorChange)); - } - watchEditorChange(editor: any, info: any) { - if (!("path" in info)) { - return; - } - if (!this.shouldBatchSave) { - return; - } - const file = info?.file as TFile; - if (!file) return; - if (this.storageAccess.isFileProcessing(file.path as FilePath)) { - // Logger(`Editor change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE); - return; - } - if (!this.isWaiting(file.path as FilePath)) { - return; - } - const data = info?.data as string; - const fi: FileEvent = { - type: "CHANGED", - file: TFileToUXFileInfoStub(file), - cachedData: data, - }; - void this.appendQueue([fi]); - } - - watchVaultCreate(file: TAbstractFile, ctx?: any) { - if (file instanceof TFolder) return; - if (this.storageAccess.isFileProcessing(file.path as FilePath)) { - // Logger(`File create skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE); - return; - } - const fileInfo = TFileToUXFileInfoStub(file); - void this.appendQueue([{ type: "CREATE", file: fileInfo }], ctx); - } - - watchVaultChange(file: TAbstractFile, ctx?: any) { - if (file instanceof TFolder) return; - if (this.storageAccess.isFileProcessing(file.path as FilePath)) { - // Logger(`File change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE); - return; - } - const fileInfo = TFileToUXFileInfoStub(file); - void this.appendQueue([{ type: "CHANGED", file: fileInfo }], ctx); - } - - watchVaultDelete(file: TAbstractFile, ctx?: any) { - if (file instanceof TFolder) return; - if (this.storageAccess.isFileProcessing(file.path as FilePath)) { - // Logger(`File delete skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE); - return; - } - const fileInfo = TFileToUXFileInfoStub(file, true); - void this.appendQueue([{ type: "DELETE", file: fileInfo }], ctx); - } - watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) { - // vault Rename will not be raised for self-events (Self-hosted LiveSync will not handle 'rename'). - if (file instanceof TFile) { - const fileInfo = TFileToUXFileInfoStub(file); - void this.appendQueue( - [ - { - type: "DELETE", - file: { - path: oldFile as FilePath, - name: file.name, - stat: { - mtime: file.stat.mtime, - ctime: file.stat.ctime, - size: file.stat.size, - type: "file", - }, - deleted: true, - }, - skipBatchWait: true, - }, - { type: "CREATE", file: fileInfo, skipBatchWait: true }, - ], - ctx - ); - } - } - // Watch raw events (Internal API) - watchVaultRawEvents(path: FilePath) { - if (this.storageAccess.isFileProcessing(path)) { - // Logger(`Raw file event skipped because the file is being processed: ${path}`, LOG_LEVEL_VERBOSE); - return; - } - // Only for internal files. - if (!this.plugin.settings) return; - // if (this.plugin.settings.useIgnoreFiles && this.plugin.ignoreFiles.some(e => path.endsWith(e.trim()))) { - if (this.plugin.settings.useIgnoreFiles) { - // If it is one of ignore files, refresh the cached one. - // (Calling$$isTargetFile will refresh the cache) - void this.services.vault.isTargetFile(path).then(() => this._watchVaultRawEvents(path)); - } else { - void this._watchVaultRawEvents(path); - } - } - - async _watchVaultRawEvents(path: FilePath) { - if (!this.plugin.settings.syncInternalFiles && !this.plugin.settings.usePluginSync) return; - if (!this.plugin.settings.watchInternalFileChanges) return; - if (!path.startsWith(this.plugin.app.vault.configDir)) return; - if (path.endsWith("/")) { - // Folder - return; - } - const isTargetFile = await this.cmdHiddenFileSync.isTargetFile(path); - if (!isTargetFile) return; - - void this.appendQueue( - [ - { - type: "INTERNAL", - file: InternalFileToUXFileInfoStub(path), - skipBatchWait: true, // Internal files should be processed immediately. - }, - ], - null - ); - } - - // Cache file and waiting to can be proceed. - async appendQueue(params: FileEvent[], ctx?: any) { - if (!this.core.settings.isConfigured) return; - if (this.core.settings.suspendFileWatching) return; - if (this.core.settings.maxMTimeForReflectEvents > 0) { - return; - } - this.core.services.vault.markFileListPossiblyChanged(); - // Flag up to be reload - for (const param of params) { - if (shouldBeIgnored(param.file.path)) { - continue; - } - const atomicKey = [0, 0, 0, 0, 0, 0].map((e) => `${Math.floor(Math.random() * 100000)}`).join("-"); - const type = param.type; - const file = param.file; - const oldPath = param.oldPath; - if (type !== "INTERNAL") { - const size = (file as UXFileInfoStub).stat.size; - if (this.services.vault.isFileSizeTooLarge(size) && (type == "CREATE" || type == "CHANGED")) { - Logger( - `The storage file has been changed but exceeds the maximum size. Skipping: ${param.file.path}`, - LOG_LEVEL_NOTICE - ); - continue; - } - } - if (file instanceof TFolder) continue; - // TODO: Confirm why only the TFolder skipping - // Possibly following line is needed... - // if (file?.isFolder) continue; - if (!(await this.services.vault.isTargetFile(file.path))) continue; - - // Stop cache using to prevent the corruption; - // let cache: null | string | ArrayBuffer; - // new file or something changed, cache the changes. - // if (file instanceof TFile && (type == "CREATE" || type == "CHANGED")) { - if (file instanceof TFile || !file.isFolder) { - if (type == "CREATE" || type == "CHANGED") { - // Wait for a bit while to let the writer has marked `touched` at the file. - await delay(10); - if (this.core.storageAccess.recentlyTouched(file.path)) { - continue; - } - } - } - - let cache: string | undefined = undefined; - if (param.cachedData) { - cache = param.cachedData; - } - void this.enqueue({ - type, - args: { - file: file, - oldPath, - cache, - ctx, - }, - skipBatchWait: param.skipBatchWait, - key: atomicKey, - }); - } - } - private bufferedQueuedItems = [] as (FileEventItem | FileEventItemSentinel)[]; - - /** - * Immediately take snapshot. - */ - private _triggerTakeSnapshot() { - void this._takeSnapshot(); - } - /** - * Trigger taking snapshot after throttled period. - */ - triggerTakeSnapshot = throttle(() => this._triggerTakeSnapshot(), 100); - - enqueue(newItem: FileEventItem) { - if (newItem.type == "DELETE") { - // If the sentinel pushed, the runQueuedEvents will wait for idle before processing delete. - this.bufferedQueuedItems.push({ - type: TYPE_SENTINEL_FLUSH, - }); - } - this.updateStatus(); - this.bufferedQueuedItems.push(newItem); - - fireAndForget(() => this._takeSnapshot().then(() => this.runQueuedEvents())); - } - - // Limit concurrent processing to reduce the IO load. file-processing + scheduler (1), so file events can be processed in 4 slots. - concurrentProcessing = Semaphore(5); - - private _waitingMap = new Map(); - private _waitForIdle: Promise | null = null; - - /** - * Wait until all queued events are processed. - * Subsequent new events will not be waited, but new events will not be added. - * @returns - */ - waitForIdle(): Promise { - if (this._waitingMap.size === 0) { - return Promise.resolve(); - } - if (this._waitForIdle) { - return this._waitForIdle; - } - const promises = [...this._waitingMap.entries()].map(([key, waitInfo]) => { - return new Promise((resolve) => { - waitInfo.canProceed.promise - .then(() => { - Logger(`Processing ${key}: Wait for idle completed`, LOG_LEVEL_DEBUG); - // No op - }) - .catch((e) => { - Logger(`Processing ${key}: Wait for idle error`, LOG_LEVEL_INFO); - Logger(e, LOG_LEVEL_VERBOSE); - //no op - }) - .finally(() => { - resolve(); - }); - this._proceedWaiting(key); - }); - }); - const waitPromise = Promise.all(promises).then(() => { - this._waitForIdle = null; - Logger(`All wait for idle completed`, LOG_LEVEL_VERBOSE); - }); - this._waitForIdle = waitPromise; - return waitPromise; - } - - /** - * Proceed waiting for the given key immediately. - */ - private _proceedWaiting(key: string) { - const waitInfo = this._waitingMap.get(key); - if (waitInfo) { - waitInfo.canProceed.resolve(true); - clearTimeout(waitInfo.timerHandler); - this._waitingMap.delete(key); - } - this.triggerTakeSnapshot(); - } - /** - * Cancel waiting for the given key. - */ - private _cancelWaiting(key: string) { - const waitInfo = this._waitingMap.get(key); - if (waitInfo) { - waitInfo.canProceed.resolve(false); - clearTimeout(waitInfo.timerHandler); - this._waitingMap.delete(key); - } - this.triggerTakeSnapshot(); - } - /** - * Add waiting for the given key. - * @param key - * @param event - * @param waitedSince Optional waited since timestamp to calculate the remaining delay. - */ - private _addWaiting(key: string, event: FileEventItem, waitedSince?: number): WaitInfo { - if (this._waitingMap.has(key)) { - // Already waiting - throw new Error(`Already waiting for key: ${key}`); - } - const resolver = promiseWithResolvers(); - const now = Date.now(); - const since = waitedSince ?? now; - const elapsed = now - since; - const maxDelay = this.batchSaveMaximumDelay * 1000; - const remainingDelay = Math.max(0, maxDelay - elapsed); - const nextDelay = Math.min(remainingDelay, this.batchSaveMinimumDelay * 1000); - // x*<------- maxDelay --------->* - // x*<-- minDelay -->* - // x* x<-- nextDelay -->* - // x* x<-- Capped-->* - // x* x.......* - // x: event - // *: save - // When at event (x) At least, save (*) within maxDelay, but maintain minimum delay between saves. - - if (elapsed >= maxDelay) { - // Already exceeded maximum delay, do not wait. - Logger(`Processing ${key}: Batch save maximum delay already exceeded: ${event.type}`, LOG_LEVEL_DEBUG); - } else { - Logger(`Processing ${key}: Adding waiting for batch save: ${event.type} (${nextDelay}ms)`, LOG_LEVEL_DEBUG); - } - const waitInfo: WaitInfo = { - since: since, - type: event.type, - event: event, - canProceed: resolver, - timerHandler: setTimeout(() => { - Logger(`Processing ${key}: Batch save timeout reached: ${event.type}`, LOG_LEVEL_DEBUG); - this._proceedWaiting(key); - }, nextDelay), - }; - this._waitingMap.set(key, waitInfo); - this.triggerTakeSnapshot(); - return waitInfo; - } - - /** - * Process the given file event. - */ - async processFileEvent(fei: FileEventItem) { - const releaser = await this.concurrentProcessing.acquire(); - try { - this.updateStatus(); - const filename = fei.args.file.path; - const waitingKey = `${filename}`; - const previous = this._waitingMap.get(waitingKey); - let isShouldBeCancelled = fei.skipBatchWait || false; - let previousPromise: Promise = Promise.resolve(true); - let waitPromise: Promise = Promise.resolve(true); - // 1. Check if there is previous waiting for the same file - if (previous) { - previousPromise = previous.canProceed.promise; - if (isShouldBeCancelled) { - Logger( - `Processing ${filename}: Requested to perform immediately, cancelling previous waiting: ${fei.type}`, - LOG_LEVEL_DEBUG - ); - } - if (!isShouldBeCancelled && fei.type === "DELETE") { - // For DELETE, cancel any previous waiting and proceed immediately - // That because when deleting, we cannot read the file anymore. - Logger( - `Processing ${filename}: DELETE requested, cancelling previous waiting: ${fei.type}`, - LOG_LEVEL_DEBUG - ); - isShouldBeCancelled = true; - } - if (!isShouldBeCancelled && previous.type === fei.type) { - // For the same type, we can cancel the previous waiting and proceed immediately. - Logger(`Processing ${filename}: Cancelling previous waiting: ${fei.type}`, LOG_LEVEL_DEBUG); - isShouldBeCancelled = true; - } - // 2. wait for the previous to complete - if (isShouldBeCancelled) { - this._cancelWaiting(waitingKey); - Logger(`Processing ${filename}: Previous cancelled: ${fei.type}`, LOG_LEVEL_DEBUG); - isShouldBeCancelled = true; - } - if (!isShouldBeCancelled) { - Logger(`Processing ${filename}: Waiting for previous to complete: ${fei.type}`, LOG_LEVEL_DEBUG); - this._proceedWaiting(waitingKey); - Logger(`Processing ${filename}: Previous completed: ${fei.type}`, LOG_LEVEL_DEBUG); - } - } - await previousPromise; - // 3. Check if shouldBatchSave is true - if (this.shouldBatchSave && !fei.skipBatchWait) { - // if type is CREATE or CHANGED, set waiting - if (fei.type == "CREATE" || fei.type == "CHANGED") { - // 3.2. If true, set the queue, and wait for the waiting, or until timeout - // (since is copied from previous waiting if exists to limit the maximum wait time) - // console.warn(`Since:`, previous?.since); - const info = this._addWaiting(waitingKey, fei, previous?.since); - waitPromise = info.canProceed.promise; - } else if (fei.type == "DELETE") { - // For DELETE, cancel any previous waiting and proceed immediately - } - Logger(`Processing ${filename}: Waiting for batch save: ${fei.type}`, LOG_LEVEL_DEBUG); - const canProceed = await waitPromise; - if (!canProceed) { - // 3.2.1. If cancelled by new queue, cancel subsequent process. - Logger(`Processing ${filename}: Cancelled by new queue: ${fei.type}`, LOG_LEVEL_DEBUG); - return; - } - } - // await this.handleFileEvent(fei); - await this.requestProcessQueue(fei); - } finally { - await this._takeSnapshot(); - releaser(); - } - } - async _takeSnapshot() { - const processingEvents = [...this._waitingMap.values()].map((e) => e.event); - const waitingEvents = this.bufferedQueuedItems; - const snapShot = [...processingEvents, ...waitingEvents]; - await this.core.kvDB.set("storage-event-manager-snapshot", snapShot); - Logger(`Storage operation snapshot taken: ${snapShot.length} items`, LOG_LEVEL_DEBUG); - this.updateStatus(); - } - async _restoreFromSnapshot() { - const snapShot = await this.core.kvDB.get<(FileEventItem | FileEventItemSentinel)[]>( - "storage-event-manager-snapshot" - ); - if (snapShot && Array.isArray(snapShot) && snapShot.length > 0) { - // console.warn(`Restoring snapshot: ${snapShot.length} items`); - Logger(`Restoring storage operation snapshot: ${snapShot.length} items`, LOG_LEVEL_VERBOSE); - // Restore the snapshot - // Note: Mark all items as skipBatchWait to prevent apply the off-line batch saving. - this.bufferedQueuedItems = snapShot.map((e) => ({ ...e, skipBatchWait: true })); - this.updateStatus(); - await this.runQueuedEvents(); - } else { - Logger(`No snapshot to restore`, LOG_LEVEL_VERBOSE); - // console.warn(`No snapshot to restore`); - } - } - runQueuedEvents() { - return skipIfDuplicated("storage-event-manager-run-queued-events", async () => { - do { - if (this.bufferedQueuedItems.length === 0) { - break; - } - // 1. Get the first queued item - - const fei = this.bufferedQueuedItems.shift()!; - await this._takeSnapshot(); - this.updateStatus(); - // 2. Consume 1 semaphore slot to enqueue processing. Then release immediately. - // (Just to limit the total concurrent processing count, because skipping batch handles at processFileEvent). - const releaser = await this.concurrentProcessing.acquire(); - releaser(); - this.updateStatus(); - // 3. Check if sentinel flush - // If sentinel, wait for idle and continue. - if (fei.type === TYPE_SENTINEL_FLUSH) { - Logger(`Waiting for idle`, LOG_LEVEL_VERBOSE); - // Flush all waiting batch queues - await this.waitForIdle(); - this.updateStatus(); - continue; - } - // 4. Process the event, this should be fire-and-forget to not block the queue processing in each file. - fireAndForget(() => this.processFileEvent(fei)); - } while (this.bufferedQueuedItems.length > 0); - }); - } - - processingCount = 0; - async requestProcessQueue(fei: FileEventItem) { - try { - this.processingCount++; - // this.bufferedQueuedItems.remove(fei); - this.updateStatus(); - // this.waitedSince.delete(fei.args.file.path); - await this.handleFileEvent(fei); - await this._takeSnapshot(); - } finally { - this.processingCount--; - this.updateStatus(); - } - } - isWaiting(filename: FilePath) { - return isWaitingForTimeout(`storage-event-manager-batchsave-${filename}`); - } - - updateStatus() { - const allFileEventItems = this.bufferedQueuedItems.filter((e): e is FileEventItem => "args" in e); - const allItems = allFileEventItems.filter((e) => !e.cancelled); - const totalItems = allItems.length + this.concurrentProcessing.waiting; - const processing = this.processingCount; - const batchedCount = this._waitingMap.size; - this.core.batched.value = batchedCount; - this.core.processing.value = processing; - this.core.totalQueued.value = totalItems + batchedCount + processing; - } - - async handleFileEvent(queue: FileEventItem): Promise { - const file = queue.args.file; - const lockKey = `handleFile:${file.path}`; - const ret = await serialized(lockKey, async () => { - if (queue.cancelled) { - Logger(`File event cancelled before processing: ${file.path}`, LOG_LEVEL_INFO); - return; - } - if (queue.type == "INTERNAL" || file.isInternal) { - await this.core.services.fileProcessing.processOptionalFileEvent(file.path as unknown as FilePath); - } else { - const key = `file-last-proc-${queue.type}-${file.path}`; - const last = Number((await this.core.kvDB.get(key)) || 0); - if (queue.type == "DELETE") { - await this.core.services.fileProcessing.processFileEvent(queue); - } else { - if (file.stat.mtime == last) { - Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL_VERBOSE); - // Should Cancel the relative operations? (e.g. rename) - // this.cancelRelativeEvent(queue); - return; - } - if (!(await this.core.services.fileProcessing.processFileEvent(queue))) { - Logger( - `STORAGE -> DB: Handler failed, cancel the relative operations: ${file.path}`, - LOG_LEVEL_INFO - ); - // cancel running queues and remove one of atomic operation (e.g. rename) - this.cancelRelativeEvent(queue); - return; - } - } - } - }); - this.updateStatus(); - return ret; - } - - cancelRelativeEvent(item: FileEventItem): void { - this._cancelWaiting(item.args.file.path); - } -} diff --git a/src/modules/coreObsidian/storageLib/utilObsidian.ts b/src/modules/coreObsidian/storageLib/utilObsidian.ts index 0158072..c4d4dfe 100644 --- a/src/modules/coreObsidian/storageLib/utilObsidian.ts +++ b/src/modules/coreObsidian/storageLib/utilObsidian.ts @@ -2,7 +2,6 @@ import { TFile, type TAbstractFile, type TFolder } from "../../../deps.ts"; import { ICHeader } from "../../../common/types.ts"; -import type { ObsidianFileAccess } from "./SerializedFileAccess.ts"; import { addPrefix, isPlainText } from "../../../lib/src/string_and_binary/path.ts"; import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger"; import { createBlob } from "../../../lib/src/common/utils.ts"; @@ -15,6 +14,7 @@ import type { UXInternalFileInfoStub, } from "../../../lib/src/common/types.ts"; import type { LiveSyncCore } from "../../../main.ts"; +import type { FileAccessObsidian } from "@/serviceModules/FileAccessObsidian.ts"; export async function TFileToUXFileInfo( core: LiveSyncCore, @@ -51,7 +51,7 @@ export async function TFileToUXFileInfo( export async function InternalFileToUXFileInfo( fullPath: string, - vaultAccess: ObsidianFileAccess, + vaultAccess: FileAccessObsidian, prefix: string = ICHeader ): Promise { const name = fullPath.split("/").pop() as string; diff --git a/src/serviceModules/FileAccessObsidian.ts b/src/serviceModules/FileAccessObsidian.ts new file mode 100644 index 0000000..4755717 --- /dev/null +++ b/src/serviceModules/FileAccessObsidian.ts @@ -0,0 +1,156 @@ +import { markChangesAreSame } from "@/common/utils"; +import type { FilePath, UXDataWriteOptions, UXFileInfoStub, UXFolderInfo } from "@lib/common/types"; + +import { TFolder, type TAbstractFile, TFile, type Stat, type App, type DataWriteOptions } from "@/deps"; +import { FileAccessBase, toArrayBuffer, type FileAccessBaseDependencies } from "@lib/serviceModules/FileAccessBase.ts"; +import { TFileToUXFileInfoStub } from "@/modules/coreObsidian/storageLib/utilObsidian"; + +declare module "obsidian" { + interface Vault { + getAbstractFileByPathInsensitive(path: string): TAbstractFile | null; + } + interface DataAdapter { + reconcileInternalFile?(path: string): Promise; + } +} + +export class FileAccessObsidian extends FileAccessBase { + app: App; + + override getPath(file: string | TAbstractFile): FilePath { + return (typeof file === "string" ? file : file.path) as FilePath; + } + + override isFile(file: TAbstractFile | null): file is TFile { + return file instanceof TFile; + } + override isFolder(file: TAbstractFile | null): file is TFolder { + return file instanceof TFolder; + } + override _statFromNative(file: TFile): Promise { + return Promise.resolve(file.stat); + } + + override nativeFileToUXFileInfoStub(file: TFile): UXFileInfoStub { + return TFileToUXFileInfoStub(file); + } + override nativeFolderToUXFolder(folder: TFolder): UXFolderInfo { + if (folder instanceof TFolder) { + return this.nativeFolderToUXFolder(folder); + } else { + throw new Error(`Not a folder: ${(folder as TAbstractFile)?.name}`); + } + } + + constructor(app: App, dependencies: FileAccessBaseDependencies) { + super({ + storageAccessManager: dependencies.storageAccessManager, + vaultService: dependencies.vaultService, + settingService: dependencies.settingService, + APIService: dependencies.APIService, + }); + this.app = app; + } + + protected async _adapterMkdir(path: string) { + await this.app.vault.adapter.mkdir(path); + } + protected _getAbstractFileByPath(path: FilePath) { + return this.app.vault.getAbstractFileByPath(path); + } + protected _getAbstractFileByPathInsensitive(path: FilePath) { + return this.app.vault.getAbstractFileByPathInsensitive(path); + } + + protected async _tryAdapterStat(path: FilePath) { + if (!(await this.app.vault.adapter.exists(path))) return null; + return await this.app.vault.adapter.stat(path); + } + + protected async _adapterStat(path: FilePath) { + return await this.app.vault.adapter.stat(path); + } + + protected async _adapterExists(path: FilePath) { + return await this.app.vault.adapter.exists(path); + } + protected async _adapterRemove(path: FilePath) { + await this.app.vault.adapter.remove(path); + } + + protected async _adapterRead(path: FilePath) { + return await this.app.vault.adapter.read(path); + } + + protected async _adapterReadBinary(path: FilePath) { + return await this.app.vault.adapter.readBinary(path); + } + + _adapterWrite(file: string, data: string, options?: UXDataWriteOptions): Promise { + return this.app.vault.adapter.write(file, data, options); + } + _adapterWriteBinary(file: string, data: ArrayBuffer, options?: UXDataWriteOptions): Promise { + return this.app.vault.adapter.writeBinary(file, toArrayBuffer(data), options); + } + + protected _adapterList(basePath: string): Promise<{ files: string[]; folders: string[] }> { + return Promise.resolve(this.app.vault.adapter.list(basePath)); + } + + async _vaultCacheRead(file: TFile) { + return await this.app.vault.cachedRead(file); + } + + protected async _vaultRead(file: TFile): Promise { + return await this.app.vault.read(file); + } + + protected async _vaultReadBinary(file: TFile): Promise { + return await this.app.vault.readBinary(file); + } + + protected override markChangesAreSame(path: string, mtime: number, newMtime: number) { + return markChangesAreSame(path, mtime, newMtime); + } + + protected override async _vaultModify(file: TFile, data: string, options?: UXDataWriteOptions): Promise { + return await this.app.vault.modify(file, data, options); + } + protected override async _vaultModifyBinary( + file: TFile, + data: ArrayBuffer, + options?: UXDataWriteOptions + ): Promise { + return await this.app.vault.modifyBinary(file, toArrayBuffer(data), options); + } + protected override async _vaultCreate(path: string, data: string, options?: UXDataWriteOptions): Promise { + return await this.app.vault.create(path, data, options); + } + protected override async _vaultCreateBinary( + path: string, + data: ArrayBuffer, + options?: UXDataWriteOptions + ): Promise { + return await this.app.vault.createBinary(path, toArrayBuffer(data), options); + } + + protected override _trigger(name: string, ...data: any[]) { + return this.app.vault.trigger(name, ...data); + } + protected override async _reconcileInternalFile(path: string) { + return await Promise.resolve(this.app.vault.adapter.reconcileInternalFile?.(path)); + } + protected override async _adapterAppend(normalizedPath: string, data: string, options?: DataWriteOptions) { + return await this.app.vault.adapter.append(normalizedPath, data, options); + } + protected override async _delete(file: TFile | TFolder, force = false) { + return await this.app.vault.delete(file, force); + } + protected override async _trash(file: TFile | TFolder, force = false) { + return await this.app.vault.trash(file, force); + } + + protected override _getFiles() { + return this.app.vault.getFiles(); + } +} diff --git a/src/serviceModules/ServiceFileAccessImpl.ts b/src/serviceModules/ServiceFileAccessImpl.ts new file mode 100644 index 0000000..686dd0b --- /dev/null +++ b/src/serviceModules/ServiceFileAccessImpl.ts @@ -0,0 +1,6 @@ +import type { TAbstractFile, TFile, TFolder, Stat } from "@/deps"; + +import { ServiceFileAccessBase } from "@lib/serviceModules/ServiceFileAccessBase"; + +// For typechecking purpose +export class ServiceFileAccessObsidian extends ServiceFileAccessBase {} diff --git a/src/serviceModules/ServiceFileAccessObsidian.ts b/src/serviceModules/ServiceFileAccessObsidian.ts deleted file mode 100644 index 95a9278..0000000 --- a/src/serviceModules/ServiceFileAccessObsidian.ts +++ /dev/null @@ -1,363 +0,0 @@ -import { TFile, TFolder, type ListedFiles } from "@/deps.ts"; -import { LOG_LEVEL_INFO, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger"; -import type { - FilePath, - FilePathWithPrefix, - UXDataWriteOptions, - UXFileInfo, - UXFileInfoStub, - UXFolderInfo, - UXStat, -} from "@lib/common/types"; - -import { ServiceModuleBase } from "@lib/serviceModules/ServiceModuleBase"; -import type { APIService } from "@lib/services/base/APIService"; -import type { IStorageAccessManager, StorageAccess } from "@lib/interfaces/StorageAccess.ts"; -import type { AppLifecycleService } from "@lib/services/base/AppLifecycleService"; -import type { FileProcessingService } from "@lib/services/base/FileProcessingService"; -import { ObsidianFileAccess } from "@/modules/coreObsidian/storageLib/SerializedFileAccess"; -import { StorageEventManager } from "@lib/interfaces/StorageEventManager.ts"; -import { TFileToUXFileInfoStub, TFolderToUXFileInfoStub } from "@/modules/coreObsidian/storageLib/utilObsidian"; -import { createBlob, type CustomRegExp } from "@lib/common/utils"; -import type { VaultService } from "@lib/services/base/VaultService"; -import type { SettingService } from "@lib/services/base/SettingService"; - -export interface StorageAccessObsidianDependencies { - API: APIService; - appLifecycle: AppLifecycleService; - fileProcessing: FileProcessingService; - vault: VaultService; - setting: SettingService; - storageEventManager: StorageEventManager; - storageAccessManager: IStorageAccessManager; - vaultAccess: ObsidianFileAccess; -} - -export class ServiceFileAccessObsidian - extends ServiceModuleBase - implements StorageAccess -{ - private vaultAccess: ObsidianFileAccess; - private vaultManager: StorageEventManager; - private vault: VaultService; - private setting: SettingService; - - constructor(services: StorageAccessObsidianDependencies) { - super(services); - // this.appLifecycle = services.appLifecycle; - this.vault = services.vault; - this.setting = services.setting; - this.vaultManager = services.storageEventManager; - this.vaultAccess = services.vaultAccess; - services.appLifecycle.onFirstInitialise.addHandler(this._everyOnFirstInitialize.bind(this)); - services.fileProcessing.commitPendingFileEvents.addHandler(this._everyCommitPendingFileEvent.bind(this)); - } - - restoreState() { - return this.vaultManager.restoreState(); - } - async _everyOnFirstInitialize(): Promise { - await this.vaultManager.beginWatch(); - return Promise.resolve(true); - } - - async _everyCommitPendingFileEvent(): Promise { - await this.vaultManager.waitForIdle(); - return Promise.resolve(true); - } - - async writeFileAuto(path: string, data: string | ArrayBuffer, opt?: UXDataWriteOptions): Promise { - const file = this.vaultAccess.getAbstractFileByPath(path); - if (file instanceof TFile) { - return this.vaultAccess.vaultModify(file, data, opt); - } else if (file === null) { - if (!path.endsWith(".md")) { - // Very rare case, we encountered this case with `writing-goals-history.csv` file. - // Indeed, that file not appears in the File Explorer, but it exists in the vault. - // Hence, we cannot retrieve the file from the vault by getAbstractFileByPath, and we cannot write it via vaultModify. - // It makes `File already exists` error. - // Therefore, we need to write it via adapterWrite. - // Maybe there are others like this, so I will write it via adapterWrite. - // This is a workaround for the issue, but I don't know if this is the right solution. - // (So limits to non-md files). - // Has Obsidian been patched?, anyway, writing directly might be a safer approach. - // However, does changes of that file trigger file-change event? - await this.vaultAccess.adapterWrite(path, data, opt); - // For safety, check existence - return await this.vaultAccess.adapterExists(path); - } else { - return (await this.vaultAccess.vaultCreate(path, data, opt)) instanceof TFile; - } - } else { - this._log(`Could not write file (Possibly already exists as a folder): ${path}`, LOG_LEVEL_VERBOSE); - return false; - } - } - readFileAuto(path: string): Promise { - const file = this.vaultAccess.getAbstractFileByPath(path); - if (file instanceof TFile) { - return this.vaultAccess.vaultRead(file); - } else { - throw new Error(`Could not read file (Possibly does not exist): ${path}`); - } - } - readFileText(path: string): Promise { - const file = this.vaultAccess.getAbstractFileByPath(path); - if (file instanceof TFile) { - return this.vaultAccess.vaultRead(file); - } else { - throw new Error(`Could not read file (Possibly does not exist): ${path}`); - } - } - isExists(path: string): Promise { - return Promise.resolve(this.vaultAccess.getAbstractFileByPath(path) instanceof TFile); - } - async writeHiddenFileAuto(path: string, data: string | ArrayBuffer, opt?: UXDataWriteOptions): Promise { - try { - await this.vaultAccess.adapterWrite(path, data, opt); - return true; - } catch (e) { - this._log(`Could not write hidden file: ${path}`, LOG_LEVEL_VERBOSE); - this._log(e, LOG_LEVEL_VERBOSE); - return false; - } - } - async appendHiddenFile(path: string, data: string, opt?: UXDataWriteOptions): Promise { - try { - await this.vaultAccess.adapterAppend(path, data, opt); - return true; - } catch (e) { - this._log(`Could not append hidden file: ${path}`, LOG_LEVEL_VERBOSE); - this._log(e, LOG_LEVEL_VERBOSE); - return false; - } - } - stat(path: string): Promise { - const file = this.vaultAccess.getAbstractFileByPath(path); - if (file === null) return Promise.resolve(null); - if (file instanceof TFile) { - return Promise.resolve({ - ctime: file.stat.ctime, - mtime: file.stat.mtime, - size: file.stat.size, - type: "file", - }); - } else { - throw new Error(`Could not stat file (Possibly does not exist): ${path}`); - } - } - statHidden(path: string): Promise { - return this.vaultAccess.tryAdapterStat(path); - } - async removeHidden(path: string): Promise { - try { - await this.vaultAccess.adapterRemove(path); - if (this.vaultAccess.tryAdapterStat(path) !== null) { - return false; - } - return true; - } catch (e) { - this._log(`Could not remove hidden file: ${path}`, LOG_LEVEL_VERBOSE); - this._log(e, LOG_LEVEL_VERBOSE); - return false; - } - } - async readHiddenFileAuto(path: string): Promise { - return await this.vaultAccess.adapterReadAuto(path); - } - async readHiddenFileText(path: string): Promise { - return await this.vaultAccess.adapterRead(path); - } - async readHiddenFileBinary(path: string): Promise { - return await this.vaultAccess.adapterReadBinary(path); - } - async isExistsIncludeHidden(path: string): Promise { - return (await this.vaultAccess.tryAdapterStat(path)) !== null; - } - async ensureDir(path: string): Promise { - try { - await this.vaultAccess.ensureDirectory(path); - return true; - } catch (e) { - this._log(`Could not ensure directory: ${path}`, LOG_LEVEL_VERBOSE); - this._log(e, LOG_LEVEL_VERBOSE); - return false; - } - } - triggerFileEvent(event: string, path: string): void { - const file = this.vaultAccess.getAbstractFileByPath(path); - if (file === null) return; - this.vaultAccess.trigger(event, file); - } - async triggerHiddenFile(path: string): Promise { - await this.vaultAccess.reconcileInternalFile(path); - } - // getFileStub(file: TFile): UXFileInfoStub { - // return TFileToUXFileInfoStub(file); - // } - getFileStub(path: string): UXFileInfoStub | null { - const file = this.vaultAccess.getAbstractFileByPath(path); - if (file instanceof TFile) { - return TFileToUXFileInfoStub(file); - } else { - return null; - } - } - - async readStubContent(stub: UXFileInfoStub): Promise { - const file = this.vaultAccess.getAbstractFileByPath(stub.path); - if (!(file instanceof TFile)) { - this._log(`Could not read file (Possibly does not exist or a folder): ${stub.path}`, LOG_LEVEL_VERBOSE); - return false; - } - const data = await this.vaultAccess.vaultReadAuto(file); - return { - ...stub, - ...TFileToUXFileInfoStub(file), - body: createBlob(data), - }; - } - getStub(path: string): UXFileInfoStub | UXFolderInfo | null { - const file = this.vaultAccess.getAbstractFileByPath(path); - if (file instanceof TFile) { - return TFileToUXFileInfoStub(file); - } else if (file instanceof TFolder) { - return TFolderToUXFileInfoStub(file); - } - return null; - } - getFiles(): UXFileInfoStub[] { - return this.vaultAccess.getFiles().map((f) => TFileToUXFileInfoStub(f)); - } - getFileNames(): FilePath[] { - return this.vaultAccess.getFiles().map((f) => f.path as FilePath); - } - - async getFilesIncludeHidden( - basePath: string, - includeFilter?: CustomRegExp[], - excludeFilter?: CustomRegExp[], - skipFolder: string[] = [".git", ".trash", "node_modules"] - ): Promise { - let w: ListedFiles; - try { - w = await this.vaultAccess.adapterList(basePath); - // w = await this.plugin.app.vault.adapter.list(basePath); - } catch (ex) { - this._log(`Could not traverse(getFilesIncludeHidden):${basePath}`, LOG_LEVEL_INFO); - this._log(ex, LOG_LEVEL_VERBOSE); - return []; - } - skipFolder = skipFolder.map((e) => e.toLowerCase()); - - let files = [] as string[]; - for (const file of w.files) { - if (includeFilter && includeFilter.length > 0) { - if (!includeFilter.some((e) => e.test(file))) continue; - } - if (excludeFilter && excludeFilter.some((ee) => ee.test(file))) { - continue; - } - if (await this.vault.isIgnoredByIgnoreFile(file)) continue; - files.push(file); - } - - for (const v of w.folders) { - const folderName = (v.split("/").pop() ?? "").toLowerCase(); - if (skipFolder.some((e) => folderName === e)) { - continue; - } - - if (excludeFilter && excludeFilter.some((e) => e.test(v))) { - continue; - } - if (await this.vault.isIgnoredByIgnoreFile(v)) { - continue; - } - // OK, deep dive! - files = files.concat(await this.getFilesIncludeHidden(v, includeFilter, excludeFilter, skipFolder)); - } - return files as FilePath[]; - } - async touched(file: UXFileInfoStub | FilePathWithPrefix): Promise { - const path = typeof file === "string" ? file : file.path; - await this.vaultAccess.touch(path as FilePath); - } - recentlyTouched(file: UXFileInfoStub | FilePathWithPrefix): boolean { - const xFile = typeof file === "string" ? (this.vaultAccess.getAbstractFileByPath(file) as TFile) : file; - if (xFile === null) return false; - if (xFile instanceof TFolder) return false; - return this.vaultAccess.recentlyTouched(xFile); - } - clearTouched(): void { - this.vaultAccess.clearTouched(); - } - - delete(file: FilePathWithPrefix | UXFileInfoStub | string, force: boolean): Promise { - const xPath = typeof file === "string" ? file : file.path; - const xFile = this.vaultAccess.getAbstractFileByPath(xPath); - if (xFile === null) return Promise.resolve(); - if (!(xFile instanceof TFile) && !(xFile instanceof TFolder)) return Promise.resolve(); - return this.vaultAccess.delete(xFile, force); - } - trash(file: FilePathWithPrefix | UXFileInfoStub | string, system: boolean): Promise { - const xPath = typeof file === "string" ? file : file.path; - const xFile = this.vaultAccess.getAbstractFileByPath(xPath); - if (xFile === null) return Promise.resolve(); - if (!(xFile instanceof TFile) && !(xFile instanceof TFolder)) return Promise.resolve(); - return this.vaultAccess.trash(xFile, system); - } - // $readFileBinary(path: string): Promise { - // const file = this.vaultAccess.getAbstractFileByPath(path); - // if (file instanceof TFile) { - // return this.vaultAccess.vaultReadBinary(file); - // } else { - // throw new Error(`Could not read file (Possibly does not exist): ${path}`); - // } - // } - // async $appendFileAuto(path: string, data: string | ArrayBuffer, opt?: DataWriteOptions): Promise { - // const file = this.vaultAccess.getAbstractFileByPath(path); - // if (file instanceof TFile) { - // return this.vaultAccess.a(file, data, opt); - // } else if (file !== null) { - // return await this.vaultAccess.vaultCreate(path, data, opt) instanceof TFile; - // } else { - // this._log(`Could not append file (Possibly already exists as a folder): ${path}`, LOG_LEVEL_VERBOSE); - // return false; - // } - // } - - async __deleteVaultItem(file: TFile | TFolder) { - if (file instanceof TFile) { - if (!(await this.vault.isTargetFile(file.path))) return; - } - const dir = file.parent; - const settings = this.setting.currentSettings(); - if (settings.trashInsteadDelete) { - await this.vaultAccess.trash(file, false); - } else { - await this.vaultAccess.delete(file, true); - } - this._log(`xxx <- STORAGE (deleted) ${file.path}`); - if (dir) { - this._log(`files: ${dir.children.length}`); - if (dir.children.length == 0) { - if (!settings.doNotDeleteFolder) { - this._log( - `All files under the parent directory (${dir.path}) have been deleted, so delete this one.` - ); - await this.__deleteVaultItem(dir); - } - } - } - } - - async deleteVaultItem(fileSrc: FilePathWithPrefix | UXFileInfoStub | UXFolderInfo): Promise { - const path = typeof fileSrc === "string" ? fileSrc : fileSrc.path; - const file = this.vaultAccess.getAbstractFileByPath(path); - if (file === null) return; - if (file instanceof TFile || file instanceof TFolder) { - return await this.__deleteVaultItem(file); - } - } -} diff --git a/updates.md b/updates.md index ac65b30..76d2a3e 100644 --- a/updates.md +++ b/updates.md @@ -3,6 +3,27 @@ Since 19th July, 2025 (beta1 in 0.25.0-beta1, 13th July, 2025) The head note of 0.25 is now in [updates_old.md](https://github.com/vrtmrz/obsidian-livesync/blob/main/updates_old.md). Because 0.25 got a lot of updates, thankfully, compatibility is kept and we do not need breaking changes! In other words, when get enough stabled. The next version will be v1.0.0. Even though it my hope. +## 0.25.43-patched-6 + +18th February, 2026 + +Let me confess that I have lied about `now all ambiguous properties`... I have found some more implicit calling. + +Note: I have not checked hidden file sync and customisation sync, yet. Please report if you find any unexpected behaviour on these features. + +### Fixed + +- Now ReplicatorService responds to database reset and database initialisation events to dispose the active replicator. + - Fixes some unlocking issues during rebuilding. + +### Refactored + +- Now `StorageEventManagerBase` is separated from `StorageEventManagerObsidian` following their concerns. + - No longer using `ObsidianFileAccess` indirectly during checking duplicated-file events. + - Last event memorisation is now moved into the StorageAccessManager, just like the file processing interlocking. + - These methods, i.e., `ObsidianFileAccess.touch`. `StorageEventManager.recentlyTouched`, and `StorageEventManager.touch` are still available, but simply call the StorageAccessManager's methods. +- Now `FileAccessBase` is separated from `FileAccessObsidian` following their concerns. + ## 0.25.43-patched-5 17th February, 2026