public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-02-21 03:28:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

## 0.25.43-patched-6

Browse Source 
### Fixed

- Unlocking the remote database after rebuilding has been fixed.

### Refactored
- Now `StorageEventManagerBase` is separated from `StorageEventManagerObsidian` following their concerns.
- Now `FileAccessBase` is separated from `FileAccessObsidian` following their concerns.
This commit is contained in:
vorotamoroz
2026-02-18 12:13:05 +00:00
parent 4658e3735d
commit 2bf1c775ee
15 changed files with 421 additions and 1274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/deps.ts
View File

@@ -40,6 +40,7 @@ export type {
MarkdownFileInfo,
ListedFiles,
ValueComponent,
Stat,
} from "obsidian";
import { normalizePath as normalizePath_ } from "obsidian";
const normalizePath = normalizePath_ as <T extends string | FilePath>(from: T) => T;

2
src/lib

Submodule src/lib updated: 56fc24e001...744a1b01eb

21
src/main.ts
View File

@@ -60,12 +60,12 @@ import type { ServiceContext } from "./lib/src/services/base/ServiceBase.ts";
import { ServiceRebuilder } from "@lib/serviceModules/Rebuilder.ts";
import type { IFileHandler } from "@lib/interfaces/FileHandler.ts";
import { ServiceDatabaseFileAccess } from "@/serviceModules/DatabaseFileAccess.ts";
import { ServiceFileAccessObsidian } from "@/serviceModules/ServiceFileAccessObsidian.ts";
import { StorageEventManagerObsidian } from "@/modules/coreObsidian/storageLib/StorageEventManager.ts";
import { ObsidianFileAccess } from "@/modules/coreObsidian/storageLib/SerializedFileAccess.ts";
import { ServiceFileAccessObsidian } from "@/serviceModules/ServiceFileAccessImpl.ts";
import { StorageAccessManager } from "@lib/managers/StorageProcessingManager.ts";
import { __$checkInstanceBinding } from "./lib/src/dev/checks.ts";
import { ServiceFileHandler } from "./serviceModules/FileHandler.ts";
import { FileAccessObsidian } from "./serviceModules/FileAccessObsidian.ts";
import { StorageEventManagerObsidian } from "./managers/StorageEventManagerObsidian.ts";
export default class ObsidianLiveSyncPlugin
extends Plugin
@@ -302,8 +302,19 @@ export default class ObsidianLiveSyncPlugin
private initialiseServiceModules() {
const storageAccessManager = new StorageAccessManager();
// If we want to implement to the other platform, implement ObsidianXXXXXService.
const vaultAccess = new ObsidianFileAccess(this.app, this, storageAccessManager);
const storageEventManager = new StorageEventManagerObsidian(this, this, storageAccessManager);
const vaultAccess = new FileAccessObsidian(this.app, {
storageAccessManager: storageAccessManager,
vaultService: this.services.vault,
settingService: this.services.setting,
APIService: this.services.API,
});
const storageEventManager = new StorageEventManagerObsidian(this, this, {
fileProcessing: this.services.fileProcessing,
setting: this.services.setting,
vaultService: this.services.vault,
storageAccessManager: storageAccessManager,
APIService: this.services.API,
});
const storageAccess = new ServiceFileAccessObsidian({
API: this.services.API,
setting: this.services.setting,

210
src/managers/StorageEventManagerObsidian.ts Normal file
View File

@@ -0,0 +1,210 @@
import type { FileEventItem } from "@/common/types";
import { HiddenFileSync } from "@/features/HiddenFileSync/CmdHiddenFileSync";
import type { FilePath, UXFileInfoStub, UXFolderInfo, UXInternalFileInfoStub } from "@lib/common/types";
import type { FileEvent } from "@lib/interfaces/StorageEventManager";
import { TFile, type TAbstractFile, TFolder } from "@/deps";
import { LOG_LEVEL_DEBUG } from "octagonal-wheels/common/logger";
import type ObsidianLiveSyncPlugin from "@/main";
import type { LiveSyncCore } from "@/main";
import {
StorageEventManagerBase,
type FileEventItemSentinel,
type StorageEventManagerBaseDependencies,
} from "@lib/managers/StorageEventManager";
import { InternalFileToUXFileInfoStub, TFileToUXFileInfoStub } from "@/modules/coreObsidian/storageLib/utilObsidian";
export class StorageEventManagerObsidian extends StorageEventManagerBase {
plugin: ObsidianLiveSyncPlugin;
core: LiveSyncCore;
// Necessary evil.
cmdHiddenFileSync: HiddenFileSync;
override isFile(file: UXFileInfoStub | UXInternalFileInfoStub | UXFolderInfo | TFile): boolean {
if (file instanceof TFile) {
return true;
}
if (super.isFile(file)) {
return true;
}
return !file.isFolder;
}
override isFolder(file: UXFileInfoStub | UXInternalFileInfoStub | UXFolderInfo | TFolder): boolean {
if (file instanceof TFolder) {
return true;
}
if (super.isFolder(file)) {
return true;
}
return !!file.isFolder;
}
constructor(plugin: ObsidianLiveSyncPlugin, core: LiveSyncCore, dependencies: StorageEventManagerBaseDependencies) {
super(dependencies);
this.plugin = plugin;
this.core = core;
this.cmdHiddenFileSync = this.plugin.getAddOn(HiddenFileSync.name) as HiddenFileSync;
}
async beginWatch() {
await this.snapShotRestored;
const plugin = this.plugin;
this.watchVaultChange = this.watchVaultChange.bind(this);
this.watchVaultCreate = this.watchVaultCreate.bind(this);
this.watchVaultDelete = this.watchVaultDelete.bind(this);
this.watchVaultRename = this.watchVaultRename.bind(this);
this.watchVaultRawEvents = this.watchVaultRawEvents.bind(this);
this.watchEditorChange = this.watchEditorChange.bind(this);
plugin.registerEvent(plugin.app.vault.on("modify", this.watchVaultChange));
plugin.registerEvent(plugin.app.vault.on("delete", this.watchVaultDelete));
plugin.registerEvent(plugin.app.vault.on("rename", this.watchVaultRename));
plugin.registerEvent(plugin.app.vault.on("create", this.watchVaultCreate));
//@ts-ignore : Internal API
plugin.registerEvent(plugin.app.vault.on("raw", this.watchVaultRawEvents));
plugin.registerEvent(plugin.app.workspace.on("editor-change", this.watchEditorChange));
}
watchEditorChange(editor: any, info: any) {
if (!("path" in info)) {
return;
}
if (!this.shouldBatchSave) {
return;
}
const file = info?.file as TFile;
if (!file) return;
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
// this._log(`Editor change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
return;
}
if (!this.isWaiting(file.path as FilePath)) {
return;
}
const data = info?.data as string;
const fi: FileEvent = {
type: "CHANGED",
file: TFileToUXFileInfoStub(file),
cachedData: data,
};
void this.appendQueue([fi]);
}
watchVaultCreate(file: TAbstractFile, ctx?: any) {
if (file instanceof TFolder) return;
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
// this._log(`File create skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
return;
}
const fileInfo = TFileToUXFileInfoStub(file);
void this.appendQueue([{ type: "CREATE", file: fileInfo }], ctx);
}
watchVaultChange(file: TAbstractFile, ctx?: any) {
if (file instanceof TFolder) return;
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
// this._log(`File change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
return;
}
const fileInfo = TFileToUXFileInfoStub(file);
void this.appendQueue([{ type: "CHANGED", file: fileInfo }], ctx);
}
watchVaultDelete(file: TAbstractFile, ctx?: any) {
if (file instanceof TFolder) return;
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
// this._log(`File delete skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
return;
}
const fileInfo = TFileToUXFileInfoStub(file, true);
void this.appendQueue([{ type: "DELETE", file: fileInfo }], ctx);
}
watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) {
// vault Rename will not be raised for self-events (Self-hosted LiveSync will not handle 'rename').
if (file instanceof TFile) {
const fileInfo = TFileToUXFileInfoStub(file);
void this.appendQueue(
[
{
type: "DELETE",
file: {
path: oldFile as FilePath,
name: file.name,
stat: {
mtime: file.stat.mtime,
ctime: file.stat.ctime,
size: file.stat.size,
type: "file",
},
deleted: true,
},
skipBatchWait: true,
},
{ type: "CREATE", file: fileInfo, skipBatchWait: true },
],
ctx
);
}
}
// Watch raw events (Internal API)
watchVaultRawEvents(path: FilePath) {
if (this.storageAccess.isFileProcessing(path)) {
// this._log(`Raw file event skipped because the file is being processed: ${path}`, LOG_LEVEL_VERBOSE);
return;
}
// Only for internal files.
if (!this.settings) return;
// if (this.plugin.settings.useIgnoreFiles && this.plugin.ignoreFiles.some(e => path.endsWith(e.trim()))) {
if (this.settings.useIgnoreFiles) {
// If it is one of ignore files, refresh the cached one.
// (Calling$$isTargetFile will refresh the cache)
void this.vaultService.isTargetFile(path).then(() => this._watchVaultRawEvents(path));
} else {
void this._watchVaultRawEvents(path);
}
}
async _watchVaultRawEvents(path: FilePath) {
if (!this.settings.syncInternalFiles && !this.settings.usePluginSync) return;
if (!this.settings.watchInternalFileChanges) return;
if (!path.startsWith(this.plugin.app.vault.configDir)) return;
if (path.endsWith("/")) {
// Folder
return;
}
const isTargetFile = await this.cmdHiddenFileSync.isTargetFile(path);
if (!isTargetFile) return;
void this.appendQueue(
[
{
type: "INTERNAL",
file: InternalFileToUXFileInfoStub(path),
skipBatchWait: true, // Internal files should be processed immediately.
},
],
null
);
}
async _saveSnapshot(snapshot: (FileEventItem | FileEventItemSentinel)[]) {
await this.core.kvDB.set("storage-event-manager-snapshot", snapshot);
this._log(`Storage operation snapshot saved: ${snapshot.length} items`, LOG_LEVEL_DEBUG);
}
async _loadSnapshot() {
const snapShot = await this.core.kvDB.get<(FileEventItem | FileEventItemSentinel)[]>(
"storage-event-manager-snapshot"
);
return snapShot;
}
updateStatus() {
const allFileEventItems = this.bufferedQueuedItems.filter((e): e is FileEventItem => "args" in e);
const allItems = allFileEventItems.filter((e) => !e.cancelled);
const totalItems = allItems.length + this.concurrentProcessing.waiting;
const processing = this.processingCount;
const batchedCount = this._waitingMap.size;
this.core.batched.value = batchedCount;
this.core.processing.value = processing;
this.core.totalQueued.value = totalItems + batchedCount + processing;
}
}

8
src/modules/core/ModuleTargetFilter.ts
View File

@@ -141,10 +141,10 @@ export class ModuleTargetFilter extends AbstractModule {
services.vault.markFileListPossiblyChanged.setHandler(this._markFileListPossiblyChanged.bind(this));
services.appLifecycle.onLoaded.addHandler(this._everyOnload.bind(this));
services.vault.isIgnoredByIgnoreFile.setHandler(this._isTargetIgnoredByIgnoreFiles.bind(this));
services.vault.isTargetFile.addHandler(this._isTargetFileByFileNameDuplication.bind(this));
services.vault.isTargetFile.addHandler(this._isTargetIgnoredByIgnoreFiles.bind(this));
services.vault.isTargetFile.addHandler(this._isTargetFileByLocalDB.bind(this));
services.vault.isTargetFile.addHandler(this._isTargetFileFinal.bind(this));
services.vault.isTargetFile.addHandler(this._isTargetFileByFileNameDuplication.bind(this), 10);
services.vault.isTargetFile.addHandler(this._isTargetIgnoredByIgnoreFiles.bind(this), 20);
services.vault.isTargetFile.addHandler(this._isTargetFileByLocalDB.bind(this), 30);
services.vault.isTargetFile.addHandler(this._isTargetFileFinal.bind(this), 100);
services.setting.onSettingRealised.addHandler(this.refreshSettings.bind(this));
}
}

264
src/modules/coreObsidian/storageLib/SerializedFileAccess.ts
View File

@@ -1,264 +0,0 @@
import { type App, TFile, type DataWriteOptions, TFolder, TAbstractFile } from "../../../deps.ts";
import { Logger } from "../../../lib/src/common/logger.ts";
import { isPlainText } from "../../../lib/src/string_and_binary/path.ts";
import type { FilePath, UXFileInfoStub } from "../../../lib/src/common/types.ts";
import { createBinaryBlob, isDocContentSame } from "../../../lib/src/common/utils.ts";
import type { InternalFileInfo } from "../../../common/types.ts";
import { markChangesAreSame } from "../../../common/utils.ts";
import type { IStorageAccessManager } from "@lib/interfaces/StorageAccess.ts";
import type { LiveSyncCore } from "@/main.ts";
function toArrayBuffer(arr: Uint8Array<ArrayBuffer> | ArrayBuffer | DataView<ArrayBuffer>): ArrayBuffer {
if (arr instanceof Uint8Array) {
return arr.buffer;
}
if (arr instanceof DataView) {
return arr.buffer;
}
return arr;
}
// TODO: add abstraction for the file access (as wrapping TFile or something similar)
export abstract class FileAccessBase<TNativeFile> {
storageAccessManager: IStorageAccessManager;
constructor(storageAccessManager: IStorageAccessManager) {
this.storageAccessManager = storageAccessManager;
}
abstract getPath(file: TNativeFile | string): FilePath;
}
export class ObsidianFileAccess extends FileAccessBase<TFile> {
app: App;
plugin: LiveSyncCore;
getPath(file: string | TFile): FilePath {
return (typeof file === "string" ? file : file.path) as FilePath;
}
constructor(app: App, plugin: LiveSyncCore, storageAccessManager: IStorageAccessManager) {
super(storageAccessManager);
this.app = app;
this.plugin = plugin;
}
async tryAdapterStat(file: TFile | string) {
const path = file instanceof TFile ? file.path : file;
return await this.storageAccessManager.processReadFile(path as FilePath, async () => {
if (!(await this.app.vault.adapter.exists(path))) return null;
return this.app.vault.adapter.stat(path);
});
}
async adapterStat(file: TFile | string) {
const path = file instanceof TFile ? file.path : file;
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
this.app.vault.adapter.stat(path)
);
}
async adapterExists(file: TFile | string) {
const path = file instanceof TFile ? file.path : file;
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
this.app.vault.adapter.exists(path)
);
}
async adapterRemove(file: TFile | string) {
const path = file instanceof TFile ? file.path : file;
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
this.app.vault.adapter.remove(path)
);
}
async adapterRead(file: TFile | string) {
const path = file instanceof TFile ? file.path : file;
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
this.app.vault.adapter.read(path)
);
}
async adapterReadBinary(file: TFile | string) {
const path = file instanceof TFile ? file.path : file;
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
this.app.vault.adapter.readBinary(path)
);
}
async adapterReadAuto(file: TFile | string) {
const path = file instanceof TFile ? file.path : file;
if (isPlainText(path)) {
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
this.app.vault.adapter.read(path)
);
}
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
this.app.vault.adapter.readBinary(path)
);
}
async adapterWrite(
file: TFile | string,
data: string | ArrayBuffer | Uint8Array<ArrayBuffer>,
options?: DataWriteOptions
) {
const path = file instanceof TFile ? file.path : file;
if (typeof data === "string") {
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
this.app.vault.adapter.write(path, data, options)
);
} else {
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
this.app.vault.adapter.writeBinary(path, toArrayBuffer(data), options)
);
}
}
adapterList(basePath: string): Promise<{ files: string[]; folders: string[] }> {
return Promise.resolve(this.app.vault.adapter.list(basePath));
}
async vaultCacheRead(file: TFile) {
return await this.storageAccessManager.processReadFile(file.path as FilePath, () =>
this.app.vault.cachedRead(file)
);
}
async vaultRead(file: TFile) {
return await this.storageAccessManager.processReadFile(file.path as FilePath, () => this.app.vault.read(file));
}
async vaultReadBinary(file: TFile) {
return await this.storageAccessManager.processReadFile(file.path as FilePath, () =>
this.app.vault.readBinary(file)
);
}
async vaultReadAuto(file: TFile) {
const path = file.path;
if (isPlainText(path)) {
return await this.storageAccessManager.processReadFile(path as FilePath, () => this.app.vault.read(file));
}
return await this.storageAccessManager.processReadFile(path as FilePath, () => this.app.vault.readBinary(file));
}
async vaultModify(file: TFile, data: string | ArrayBuffer | Uint8Array<ArrayBuffer>, options?: DataWriteOptions) {
if (typeof data === "string") {
return await this.storageAccessManager.processWriteFile(file.path as FilePath, async () => {
const oldData = await this.app.vault.read(file);
if (data === oldData) {
if (options && options.mtime) markChangesAreSame(file.path, file.stat.mtime, options.mtime);
return true;
}
await this.app.vault.modify(file, data, options);
return true;
});
} else {
return await this.storageAccessManager.processWriteFile(file.path as FilePath, async () => {
const oldData = await this.app.vault.readBinary(file);
if (await isDocContentSame(createBinaryBlob(oldData), createBinaryBlob(data))) {
if (options && options.mtime) markChangesAreSame(file.path, file.stat.mtime, options.mtime);
return true;
}
await this.app.vault.modifyBinary(file, toArrayBuffer(data), options);
return true;
});
}
}
async vaultCreate(
path: string,
data: string | ArrayBuffer | Uint8Array<ArrayBuffer>,
options?: DataWriteOptions
): Promise<TFile> {
if (typeof data === "string") {
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
this.app.vault.create(path, data, options)
);
} else {
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
this.app.vault.createBinary(path, toArrayBuffer(data), options)
);
}
}
trigger(name: string, ...data: any[]) {
return this.app.vault.trigger(name, ...data);
}
async reconcileInternalFile(path: string) {
await (this.app.vault.adapter as any)?.reconcileInternalFile(path);
}
async adapterAppend(normalizedPath: string, data: string, options?: DataWriteOptions) {
return await this.app.vault.adapter.append(normalizedPath, data, options);
}
async delete(file: TFile | TFolder, force = false) {
return await this.storageAccessManager.processWriteFile(file.path as FilePath, () =>
this.app.vault.delete(file, force)
);
}
async trash(file: TFile | TFolder, force = false) {
return await this.storageAccessManager.processWriteFile(file.path as FilePath, () =>
this.app.vault.trash(file, force)
);
}
isStorageInsensitive(): boolean {
return this.plugin.services.vault.isStorageInsensitive();
}
getAbstractFileByPathInsensitive(path: FilePath | string): TAbstractFile | null {
//@ts-ignore
return this.app.vault.getAbstractFileByPathInsensitive(path);
}
getAbstractFileByPath(path: FilePath | string): TAbstractFile | null {
if (!this.plugin.settings.handleFilenameCaseSensitive || this.isStorageInsensitive()) {
return this.getAbstractFileByPathInsensitive(path);
}
return this.app.vault.getAbstractFileByPath(path);
}
getFiles() {
return this.app.vault.getFiles();
}
async ensureDirectory(fullPath: string) {
const pathElements = fullPath.split("/");
pathElements.pop();
let c = "";
for (const v of pathElements) {
c += v;
try {
await this.app.vault.adapter.mkdir(c);
} catch (ex: any) {
if (ex?.message == "Folder already exists.") {
// Skip if already exists.
} else {
Logger("Folder Create Error");
Logger(ex);
}
}
c += "/";
}
}
touchedFiles: string[] = [];
_statInternal(file: FilePath) {
return this.app.vault.adapter.stat(file);
}
async touch(file: TFile | FilePath) {
const path = file instanceof TFile ? (file.path as FilePath) : file;
const statOrg = file instanceof TFile ? file.stat : await this._statInternal(path);
const stat = statOrg || { mtime: 0, size: 0 };
const key = `${path}-${stat.mtime}-${stat.size}`;
this.touchedFiles.unshift(key);
this.touchedFiles = this.touchedFiles.slice(0, 100);
}
recentlyTouched(file: TFile | InternalFileInfo | UXFileInfoStub) {
const key =
"stat" in file
? `${file.path}-${file.stat.mtime}-${file.stat.size}`
: `${file.path}-${file.mtime}-${file.size}`;
if (this.touchedFiles.indexOf(key) == -1) return false;
return true;
}
clearTouched() {
this.touchedFiles = [];
}
}

631
src/modules/coreObsidian/storageLib/StorageEventManager.ts
View File

@@ -1,631 +0,0 @@
import { TAbstractFile, TFile, TFolder } from "../../../deps.ts";
import { Logger } from "../../../lib/src/common/logger.ts";
import { shouldBeIgnored } from "../../../lib/src/string_and_binary/path.ts";
import {
DEFAULT_SETTINGS,
LOG_LEVEL_DEBUG,
LOG_LEVEL_INFO,
LOG_LEVEL_NOTICE,
LOG_LEVEL_VERBOSE,
type FileEventType,
type FilePath,
type UXFileInfoStub,
} from "../../../lib/src/common/types.ts";
import { delay, fireAndForget, throttle } from "../../../lib/src/common/utils.ts";
import { type FileEventItem } from "../../../common/types.ts";
import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
import { isWaitingForTimeout } from "octagonal-wheels/concurrency/task";
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
import type { LiveSyncCore } from "../../../main.ts";
import { InternalFileToUXFileInfoStub, TFileToUXFileInfoStub } from "./utilObsidian.ts";
import ObsidianLiveSyncPlugin from "../../../main.ts";
import type { IStorageAccessManager } from "@lib/interfaces/StorageAccess.ts";
import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts";
import { promiseWithResolvers, type PromiseWithResolvers } from "octagonal-wheels/promises";
import { StorageEventManager, type FileEvent } from "@lib/interfaces/StorageEventManager.ts";
type WaitInfo = {
since: number;
type: FileEventType;
canProceed: PromiseWithResolvers<boolean>;
timerHandler: ReturnType<typeof setTimeout>;
event: FileEventItem;
};
const TYPE_SENTINEL_FLUSH = "SENTINEL_FLUSH";
type FileEventItemSentinelFlush = {
type: typeof TYPE_SENTINEL_FLUSH;
};
type FileEventItemSentinel = FileEventItemSentinelFlush;
export class StorageEventManagerObsidian extends StorageEventManager {
plugin: ObsidianLiveSyncPlugin;
core: LiveSyncCore;
storageAccess: IStorageAccessManager;
get services() {
return this.core.services;
}
get shouldBatchSave() {
return this.core.settings?.batchSave && this.core.settings?.liveSync != true;
}
get batchSaveMinimumDelay(): number {
return this.core.settings?.batchSaveMinimumDelay ?? DEFAULT_SETTINGS.batchSaveMinimumDelay;
}
get batchSaveMaximumDelay(): number {
return this.core.settings?.batchSaveMaximumDelay ?? DEFAULT_SETTINGS.batchSaveMaximumDelay;
}
// Necessary evil.
cmdHiddenFileSync: HiddenFileSync;
/**
* Snapshot restoration promise.
* Snapshot will be restored before starting to watch vault changes.
* In designed time, this has been called from Initialisation process, which has been implemented on `ModuleInitializerFile.ts`.
*/
snapShotRestored: Promise<void> | null = null;
constructor(plugin: ObsidianLiveSyncPlugin, core: LiveSyncCore, storageAccessManager: IStorageAccessManager) {
super();
this.storageAccess = storageAccessManager;
this.plugin = plugin;
this.core = core;
this.cmdHiddenFileSync = this.plugin.getAddOn(HiddenFileSync.name) as HiddenFileSync;
}
/**
* Restore the previous snapshot if exists.
* @returns
*/
restoreState(): Promise<void> {
this.snapShotRestored = this._restoreFromSnapshot();
return this.snapShotRestored;
}
async beginWatch() {
await this.snapShotRestored;
const plugin = this.plugin;
this.watchVaultChange = this.watchVaultChange.bind(this);
this.watchVaultCreate = this.watchVaultCreate.bind(this);
this.watchVaultDelete = this.watchVaultDelete.bind(this);
this.watchVaultRename = this.watchVaultRename.bind(this);
this.watchVaultRawEvents = this.watchVaultRawEvents.bind(this);
this.watchEditorChange = this.watchEditorChange.bind(this);
plugin.registerEvent(plugin.app.vault.on("modify", this.watchVaultChange));
plugin.registerEvent(plugin.app.vault.on("delete", this.watchVaultDelete));
plugin.registerEvent(plugin.app.vault.on("rename", this.watchVaultRename));
plugin.registerEvent(plugin.app.vault.on("create", this.watchVaultCreate));
//@ts-ignore : Internal API
plugin.registerEvent(plugin.app.vault.on("raw", this.watchVaultRawEvents));
plugin.registerEvent(plugin.app.workspace.on("editor-change", this.watchEditorChange));
}
watchEditorChange(editor: any, info: any) {
if (!("path" in info)) {
return;
}
if (!this.shouldBatchSave) {
return;
}
const file = info?.file as TFile;
if (!file) return;
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
// Logger(`Editor change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
return;
}
if (!this.isWaiting(file.path as FilePath)) {
return;
}
const data = info?.data as string;
const fi: FileEvent = {
type: "CHANGED",
file: TFileToUXFileInfoStub(file),
cachedData: data,
};
void this.appendQueue([fi]);
}
watchVaultCreate(file: TAbstractFile, ctx?: any) {
if (file instanceof TFolder) return;
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
// Logger(`File create skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
return;
}
const fileInfo = TFileToUXFileInfoStub(file);
void this.appendQueue([{ type: "CREATE", file: fileInfo }], ctx);
}
watchVaultChange(file: TAbstractFile, ctx?: any) {
if (file instanceof TFolder) return;
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
// Logger(`File change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
return;
}
const fileInfo = TFileToUXFileInfoStub(file);
void this.appendQueue([{ type: "CHANGED", file: fileInfo }], ctx);
}
watchVaultDelete(file: TAbstractFile, ctx?: any) {
if (file instanceof TFolder) return;
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
// Logger(`File delete skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
return;
}
const fileInfo = TFileToUXFileInfoStub(file, true);
void this.appendQueue([{ type: "DELETE", file: fileInfo }], ctx);
}
watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) {
// vault Rename will not be raised for self-events (Self-hosted LiveSync will not handle 'rename').
if (file instanceof TFile) {
const fileInfo = TFileToUXFileInfoStub(file);
void this.appendQueue(
[
{
type: "DELETE",
file: {
path: oldFile as FilePath,
name: file.name,
stat: {
mtime: file.stat.mtime,
ctime: file.stat.ctime,
size: file.stat.size,
type: "file",
},
deleted: true,
},
skipBatchWait: true,
},
{ type: "CREATE", file: fileInfo, skipBatchWait: true },
],
ctx
);
}
}
// Watch raw events (Internal API)
watchVaultRawEvents(path: FilePath) {
if (this.storageAccess.isFileProcessing(path)) {
// Logger(`Raw file event skipped because the file is being processed: ${path}`, LOG_LEVEL_VERBOSE);
return;
}
// Only for internal files.
if (!this.plugin.settings) return;
// if (this.plugin.settings.useIgnoreFiles && this.plugin.ignoreFiles.some(e => path.endsWith(e.trim()))) {
if (this.plugin.settings.useIgnoreFiles) {
// If it is one of ignore files, refresh the cached one.
// (Calling$$isTargetFile will refresh the cache)
void this.services.vault.isTargetFile(path).then(() => this._watchVaultRawEvents(path));
} else {
void this._watchVaultRawEvents(path);
}
}
async _watchVaultRawEvents(path: FilePath) {
if (!this.plugin.settings.syncInternalFiles && !this.plugin.settings.usePluginSync) return;
if (!this.plugin.settings.watchInternalFileChanges) return;
if (!path.startsWith(this.plugin.app.vault.configDir)) return;
if (path.endsWith("/")) {
// Folder
return;
}
const isTargetFile = await this.cmdHiddenFileSync.isTargetFile(path);
if (!isTargetFile) return;
void this.appendQueue(
[
{
type: "INTERNAL",
file: InternalFileToUXFileInfoStub(path),
skipBatchWait: true, // Internal files should be processed immediately.
},
],
null
);
}
// Cache file and waiting to can be proceed.
async appendQueue(params: FileEvent[], ctx?: any) {
if (!this.core.settings.isConfigured) return;
if (this.core.settings.suspendFileWatching) return;
if (this.core.settings.maxMTimeForReflectEvents > 0) {
return;
}
this.core.services.vault.markFileListPossiblyChanged();
// Flag up to be reload
for (const param of params) {
if (shouldBeIgnored(param.file.path)) {
continue;
}
const atomicKey = [0, 0, 0, 0, 0, 0].map((e) => `${Math.floor(Math.random() * 100000)}`).join("-");
const type = param.type;
const file = param.file;
const oldPath = param.oldPath;
if (type !== "INTERNAL") {
const size = (file as UXFileInfoStub).stat.size;
if (this.services.vault.isFileSizeTooLarge(size) && (type == "CREATE" || type == "CHANGED")) {
Logger(
`The storage file has been changed but exceeds the maximum size. Skipping: ${param.file.path}`,
LOG_LEVEL_NOTICE
);
continue;
}
}
if (file instanceof TFolder) continue;
// TODO: Confirm why only the TFolder skipping
// Possibly following line is needed...
// if (file?.isFolder) continue;
if (!(await this.services.vault.isTargetFile(file.path))) continue;
// Stop cache using to prevent the corruption;
// let cache: null | string | ArrayBuffer;
// new file or something changed, cache the changes.
// if (file instanceof TFile && (type == "CREATE" || type == "CHANGED")) {
if (file instanceof TFile || !file.isFolder) {
if (type == "CREATE" || type == "CHANGED") {
// Wait for a bit while to let the writer has marked `touched` at the file.
await delay(10);
if (this.core.storageAccess.recentlyTouched(file.path)) {
continue;
}
}
}
let cache: string | undefined = undefined;
if (param.cachedData) {
cache = param.cachedData;
}
void this.enqueue({
type,
args: {
file: file,
oldPath,
cache,
ctx,
},
skipBatchWait: param.skipBatchWait,
key: atomicKey,
});
}
}
private bufferedQueuedItems = [] as (FileEventItem | FileEventItemSentinel)[];
/**
* Immediately take snapshot.
*/
private _triggerTakeSnapshot() {
void this._takeSnapshot();
}
/**
* Trigger taking snapshot after throttled period.
*/
triggerTakeSnapshot = throttle(() => this._triggerTakeSnapshot(), 100);
enqueue(newItem: FileEventItem) {
if (newItem.type == "DELETE") {
// If the sentinel pushed, the runQueuedEvents will wait for idle before processing delete.
this.bufferedQueuedItems.push({
type: TYPE_SENTINEL_FLUSH,
});
}
this.updateStatus();
this.bufferedQueuedItems.push(newItem);
fireAndForget(() => this._takeSnapshot().then(() => this.runQueuedEvents()));
}
// Limit concurrent processing to reduce the IO load. file-processing + scheduler (1), so file events can be processed in 4 slots.
concurrentProcessing = Semaphore(5);
private _waitingMap = new Map<string, WaitInfo>();
private _waitForIdle: Promise<void> | null = null;
/**
* Wait until all queued events are processed.
* Subsequent new events will not be waited, but new events will not be added.
* @returns
*/
waitForIdle(): Promise<void> {
if (this._waitingMap.size === 0) {
return Promise.resolve();
}
if (this._waitForIdle) {
return this._waitForIdle;
}
const promises = [...this._waitingMap.entries()].map(([key, waitInfo]) => {
return new Promise<void>((resolve) => {
waitInfo.canProceed.promise
.then(() => {
Logger(`Processing ${key}: Wait for idle completed`, LOG_LEVEL_DEBUG);
// No op
})
.catch((e) => {
Logger(`Processing ${key}: Wait for idle error`, LOG_LEVEL_INFO);
Logger(e, LOG_LEVEL_VERBOSE);
//no op
})
.finally(() => {
resolve();
});
this._proceedWaiting(key);
});
});
const waitPromise = Promise.all(promises).then(() => {
this._waitForIdle = null;
Logger(`All wait for idle completed`, LOG_LEVEL_VERBOSE);
});
this._waitForIdle = waitPromise;
return waitPromise;
}
/**
* Proceed waiting for the given key immediately.
*/
private _proceedWaiting(key: string) {
const waitInfo = this._waitingMap.get(key);
if (waitInfo) {
waitInfo.canProceed.resolve(true);
clearTimeout(waitInfo.timerHandler);
this._waitingMap.delete(key);
}
this.triggerTakeSnapshot();
}
/**
* Cancel waiting for the given key.
*/
private _cancelWaiting(key: string) {
const waitInfo = this._waitingMap.get(key);
if (waitInfo) {
waitInfo.canProceed.resolve(false);
clearTimeout(waitInfo.timerHandler);
this._waitingMap.delete(key);
}
this.triggerTakeSnapshot();
}
/**
* Add waiting for the given key.
* @param key
* @param event
* @param waitedSince Optional waited since timestamp to calculate the remaining delay.
*/
private _addWaiting(key: string, event: FileEventItem, waitedSince?: number): WaitInfo {
if (this._waitingMap.has(key)) {
// Already waiting
throw new Error(`Already waiting for key: ${key}`);
}
const resolver = promiseWithResolvers<boolean>();
const now = Date.now();
const since = waitedSince ?? now;
const elapsed = now - since;
const maxDelay = this.batchSaveMaximumDelay * 1000;
const remainingDelay = Math.max(0, maxDelay - elapsed);
const nextDelay = Math.min(remainingDelay, this.batchSaveMinimumDelay * 1000);
// x*<------- maxDelay --------->*
// x*<-- minDelay -->*
// x* x<-- nextDelay -->*
// x* x<-- Capped-->*
// x* x.......*
// x: event
// *: save
// When at event (x) At least, save (*) within maxDelay, but maintain minimum delay between saves.
if (elapsed >= maxDelay) {
// Already exceeded maximum delay, do not wait.
Logger(`Processing ${key}: Batch save maximum delay already exceeded: ${event.type}`, LOG_LEVEL_DEBUG);
} else {
Logger(`Processing ${key}: Adding waiting for batch save: ${event.type} (${nextDelay}ms)`, LOG_LEVEL_DEBUG);
}
const waitInfo: WaitInfo = {
since: since,
type: event.type,
event: event,
canProceed: resolver,
timerHandler: setTimeout(() => {
Logger(`Processing ${key}: Batch save timeout reached: ${event.type}`, LOG_LEVEL_DEBUG);
this._proceedWaiting(key);
}, nextDelay),
};
this._waitingMap.set(key, waitInfo);
this.triggerTakeSnapshot();
return waitInfo;
}
/**
* Process the given file event.
*/
async processFileEvent(fei: FileEventItem) {
const releaser = await this.concurrentProcessing.acquire();
try {
this.updateStatus();
const filename = fei.args.file.path;
const waitingKey = `${filename}`;
const previous = this._waitingMap.get(waitingKey);
let isShouldBeCancelled = fei.skipBatchWait || false;
let previousPromise: Promise<boolean> = Promise.resolve(true);
let waitPromise: Promise<boolean> = Promise.resolve(true);
// 1. Check if there is previous waiting for the same file
if (previous) {
previousPromise = previous.canProceed.promise;
if (isShouldBeCancelled) {
Logger(
`Processing ${filename}: Requested to perform immediately, cancelling previous waiting: ${fei.type}`,
LOG_LEVEL_DEBUG
);
}
if (!isShouldBeCancelled && fei.type === "DELETE") {
// For DELETE, cancel any previous waiting and proceed immediately
// That because when deleting, we cannot read the file anymore.
Logger(
`Processing ${filename}: DELETE requested, cancelling previous waiting: ${fei.type}`,
LOG_LEVEL_DEBUG
);
isShouldBeCancelled = true;
}
if (!isShouldBeCancelled && previous.type === fei.type) {
// For the same type, we can cancel the previous waiting and proceed immediately.
Logger(`Processing ${filename}: Cancelling previous waiting: ${fei.type}`, LOG_LEVEL_DEBUG);
isShouldBeCancelled = true;
}
// 2. wait for the previous to complete
if (isShouldBeCancelled) {
this._cancelWaiting(waitingKey);
Logger(`Processing ${filename}: Previous cancelled: ${fei.type}`, LOG_LEVEL_DEBUG);
isShouldBeCancelled = true;
}
if (!isShouldBeCancelled) {
Logger(`Processing ${filename}: Waiting for previous to complete: ${fei.type}`, LOG_LEVEL_DEBUG);
this._proceedWaiting(waitingKey);
Logger(`Processing ${filename}: Previous completed: ${fei.type}`, LOG_LEVEL_DEBUG);
}
}
await previousPromise;
// 3. Check if shouldBatchSave is true
if (this.shouldBatchSave && !fei.skipBatchWait) {
// if type is CREATE or CHANGED, set waiting
if (fei.type == "CREATE" || fei.type == "CHANGED") {
// 3.2. If true, set the queue, and wait for the waiting, or until timeout
// (since is copied from previous waiting if exists to limit the maximum wait time)
// console.warn(`Since:`, previous?.since);
const info = this._addWaiting(waitingKey, fei, previous?.since);
waitPromise = info.canProceed.promise;
} else if (fei.type == "DELETE") {
// For DELETE, cancel any previous waiting and proceed immediately
}
Logger(`Processing ${filename}: Waiting for batch save: ${fei.type}`, LOG_LEVEL_DEBUG);
const canProceed = await waitPromise;
if (!canProceed) {
// 3.2.1. If cancelled by new queue, cancel subsequent process.
Logger(`Processing ${filename}: Cancelled by new queue: ${fei.type}`, LOG_LEVEL_DEBUG);
return;
}
}
// await this.handleFileEvent(fei);
await this.requestProcessQueue(fei);
} finally {
await this._takeSnapshot();
releaser();
}
}
async _takeSnapshot() {
const processingEvents = [...this._waitingMap.values()].map((e) => e.event);
const waitingEvents = this.bufferedQueuedItems;
const snapShot = [...processingEvents, ...waitingEvents];
await this.core.kvDB.set("storage-event-manager-snapshot", snapShot);
Logger(`Storage operation snapshot taken: ${snapShot.length} items`, LOG_LEVEL_DEBUG);
this.updateStatus();
}
async _restoreFromSnapshot() {
const snapShot = await this.core.kvDB.get<(FileEventItem | FileEventItemSentinel)[]>(
"storage-event-manager-snapshot"
);
if (snapShot && Array.isArray(snapShot) && snapShot.length > 0) {
// console.warn(`Restoring snapshot: ${snapShot.length} items`);
Logger(`Restoring storage operation snapshot: ${snapShot.length} items`, LOG_LEVEL_VERBOSE);
// Restore the snapshot
// Note: Mark all items as skipBatchWait to prevent apply the off-line batch saving.
this.bufferedQueuedItems = snapShot.map((e) => ({ ...e, skipBatchWait: true }));
this.updateStatus();
await this.runQueuedEvents();
} else {
Logger(`No snapshot to restore`, LOG_LEVEL_VERBOSE);
// console.warn(`No snapshot to restore`);
}
}
runQueuedEvents() {
return skipIfDuplicated("storage-event-manager-run-queued-events", async () => {
do {
if (this.bufferedQueuedItems.length === 0) {
break;
}
// 1. Get the first queued item
const fei = this.bufferedQueuedItems.shift()!;
await this._takeSnapshot();
this.updateStatus();
// 2. Consume 1 semaphore slot to enqueue processing. Then release immediately.
// (Just to limit the total concurrent processing count, because skipping batch handles at processFileEvent).
const releaser = await this.concurrentProcessing.acquire();
releaser();
this.updateStatus();
// 3. Check if sentinel flush
// If sentinel, wait for idle and continue.
if (fei.type === TYPE_SENTINEL_FLUSH) {
Logger(`Waiting for idle`, LOG_LEVEL_VERBOSE);
// Flush all waiting batch queues
await this.waitForIdle();
this.updateStatus();
continue;
}
// 4. Process the event, this should be fire-and-forget to not block the queue processing in each file.
fireAndForget(() => this.processFileEvent(fei));
} while (this.bufferedQueuedItems.length > 0);
});
}
processingCount = 0;
async requestProcessQueue(fei: FileEventItem) {
try {
this.processingCount++;
// this.bufferedQueuedItems.remove(fei);
this.updateStatus();
// this.waitedSince.delete(fei.args.file.path);
await this.handleFileEvent(fei);
await this._takeSnapshot();
} finally {
this.processingCount--;
this.updateStatus();
}
}
isWaiting(filename: FilePath) {
return isWaitingForTimeout(`storage-event-manager-batchsave-${filename}`);
}
updateStatus() {
const allFileEventItems = this.bufferedQueuedItems.filter((e): e is FileEventItem => "args" in e);
const allItems = allFileEventItems.filter((e) => !e.cancelled);
const totalItems = allItems.length + this.concurrentProcessing.waiting;
const processing = this.processingCount;
const batchedCount = this._waitingMap.size;
this.core.batched.value = batchedCount;
this.core.processing.value = processing;
this.core.totalQueued.value = totalItems + batchedCount + processing;
}
async handleFileEvent(queue: FileEventItem): Promise<any> {
const file = queue.args.file;
const lockKey = `handleFile:${file.path}`;
const ret = await serialized(lockKey, async () => {
if (queue.cancelled) {
Logger(`File event cancelled before processing: ${file.path}`, LOG_LEVEL_INFO);
return;
}
if (queue.type == "INTERNAL" || file.isInternal) {
await this.core.services.fileProcessing.processOptionalFileEvent(file.path as unknown as FilePath);
} else {
const key = `file-last-proc-${queue.type}-${file.path}`;
const last = Number((await this.core.kvDB.get(key)) || 0);
if (queue.type == "DELETE") {
await this.core.services.fileProcessing.processFileEvent(queue);
} else {
if (file.stat.mtime == last) {
Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL_VERBOSE);
// Should Cancel the relative operations? (e.g. rename)
// this.cancelRelativeEvent(queue);
return;
}
if (!(await this.core.services.fileProcessing.processFileEvent(queue))) {
Logger(
`STORAGE -> DB: Handler failed, cancel the relative operations: ${file.path}`,
LOG_LEVEL_INFO
);
// cancel running queues and remove one of atomic operation (e.g. rename)
this.cancelRelativeEvent(queue);
return;
}
}
}
});
this.updateStatus();
return ret;
}
cancelRelativeEvent(item: FileEventItem): void {
this._cancelWaiting(item.args.file.path);
}
}

4
src/modules/coreObsidian/storageLib/utilObsidian.ts
View File

@@ -2,7 +2,6 @@
import { TFile, type TAbstractFile, type TFolder } from "../../../deps.ts";
import { ICHeader } from "../../../common/types.ts";
import type { ObsidianFileAccess } from "./SerializedFileAccess.ts";
import { addPrefix, isPlainText } from "../../../lib/src/string_and_binary/path.ts";
import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
import { createBlob } from "../../../lib/src/common/utils.ts";
@@ -15,6 +14,7 @@ import type {
UXInternalFileInfoStub,
} from "../../../lib/src/common/types.ts";
import type { LiveSyncCore } from "../../../main.ts";
import type { FileAccessObsidian } from "@/serviceModules/FileAccessObsidian.ts";
export async function TFileToUXFileInfo(
core: LiveSyncCore,
@@ -51,7 +51,7 @@ export async function TFileToUXFileInfo(
export async function InternalFileToUXFileInfo(
fullPath: string,
vaultAccess: ObsidianFileAccess,
vaultAccess: FileAccessObsidian,
prefix: string = ICHeader
): Promise<UXFileInfo> {
const name = fullPath.split("/").pop() as string;

156
src/serviceModules/FileAccessObsidian.ts Normal file
View File

@@ -0,0 +1,156 @@
import { markChangesAreSame } from "@/common/utils";
import type { FilePath, UXDataWriteOptions, UXFileInfoStub, UXFolderInfo } from "@lib/common/types";
import { TFolder, type TAbstractFile, TFile, type Stat, type App, type DataWriteOptions } from "@/deps";
import { FileAccessBase, toArrayBuffer, type FileAccessBaseDependencies } from "@lib/serviceModules/FileAccessBase.ts";
import { TFileToUXFileInfoStub } from "@/modules/coreObsidian/storageLib/utilObsidian";
declare module "obsidian" {
interface Vault {
getAbstractFileByPathInsensitive(path: string): TAbstractFile | null;
}
interface DataAdapter {
reconcileInternalFile?(path: string): Promise<void>;
}
}
export class FileAccessObsidian extends FileAccessBase<TAbstractFile, TFile, TFolder, Stat> {
app: App;
override getPath(file: string | TAbstractFile): FilePath {
return (typeof file === "string" ? file : file.path) as FilePath;
}
override isFile(file: TAbstractFile | null): file is TFile {
return file instanceof TFile;
}
override isFolder(file: TAbstractFile | null): file is TFolder {
return file instanceof TFolder;
}
override _statFromNative(file: TFile): Promise<TFile["stat"]> {
return Promise.resolve(file.stat);
}
override nativeFileToUXFileInfoStub(file: TFile): UXFileInfoStub {
return TFileToUXFileInfoStub(file);
}
override nativeFolderToUXFolder(folder: TFolder): UXFolderInfo {
if (folder instanceof TFolder) {
return this.nativeFolderToUXFolder(folder);
} else {
throw new Error(`Not a folder: ${(folder as TAbstractFile)?.name}`);
}
}
constructor(app: App, dependencies: FileAccessBaseDependencies) {
super({
storageAccessManager: dependencies.storageAccessManager,
vaultService: dependencies.vaultService,
settingService: dependencies.settingService,
APIService: dependencies.APIService,
});
this.app = app;
}
protected async _adapterMkdir(path: string) {
await this.app.vault.adapter.mkdir(path);
}
protected _getAbstractFileByPath(path: FilePath) {
return this.app.vault.getAbstractFileByPath(path);
}
protected _getAbstractFileByPathInsensitive(path: FilePath) {
return this.app.vault.getAbstractFileByPathInsensitive(path);
}
protected async _tryAdapterStat(path: FilePath) {
if (!(await this.app.vault.adapter.exists(path))) return null;
return await this.app.vault.adapter.stat(path);
}
protected async _adapterStat(path: FilePath) {
return await this.app.vault.adapter.stat(path);
}
protected async _adapterExists(path: FilePath) {
return await this.app.vault.adapter.exists(path);
}
protected async _adapterRemove(path: FilePath) {
await this.app.vault.adapter.remove(path);
}
protected async _adapterRead(path: FilePath) {
return await this.app.vault.adapter.read(path);
}
protected async _adapterReadBinary(path: FilePath) {
return await this.app.vault.adapter.readBinary(path);
}
_adapterWrite(file: string, data: string, options?: UXDataWriteOptions): Promise<void> {
return this.app.vault.adapter.write(file, data, options);
}
_adapterWriteBinary(file: string, data: ArrayBuffer, options?: UXDataWriteOptions): Promise<void> {
return this.app.vault.adapter.writeBinary(file, toArrayBuffer(data), options);
}
protected _adapterList(basePath: string): Promise<{ files: string[]; folders: string[] }> {
return Promise.resolve(this.app.vault.adapter.list(basePath));
}
async _vaultCacheRead(file: TFile) {
return await this.app.vault.cachedRead(file);
}
protected async _vaultRead(file: TFile): Promise<string> {
return await this.app.vault.read(file);
}
protected async _vaultReadBinary(file: TFile): Promise<ArrayBuffer> {
return await this.app.vault.readBinary(file);
}
protected override markChangesAreSame(path: string, mtime: number, newMtime: number) {
return markChangesAreSame(path, mtime, newMtime);
}
protected override async _vaultModify(file: TFile, data: string, options?: UXDataWriteOptions): Promise<void> {
return await this.app.vault.modify(file, data, options);
}
protected override async _vaultModifyBinary(
file: TFile,
data: ArrayBuffer,
options?: UXDataWriteOptions
): Promise<void> {
return await this.app.vault.modifyBinary(file, toArrayBuffer(data), options);
}
protected override async _vaultCreate(path: string, data: string, options?: UXDataWriteOptions): Promise<TFile> {
return await this.app.vault.create(path, data, options);
}
protected override async _vaultCreateBinary(
path: string,
data: ArrayBuffer,
options?: UXDataWriteOptions
): Promise<TFile> {
return await this.app.vault.createBinary(path, toArrayBuffer(data), options);
}
protected override _trigger(name: string, ...data: any[]) {
return this.app.vault.trigger(name, ...data);
}
protected override async _reconcileInternalFile(path: string) {
return await Promise.resolve(this.app.vault.adapter.reconcileInternalFile?.(path));
}
protected override async _adapterAppend(normalizedPath: string, data: string, options?: DataWriteOptions) {
return await this.app.vault.adapter.append(normalizedPath, data, options);
}
protected override async _delete(file: TFile | TFolder, force = false) {
return await this.app.vault.delete(file, force);
}
protected override async _trash(file: TFile | TFolder, force = false) {
return await this.app.vault.trash(file, force);
}
protected override _getFiles() {
return this.app.vault.getFiles();
}
}

6
src/serviceModules/ServiceFileAccessImpl.ts Normal file
View File

@@ -0,0 +1,6 @@
import type { TAbstractFile, TFile, TFolder, Stat } from "@/deps";
import { ServiceFileAccessBase } from "@lib/serviceModules/ServiceFileAccessBase";
// For typechecking purpose
export class ServiceFileAccessObsidian extends ServiceFileAccessBase<TAbstractFile, TFile, TFolder, Stat> {}

363
src/serviceModules/ServiceFileAccessObsidian.ts
View File

@@ -1,363 +0,0 @@
import { TFile, TFolder, type ListedFiles } from "@/deps.ts";
import { LOG_LEVEL_INFO, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import type {
FilePath,
FilePathWithPrefix,
UXDataWriteOptions,
UXFileInfo,
UXFileInfoStub,
UXFolderInfo,
UXStat,
} from "@lib/common/types";
import { ServiceModuleBase } from "@lib/serviceModules/ServiceModuleBase";
import type { APIService } from "@lib/services/base/APIService";
import type { IStorageAccessManager, StorageAccess } from "@lib/interfaces/StorageAccess.ts";
import type { AppLifecycleService } from "@lib/services/base/AppLifecycleService";
import type { FileProcessingService } from "@lib/services/base/FileProcessingService";
import { ObsidianFileAccess } from "@/modules/coreObsidian/storageLib/SerializedFileAccess";
import { StorageEventManager } from "@lib/interfaces/StorageEventManager.ts";
import { TFileToUXFileInfoStub, TFolderToUXFileInfoStub } from "@/modules/coreObsidian/storageLib/utilObsidian";
import { createBlob, type CustomRegExp } from "@lib/common/utils";
import type { VaultService } from "@lib/services/base/VaultService";
import type { SettingService } from "@lib/services/base/SettingService";
export interface StorageAccessObsidianDependencies {
API: APIService;
appLifecycle: AppLifecycleService;
fileProcessing: FileProcessingService;
vault: VaultService;
setting: SettingService;
storageEventManager: StorageEventManager;
storageAccessManager: IStorageAccessManager;
vaultAccess: ObsidianFileAccess;
}
export class ServiceFileAccessObsidian
extends ServiceModuleBase<StorageAccessObsidianDependencies>
implements StorageAccess
{
private vaultAccess: ObsidianFileAccess;
private vaultManager: StorageEventManager;
private vault: VaultService;
private setting: SettingService;
constructor(services: StorageAccessObsidianDependencies) {
super(services);
// this.appLifecycle = services.appLifecycle;
this.vault = services.vault;
this.setting = services.setting;
this.vaultManager = services.storageEventManager;
this.vaultAccess = services.vaultAccess;
services.appLifecycle.onFirstInitialise.addHandler(this._everyOnFirstInitialize.bind(this));
services.fileProcessing.commitPendingFileEvents.addHandler(this._everyCommitPendingFileEvent.bind(this));
}
restoreState() {
return this.vaultManager.restoreState();
}
async _everyOnFirstInitialize(): Promise<boolean> {
await this.vaultManager.beginWatch();
return Promise.resolve(true);
}
async _everyCommitPendingFileEvent(): Promise<boolean> {
await this.vaultManager.waitForIdle();
return Promise.resolve(true);
}
async writeFileAuto(path: string, data: string | ArrayBuffer, opt?: UXDataWriteOptions): Promise<boolean> {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return this.vaultAccess.vaultModify(file, data, opt);
} else if (file === null) {
if (!path.endsWith(".md")) {
// Very rare case, we encountered this case with `writing-goals-history.csv` file.
// Indeed, that file not appears in the File Explorer, but it exists in the vault.
// Hence, we cannot retrieve the file from the vault by getAbstractFileByPath, and we cannot write it via vaultModify.
// It makes `File already exists` error.
// Therefore, we need to write it via adapterWrite.
// Maybe there are others like this, so I will write it via adapterWrite.
// This is a workaround for the issue, but I don't know if this is the right solution.
// (So limits to non-md files).
// Has Obsidian been patched?, anyway, writing directly might be a safer approach.
// However, does changes of that file trigger file-change event?
await this.vaultAccess.adapterWrite(path, data, opt);
// For safety, check existence
return await this.vaultAccess.adapterExists(path);
} else {
return (await this.vaultAccess.vaultCreate(path, data, opt)) instanceof TFile;
}
} else {
this._log(`Could not write file (Possibly already exists as a folder): ${path}`, LOG_LEVEL_VERBOSE);
return false;
}
}
readFileAuto(path: string): Promise<string | ArrayBuffer> {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return this.vaultAccess.vaultRead(file);
} else {
throw new Error(`Could not read file (Possibly does not exist): ${path}`);
}
}
readFileText(path: string): Promise<string> {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return this.vaultAccess.vaultRead(file);
} else {
throw new Error(`Could not read file (Possibly does not exist): ${path}`);
}
}
isExists(path: string): Promise<boolean> {
return Promise.resolve(this.vaultAccess.getAbstractFileByPath(path) instanceof TFile);
}
async writeHiddenFileAuto(path: string, data: string | ArrayBuffer, opt?: UXDataWriteOptions): Promise<boolean> {
try {
await this.vaultAccess.adapterWrite(path, data, opt);
return true;
} catch (e) {
this._log(`Could not write hidden file: ${path}`, LOG_LEVEL_VERBOSE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
}
async appendHiddenFile(path: string, data: string, opt?: UXDataWriteOptions): Promise<boolean> {
try {
await this.vaultAccess.adapterAppend(path, data, opt);
return true;
} catch (e) {
this._log(`Could not append hidden file: ${path}`, LOG_LEVEL_VERBOSE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
}
stat(path: string): Promise<UXStat | null> {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file === null) return Promise.resolve(null);
if (file instanceof TFile) {
return Promise.resolve({
ctime: file.stat.ctime,
mtime: file.stat.mtime,
size: file.stat.size,
type: "file",
});
} else {
throw new Error(`Could not stat file (Possibly does not exist): ${path}`);
}
}
statHidden(path: string): Promise<UXStat | null> {
return this.vaultAccess.tryAdapterStat(path);
}
async removeHidden(path: string): Promise<boolean> {
try {
await this.vaultAccess.adapterRemove(path);
if (this.vaultAccess.tryAdapterStat(path) !== null) {
return false;
}
return true;
} catch (e) {
this._log(`Could not remove hidden file: ${path}`, LOG_LEVEL_VERBOSE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
}
async readHiddenFileAuto(path: string): Promise<string | ArrayBuffer> {
return await this.vaultAccess.adapterReadAuto(path);
}
async readHiddenFileText(path: string): Promise<string> {
return await this.vaultAccess.adapterRead(path);
}
async readHiddenFileBinary(path: string): Promise<ArrayBuffer> {
return await this.vaultAccess.adapterReadBinary(path);
}
async isExistsIncludeHidden(path: string): Promise<boolean> {
return (await this.vaultAccess.tryAdapterStat(path)) !== null;
}
async ensureDir(path: string): Promise<boolean> {
try {
await this.vaultAccess.ensureDirectory(path);
return true;
} catch (e) {
this._log(`Could not ensure directory: ${path}`, LOG_LEVEL_VERBOSE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
}
triggerFileEvent(event: string, path: string): void {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file === null) return;
this.vaultAccess.trigger(event, file);
}
async triggerHiddenFile(path: string): Promise<void> {
await this.vaultAccess.reconcileInternalFile(path);
}
// getFileStub(file: TFile): UXFileInfoStub {
// return TFileToUXFileInfoStub(file);
// }
getFileStub(path: string): UXFileInfoStub | null {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return TFileToUXFileInfoStub(file);
} else {
return null;
}
}
async readStubContent(stub: UXFileInfoStub): Promise<UXFileInfo | false> {
const file = this.vaultAccess.getAbstractFileByPath(stub.path);
if (!(file instanceof TFile)) {
this._log(`Could not read file (Possibly does not exist or a folder): ${stub.path}`, LOG_LEVEL_VERBOSE);
return false;
}
const data = await this.vaultAccess.vaultReadAuto(file);
return {
...stub,
...TFileToUXFileInfoStub(file),
body: createBlob(data),
};
}
getStub(path: string): UXFileInfoStub | UXFolderInfo | null {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return TFileToUXFileInfoStub(file);
} else if (file instanceof TFolder) {
return TFolderToUXFileInfoStub(file);
}
return null;
}
getFiles(): UXFileInfoStub[] {
return this.vaultAccess.getFiles().map((f) => TFileToUXFileInfoStub(f));
}
getFileNames(): FilePath[] {
return this.vaultAccess.getFiles().map((f) => f.path as FilePath);
}
async getFilesIncludeHidden(
basePath: string,
includeFilter?: CustomRegExp[],
excludeFilter?: CustomRegExp[],
skipFolder: string[] = [".git", ".trash", "node_modules"]
): Promise<FilePath[]> {
let w: ListedFiles;
try {
w = await this.vaultAccess.adapterList(basePath);
// w = await this.plugin.app.vault.adapter.list(basePath);
} catch (ex) {
this._log(`Could not traverse(getFilesIncludeHidden):${basePath}`, LOG_LEVEL_INFO);
this._log(ex, LOG_LEVEL_VERBOSE);
return [];
}
skipFolder = skipFolder.map((e) => e.toLowerCase());
let files = [] as string[];
for (const file of w.files) {
if (includeFilter && includeFilter.length > 0) {
if (!includeFilter.some((e) => e.test(file))) continue;
}
if (excludeFilter && excludeFilter.some((ee) => ee.test(file))) {
continue;
}
if (await this.vault.isIgnoredByIgnoreFile(file)) continue;
files.push(file);
}
for (const v of w.folders) {
const folderName = (v.split("/").pop() ?? "").toLowerCase();
if (skipFolder.some((e) => folderName === e)) {
continue;
}
if (excludeFilter && excludeFilter.some((e) => e.test(v))) {
continue;
}
if (await this.vault.isIgnoredByIgnoreFile(v)) {
continue;
}
// OK, deep dive!
files = files.concat(await this.getFilesIncludeHidden(v, includeFilter, excludeFilter, skipFolder));
}
return files as FilePath[];
}
async touched(file: UXFileInfoStub | FilePathWithPrefix): Promise<void> {
const path = typeof file === "string" ? file : file.path;
await this.vaultAccess.touch(path as FilePath);
}
recentlyTouched(file: UXFileInfoStub | FilePathWithPrefix): boolean {
const xFile = typeof file === "string" ? (this.vaultAccess.getAbstractFileByPath(file) as TFile) : file;
if (xFile === null) return false;
if (xFile instanceof TFolder) return false;
return this.vaultAccess.recentlyTouched(xFile);
}
clearTouched(): void {
this.vaultAccess.clearTouched();
}
delete(file: FilePathWithPrefix | UXFileInfoStub | string, force: boolean): Promise<void> {
const xPath = typeof file === "string" ? file : file.path;
const xFile = this.vaultAccess.getAbstractFileByPath(xPath);
if (xFile === null) return Promise.resolve();
if (!(xFile instanceof TFile) && !(xFile instanceof TFolder)) return Promise.resolve();
return this.vaultAccess.delete(xFile, force);
}
trash(file: FilePathWithPrefix | UXFileInfoStub | string, system: boolean): Promise<void> {
const xPath = typeof file === "string" ? file : file.path;
const xFile = this.vaultAccess.getAbstractFileByPath(xPath);
if (xFile === null) return Promise.resolve();
if (!(xFile instanceof TFile) && !(xFile instanceof TFolder)) return Promise.resolve();
return this.vaultAccess.trash(xFile, system);
}
// $readFileBinary(path: string): Promise<ArrayBuffer> {
// const file = this.vaultAccess.getAbstractFileByPath(path);
// if (file instanceof TFile) {
// return this.vaultAccess.vaultReadBinary(file);
// } else {
// throw new Error(`Could not read file (Possibly does not exist): ${path}`);
// }
// }
// async $appendFileAuto(path: string, data: string | ArrayBuffer, opt?: DataWriteOptions): Promise<boolean> {
// const file = this.vaultAccess.getAbstractFileByPath(path);
// if (file instanceof TFile) {
// return this.vaultAccess.a(file, data, opt);
// } else if (file !== null) {
// return await this.vaultAccess.vaultCreate(path, data, opt) instanceof TFile;
// } else {
// this._log(`Could not append file (Possibly already exists as a folder): ${path}`, LOG_LEVEL_VERBOSE);
// return false;
// }
// }
async __deleteVaultItem(file: TFile | TFolder) {
if (file instanceof TFile) {
if (!(await this.vault.isTargetFile(file.path))) return;
}
const dir = file.parent;
const settings = this.setting.currentSettings();
if (settings.trashInsteadDelete) {
await this.vaultAccess.trash(file, false);
} else {
await this.vaultAccess.delete(file, true);
}
this._log(`xxx <- STORAGE (deleted) ${file.path}`);
if (dir) {
this._log(`files: ${dir.children.length}`);
if (dir.children.length == 0) {
if (!settings.doNotDeleteFolder) {
this._log(
`All files under the parent directory (${dir.path}) have been deleted, so delete this one.`
);
await this.__deleteVaultItem(dir);
}
}
}
}
async deleteVaultItem(fileSrc: FilePathWithPrefix | UXFileInfoStub | UXFolderInfo): Promise<void> {
const path = typeof fileSrc === "string" ? fileSrc : fileSrc.path;
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file === null) return;
if (file instanceof TFile || file instanceof TFolder) {
return await this.__deleteVaultItem(file);
}
}
}