mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-04-12 11:58:45 +00:00
## 0.25.43-patched-6
### Fixed - Unlocking the remote database after rebuilding has been fixed. ### Refactored - Now `StorageEventManagerBase` is separated from `StorageEventManagerObsidian` following their concerns. - Now `FileAccessBase` is separated from `FileAccessObsidian` following their concerns.
This commit is contained in:
vorotamoroz
@@ -141,10 +141,10 @@ export class ModuleTargetFilter extends AbstractModule {
|
||||
services.vault.markFileListPossiblyChanged.setHandler(this._markFileListPossiblyChanged.bind(this));
|
||||
services.appLifecycle.onLoaded.addHandler(this._everyOnload.bind(this));
|
||||
services.vault.isIgnoredByIgnoreFile.setHandler(this._isTargetIgnoredByIgnoreFiles.bind(this));
|
||||
services.vault.isTargetFile.addHandler(this._isTargetFileByFileNameDuplication.bind(this));
|
||||
services.vault.isTargetFile.addHandler(this._isTargetIgnoredByIgnoreFiles.bind(this));
|
||||
services.vault.isTargetFile.addHandler(this._isTargetFileByLocalDB.bind(this));
|
||||
services.vault.isTargetFile.addHandler(this._isTargetFileFinal.bind(this));
|
||||
services.vault.isTargetFile.addHandler(this._isTargetFileByFileNameDuplication.bind(this), 10);
|
||||
services.vault.isTargetFile.addHandler(this._isTargetIgnoredByIgnoreFiles.bind(this), 20);
|
||||
services.vault.isTargetFile.addHandler(this._isTargetFileByLocalDB.bind(this), 30);
|
||||
services.vault.isTargetFile.addHandler(this._isTargetFileFinal.bind(this), 100);
|
||||
services.setting.onSettingRealised.addHandler(this.refreshSettings.bind(this));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,264 +0,0 @@
|
||||
import { type App, TFile, type DataWriteOptions, TFolder, TAbstractFile } from "../../../deps.ts";
|
||||
import { Logger } from "../../../lib/src/common/logger.ts";
|
||||
import { isPlainText } from "../../../lib/src/string_and_binary/path.ts";
|
||||
import type { FilePath, UXFileInfoStub } from "../../../lib/src/common/types.ts";
|
||||
import { createBinaryBlob, isDocContentSame } from "../../../lib/src/common/utils.ts";
|
||||
import type { InternalFileInfo } from "../../../common/types.ts";
|
||||
import { markChangesAreSame } from "../../../common/utils.ts";
|
||||
import type { IStorageAccessManager } from "@lib/interfaces/StorageAccess.ts";
|
||||
import type { LiveSyncCore } from "@/main.ts";
|
||||
function toArrayBuffer(arr: Uint8Array<ArrayBuffer> | ArrayBuffer | DataView<ArrayBuffer>): ArrayBuffer {
|
||||
if (arr instanceof Uint8Array) {
|
||||
return arr.buffer;
|
||||
}
|
||||
if (arr instanceof DataView) {
|
||||
return arr.buffer;
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
// TODO: add abstraction for the file access (as wrapping TFile or something similar)
|
||||
export abstract class FileAccessBase<TNativeFile> {
|
||||
storageAccessManager: IStorageAccessManager;
|
||||
constructor(storageAccessManager: IStorageAccessManager) {
|
||||
this.storageAccessManager = storageAccessManager;
|
||||
}
|
||||
abstract getPath(file: TNativeFile | string): FilePath;
|
||||
}
|
||||
|
||||
export class ObsidianFileAccess extends FileAccessBase<TFile> {
|
||||
app: App;
|
||||
plugin: LiveSyncCore;
|
||||
|
||||
getPath(file: string | TFile): FilePath {
|
||||
return (typeof file === "string" ? file : file.path) as FilePath;
|
||||
}
|
||||
|
||||
constructor(app: App, plugin: LiveSyncCore, storageAccessManager: IStorageAccessManager) {
|
||||
super(storageAccessManager);
|
||||
this.app = app;
|
||||
this.plugin = plugin;
|
||||
}
|
||||
|
||||
async tryAdapterStat(file: TFile | string) {
|
||||
const path = file instanceof TFile ? file.path : file;
|
||||
return await this.storageAccessManager.processReadFile(path as FilePath, async () => {
|
||||
if (!(await this.app.vault.adapter.exists(path))) return null;
|
||||
return this.app.vault.adapter.stat(path);
|
||||
});
|
||||
}
|
||||
async adapterStat(file: TFile | string) {
|
||||
const path = file instanceof TFile ? file.path : file;
|
||||
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
|
||||
this.app.vault.adapter.stat(path)
|
||||
);
|
||||
}
|
||||
async adapterExists(file: TFile | string) {
|
||||
const path = file instanceof TFile ? file.path : file;
|
||||
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
|
||||
this.app.vault.adapter.exists(path)
|
||||
);
|
||||
}
|
||||
async adapterRemove(file: TFile | string) {
|
||||
const path = file instanceof TFile ? file.path : file;
|
||||
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
|
||||
this.app.vault.adapter.remove(path)
|
||||
);
|
||||
}
|
||||
|
||||
async adapterRead(file: TFile | string) {
|
||||
const path = file instanceof TFile ? file.path : file;
|
||||
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
|
||||
this.app.vault.adapter.read(path)
|
||||
);
|
||||
}
|
||||
async adapterReadBinary(file: TFile | string) {
|
||||
const path = file instanceof TFile ? file.path : file;
|
||||
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
|
||||
this.app.vault.adapter.readBinary(path)
|
||||
);
|
||||
}
|
||||
|
||||
async adapterReadAuto(file: TFile | string) {
|
||||
const path = file instanceof TFile ? file.path : file;
|
||||
if (isPlainText(path)) {
|
||||
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
|
||||
this.app.vault.adapter.read(path)
|
||||
);
|
||||
}
|
||||
return await this.storageAccessManager.processReadFile(path as FilePath, () =>
|
||||
this.app.vault.adapter.readBinary(path)
|
||||
);
|
||||
}
|
||||
|
||||
async adapterWrite(
|
||||
file: TFile | string,
|
||||
data: string | ArrayBuffer | Uint8Array<ArrayBuffer>,
|
||||
options?: DataWriteOptions
|
||||
) {
|
||||
const path = file instanceof TFile ? file.path : file;
|
||||
if (typeof data === "string") {
|
||||
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
|
||||
this.app.vault.adapter.write(path, data, options)
|
||||
);
|
||||
} else {
|
||||
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
|
||||
this.app.vault.adapter.writeBinary(path, toArrayBuffer(data), options)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
adapterList(basePath: string): Promise<{ files: string[]; folders: string[] }> {
|
||||
return Promise.resolve(this.app.vault.adapter.list(basePath));
|
||||
}
|
||||
|
||||
async vaultCacheRead(file: TFile) {
|
||||
return await this.storageAccessManager.processReadFile(file.path as FilePath, () =>
|
||||
this.app.vault.cachedRead(file)
|
||||
);
|
||||
}
|
||||
|
||||
async vaultRead(file: TFile) {
|
||||
return await this.storageAccessManager.processReadFile(file.path as FilePath, () => this.app.vault.read(file));
|
||||
}
|
||||
|
||||
async vaultReadBinary(file: TFile) {
|
||||
return await this.storageAccessManager.processReadFile(file.path as FilePath, () =>
|
||||
this.app.vault.readBinary(file)
|
||||
);
|
||||
}
|
||||
|
||||
async vaultReadAuto(file: TFile) {
|
||||
const path = file.path;
|
||||
if (isPlainText(path)) {
|
||||
return await this.storageAccessManager.processReadFile(path as FilePath, () => this.app.vault.read(file));
|
||||
}
|
||||
return await this.storageAccessManager.processReadFile(path as FilePath, () => this.app.vault.readBinary(file));
|
||||
}
|
||||
|
||||
async vaultModify(file: TFile, data: string | ArrayBuffer | Uint8Array<ArrayBuffer>, options?: DataWriteOptions) {
|
||||
if (typeof data === "string") {
|
||||
return await this.storageAccessManager.processWriteFile(file.path as FilePath, async () => {
|
||||
const oldData = await this.app.vault.read(file);
|
||||
if (data === oldData) {
|
||||
if (options && options.mtime) markChangesAreSame(file.path, file.stat.mtime, options.mtime);
|
||||
return true;
|
||||
}
|
||||
await this.app.vault.modify(file, data, options);
|
||||
return true;
|
||||
});
|
||||
} else {
|
||||
return await this.storageAccessManager.processWriteFile(file.path as FilePath, async () => {
|
||||
const oldData = await this.app.vault.readBinary(file);
|
||||
if (await isDocContentSame(createBinaryBlob(oldData), createBinaryBlob(data))) {
|
||||
if (options && options.mtime) markChangesAreSame(file.path, file.stat.mtime, options.mtime);
|
||||
return true;
|
||||
}
|
||||
await this.app.vault.modifyBinary(file, toArrayBuffer(data), options);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
async vaultCreate(
|
||||
path: string,
|
||||
data: string | ArrayBuffer | Uint8Array<ArrayBuffer>,
|
||||
options?: DataWriteOptions
|
||||
): Promise<TFile> {
|
||||
if (typeof data === "string") {
|
||||
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
|
||||
this.app.vault.create(path, data, options)
|
||||
);
|
||||
} else {
|
||||
return await this.storageAccessManager.processWriteFile(path as FilePath, () =>
|
||||
this.app.vault.createBinary(path, toArrayBuffer(data), options)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
trigger(name: string, ...data: any[]) {
|
||||
return this.app.vault.trigger(name, ...data);
|
||||
}
|
||||
async reconcileInternalFile(path: string) {
|
||||
await (this.app.vault.adapter as any)?.reconcileInternalFile(path);
|
||||
}
|
||||
|
||||
async adapterAppend(normalizedPath: string, data: string, options?: DataWriteOptions) {
|
||||
return await this.app.vault.adapter.append(normalizedPath, data, options);
|
||||
}
|
||||
|
||||
async delete(file: TFile | TFolder, force = false) {
|
||||
return await this.storageAccessManager.processWriteFile(file.path as FilePath, () =>
|
||||
this.app.vault.delete(file, force)
|
||||
);
|
||||
}
|
||||
async trash(file: TFile | TFolder, force = false) {
|
||||
return await this.storageAccessManager.processWriteFile(file.path as FilePath, () =>
|
||||
this.app.vault.trash(file, force)
|
||||
);
|
||||
}
|
||||
|
||||
isStorageInsensitive(): boolean {
|
||||
return this.plugin.services.vault.isStorageInsensitive();
|
||||
}
|
||||
|
||||
getAbstractFileByPathInsensitive(path: FilePath | string): TAbstractFile | null {
|
||||
//@ts-ignore
|
||||
return this.app.vault.getAbstractFileByPathInsensitive(path);
|
||||
}
|
||||
|
||||
getAbstractFileByPath(path: FilePath | string): TAbstractFile | null {
|
||||
if (!this.plugin.settings.handleFilenameCaseSensitive || this.isStorageInsensitive()) {
|
||||
return this.getAbstractFileByPathInsensitive(path);
|
||||
}
|
||||
return this.app.vault.getAbstractFileByPath(path);
|
||||
}
|
||||
|
||||
getFiles() {
|
||||
return this.app.vault.getFiles();
|
||||
}
|
||||
|
||||
async ensureDirectory(fullPath: string) {
|
||||
const pathElements = fullPath.split("/");
|
||||
pathElements.pop();
|
||||
let c = "";
|
||||
for (const v of pathElements) {
|
||||
c += v;
|
||||
try {
|
||||
await this.app.vault.adapter.mkdir(c);
|
||||
} catch (ex: any) {
|
||||
if (ex?.message == "Folder already exists.") {
|
||||
// Skip if already exists.
|
||||
} else {
|
||||
Logger("Folder Create Error");
|
||||
Logger(ex);
|
||||
}
|
||||
}
|
||||
c += "/";
|
||||
}
|
||||
}
|
||||
|
||||
touchedFiles: string[] = [];
|
||||
|
||||
_statInternal(file: FilePath) {
|
||||
return this.app.vault.adapter.stat(file);
|
||||
}
|
||||
|
||||
async touch(file: TFile | FilePath) {
|
||||
const path = file instanceof TFile ? (file.path as FilePath) : file;
|
||||
const statOrg = file instanceof TFile ? file.stat : await this._statInternal(path);
|
||||
const stat = statOrg || { mtime: 0, size: 0 };
|
||||
const key = `${path}-${stat.mtime}-${stat.size}`;
|
||||
this.touchedFiles.unshift(key);
|
||||
this.touchedFiles = this.touchedFiles.slice(0, 100);
|
||||
}
|
||||
recentlyTouched(file: TFile | InternalFileInfo | UXFileInfoStub) {
|
||||
const key =
|
||||
"stat" in file
|
||||
? `${file.path}-${file.stat.mtime}-${file.stat.size}`
|
||||
: `${file.path}-${file.mtime}-${file.size}`;
|
||||
if (this.touchedFiles.indexOf(key) == -1) return false;
|
||||
return true;
|
||||
}
|
||||
clearTouched() {
|
||||
this.touchedFiles = [];
|
||||
}
|
||||
}
|
||||
@@ -1,631 +0,0 @@
|
||||
import { TAbstractFile, TFile, TFolder } from "../../../deps.ts";
|
||||
import { Logger } from "../../../lib/src/common/logger.ts";
|
||||
import { shouldBeIgnored } from "../../../lib/src/string_and_binary/path.ts";
|
||||
import {
|
||||
DEFAULT_SETTINGS,
|
||||
LOG_LEVEL_DEBUG,
|
||||
LOG_LEVEL_INFO,
|
||||
LOG_LEVEL_NOTICE,
|
||||
LOG_LEVEL_VERBOSE,
|
||||
type FileEventType,
|
||||
type FilePath,
|
||||
type UXFileInfoStub,
|
||||
} from "../../../lib/src/common/types.ts";
|
||||
import { delay, fireAndForget, throttle } from "../../../lib/src/common/utils.ts";
|
||||
import { type FileEventItem } from "../../../common/types.ts";
|
||||
import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
|
||||
import { isWaitingForTimeout } from "octagonal-wheels/concurrency/task";
|
||||
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
|
||||
import type { LiveSyncCore } from "../../../main.ts";
|
||||
import { InternalFileToUXFileInfoStub, TFileToUXFileInfoStub } from "./utilObsidian.ts";
|
||||
import ObsidianLiveSyncPlugin from "../../../main.ts";
|
||||
import type { IStorageAccessManager } from "@lib/interfaces/StorageAccess.ts";
|
||||
import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts";
|
||||
import { promiseWithResolvers, type PromiseWithResolvers } from "octagonal-wheels/promises";
|
||||
import { StorageEventManager, type FileEvent } from "@lib/interfaces/StorageEventManager.ts";
|
||||
|
||||
type WaitInfo = {
|
||||
since: number;
|
||||
type: FileEventType;
|
||||
canProceed: PromiseWithResolvers<boolean>;
|
||||
timerHandler: ReturnType<typeof setTimeout>;
|
||||
event: FileEventItem;
|
||||
};
|
||||
const TYPE_SENTINEL_FLUSH = "SENTINEL_FLUSH";
|
||||
type FileEventItemSentinelFlush = {
|
||||
type: typeof TYPE_SENTINEL_FLUSH;
|
||||
};
|
||||
type FileEventItemSentinel = FileEventItemSentinelFlush;
|
||||
|
||||
export class StorageEventManagerObsidian extends StorageEventManager {
|
||||
plugin: ObsidianLiveSyncPlugin;
|
||||
core: LiveSyncCore;
|
||||
storageAccess: IStorageAccessManager;
|
||||
get services() {
|
||||
return this.core.services;
|
||||
}
|
||||
|
||||
get shouldBatchSave() {
|
||||
return this.core.settings?.batchSave && this.core.settings?.liveSync != true;
|
||||
}
|
||||
get batchSaveMinimumDelay(): number {
|
||||
return this.core.settings?.batchSaveMinimumDelay ?? DEFAULT_SETTINGS.batchSaveMinimumDelay;
|
||||
}
|
||||
get batchSaveMaximumDelay(): number {
|
||||
return this.core.settings?.batchSaveMaximumDelay ?? DEFAULT_SETTINGS.batchSaveMaximumDelay;
|
||||
}
|
||||
// Necessary evil.
|
||||
cmdHiddenFileSync: HiddenFileSync;
|
||||
|
||||
/**
|
||||
* Snapshot restoration promise.
|
||||
* Snapshot will be restored before starting to watch vault changes.
|
||||
* In designed time, this has been called from Initialisation process, which has been implemented on `ModuleInitializerFile.ts`.
|
||||
*/
|
||||
snapShotRestored: Promise<void> | null = null;
|
||||
|
||||
constructor(plugin: ObsidianLiveSyncPlugin, core: LiveSyncCore, storageAccessManager: IStorageAccessManager) {
|
||||
super();
|
||||
this.storageAccess = storageAccessManager;
|
||||
this.plugin = plugin;
|
||||
this.core = core;
|
||||
this.cmdHiddenFileSync = this.plugin.getAddOn(HiddenFileSync.name) as HiddenFileSync;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore the previous snapshot if exists.
|
||||
* @returns
|
||||
*/
|
||||
restoreState(): Promise<void> {
|
||||
this.snapShotRestored = this._restoreFromSnapshot();
|
||||
return this.snapShotRestored;
|
||||
}
|
||||
|
||||
async beginWatch() {
|
||||
await this.snapShotRestored;
|
||||
const plugin = this.plugin;
|
||||
this.watchVaultChange = this.watchVaultChange.bind(this);
|
||||
this.watchVaultCreate = this.watchVaultCreate.bind(this);
|
||||
this.watchVaultDelete = this.watchVaultDelete.bind(this);
|
||||
this.watchVaultRename = this.watchVaultRename.bind(this);
|
||||
this.watchVaultRawEvents = this.watchVaultRawEvents.bind(this);
|
||||
this.watchEditorChange = this.watchEditorChange.bind(this);
|
||||
plugin.registerEvent(plugin.app.vault.on("modify", this.watchVaultChange));
|
||||
plugin.registerEvent(plugin.app.vault.on("delete", this.watchVaultDelete));
|
||||
plugin.registerEvent(plugin.app.vault.on("rename", this.watchVaultRename));
|
||||
plugin.registerEvent(plugin.app.vault.on("create", this.watchVaultCreate));
|
||||
//@ts-ignore : Internal API
|
||||
plugin.registerEvent(plugin.app.vault.on("raw", this.watchVaultRawEvents));
|
||||
plugin.registerEvent(plugin.app.workspace.on("editor-change", this.watchEditorChange));
|
||||
}
|
||||
watchEditorChange(editor: any, info: any) {
|
||||
if (!("path" in info)) {
|
||||
return;
|
||||
}
|
||||
if (!this.shouldBatchSave) {
|
||||
return;
|
||||
}
|
||||
const file = info?.file as TFile;
|
||||
if (!file) return;
|
||||
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
|
||||
// Logger(`Editor change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
|
||||
return;
|
||||
}
|
||||
if (!this.isWaiting(file.path as FilePath)) {
|
||||
return;
|
||||
}
|
||||
const data = info?.data as string;
|
||||
const fi: FileEvent = {
|
||||
type: "CHANGED",
|
||||
file: TFileToUXFileInfoStub(file),
|
||||
cachedData: data,
|
||||
};
|
||||
void this.appendQueue([fi]);
|
||||
}
|
||||
|
||||
watchVaultCreate(file: TAbstractFile, ctx?: any) {
|
||||
if (file instanceof TFolder) return;
|
||||
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
|
||||
// Logger(`File create skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
|
||||
return;
|
||||
}
|
||||
const fileInfo = TFileToUXFileInfoStub(file);
|
||||
void this.appendQueue([{ type: "CREATE", file: fileInfo }], ctx);
|
||||
}
|
||||
|
||||
watchVaultChange(file: TAbstractFile, ctx?: any) {
|
||||
if (file instanceof TFolder) return;
|
||||
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
|
||||
// Logger(`File change skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
|
||||
return;
|
||||
}
|
||||
const fileInfo = TFileToUXFileInfoStub(file);
|
||||
void this.appendQueue([{ type: "CHANGED", file: fileInfo }], ctx);
|
||||
}
|
||||
|
||||
watchVaultDelete(file: TAbstractFile, ctx?: any) {
|
||||
if (file instanceof TFolder) return;
|
||||
if (this.storageAccess.isFileProcessing(file.path as FilePath)) {
|
||||
// Logger(`File delete skipped because the file is being processed: ${file.path}`, LOG_LEVEL_VERBOSE);
|
||||
return;
|
||||
}
|
||||
const fileInfo = TFileToUXFileInfoStub(file, true);
|
||||
void this.appendQueue([{ type: "DELETE", file: fileInfo }], ctx);
|
||||
}
|
||||
watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) {
|
||||
// vault Rename will not be raised for self-events (Self-hosted LiveSync will not handle 'rename').
|
||||
if (file instanceof TFile) {
|
||||
const fileInfo = TFileToUXFileInfoStub(file);
|
||||
void this.appendQueue(
|
||||
[
|
||||
{
|
||||
type: "DELETE",
|
||||
file: {
|
||||
path: oldFile as FilePath,
|
||||
name: file.name,
|
||||
stat: {
|
||||
mtime: file.stat.mtime,
|
||||
ctime: file.stat.ctime,
|
||||
size: file.stat.size,
|
||||
type: "file",
|
||||
},
|
||||
deleted: true,
|
||||
},
|
||||
skipBatchWait: true,
|
||||
},
|
||||
{ type: "CREATE", file: fileInfo, skipBatchWait: true },
|
||||
],
|
||||
ctx
|
||||
);
|
||||
}
|
||||
}
|
||||
// Watch raw events (Internal API)
|
||||
watchVaultRawEvents(path: FilePath) {
|
||||
if (this.storageAccess.isFileProcessing(path)) {
|
||||
// Logger(`Raw file event skipped because the file is being processed: ${path}`, LOG_LEVEL_VERBOSE);
|
||||
return;
|
||||
}
|
||||
// Only for internal files.
|
||||
if (!this.plugin.settings) return;
|
||||
// if (this.plugin.settings.useIgnoreFiles && this.plugin.ignoreFiles.some(e => path.endsWith(e.trim()))) {
|
||||
if (this.plugin.settings.useIgnoreFiles) {
|
||||
// If it is one of ignore files, refresh the cached one.
|
||||
// (Calling$$isTargetFile will refresh the cache)
|
||||
void this.services.vault.isTargetFile(path).then(() => this._watchVaultRawEvents(path));
|
||||
} else {
|
||||
void this._watchVaultRawEvents(path);
|
||||
}
|
||||
}
|
||||
|
||||
async _watchVaultRawEvents(path: FilePath) {
|
||||
if (!this.plugin.settings.syncInternalFiles && !this.plugin.settings.usePluginSync) return;
|
||||
if (!this.plugin.settings.watchInternalFileChanges) return;
|
||||
if (!path.startsWith(this.plugin.app.vault.configDir)) return;
|
||||
if (path.endsWith("/")) {
|
||||
// Folder
|
||||
return;
|
||||
}
|
||||
const isTargetFile = await this.cmdHiddenFileSync.isTargetFile(path);
|
||||
if (!isTargetFile) return;
|
||||
|
||||
void this.appendQueue(
|
||||
[
|
||||
{
|
||||
type: "INTERNAL",
|
||||
file: InternalFileToUXFileInfoStub(path),
|
||||
skipBatchWait: true, // Internal files should be processed immediately.
|
||||
},
|
||||
],
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
// Cache file and waiting to can be proceed.
|
||||
async appendQueue(params: FileEvent[], ctx?: any) {
|
||||
if (!this.core.settings.isConfigured) return;
|
||||
if (this.core.settings.suspendFileWatching) return;
|
||||
if (this.core.settings.maxMTimeForReflectEvents > 0) {
|
||||
return;
|
||||
}
|
||||
this.core.services.vault.markFileListPossiblyChanged();
|
||||
// Flag up to be reload
|
||||
for (const param of params) {
|
||||
if (shouldBeIgnored(param.file.path)) {
|
||||
continue;
|
||||
}
|
||||
const atomicKey = [0, 0, 0, 0, 0, 0].map((e) => `${Math.floor(Math.random() * 100000)}`).join("-");
|
||||
const type = param.type;
|
||||
const file = param.file;
|
||||
const oldPath = param.oldPath;
|
||||
if (type !== "INTERNAL") {
|
||||
const size = (file as UXFileInfoStub).stat.size;
|
||||
if (this.services.vault.isFileSizeTooLarge(size) && (type == "CREATE" || type == "CHANGED")) {
|
||||
Logger(
|
||||
`The storage file has been changed but exceeds the maximum size. Skipping: ${param.file.path}`,
|
||||
LOG_LEVEL_NOTICE
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (file instanceof TFolder) continue;
|
||||
// TODO: Confirm why only the TFolder skipping
|
||||
// Possibly following line is needed...
|
||||
// if (file?.isFolder) continue;
|
||||
if (!(await this.services.vault.isTargetFile(file.path))) continue;
|
||||
|
||||
// Stop cache using to prevent the corruption;
|
||||
// let cache: null | string | ArrayBuffer;
|
||||
// new file or something changed, cache the changes.
|
||||
// if (file instanceof TFile && (type == "CREATE" || type == "CHANGED")) {
|
||||
if (file instanceof TFile || !file.isFolder) {
|
||||
if (type == "CREATE" || type == "CHANGED") {
|
||||
// Wait for a bit while to let the writer has marked `touched` at the file.
|
||||
await delay(10);
|
||||
if (this.core.storageAccess.recentlyTouched(file.path)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let cache: string | undefined = undefined;
|
||||
if (param.cachedData) {
|
||||
cache = param.cachedData;
|
||||
}
|
||||
void this.enqueue({
|
||||
type,
|
||||
args: {
|
||||
file: file,
|
||||
oldPath,
|
||||
cache,
|
||||
ctx,
|
||||
},
|
||||
skipBatchWait: param.skipBatchWait,
|
||||
key: atomicKey,
|
||||
});
|
||||
}
|
||||
}
|
||||
private bufferedQueuedItems = [] as (FileEventItem | FileEventItemSentinel)[];
|
||||
|
||||
/**
|
||||
* Immediately take snapshot.
|
||||
*/
|
||||
private _triggerTakeSnapshot() {
|
||||
void this._takeSnapshot();
|
||||
}
|
||||
/**
|
||||
* Trigger taking snapshot after throttled period.
|
||||
*/
|
||||
triggerTakeSnapshot = throttle(() => this._triggerTakeSnapshot(), 100);
|
||||
|
||||
enqueue(newItem: FileEventItem) {
|
||||
if (newItem.type == "DELETE") {
|
||||
// If the sentinel pushed, the runQueuedEvents will wait for idle before processing delete.
|
||||
this.bufferedQueuedItems.push({
|
||||
type: TYPE_SENTINEL_FLUSH,
|
||||
});
|
||||
}
|
||||
this.updateStatus();
|
||||
this.bufferedQueuedItems.push(newItem);
|
||||
|
||||
fireAndForget(() => this._takeSnapshot().then(() => this.runQueuedEvents()));
|
||||
}
|
||||
|
||||
// Limit concurrent processing to reduce the IO load. file-processing + scheduler (1), so file events can be processed in 4 slots.
|
||||
concurrentProcessing = Semaphore(5);
|
||||
|
||||
private _waitingMap = new Map<string, WaitInfo>();
|
||||
private _waitForIdle: Promise<void> | null = null;
|
||||
|
||||
/**
|
||||
* Wait until all queued events are processed.
|
||||
* Subsequent new events will not be waited, but new events will not be added.
|
||||
* @returns
|
||||
*/
|
||||
waitForIdle(): Promise<void> {
|
||||
if (this._waitingMap.size === 0) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
if (this._waitForIdle) {
|
||||
return this._waitForIdle;
|
||||
}
|
||||
const promises = [...this._waitingMap.entries()].map(([key, waitInfo]) => {
|
||||
return new Promise<void>((resolve) => {
|
||||
waitInfo.canProceed.promise
|
||||
.then(() => {
|
||||
Logger(`Processing ${key}: Wait for idle completed`, LOG_LEVEL_DEBUG);
|
||||
// No op
|
||||
})
|
||||
.catch((e) => {
|
||||
Logger(`Processing ${key}: Wait for idle error`, LOG_LEVEL_INFO);
|
||||
Logger(e, LOG_LEVEL_VERBOSE);
|
||||
//no op
|
||||
})
|
||||
.finally(() => {
|
||||
resolve();
|
||||
});
|
||||
this._proceedWaiting(key);
|
||||
});
|
||||
});
|
||||
const waitPromise = Promise.all(promises).then(() => {
|
||||
this._waitForIdle = null;
|
||||
Logger(`All wait for idle completed`, LOG_LEVEL_VERBOSE);
|
||||
});
|
||||
this._waitForIdle = waitPromise;
|
||||
return waitPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Proceed waiting for the given key immediately.
|
||||
*/
|
||||
private _proceedWaiting(key: string) {
|
||||
const waitInfo = this._waitingMap.get(key);
|
||||
if (waitInfo) {
|
||||
waitInfo.canProceed.resolve(true);
|
||||
clearTimeout(waitInfo.timerHandler);
|
||||
this._waitingMap.delete(key);
|
||||
}
|
||||
this.triggerTakeSnapshot();
|
||||
}
|
||||
/**
|
||||
* Cancel waiting for the given key.
|
||||
*/
|
||||
private _cancelWaiting(key: string) {
|
||||
const waitInfo = this._waitingMap.get(key);
|
||||
if (waitInfo) {
|
||||
waitInfo.canProceed.resolve(false);
|
||||
clearTimeout(waitInfo.timerHandler);
|
||||
this._waitingMap.delete(key);
|
||||
}
|
||||
this.triggerTakeSnapshot();
|
||||
}
|
||||
/**
|
||||
* Add waiting for the given key.
|
||||
* @param key
|
||||
* @param event
|
||||
* @param waitedSince Optional waited since timestamp to calculate the remaining delay.
|
||||
*/
|
||||
private _addWaiting(key: string, event: FileEventItem, waitedSince?: number): WaitInfo {
|
||||
if (this._waitingMap.has(key)) {
|
||||
// Already waiting
|
||||
throw new Error(`Already waiting for key: ${key}`);
|
||||
}
|
||||
const resolver = promiseWithResolvers<boolean>();
|
||||
const now = Date.now();
|
||||
const since = waitedSince ?? now;
|
||||
const elapsed = now - since;
|
||||
const maxDelay = this.batchSaveMaximumDelay * 1000;
|
||||
const remainingDelay = Math.max(0, maxDelay - elapsed);
|
||||
const nextDelay = Math.min(remainingDelay, this.batchSaveMinimumDelay * 1000);
|
||||
// x*<------- maxDelay --------->*
|
||||
// x*<-- minDelay -->*
|
||||
// x* x<-- nextDelay -->*
|
||||
// x* x<-- Capped-->*
|
||||
// x* x.......*
|
||||
// x: event
|
||||
// *: save
|
||||
// When at event (x) At least, save (*) within maxDelay, but maintain minimum delay between saves.
|
||||
|
||||
if (elapsed >= maxDelay) {
|
||||
// Already exceeded maximum delay, do not wait.
|
||||
Logger(`Processing ${key}: Batch save maximum delay already exceeded: ${event.type}`, LOG_LEVEL_DEBUG);
|
||||
} else {
|
||||
Logger(`Processing ${key}: Adding waiting for batch save: ${event.type} (${nextDelay}ms)`, LOG_LEVEL_DEBUG);
|
||||
}
|
||||
const waitInfo: WaitInfo = {
|
||||
since: since,
|
||||
type: event.type,
|
||||
event: event,
|
||||
canProceed: resolver,
|
||||
timerHandler: setTimeout(() => {
|
||||
Logger(`Processing ${key}: Batch save timeout reached: ${event.type}`, LOG_LEVEL_DEBUG);
|
||||
this._proceedWaiting(key);
|
||||
}, nextDelay),
|
||||
};
|
||||
this._waitingMap.set(key, waitInfo);
|
||||
this.triggerTakeSnapshot();
|
||||
return waitInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the given file event.
|
||||
*/
|
||||
async processFileEvent(fei: FileEventItem) {
|
||||
const releaser = await this.concurrentProcessing.acquire();
|
||||
try {
|
||||
this.updateStatus();
|
||||
const filename = fei.args.file.path;
|
||||
const waitingKey = `${filename}`;
|
||||
const previous = this._waitingMap.get(waitingKey);
|
||||
let isShouldBeCancelled = fei.skipBatchWait || false;
|
||||
let previousPromise: Promise<boolean> = Promise.resolve(true);
|
||||
let waitPromise: Promise<boolean> = Promise.resolve(true);
|
||||
// 1. Check if there is previous waiting for the same file
|
||||
if (previous) {
|
||||
previousPromise = previous.canProceed.promise;
|
||||
if (isShouldBeCancelled) {
|
||||
Logger(
|
||||
`Processing ${filename}: Requested to perform immediately, cancelling previous waiting: ${fei.type}`,
|
||||
LOG_LEVEL_DEBUG
|
||||
);
|
||||
}
|
||||
if (!isShouldBeCancelled && fei.type === "DELETE") {
|
||||
// For DELETE, cancel any previous waiting and proceed immediately
|
||||
// That because when deleting, we cannot read the file anymore.
|
||||
Logger(
|
||||
`Processing ${filename}: DELETE requested, cancelling previous waiting: ${fei.type}`,
|
||||
LOG_LEVEL_DEBUG
|
||||
);
|
||||
isShouldBeCancelled = true;
|
||||
}
|
||||
if (!isShouldBeCancelled && previous.type === fei.type) {
|
||||
// For the same type, we can cancel the previous waiting and proceed immediately.
|
||||
Logger(`Processing ${filename}: Cancelling previous waiting: ${fei.type}`, LOG_LEVEL_DEBUG);
|
||||
isShouldBeCancelled = true;
|
||||
}
|
||||
// 2. wait for the previous to complete
|
||||
if (isShouldBeCancelled) {
|
||||
this._cancelWaiting(waitingKey);
|
||||
Logger(`Processing ${filename}: Previous cancelled: ${fei.type}`, LOG_LEVEL_DEBUG);
|
||||
isShouldBeCancelled = true;
|
||||
}
|
||||
if (!isShouldBeCancelled) {
|
||||
Logger(`Processing ${filename}: Waiting for previous to complete: ${fei.type}`, LOG_LEVEL_DEBUG);
|
||||
this._proceedWaiting(waitingKey);
|
||||
Logger(`Processing ${filename}: Previous completed: ${fei.type}`, LOG_LEVEL_DEBUG);
|
||||
}
|
||||
}
|
||||
await previousPromise;
|
||||
// 3. Check if shouldBatchSave is true
|
||||
if (this.shouldBatchSave && !fei.skipBatchWait) {
|
||||
// if type is CREATE or CHANGED, set waiting
|
||||
if (fei.type == "CREATE" || fei.type == "CHANGED") {
|
||||
// 3.2. If true, set the queue, and wait for the waiting, or until timeout
|
||||
// (since is copied from previous waiting if exists to limit the maximum wait time)
|
||||
// console.warn(`Since:`, previous?.since);
|
||||
const info = this._addWaiting(waitingKey, fei, previous?.since);
|
||||
waitPromise = info.canProceed.promise;
|
||||
} else if (fei.type == "DELETE") {
|
||||
// For DELETE, cancel any previous waiting and proceed immediately
|
||||
}
|
||||
Logger(`Processing ${filename}: Waiting for batch save: ${fei.type}`, LOG_LEVEL_DEBUG);
|
||||
const canProceed = await waitPromise;
|
||||
if (!canProceed) {
|
||||
// 3.2.1. If cancelled by new queue, cancel subsequent process.
|
||||
Logger(`Processing ${filename}: Cancelled by new queue: ${fei.type}`, LOG_LEVEL_DEBUG);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// await this.handleFileEvent(fei);
|
||||
await this.requestProcessQueue(fei);
|
||||
} finally {
|
||||
await this._takeSnapshot();
|
||||
releaser();
|
||||
}
|
||||
}
|
||||
async _takeSnapshot() {
|
||||
const processingEvents = [...this._waitingMap.values()].map((e) => e.event);
|
||||
const waitingEvents = this.bufferedQueuedItems;
|
||||
const snapShot = [...processingEvents, ...waitingEvents];
|
||||
await this.core.kvDB.set("storage-event-manager-snapshot", snapShot);
|
||||
Logger(`Storage operation snapshot taken: ${snapShot.length} items`, LOG_LEVEL_DEBUG);
|
||||
this.updateStatus();
|
||||
}
|
||||
async _restoreFromSnapshot() {
|
||||
const snapShot = await this.core.kvDB.get<(FileEventItem | FileEventItemSentinel)[]>(
|
||||
"storage-event-manager-snapshot"
|
||||
);
|
||||
if (snapShot && Array.isArray(snapShot) && snapShot.length > 0) {
|
||||
// console.warn(`Restoring snapshot: ${snapShot.length} items`);
|
||||
Logger(`Restoring storage operation snapshot: ${snapShot.length} items`, LOG_LEVEL_VERBOSE);
|
||||
// Restore the snapshot
|
||||
// Note: Mark all items as skipBatchWait to prevent apply the off-line batch saving.
|
||||
this.bufferedQueuedItems = snapShot.map((e) => ({ ...e, skipBatchWait: true }));
|
||||
this.updateStatus();
|
||||
await this.runQueuedEvents();
|
||||
} else {
|
||||
Logger(`No snapshot to restore`, LOG_LEVEL_VERBOSE);
|
||||
// console.warn(`No snapshot to restore`);
|
||||
}
|
||||
}
|
||||
runQueuedEvents() {
|
||||
return skipIfDuplicated("storage-event-manager-run-queued-events", async () => {
|
||||
do {
|
||||
if (this.bufferedQueuedItems.length === 0) {
|
||||
break;
|
||||
}
|
||||
// 1. Get the first queued item
|
||||
|
||||
const fei = this.bufferedQueuedItems.shift()!;
|
||||
await this._takeSnapshot();
|
||||
this.updateStatus();
|
||||
// 2. Consume 1 semaphore slot to enqueue processing. Then release immediately.
|
||||
// (Just to limit the total concurrent processing count, because skipping batch handles at processFileEvent).
|
||||
const releaser = await this.concurrentProcessing.acquire();
|
||||
releaser();
|
||||
this.updateStatus();
|
||||
// 3. Check if sentinel flush
|
||||
// If sentinel, wait for idle and continue.
|
||||
if (fei.type === TYPE_SENTINEL_FLUSH) {
|
||||
Logger(`Waiting for idle`, LOG_LEVEL_VERBOSE);
|
||||
// Flush all waiting batch queues
|
||||
await this.waitForIdle();
|
||||
this.updateStatus();
|
||||
continue;
|
||||
}
|
||||
// 4. Process the event, this should be fire-and-forget to not block the queue processing in each file.
|
||||
fireAndForget(() => this.processFileEvent(fei));
|
||||
} while (this.bufferedQueuedItems.length > 0);
|
||||
});
|
||||
}
|
||||
|
||||
processingCount = 0;
|
||||
async requestProcessQueue(fei: FileEventItem) {
|
||||
try {
|
||||
this.processingCount++;
|
||||
// this.bufferedQueuedItems.remove(fei);
|
||||
this.updateStatus();
|
||||
// this.waitedSince.delete(fei.args.file.path);
|
||||
await this.handleFileEvent(fei);
|
||||
await this._takeSnapshot();
|
||||
} finally {
|
||||
this.processingCount--;
|
||||
this.updateStatus();
|
||||
}
|
||||
}
|
||||
isWaiting(filename: FilePath) {
|
||||
return isWaitingForTimeout(`storage-event-manager-batchsave-${filename}`);
|
||||
}
|
||||
|
||||
updateStatus() {
|
||||
const allFileEventItems = this.bufferedQueuedItems.filter((e): e is FileEventItem => "args" in e);
|
||||
const allItems = allFileEventItems.filter((e) => !e.cancelled);
|
||||
const totalItems = allItems.length + this.concurrentProcessing.waiting;
|
||||
const processing = this.processingCount;
|
||||
const batchedCount = this._waitingMap.size;
|
||||
this.core.batched.value = batchedCount;
|
||||
this.core.processing.value = processing;
|
||||
this.core.totalQueued.value = totalItems + batchedCount + processing;
|
||||
}
|
||||
|
||||
async handleFileEvent(queue: FileEventItem): Promise<any> {
|
||||
const file = queue.args.file;
|
||||
const lockKey = `handleFile:${file.path}`;
|
||||
const ret = await serialized(lockKey, async () => {
|
||||
if (queue.cancelled) {
|
||||
Logger(`File event cancelled before processing: ${file.path}`, LOG_LEVEL_INFO);
|
||||
return;
|
||||
}
|
||||
if (queue.type == "INTERNAL" || file.isInternal) {
|
||||
await this.core.services.fileProcessing.processOptionalFileEvent(file.path as unknown as FilePath);
|
||||
} else {
|
||||
const key = `file-last-proc-${queue.type}-${file.path}`;
|
||||
const last = Number((await this.core.kvDB.get(key)) || 0);
|
||||
if (queue.type == "DELETE") {
|
||||
await this.core.services.fileProcessing.processFileEvent(queue);
|
||||
} else {
|
||||
if (file.stat.mtime == last) {
|
||||
Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL_VERBOSE);
|
||||
// Should Cancel the relative operations? (e.g. rename)
|
||||
// this.cancelRelativeEvent(queue);
|
||||
return;
|
||||
}
|
||||
if (!(await this.core.services.fileProcessing.processFileEvent(queue))) {
|
||||
Logger(
|
||||
`STORAGE -> DB: Handler failed, cancel the relative operations: ${file.path}`,
|
||||
LOG_LEVEL_INFO
|
||||
);
|
||||
// cancel running queues and remove one of atomic operation (e.g. rename)
|
||||
this.cancelRelativeEvent(queue);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
this.updateStatus();
|
||||
return ret;
|
||||
}
|
||||
|
||||
cancelRelativeEvent(item: FileEventItem): void {
|
||||
this._cancelWaiting(item.args.file.path);
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import { TFile, type TAbstractFile, type TFolder } from "../../../deps.ts";
|
||||
import { ICHeader } from "../../../common/types.ts";
|
||||
import type { ObsidianFileAccess } from "./SerializedFileAccess.ts";
|
||||
import { addPrefix, isPlainText } from "../../../lib/src/string_and_binary/path.ts";
|
||||
import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
|
||||
import { createBlob } from "../../../lib/src/common/utils.ts";
|
||||
@@ -15,6 +14,7 @@ import type {
|
||||
UXInternalFileInfoStub,
|
||||
} from "../../../lib/src/common/types.ts";
|
||||
import type { LiveSyncCore } from "../../../main.ts";
|
||||
import type { FileAccessObsidian } from "@/serviceModules/FileAccessObsidian.ts";
|
||||
|
||||
export async function TFileToUXFileInfo(
|
||||
core: LiveSyncCore,
|
||||
@@ -51,7 +51,7 @@ export async function TFileToUXFileInfo(
|
||||
|
||||
export async function InternalFileToUXFileInfo(
|
||||
fullPath: string,
|
||||
vaultAccess: ObsidianFileAccess,
|
||||
vaultAccess: FileAccessObsidian,
|
||||
prefix: string = ICHeader
|
||||
): Promise<UXFileInfo> {
|
||||
const name = fullPath.split("/").pop() as string;
|
||||
|
||||
Reference in New Issue
Block a user