public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2025-12-12 17:25:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

### Behaviour change

Browse Source 
- The plug-in automatically fetches the missing chunks even if `Fetch chunks on demand` is disabled.
    - This change is to avoid loss of data when receiving a bulk of revisions.
    - This can be prevented by enabling `Use Only Local Chunks` in the settings.
- Storage application now saved during each event and restored on startup.
- Synchronisation result application is also now saved during each event and restored on startup.
    - These may avoid some unexpected loss of data when the editor crashes.

### Fixed

- Now the plug-in waits for the application of pended batch changes before the synchronisation starts.
    - This may avoid some unexpected loss or unexpected conflicts.
      Plug-in sends custom headers correctly when RequestAPI is used.
- No longer causing unexpected chunk creation during `Reset synchronisation on This Device` with bucket sync.

### Refactored

- Synchronisation result application process has been refactored.
- Storage application process has been refactored.
    - Please report if you find any unexpected behaviour after this update. A bit of large refactoring.
This commit is contained in:
vorotamoroz
2025-12-10 09:45:57 +00:00
parent f06f8d1eb6
commit ac0378ca4b
12 changed files with 848 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/lib

Submodule src/lib updated: 9393ff1e08...da14678633

4
src/modules/core/ModuleDatabaseFileAccess.ts
View File

@@ -76,11 +76,11 @@ export class ModuleDatabaseFileAccess extends AbstractModule implements Database
async checkIsTargetFile(file: UXFileInfoStub | FilePathWithPrefix): Promise<boolean> {
const path = getStoragePathFromUXFileInfo(file);
if (!(await this.services.vault.isTargetFile(path))) {
this._log(`File is not target`, LOG_LEVEL_VERBOSE);
this._log(`File is not target: ${path}`, LOG_LEVEL_VERBOSE);
return false;
}
if (shouldBeIgnored(path)) {
this._log(`File should be ignored`, LOG_LEVEL_VERBOSE);
this._log(`File should be ignored: ${path}`, LOG_LEVEL_VERBOSE);
return false;
}
return true;

2
src/modules/core/ModuleRebuilder.ts
View File

@@ -240,7 +240,7 @@ Please enable them from the settings screen after setup is complete.`,
async fetchRemoteChunks() {
if (
!this.core.settings.doNotSuspendOnFetching &&
this.core.settings.readChunksOnline &&
!this.core.settings.useOnlyLocalChunk &&
this.core.settings.remoteType == REMOTE_COUCHDB
) {
this._log(`Fetching chunks`, LOG_LEVEL_NOTICE);

281
src/modules/core/ModuleReplicator.ts
View File

@@ -14,34 +14,15 @@ import { isLockAcquired, shareRunningResult, skipIfDuplicated } from "octagonal-
import { balanceChunkPurgedDBs } from "@/lib/src/pouchdb/chunks";
import { purgeUnreferencedChunks } from "@/lib/src/pouchdb/chunks";
import { LiveSyncCouchDBReplicator } from "../../lib/src/replication/couchdb/LiveSyncReplicator";
import { throttle } from "octagonal-wheels/function";
import { arrayToChunkedArray } from "octagonal-wheels/collection";
import {
SYNCINFO_ID,
VER,
type EntryBody,
type EntryDoc,
type EntryLeaf,
type LoadedEntry,
type MetaEntry,
type RemoteType,
} from "../../lib/src/common/types";
import { QueueProcessor } from "octagonal-wheels/concurrency/processor";
import {
getPath,
isChunk,
isValidPath,
rateLimitedSharedExecution,
scheduleTask,
updatePreviousExecutionTime,
} from "../../common/utils";
import { isAnyNote } from "../../lib/src/common/utils";
import { type EntryDoc, type RemoteType } from "../../lib/src/common/types";
import { rateLimitedSharedExecution, scheduleTask, updatePreviousExecutionTime } from "../../common/utils";
import { EVENT_FILE_SAVED, EVENT_ON_UNRESOLVED_ERROR, EVENT_SETTING_SAVED, eventHub } from "../../common/events";
import type { LiveSyncAbstractReplicator } from "../../lib/src/replication/LiveSyncAbstractReplicator";
import { $msg } from "../../lib/src/common/i18n";
import { clearHandlers } from "../../lib/src/replication/SyncParamsHandler";
import type { LiveSyncCore } from "../../main";
import { ReplicateResultProcessor } from "./ReplicateResultProcessor";
const KEY_REPLICATION_ON_EVENT = "replicationOnEvent";
const REPLICATION_ON_EVENT_FORECASTED_TIME = 5000;
@@ -49,6 +30,7 @@ const REPLICATION_ON_EVENT_FORECASTED_TIME = 5000;
export class ModuleReplicator extends AbstractModule {
_replicatorType?: RemoteType;
_previousErrors = new Set<string>();
processor: ReplicateResultProcessor = new ReplicateResultProcessor(this);
showError(msg: string, max_log_level: LOG_LEVEL = LEVEL_NOTICE) {
const level = this._previousErrors.has(msg) ? LEVEL_INFO : max_log_level;
@@ -73,6 +55,11 @@ export class ModuleReplicator extends AbstractModule {
if (this._replicatorType !== setting.remoteType) {
void this.setReplicator();
}
if (this.core.settings.suspendParseReplicationResult) {
this.processor.suspend();
} else {
this.processor.resume();
}
});
return Promise.resolve(true);
@@ -103,6 +90,10 @@ export class ModuleReplicator extends AbstractModule {
_everyOnInitializeDatabase(db: LiveSyncLocalDB): Promise<boolean> {
return this.setReplicator();
}
_everyOnDatabaseInitialized(showNotice: boolean): Promise<boolean> {
fireAndForget(() => this.processor.restoreFromSnapshotOnce());
return Promise.resolve(true);
}
_everyOnResetDatabase(db: LiveSyncLocalDB): Promise<boolean> {
return this.setReplicator();
@@ -128,7 +119,7 @@ export class ModuleReplicator extends AbstractModule {
this.showError("Failed to initialise the encryption key, preventing replication.");
return false;
}
await this.loadQueuedFiles();
await this.processor.restoreFromSnapshotOnce();
this.clearErrors();
return true;
}
@@ -287,249 +278,10 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
}
return await shareRunningResult(`replication`, () => this.services.replication.replicate());
}
_parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>): void {
if (this.settings.suspendParseReplicationResult && !this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.suspend();
}
this.replicationResultProcessor.enqueueAll(docs);
if (!this.settings.suspendParseReplicationResult && this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.resume();
}
this.processor.enqueueAll(docs);
}
_saveQueuedFiles = throttle(() => {
const saveData = this.replicationResultProcessor._queue
.filter((e) => e !== undefined && e !== null)
.map((e) => e?._id ?? ("" as string)) as string[];
const kvDBKey = "queued-files";
// localStorage.setItem(lsKey, saveData);
fireAndForget(() => this.core.kvDB.set(kvDBKey, saveData));
}, 100);
saveQueuedFiles() {
this._saveQueuedFiles();
}
async loadQueuedFiles() {
if (this.settings.suspendParseReplicationResult) return;
if (!this.settings.isConfigured) return;
try {
const kvDBKey = "queued-files";
// const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[];
const ids = [...new Set((await this.core.kvDB.get<string[]>(kvDBKey)) ?? [])];
const batchSize = 100;
const chunkedIds = arrayToChunkedArray(ids, batchSize);
// suspendParseReplicationResult is true, so we have to resume it if it is suspended.
if (this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.resume();
}
for await (const idsBatch of chunkedIds) {
const ret = await this.localDatabase.allDocsRaw<EntryDoc>({
keys: idsBatch,
include_docs: true,
limit: 100,
});
const docs = ret.rows
.filter((e) => e.doc)
.map((e) => e.doc) as PouchDB.Core.ExistingDocument<EntryDoc>[];
const errors = ret.rows.filter((e) => !e.doc && !e.value.deleted);
if (errors.length > 0) {
Logger("Some queued processes were not resurrected");
Logger(JSON.stringify(errors), LOG_LEVEL_VERBOSE);
}
this.replicationResultProcessor.enqueueAll(docs);
}
} catch (e) {
Logger(`Failed to load queued files.`, LOG_LEVEL_NOTICE);
Logger(e, LOG_LEVEL_VERBOSE);
} finally {
// Check again before awaiting,
if (this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.resume();
}
}
// Wait for all queued files to be processed.
try {
await this.replicationResultProcessor.waitForAllProcessed();
} catch (e) {
Logger(`Failed to wait for all queued files to be processed.`, LOG_LEVEL_NOTICE);
Logger(e, LOG_LEVEL_VERBOSE);
}
}
replicationResultProcessor = new QueueProcessor(
async (docs: PouchDB.Core.ExistingDocument<EntryDoc>[]) => {
if (this.settings.suspendParseReplicationResult) return;
const change = docs[0];
if (!change) return;
if (isChunk(change._id)) {
this.localDatabase.onNewLeaf(change as EntryLeaf);
return;
}
if (await this.services.replication.processVirtualDocument(change)) return;
// any addon needs this item?
// for (const proc of this.core.addOns) {
// if (await proc.parseReplicationResultItem(change)) {
// return;
// }
// }
if (change.type == "versioninfo") {
if (change.version > VER) {
this.core.replicator.closeReplication();
Logger(
`Remote database updated to incompatible version. update your Self-hosted LiveSync plugin.`,
LOG_LEVEL_NOTICE
);
}
return;
}
if (
change._id == SYNCINFO_ID || // Synchronisation information data
change._id.startsWith("_design") //design document
) {
return;
}
if (isAnyNote(change)) {
const docPath = getPath(change);
if (!(await this.services.vault.isTargetFile(docPath))) {
Logger(`Skipped: ${docPath}`, LOG_LEVEL_VERBOSE);
return;
}
if (this.databaseQueuedProcessor._isSuspended) {
Logger(`Processing scheduled: ${docPath}`, LOG_LEVEL_INFO);
}
const size = change.size;
if (this.services.vault.isFileSizeTooLarge(size)) {
Logger(
`Processing ${docPath} has been skipped due to file size exceeding the limit`,
LOG_LEVEL_NOTICE
);
return;
}
this.databaseQueuedProcessor.enqueue(change);
}
return;
},
{
batchSize: 1,
suspended: true,
concurrentLimit: 100,
delay: 0,
totalRemainingReactiveSource: this.core.replicationResultCount,
}
)
.replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter((e) => e._id != newItem._id);
return [...q, newItem];
})
.startPipeline()
.onUpdateProgress(() => {
this.saveQueuedFiles();
});
async checkIsChangeRequiredForDatabaseProcessing(dbDoc: LoadedEntry): Promise<boolean> {
const path = getPath(dbDoc);
try {
const savedDoc = await this.localDatabase.getRaw<LoadedEntry>(dbDoc._id, {
conflicts: true,
revs_info: true,
});
const newRev = dbDoc._rev ?? "";
const latestRev = savedDoc._rev ?? "";
const revisions = savedDoc._revs_info?.map((e) => e.rev) ?? [];
if (savedDoc._conflicts && savedDoc._conflicts.length > 0) {
// There are conflicts, so we have to process it.
return true;
}
if (newRev == latestRev) {
// The latest revision. We need to process it.
return true;
}
const index = revisions.indexOf(newRev);
if (index >= 0) {
// the revision has been inserted before.
return false; // Already processed.
}
return true; // This mostly should not happen, but we have to process it just in case.
} catch (e: any) {
if ("status" in e && e.status == 404) {
return true;
// Not existing, so we have to process it.
} else {
Logger(
`Failed to get existing document for ${path} (${dbDoc._id.substring(0, 8)}, ${dbDoc._rev?.substring(0, 10)}) `,
LOG_LEVEL_NOTICE
);
Logger(e, LOG_LEVEL_VERBOSE);
return true;
}
}
return true;
}
databaseQueuedProcessor = new QueueProcessor(
async (docs: EntryBody[]) => {
const dbDoc = docs[0] as LoadedEntry; // It has no `data`
const path = getPath(dbDoc);
// If the document is existing with any revision, confirm that we have to process it.
const isRequired = await this.checkIsChangeRequiredForDatabaseProcessing(dbDoc);
if (!isRequired) {
Logger(`Skipped (Not latest): ${path} (${dbDoc._id.substring(0, 8)})`, LOG_LEVEL_VERBOSE);
return;
}
// If `Read chunks online` is disabled, chunks should be transferred before here.
// However, in some cases, chunks are after that. So, if missing chunks exist, we have to wait for them.
const doc = await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, false, true);
if (!doc) {
Logger(
`Something went wrong while gathering content of ${path} (${dbDoc._id.substring(0, 8)}, ${dbDoc._rev?.substring(0, 10)}) `,
LOG_LEVEL_NOTICE
);
return;
}
if (await this.services.replication.processOptionalSynchroniseResult(dbDoc)) {
// Already processed
} else if (isValidPath(getPath(doc))) {
this.storageApplyingProcessor.enqueue(doc as MetaEntry);
} else {
Logger(`Skipped: ${path} (${doc._id.substring(0, 8)})`, LOG_LEVEL_VERBOSE);
}
return;
},
{
suspended: true,
batchSize: 1,
concurrentLimit: 10,
yieldThreshold: 1,
delay: 0,
totalRemainingReactiveSource: this.core.databaseQueueCount,
}
)
.replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter((e) => e._id != newItem._id);
return [...q, newItem];
})
.startPipeline();
storageApplyingProcessor = new QueueProcessor(
async (docs: MetaEntry[]) => {
const entry = docs[0];
await this.services.replication.processSynchroniseResult(entry);
return;
},
{
suspended: true,
batchSize: 1,
concurrentLimit: 6,
yieldThreshold: 1,
delay: 0,
totalRemainingReactiveSource: this.core.storageApplyingCount,
}
)
.replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter((e) => e._id != newItem._id);
return [...q, newItem];
})
.startPipeline();
_everyBeforeSuspendProcess(): Promise<boolean> {
this.core.replicator?.closeReplication();
@@ -579,6 +331,7 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
onBindFunction(core: LiveSyncCore, services: typeof core.services): void {
services.replicator.handleGetActiveReplicator(this._getReplicator.bind(this));
services.databaseEvents.handleOnDatabaseInitialisation(this._everyOnInitializeDatabase.bind(this));
services.databaseEvents.handleDatabaseInitialised(this._everyOnDatabaseInitialized.bind(this));
services.databaseEvents.handleOnResetDatabase(this._everyOnResetDatabase.bind(this));
services.appLifecycle.handleOnSettingLoaded(this._everyOnloadAfterLoadSettings.bind(this));
services.replication.handleParseSynchroniseResult(this._parseReplicationResult.bind(this));

466
src/modules/core/ReplicateResultProcessor.ts Normal file
View File

@@ -0,0 +1,466 @@
import {
SYNCINFO_ID,
VER,
type AnyEntry,
type EntryDoc,
type EntryLeaf,
type LoadedEntry,
type MetaEntry,
} from "@/lib/src/common/types";
import type { ModuleReplicator } from "./ModuleReplicator";
import { getPath, isChunk, isValidPath } from "@/common/utils";
import type { LiveSyncCore } from "@/main";
import {
LOG_LEVEL_DEBUG,
LOG_LEVEL_INFO,
LOG_LEVEL_NOTICE,
LOG_LEVEL_VERBOSE,
Logger,
type LOG_LEVEL,
} from "@/lib/src/common/logger";
import { fireAndForget, isAnyNote, throttle } from "@/lib/src/common/utils";
import { Semaphore } from "octagonal-wheels/concurrency/semaphore_v2";
import { serialized } from "octagonal-wheels/concurrency/lock";
import type { ReactiveSource } from "octagonal-wheels/dataobject/reactive_v2";
const KV_KEY_REPLICATION_RESULT_PROCESSOR_SNAPSHOT = "replicationResultProcessorSnapshot";
type ReplicateResultProcessorState = {
queued: PouchDB.Core.ExistingDocument<EntryDoc>[];
processing: PouchDB.Core.ExistingDocument<EntryDoc>[];
};
function shortenId(id: string): string {
return id.length > 10 ? id.substring(0, 10) : id;
}
function shortenRev(rev: string | undefined): string {
if (!rev) return "undefined";
return rev.length > 10 ? rev.substring(0, 10) : rev;
}
export class ReplicateResultProcessor {
private log(message: string, level: LOG_LEVEL = LOG_LEVEL_INFO) {
Logger(`[ReplicateResultProcessor] ${message}`, level);
}
private logError(e: any) {
Logger(e, LOG_LEVEL_VERBOSE);
}
private replicator: ModuleReplicator;
constructor(replicator: ModuleReplicator) {
this.replicator = replicator;
}
get localDatabase() {
return this.replicator.core.localDatabase;
}
get services() {
return this.replicator.core.services;
}
get core(): LiveSyncCore {
return this.replicator.core;
}
public suspend() {
this._suspended = true;
}
public resume() {
this._suspended = false;
fireAndForget(() => this.runProcessQueue());
}
// Whether the processing is suspended
// If true, the processing queue processor bails the loop.
private _suspended: boolean = false;
public get isSuspended() {
return (
this._suspended ||
!this.core.services.appLifecycle.isReady ||
this.replicator.settings.suspendParseReplicationResult ||
this.core.services.appLifecycle.isSuspended()
);
}
/**
* Take a snapshot of the current processing state.
* This snapshot is stored in the KV database for recovery on restart.
*/
protected async _takeSnapshot() {
const snapshot = {
queued: this._queuedChanges.slice(),
processing: this._processingChanges.slice(),
} satisfies ReplicateResultProcessorState;
await this.core.kvDB.set(KV_KEY_REPLICATION_RESULT_PROCESSOR_SNAPSHOT, snapshot);
this.log(
`Snapshot taken. Queued: ${snapshot.queued.length}, Processing: ${snapshot.processing.length}`,
LOG_LEVEL_DEBUG
);
this.reportStatus();
}
/**
* Trigger taking a snapshot.
*/
protected _triggerTakeSnapshot() {
fireAndForget(() => this._takeSnapshot());
}
/**
* Throttled version of triggerTakeSnapshot.
*/
protected triggerTakeSnapshot = throttle(() => this._triggerTakeSnapshot(), 50);
/**
* Restore from snapshot.
*/
public async restoreFromSnapshot() {
const snapshot = await this.core.kvDB.get<ReplicateResultProcessorState>(
KV_KEY_REPLICATION_RESULT_PROCESSOR_SNAPSHOT
);
if (snapshot) {
// Restoring the snapshot re-runs processing for both queued and processing items.
const newQueue = [...snapshot.processing, ...snapshot.queued, ...this._queuedChanges];
this._queuedChanges = [];
this.enqueueAll(newQueue);
this.log(
`Restored from snapshot (${snapshot.processing.length + snapshot.queued.length} items)`,
LOG_LEVEL_INFO
);
// await this._takeSnapshot();
}
}
private _restoreFromSnapshot: Promise<void> | undefined = undefined;
/**
* Restore from snapshot only once.
* @returns Promise that resolves when restoration is complete.
*/
public restoreFromSnapshotOnce() {
if (!this._restoreFromSnapshot) {
this._restoreFromSnapshot = this.restoreFromSnapshot();
}
return this._restoreFromSnapshot;
}
/**
* Perform the given procedure while counting the concurrency.
* @param proc async procedure to perform
* @param countValue reactive source to count concurrency
* @returns result of the procedure
*/
async withCounting<T>(proc: () => Promise<T>, countValue: ReactiveSource<number>) {
countValue.value++;
try {
return await proc();
} finally {
countValue.value--;
}
}
/**
* Report the current status.
*/
protected reportStatus() {
this.core.replicationResultCount.value = this._queuedChanges.length + this._processingChanges.length;
}
/**
* Enqueue all the given changes for processing.
* @param changes Changes to enqueue
*/
public enqueueAll(changes: PouchDB.Core.ExistingDocument<EntryDoc>[]) {
for (const change of changes) {
// Check if the change is not a document change (e.g., chunk, versioninfo, syncinfo), and processed it directly.
const isProcessed = this.processIfNonDocumentChange(change);
if (!isProcessed) {
this.enqueueChange(change);
}
}
}
/**
* Process the change if it is not a document change.
* @param change Change to process
* @returns True if the change was processed; false otherwise
*/
protected processIfNonDocumentChange(change: PouchDB.Core.ExistingDocument<EntryDoc>) {
if (!change) {
this.log(`Received empty change`, LOG_LEVEL_VERBOSE);
return true;
}
if (isChunk(change._id)) {
// Emit event for new chunk
this.localDatabase.onNewLeaf(change as EntryLeaf);
this.log(`Processed chunk: ${shortenId(change._id)}`, LOG_LEVEL_DEBUG);
return true;
}
if (change.type == "versioninfo") {
this.log(`Version info document received: ${change._id}`, LOG_LEVEL_VERBOSE);
if (change.version > VER) {
// Incompatible version, stop replication.
this.core.replicator.closeReplication();
this.log(
`Remote database updated to incompatible version. update your Self-hosted LiveSync plugin.`,
LOG_LEVEL_NOTICE
);
}
return true;
}
if (
change._id == SYNCINFO_ID || // Synchronisation information data
change._id.startsWith("_design") //design document
) {
this.log(`Skipped system document: ${change._id}`, LOG_LEVEL_VERBOSE);
return true;
}
return false;
}
/**
* Queue of changes to be processed.
*/
private _queuedChanges: PouchDB.Core.ExistingDocument<EntryDoc>[] = [];
/**
* List of changes being processed.
*/
private _processingChanges: PouchDB.Core.ExistingDocument<EntryDoc>[] = [];
/**
* Enqueue the given document change for processing.
* @param doc Document change to enqueue
* @returns
*/
protected enqueueChange(doc: PouchDB.Core.ExistingDocument<EntryDoc>) {
const old = this._queuedChanges.find((e) => e._id == doc._id);
const path = "path" in doc ? getPath(doc) : "<unknown>";
const docNote = `${path} (${shortenId(doc._id)}, ${shortenRev(doc._rev)})`;
if (old) {
if (old._rev == doc._rev) {
this.log(`[Enqueue] skipped (Already queued): ${docNote}`, LOG_LEVEL_VERBOSE);
return;
}
const oldRev = old._rev ?? "";
const isDeletedBefore = old._deleted === true || ("deleted" in old && old.deleted === true);
const isDeletedNow = doc._deleted === true || ("deleted" in doc && doc.deleted === true);
// Replace the old queued change (This may performed batched updates, actually process performed always with the latest version, hence we can simply replace it if the change is the same type).
if (isDeletedBefore === isDeletedNow) {
this._queuedChanges = this._queuedChanges.filter((e) => e._id != doc._id);
this.log(`[Enqueue] requeued: ${docNote} (from rev: ${shortenRev(oldRev)})`, LOG_LEVEL_VERBOSE);
}
}
// Enqueue the change
this._queuedChanges.push(doc);
this.triggerTakeSnapshot();
this.triggerProcessQueue();
}
/**
* Trigger processing of the queued changes.
*/
protected triggerProcessQueue() {
fireAndForget(() => this.runProcessQueue());
}
/**
* Semaphore to limit concurrent processing.
* This is the per-id semaphore + concurrency-control (max 10 concurrent = 10 documents being processed at the same time).
*/
private _semaphore = Semaphore(10);
/**
* Flag indicating whether the process queue is currently running.
*/
private _isRunningProcessQueue: boolean = false;
/**
* Process the queued changes.
*/
private async runProcessQueue() {
// Avoid re-entrance, suspend processing, or empty queue loop consumption.
if (this._isRunningProcessQueue) return;
if (this.isSuspended) return;
if (this._queuedChanges.length == 0) return;
try {
this._isRunningProcessQueue = true;
while (this._queuedChanges.length > 0) {
// If getting suspended, bail the loop. Some concurrent tasks may still be running.
if (this.isSuspended) {
this.log(
`Processing has got suspended. Remaining items in queue: ${this._queuedChanges.length}`,
LOG_LEVEL_INFO
);
break;
}
// Acquire semaphore for new processing slot
// (per-document serialisation caps concurrency).
const releaser = await this._semaphore.acquire();
releaser();
// Dequeue the next change
const doc = this._queuedChanges.shift();
if (doc) {
this._processingChanges.push(doc);
void this.parseDocumentChange(doc);
}
// Take snapshot (to be restored on next startup if needed)
this.triggerTakeSnapshot();
}
} finally {
this._isRunningProcessQueue = false;
}
}
// Phase 1: parse replication result
/**
* Parse the given document change.
* @param change
* @returns
*/
async parseDocumentChange(change: PouchDB.Core.ExistingDocument<EntryDoc>) {
try {
// If the document is a virtual document, process it in the virtual document processor.
if (await this.services.replication.processVirtualDocument(change)) return;
// If the document is version info, check compatibility and return.
if (isAnyNote(change)) {
const docPath = getPath(change);
if (!(await this.services.vault.isTargetFile(docPath))) {
this.log(`Skipped: ${docPath}`, LOG_LEVEL_VERBOSE);
return;
}
const size = change.size;
// Note that this size check depends size that in metadata, not the actual content size.
if (this.services.vault.isFileSizeTooLarge(size)) {
this.log(
`Processing ${docPath} has been skipped due to file size exceeding the limit`,
LOG_LEVEL_NOTICE
);
return;
}
return await this.applyToDatabase(change);
}
this.log(`Skipped unexpected non-note document: ${change._id}`, LOG_LEVEL_INFO);
return;
} finally {
// Remove from processing queue
this._processingChanges.remove(change);
this.triggerTakeSnapshot();
}
}
// Phase 2: apply the document to database
protected applyToDatabase(doc: PouchDB.Core.ExistingDocument<AnyEntry>) {
return this.withCounting(async () => {
let releaser: Awaited<ReturnType<typeof this._semaphore.acquire>> | undefined = undefined;
try {
releaser = await this._semaphore.acquire();
await this._applyToDatabase(doc);
} catch (e) {
this.log(`Error while processing replication result`, LOG_LEVEL_NOTICE);
this.logError(e);
} finally {
// Remove from processing queue (To remove from "in-progress" list, and snapshot will not include it)
if (releaser) {
releaser();
}
}
}, this.replicator.core.databaseQueueCount);
}
// Phase 2.1: process the document and apply to storage
// This function is serialized per document to avoid race-condition for the same document.
private _applyToDatabase(doc_: PouchDB.Core.ExistingDocument<AnyEntry>) {
const dbDoc = doc_ as LoadedEntry; // It has no `data`
const path = getPath(dbDoc);
return serialized(`replication-process:${dbDoc._id}`, async () => {
const docNote = `${path} (${shortenId(dbDoc._id)}, ${shortenRev(dbDoc._rev)})`;
const isRequired = await this.checkIsChangeRequiredForDatabaseProcessing(dbDoc);
if (!isRequired) {
this.log(`Skipped (Not latest): ${docNote}`, LOG_LEVEL_VERBOSE);
return;
}
// If `Read chunks online` is disabled, chunks should be transferred before here.
// However, in some cases, chunks are after that. So, if missing chunks exist, we have to wait for them.
// (If `Use Only Local Chunks` is enabled, we should not attempt to fetch chunks online automatically).
const isDeleted = dbDoc._deleted === true || ("deleted" in dbDoc && dbDoc.deleted === true);
// Gather full document if not deleted
const doc = isDeleted
? { ...dbDoc, data: "" }
: await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, false, true);
if (!doc) {
// Failed to gather content
this.log(`Failed to gather content of ${docNote}`, LOG_LEVEL_NOTICE);
return;
}
// Check if other processor wants to process this document, if so, skip processing here.
if (await this.services.replication.processOptionalSynchroniseResult(dbDoc)) {
// Already processed
this.log(`Processed by other processor: ${docNote}`, LOG_LEVEL_DEBUG);
} else if (isValidPath(getPath(doc))) {
// Apply to storage if the path is valid
await this.applyToStorage(doc as MetaEntry);
this.log(`Processed: ${docNote}`, LOG_LEVEL_DEBUG);
} else {
// Should process, but have an invalid path
this.log(`Unprocessed (Invalid path): ${docNote}`, LOG_LEVEL_VERBOSE);
}
return;
});
}
/**
* Phase 3: Apply the given entry to storage.
* @param entry
* @returns
*/
protected applyToStorage(entry: MetaEntry) {
return this.withCounting(async () => {
await this.services.replication.processSynchroniseResult(entry);
}, this.replicator.core.storageApplyingCount);
}
/**
* Check whether processing is required for the given document.
* @param dbDoc Document to check
* @returns True if processing is required; false otherwise
*/
protected async checkIsChangeRequiredForDatabaseProcessing(dbDoc: LoadedEntry): Promise<boolean> {
const path = getPath(dbDoc);
try {
const savedDoc = await this.localDatabase.getRaw<LoadedEntry>(dbDoc._id, {
conflicts: true,
revs_info: true,
});
const newRev = dbDoc._rev ?? "";
const latestRev = savedDoc._rev ?? "";
const revisions = savedDoc._revs_info?.map((e) => e.rev) ?? [];
if (savedDoc._conflicts && savedDoc._conflicts.length > 0) {
// There are conflicts, so we have to process it.
// (May auto-resolve or user intervention will be occurred).
return true;
}
if (newRev == latestRev) {
// The latest revision. Simply we can process it.
return true;
}
const index = revisions.indexOf(newRev);
if (index >= 0) {
// The revision has been inserted before.
return false; // This means that the document already processed (While no conflict existed).
}
return true; // This mostly should not happen, but we have to process it just in case.
} catch (e: any) {
if ("status" in e && e.status == 404) {
// getRaw failed due to not existing, it may not be happened normally especially on replication.
// If the process caused by some other reason, we **probably** have to process it.
// Note that this is not a common case.
return true;
} else {
this.log(
`Failed to get existing document for ${path} (${shortenId(dbDoc._id)}, ${shortenRev(dbDoc._rev)}) `,
LOG_LEVEL_NOTICE
);
this.logError(e);
return false;
}
}
}
}

8
src/modules/coreFeatures/ModuleRedFlag.ts
View File

@@ -3,6 +3,7 @@ import { normalizePath } from "../../deps.ts";
import {
FlagFilesHumanReadable,
FlagFilesOriginal,
REMOTE_MINIO,
TweakValuesShouldMatchedTemplate,
type ObsidianLiveSyncSettings,
} from "../../lib/src/common/types.ts";
@@ -86,7 +87,7 @@ export class ModuleRedFlag extends AbstractModule {
const remoteTweaks = await this.services.tweakValue.fetchRemotePreferred(config);
if (!remoteTweaks) {
const choice = await this.core.confirm.askSelectStringDialogue(
"Could not fetch remote configuration. What do you want to do?",
"Could not fetch configuration from remote. If you are new to the Self-hosted LiveSync, this might be expected. If not, you should check your network or server settings.",
[SKIP_FETCH, RETRY_FETCH] as const,
{
defaultAction: RETRY_FETCH,
@@ -203,12 +204,13 @@ export class ModuleRedFlag extends AbstractModule {
return false;
}
const { vault, extra } = method;
// If remote is MinIO, makeLocalChunkBeforeSync is not available. (because no-deduplication on sending).
const makeLocalChunkBeforeSyncAvailable = this.settings.remoteType !== REMOTE_MINIO;
const mapVaultStateToAction = {
identical: {
// If both are identical, no need to make local files/chunks before sync,
// Just for the efficiency, chunks should be made before sync.
makeLocalChunkBeforeSync: true,
makeLocalChunkBeforeSync: makeLocalChunkBeforeSyncAvailable,
makeLocalFilesBeforeSync: false,
},
independent: {

12
src/modules/coreObsidian/ModuleFileAccessObsidian.ts
View File

@@ -52,12 +52,16 @@ export class ModuleFileAccessObsidian extends AbstractObsidianModule implements
}
vaultAccess!: SerializedFileAccess;
vaultManager: StorageEventManager = new StorageEventManagerObsidian(this.plugin, this.core, this);
restoreState() {
return this.vaultManager.restoreState();
}
private _everyOnload(): Promise<boolean> {
this.core.storageAccess = this;
return Promise.resolve(true);
}
_everyOnFirstInitialize(): Promise<boolean> {
this.vaultManager.beginWatch();
async _everyOnFirstInitialize(): Promise<boolean> {
await this.vaultManager.beginWatch();
return Promise.resolve(true);
}
@@ -65,8 +69,8 @@ export class ModuleFileAccessObsidian extends AbstractObsidianModule implements
// this.vaultManager.flushQueue();
// }
_everyCommitPendingFileEvent(): Promise<boolean> {
this.vaultManager.flushQueue();
async _everyCommitPendingFileEvent(): Promise<boolean> {
await this.vaultManager.waitForIdle();
return Promise.resolve(true);
}

457
src/modules/coreObsidian/storageLib/StorageEventManager.ts
View File

@@ -9,25 +9,20 @@ import {
LOG_LEVEL_VERBOSE,
type FileEventType,
type FilePath,
type FilePathWithPrefix,
type UXFileInfoStub,
type UXInternalFileInfoStub,
} from "../../../lib/src/common/types.ts";
import { delay, fireAndForget } from "../../../lib/src/common/utils.ts";
import { delay, fireAndForget, throttle } from "../../../lib/src/common/utils.ts";
import { type FileEventItem } from "../../../common/types.ts";
import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
import {
finishAllWaitingForTimeout,
finishWaitingForTimeout,
isWaitingForTimeout,
waitForTimeout,
} from "octagonal-wheels/concurrency/task";
import { isWaitingForTimeout } from "octagonal-wheels/concurrency/task";
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
import type { LiveSyncCore } from "../../../main.ts";
import { InternalFileToUXFileInfoStub, TFileToUXFileInfoStub } from "./utilObsidian.ts";
import ObsidianLiveSyncPlugin from "../../../main.ts";
import type { StorageAccess } from "../../interfaces/StorageAccess.ts";
import { HiddenFileSync } from "../../../features/HiddenFileSync/CmdHiddenFileSync.ts";
import { promiseWithResolvers, type PromiseWithResolvers } from "octagonal-wheels/promises";
// import { InternalFileToUXFileInfo } from "../platforms/obsidian.ts";
export type FileEvent = {
@@ -36,14 +31,29 @@ export type FileEvent = {
oldPath?: string;
cachedData?: string;
skipBatchWait?: boolean;
cancelled?: boolean;
};
type WaitInfo = {
since: number;
type: FileEventType;
canProceed: PromiseWithResolvers<boolean>;
timerHandler: ReturnType<typeof setTimeout>;
event: FileEventItem;
};
const TYPE_SENTINEL_FLUSH = "SENTINEL_FLUSH";
type FileEventItemSentinelFlush = {
type: typeof TYPE_SENTINEL_FLUSH;
};
type FileEventItemSentinel = FileEventItemSentinelFlush;
export abstract class StorageEventManager {
abstract beginWatch(): void;
abstract flushQueue(): void;
abstract beginWatch(): Promise<void>;
abstract appendQueue(items: FileEvent[], ctx?: any): Promise<void>;
abstract cancelQueue(key: string): void;
abstract isWaiting(filename: FilePath): boolean;
abstract waitForIdle(): Promise<void>;
abstract restoreState(): Promise<void>;
}
export class StorageEventManagerObsidian extends StorageEventManager {
@@ -66,6 +76,13 @@ export class StorageEventManagerObsidian extends StorageEventManager {
// Necessary evil.
cmdHiddenFileSync: HiddenFileSync;
/**
* Snapshot restoration promise.
* Snapshot will be restored before starting to watch vault changes.
* In designed time, this has been called from Initialisation process, which has been implemented on `ModuleInitializerFile.ts`.
*/
snapShotRestored: Promise<void> | null = null;
constructor(plugin: ObsidianLiveSyncPlugin, core: LiveSyncCore, storageAccess: StorageAccess) {
super();
this.storageAccess = storageAccess;
@@ -73,7 +90,18 @@ export class StorageEventManagerObsidian extends StorageEventManager {
this.core = core;
this.cmdHiddenFileSync = this.plugin.getAddOn(HiddenFileSync.name) as HiddenFileSync;
}
beginWatch() {
/**
* Restore the previous snapshot if exists.
* @returns
*/
restoreState(): Promise<void> {
this.snapShotRestored = this._restoreFromSnapshot();
return this.snapShotRestored;
}
async beginWatch() {
await this.snapShotRestored;
const plugin = this.plugin;
this.watchVaultChange = this.watchVaultChange.bind(this);
this.watchVaultCreate = this.watchVaultCreate.bind(this);
@@ -88,8 +116,6 @@ export class StorageEventManagerObsidian extends StorageEventManager {
//@ts-ignore : Internal API
plugin.registerEvent(plugin.app.vault.on("raw", this.watchVaultRawEvents));
plugin.registerEvent(plugin.app.workspace.on("editor-change", this.watchEditorChange));
// plugin.fileEventQueue.startPipeline();
}
watchEditorChange(editor: any, info: any) {
if (!("path" in info)) {
@@ -212,13 +238,13 @@ export class StorageEventManagerObsidian extends StorageEventManager {
null
);
}
// Cache file and waiting to can be proceed.
async appendQueue(params: FileEvent[], ctx?: any) {
if (!this.core.settings.isConfigured) return;
if (this.core.settings.suspendFileWatching) return;
this.core.services.vault.markFileListPossiblyChanged();
// Flag up to be reload
const processFiles = new Set<FilePath>();
for (const param of params) {
if (shouldBeIgnored(param.file.path)) {
continue;
@@ -261,7 +287,7 @@ export class StorageEventManagerObsidian extends StorageEventManager {
if (param.cachedData) {
cache = param.cachedData;
}
this.enqueue({
void this.enqueue({
type,
args: {
file: file,
@@ -272,123 +298,291 @@ export class StorageEventManagerObsidian extends StorageEventManager {
skipBatchWait: param.skipBatchWait,
key: atomicKey,
});
processFiles.add(file.path as FilePath);
if (oldPath) {
processFiles.add(oldPath as FilePath);
}
}
for (const path of processFiles) {
fireAndForget(() => this.startStandingBy(path));
}
}
bufferedQueuedItems = [] as FileEventItem[];
private bufferedQueuedItems = [] as (FileEventItem | FileEventItemSentinel)[];
/**
* Immediately take snapshot.
*/
private _triggerTakeSnapshot() {
void this._takeSnapshot();
}
/**
* Trigger taking snapshot after throttled period.
*/
triggerTakeSnapshot = throttle(() => this._triggerTakeSnapshot(), 100);
enqueue(newItem: FileEventItem) {
const filename = newItem.args.file.path;
if (this.shouldBatchSave) {
Logger(`Request cancel for waiting of previous ${filename}`, LOG_LEVEL_DEBUG);
finishWaitingForTimeout(`storage-event-manager-batchsave-${filename}`);
}
this.bufferedQueuedItems.push(newItem);
// When deleting or renaming, the queue must be flushed once before processing subsequent processes to prevent unexpected race condition.
if (newItem.type == "DELETE") {
return this.flushQueue();
// If the sentinel pushed, the runQueuedEvents will wait for idle before processing delete.
this.bufferedQueuedItems.push({
type: TYPE_SENTINEL_FLUSH,
});
}
this.updateStatus();
this.bufferedQueuedItems.push(newItem);
fireAndForget(() => this._takeSnapshot().then(() => this.runQueuedEvents()));
}
// Limit concurrent processing to reduce the IO load. file-processing + scheduler (1), so file events can be processed in 4 slots.
concurrentProcessing = Semaphore(5);
private _waitingMap = new Map<string, WaitInfo>();
private _waitForIdle: Promise<void> | null = null;
/**
* Wait until all queued events are processed.
* Subsequent new events will not be waited, but new events will not be added.
* @returns
*/
waitForIdle(): Promise<void> {
if (this._waitingMap.size === 0) {
return Promise.resolve();
}
if (this._waitForIdle) {
return this._waitForIdle;
}
const promises = [...this._waitingMap.entries()].map(([key, waitInfo]) => {
return new Promise<void>((resolve) => {
waitInfo.canProceed.promise
.then(() => {
Logger(`Processing ${key}: Wait for idle completed`, LOG_LEVEL_DEBUG);
// No op
})
.catch((e) => {
Logger(`Processing ${key}: Wait for idle error`, LOG_LEVEL_INFO);
Logger(e, LOG_LEVEL_VERBOSE);
//no op
})
.finally(() => {
resolve();
});
this._proceedWaiting(key);
});
});
const waitPromise = Promise.all(promises).then(() => {
this._waitForIdle = null;
Logger(`All wait for idle completed`, LOG_LEVEL_VERBOSE);
});
this._waitForIdle = waitPromise;
return waitPromise;
}
/**
* Proceed waiting for the given key immediately.
*/
private _proceedWaiting(key: string) {
const waitInfo = this._waitingMap.get(key);
if (waitInfo) {
waitInfo.canProceed.resolve(true);
clearTimeout(waitInfo.timerHandler);
this._waitingMap.delete(key);
}
this.triggerTakeSnapshot();
}
/**
* Cancel waiting for the given key.
*/
private _cancelWaiting(key: string) {
const waitInfo = this._waitingMap.get(key);
if (waitInfo) {
waitInfo.canProceed.resolve(false);
clearTimeout(waitInfo.timerHandler);
this._waitingMap.delete(key);
}
this.triggerTakeSnapshot();
}
/**
* Add waiting for the given key.
* @param key
* @param event
* @param waitedSince Optional waited since timestamp to calculate the remaining delay.
*/
private _addWaiting(key: string, event: FileEventItem, waitedSince?: number): WaitInfo {
if (this._waitingMap.has(key)) {
// Already waiting
throw new Error(`Already waiting for key: ${key}`);
}
const resolver = promiseWithResolvers<boolean>();
const now = Date.now();
const since = waitedSince ?? now;
const elapsed = now - since;
const maxDelay = this.batchSaveMaximumDelay * 1000;
const remainingDelay = Math.max(0, maxDelay - elapsed);
const nextDelay = Math.min(remainingDelay, this.batchSaveMinimumDelay * 1000);
// x*<------- maxDelay --------->*
// x*<-- minDelay -->*
// x* x<-- nextDelay -->*
// x* x<-- Capped-->*
// x* x.......*
// x: event
// *: save
// When at event (x) At least, save (*) within maxDelay, but maintain minimum delay between saves.
if (elapsed >= maxDelay) {
// Already exceeded maximum delay, do not wait.
Logger(`Processing ${key}: Batch save maximum delay already exceeded: ${event.type}`, LOG_LEVEL_DEBUG);
} else {
Logger(`Processing ${key}: Adding waiting for batch save: ${event.type} (${nextDelay}ms)`, LOG_LEVEL_DEBUG);
}
const waitInfo: WaitInfo = {
since: since,
type: event.type,
event: event,
canProceed: resolver,
timerHandler: setTimeout(() => {
Logger(`Processing ${key}: Batch save timeout reached: ${event.type}`, LOG_LEVEL_DEBUG);
this._proceedWaiting(key);
}, nextDelay),
};
this._waitingMap.set(key, waitInfo);
this.triggerTakeSnapshot();
return waitInfo;
}
/**
* Process the given file event.
*/
async processFileEvent(fei: FileEventItem) {
const releaser = await this.concurrentProcessing.acquire();
try {
this.updateStatus();
const filename = fei.args.file.path;
const waitingKey = `${filename}`;
const previous = this._waitingMap.get(waitingKey);
let isShouldBeCancelled = fei.skipBatchWait || false;
let previousPromise: Promise<boolean> = Promise.resolve(true);
let waitPromise: Promise<boolean> = Promise.resolve(true);
// 1. Check if there is previous waiting for the same file
if (previous) {
previousPromise = previous.canProceed.promise;
if (isShouldBeCancelled) {
Logger(
`Processing ${filename}: Requested to perform immediately, cancelling previous waiting: ${fei.type}`,
LOG_LEVEL_DEBUG
);
}
if (!isShouldBeCancelled && fei.type === "DELETE") {
// For DELETE, cancel any previous waiting and proceed immediately
// That because when deleting, we cannot read the file anymore.
Logger(
`Processing ${filename}: DELETE requested, cancelling previous waiting: ${fei.type}`,
LOG_LEVEL_DEBUG
);
isShouldBeCancelled = true;
}
if (!isShouldBeCancelled && previous.type === fei.type) {
// For the same type, we can cancel the previous waiting and proceed immediately.
Logger(`Processing ${filename}: Cancelling previous waiting: ${fei.type}`, LOG_LEVEL_DEBUG);
isShouldBeCancelled = true;
}
// 2. wait for the previous to complete
if (isShouldBeCancelled) {
this._cancelWaiting(waitingKey);
Logger(`Processing ${filename}: Previous cancelled: ${fei.type}`, LOG_LEVEL_DEBUG);
isShouldBeCancelled = true;
}
if (!isShouldBeCancelled) {
Logger(`Processing ${filename}: Waiting for previous to complete: ${fei.type}`, LOG_LEVEL_DEBUG);
this._proceedWaiting(waitingKey);
Logger(`Processing ${filename}: Previous completed: ${fei.type}`, LOG_LEVEL_DEBUG);
}
}
await previousPromise;
// 3. Check if shouldBatchSave is true
if (this.shouldBatchSave && !fei.skipBatchWait) {
// if type is CREATE or CHANGED, set waiting
if (fei.type == "CREATE" || fei.type == "CHANGED") {
// 3.2. If true, set the queue, and wait for the waiting, or until timeout
// (since is copied from previous waiting if exists to limit the maximum wait time)
console.warn(`Since:`, previous?.since);
const info = this._addWaiting(waitingKey, fei, previous?.since);
waitPromise = info.canProceed.promise;
} else if (fei.type == "DELETE") {
// For DELETE, cancel any previous waiting and proceed immediately
}
Logger(`Processing ${filename}: Waiting for batch save: ${fei.type}`, LOG_LEVEL_DEBUG);
const canProceed = await waitPromise;
if (!canProceed) {
// 3.2.1. If cancelled by new queue, cancel subsequent process.
Logger(`Processing ${filename}: Cancelled by new queue: ${fei.type}`, LOG_LEVEL_DEBUG);
return;
}
}
// await this.handleFileEvent(fei);
await this.requestProcessQueue(fei);
} finally {
await this._takeSnapshot();
releaser();
}
}
concurrentProcessing = Semaphore(5);
waitedSince = new Map<FilePath | FilePathWithPrefix, number>();
async startStandingBy(filename: FilePath) {
// If waited, no need to start again (looping inside the function)
await skipIfDuplicated(`storage-event-manager-${filename}`, async () => {
Logger(`Processing ${filename}: Starting`, LOG_LEVEL_DEBUG);
const release = await this.concurrentProcessing.acquire();
try {
Logger(`Processing ${filename}: Started`, LOG_LEVEL_DEBUG);
let noMoreFiles = false;
do {
const target = this.bufferedQueuedItems.find((e) => e.args.file.path == filename);
if (target === undefined) {
noMoreFiles = true;
break;
}
const operationType = target.type;
async _takeSnapshot() {
const processingEvents = [...this._waitingMap.values()].map((e) => e.event);
const waitingEvents = this.bufferedQueuedItems;
const snapShot = [...processingEvents, ...waitingEvents];
await this.core.kvDB.set("storage-event-manager-snapshot", snapShot);
Logger(`Storage operation snapshot taken: ${snapShot.length} items`, LOG_LEVEL_DEBUG);
this.updateStatus();
}
async _restoreFromSnapshot() {
const snapShot = await this.core.kvDB.get<(FileEventItem | FileEventItemSentinel)[]>(
"storage-event-manager-snapshot"
);
if (snapShot && Array.isArray(snapShot) && snapShot.length > 0) {
console.warn(`Restoring snapshot: ${snapShot.length} items`);
Logger(`Restoring storage operation snapshot: ${snapShot.length} items`, LOG_LEVEL_VERBOSE);
// Restore the snapshot
// Note: Mark all items as skipBatchWait to prevent apply the off-line batch saving.
this.bufferedQueuedItems = snapShot.map((e) => ({ ...e, skipBatchWait: true }));
this.updateStatus();
await this.runQueuedEvents();
} else {
Logger(`No snapshot to restore`, LOG_LEVEL_VERBOSE);
// console.warn(`No snapshot to restore`);
}
}
runQueuedEvents() {
return skipIfDuplicated("storage-event-manager-run-queued-events", async () => {
do {
if (this.bufferedQueuedItems.length === 0) {
break;
}
// 1. Get the first queued item
// if (target.waitedFrom + this.batchSaveMaximumDelay > now) {
// this.requestProcessQueue(target);
// continue;
// }
const type = target.type;
// If already cancelled by other operation, skip this.
if (target.cancelled) {
Logger(`Processing ${filename}: Cancelled (scheduled): ${operationType}`, LOG_LEVEL_DEBUG);
this.cancelStandingBy(target);
continue;
}
if (!target.skipBatchWait) {
if (this.shouldBatchSave && (type == "CREATE" || type == "CHANGED")) {
const waitedSince = this.waitedSince.get(filename);
let canWait = true;
const now = Date.now();
if (waitedSince !== undefined) {
if (waitedSince + this.batchSaveMaximumDelay * 1000 < now) {
Logger(
`Processing ${filename}: Could not wait no more: ${operationType}`,
LOG_LEVEL_INFO
);
canWait = false;
}
}
if (canWait) {
if (waitedSince === undefined) this.waitedSince.set(filename, now);
target.batched = true;
Logger(
`Processing ${filename}: Waiting for batch save delay: ${operationType}`,
LOG_LEVEL_DEBUG
);
this.updateStatus();
const result = await waitForTimeout(
`storage-event-manager-batchsave-${filename}`,
this.batchSaveMinimumDelay * 1000
);
if (!result) {
Logger(
`Processing ${filename}: Cancelled by new queue: ${operationType}`,
LOG_LEVEL_DEBUG
);
// If could not wait for the timeout, possibly we got a new queue. therefore, currently processing one should be cancelled
this.cancelStandingBy(target);
continue;
}
}
}
} else {
Logger(
`Processing ${filename}:Requested to perform immediately ${filename}: ${operationType}`,
LOG_LEVEL_DEBUG
);
}
Logger(`Processing ${filename}: Request main to process: ${operationType}`, LOG_LEVEL_DEBUG);
await this.requestProcessQueue(target);
} while (!noMoreFiles);
} finally {
release();
}
Logger(`Processing ${filename}: Finished`, LOG_LEVEL_DEBUG);
const fei = this.bufferedQueuedItems.shift()!;
await this._takeSnapshot();
this.updateStatus();
// 2. Consume 1 semaphore slot to enqueue processing. Then release immediately.
// (Just to limit the total concurrent processing count, because skipping batch handles at processFileEvent).
const releaser = await this.concurrentProcessing.acquire();
releaser();
this.updateStatus();
// 3. Check if sentinel flush
// If sentinel, wait for idle and continue.
if (fei.type === TYPE_SENTINEL_FLUSH) {
Logger(`Waiting for idle`, LOG_LEVEL_VERBOSE);
// Flush all waiting batch queues
await this.waitForIdle();
this.updateStatus();
continue;
}
// 4. Process the event, this should be fire-and-forget to not block the queue processing in each file.
fireAndForget(() => this.processFileEvent(fei));
} while (this.bufferedQueuedItems.length > 0);
});
}
cancelStandingBy(fei: FileEventItem) {
this.bufferedQueuedItems.remove(fei);
this.updateStatus();
}
processingCount = 0;
async requestProcessQueue(fei: FileEventItem) {
try {
this.processingCount++;
this.bufferedQueuedItems.remove(fei);
// this.bufferedQueuedItems.remove(fei);
this.updateStatus();
this.waitedSince.delete(fei.args.file.path);
// this.waitedSince.delete(fei.args.file.path);
await this.handleFileEvent(fei);
await this._takeSnapshot();
} finally {
this.processingCount--;
this.updateStatus();
@@ -397,27 +591,26 @@ export class StorageEventManagerObsidian extends StorageEventManager {
isWaiting(filename: FilePath) {
return isWaitingForTimeout(`storage-event-manager-batchsave-${filename}`);
}
flushQueue() {
this.bufferedQueuedItems.forEach((e) => (e.skipBatchWait = true));
finishAllWaitingForTimeout("storage-event-manager-batchsave-", true);
}
cancelQueue(key: string) {
this.bufferedQueuedItems.forEach((e) => {
if (e.key === key) e.skipBatchWait = true;
});
}
updateStatus() {
const allItems = this.bufferedQueuedItems.filter((e) => !e.cancelled);
const batchedCount = allItems.filter((e) => e.batched && !e.skipBatchWait).length;
const allFileEventItems = this.bufferedQueuedItems.filter((e): e is FileEventItem => "args" in e);
const allItems = allFileEventItems.filter((e) => !e.cancelled);
const totalItems = allItems.length + this.concurrentProcessing.waiting;
const processing = this.processingCount;
const batchedCount = this._waitingMap.size;
this.core.batched.value = batchedCount;
this.core.processing.value = this.processingCount;
this.core.totalQueued.value = allItems.length - batchedCount;
this.core.processing.value = processing;
this.core.totalQueued.value = totalItems + batchedCount + processing;
}
async handleFileEvent(queue: FileEventItem): Promise<any> {
const file = queue.args.file;
const lockKey = `handleFile:${file.path}`;
return await serialized(lockKey, async () => {
const ret = await serialized(lockKey, async () => {
if (queue.cancelled) {
Logger(`File event cancelled before processing: ${file.path}`, LOG_LEVEL_INFO);
return;
}
if (queue.type == "INTERNAL" || file.isInternal) {
await this.core.services.fileProcessing.processOptionalFileEvent(file.path as unknown as FilePath);
} else {
@@ -444,9 +637,11 @@ export class StorageEventManagerObsidian extends StorageEventManager {
}
}
});
this.updateStatus();
return ret;
}
cancelRelativeEvent(item: FileEventItem): void {
this.cancelQueue(item.key);
this._cancelWaiting(item.args.file.path);
}
}

4
src/modules/essential/ModuleInitializerFile.ts
View File

@@ -49,6 +49,10 @@ export class ModuleInitializerFile extends AbstractModule {
if (showingNotice) {
this._log("Initializing", LOG_LEVEL_NOTICE, "syncAll");
}
if (isInitialized) {
this._log("Restoring storage state", LOG_LEVEL_VERBOSE);
await this.core.storageAccess.restoreState();
}
this._log("Initialize and checking database files");
this._log("Checking deleted files");

17
src/modules/essentialObsidian/ModuleObsidianAPI.ts
View File

@@ -58,7 +58,20 @@ export class ModuleObsidianAPI extends AbstractObsidianModule {
async __fetchByAPI(url: string, authHeader: string, opts?: RequestInit): Promise<Response> {
const body = opts?.body as string;
const transformedHeaders = { ...(opts?.headers as Record<string, string>) };
const optHeaders = {} as Record<string, string>;
if (opts && "headers" in opts) {
if (opts.headers instanceof Headers) {
// For Compatibility, mostly headers.entries() is supported, but not all environments.
opts.headers.forEach((value, key) => {
optHeaders[key] = value;
});
} else {
for (const [key, value] of Object.entries(opts.headers as Record<string, string>)) {
optHeaders[key] = value;
}
}
}
const transformedHeaders = { ...optHeaders };
if (authHeader != "") transformedHeaders["authorization"] = authHeader;
delete transformedHeaders["host"];
delete transformedHeaders["Host"];
@@ -132,7 +145,6 @@ export class ModuleObsidianAPI extends AbstractObsidianModule {
return "Network is offline";
}
// let authHeader = await this._authHeader.getAuthorizationHeader(auth);
const conf: PouchDB.HttpAdapter.HttpAdapterConfiguration = {
adapter: "http",
auth: "username" in auth ? auth : undefined,
@@ -166,7 +178,6 @@ export class ModuleObsidianAPI extends AbstractObsidianModule {
if (!("username" in auth)) {
headers.append("authorization", authHeader);
}
try {
this.plugin.requestCount.value = this.plugin.requestCount.value + 1;
const response: Response = await (useRequestAPI

3
src/modules/features/SettingDialogue/PaneAdvanced.ts
View File

@@ -22,6 +22,9 @@ export function paneAdvanced(this: ObsidianLiveSyncSettingTab, paneEl: HTMLEleme
new Setting(paneEl)
.setClass("wizardHidden")
.autoWireToggle("readChunksOnline", { onUpdate: this.onlyOnCouchDB });
new Setting(paneEl)
.setClass("wizardHidden")
.autoWireToggle("useOnlyLocalChunk", { onUpdate: this.onlyOnCouchDB });
new Setting(paneEl).setClass("wizardHidden").autoWireNumeric("concurrencyOfReadChunksOnline", {
clampMin: 10,

1
src/modules/interfaces/StorageAccess.ts
View File

@@ -10,6 +10,7 @@ import type {
import type { CustomRegExp } from "../../lib/src/common/utils";
export interface StorageAccess {
restoreState(): Promise<void>;
processWriteFile<T>(file: UXFileInfoStub | FilePathWithPrefix, proc: () => Promise<T>): Promise<T>;
processReadFile<T>(file: UXFileInfoStub | FilePathWithPrefix, proc: () => Promise<T>): Promise<T>;
isFileProcessing(file: UXFileInfoStub | FilePathWithPrefix): boolean;