bump

- Now the result of conflict resolution could be surely written into the storage.
- Deleted files can be handled correctly again in the history dialogue and conflict dialogue.
- Some wrong log messages were fixed.
- Change handling now has become more stable.
- Some event handling became to be safer.
Improved:
- Dumping document information shows conflicts and revisions.
- The timestamp-only differences can be surely cached.
- Timestamp difference detection can be rounded by two seconds.
Refactored:
- A bit of organisation to write the test.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.22.2",
+    "version": "0.22.4",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.22.1",
+    "version": "0.22.4",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.22.1",
+            "version": "0.22.4",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.22.2",
+    "version": "0.22.4",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/CmdConfigSync.ts

@@ -4,7 +4,7 @@ import { Notice, type PluginManifest, parseYaml, normalizePath } from "./deps";
 import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, DocumentID, AnyEntry, SavingEntry } from "./lib/src/types";
 import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE } from "./lib/src/types";
 import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types";
-import { createTextBlob, delay, getDocData } from "./lib/src/utils";
+import { createTextBlob, delay, getDocData, sendSignal, waitForSignal } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { WrappedNotice } from "./lib/src/wrapper";
 import { readString, decodeBinary, arrayBufferToBase64, sha1 } from "./lib/src/strbin";
@@ -335,7 +335,9 @@ export class ConfigSync extends LiveSyncCommands {
         return;
     }, { suspended: true, batchSize: 1, concurrentLimit: 5, delay: 300, yieldThreshold: 10 }).pipeTo(
         new QueueProcessor(
-            (pluginDataList) => {
+            async (pluginDataList) => {
+                // Concurrency is two, therefore, we can unlock the previous awaiting.
+                sendSignal("plugin-next-load");
                 let newList = [...this.pluginList];
                 for (const item of pluginDataList) {
                     newList = newList.filter(x => x.documentPath != item.documentPath);
@@ -343,9 +345,13 @@ export class ConfigSync extends LiveSyncCommands {
                 }
                 this.pluginList = newList;
                 pluginList.set(newList);
+                if (pluginDataList.length != 10) {
+                    // If the queue is going to be empty, await subsequent for a while.
+                    await waitForSignal("plugin-next-load", 1000);
+                }
                 return;
             }
-            , { suspended: true, batchSize: 1000, concurrentLimit: 10, delay: 200, yieldThreshold: 25, totalRemainingReactiveSource: pluginScanningCount })).startPipeline().root.onIdle(() => {
+            , { suspended: true, batchSize: 10, concurrentLimit: 2, delay: 250, yieldThreshold: 25, totalRemainingReactiveSource: pluginScanningCount })).startPipeline().root.onIdle(() => {
                 Logger(`All files enumerated`, LOG_LEVEL_INFO, "get-plugins");
                 this.createMissingConfigurationEntry();
             });

src/SerializedFileAccess.ts

@@ -3,6 +3,8 @@ import { serialized } from "./lib/src/lock";
 import type { FilePath } from "./lib/src/types";
 import { createBinaryBlob, isDocContentSame } from "./lib/src/utils";
 import type { InternalFileInfo } from "./types";
+import { markChangesAreSame } from "./utils";
+
 function getFileLockKey(file: TFile | TFolder | string) {
     return `fl:${typeof (file) == "string" ? file : file.path}`;
 }
@@ -16,6 +18,15 @@ function toArrayBuffer(arr: Uint8Array | ArrayBuffer | DataView): ArrayBufferLik
     return arr;
 }
 
+
+async function processReadFile<T>(file: TFile | TFolder | string, proc: () => Promise<T>) {
+    const ret = await serialized(getFileLockKey(file), () => proc());
+    return ret;
+}
+async function processWriteFile<T>(file: TFile | TFolder | string, proc: () => Promise<T>) {
+    const ret = await serialized(getFileLockKey(file), () => proc());
+    return ret;
+}
 export class SerializedFileAccess {
     app: App
     constructor(app: App) {
@@ -24,60 +35,64 @@ export class SerializedFileAccess {
 
     async adapterStat(file: TFile | string) {
         const path = file instanceof TFile ? file.path : file;
-        return await serialized(getFileLockKey(path), () => this.app.vault.adapter.stat(path));
+        return await processReadFile(file, () => this.app.vault.adapter.stat(path));
     }
     async adapterExists(file: TFile | string) {
         const path = file instanceof TFile ? file.path : file;
-        return await serialized(getFileLockKey(path), () => this.app.vault.adapter.exists(path));
+        return await processReadFile(file, () => this.app.vault.adapter.exists(path));
     }
     async adapterRemove(file: TFile | string) {
         const path = file instanceof TFile ? file.path : file;
-        return await serialized(getFileLockKey(path), () => this.app.vault.adapter.remove(path));
+        return await processReadFile(file, () => this.app.vault.adapter.remove(path));
     }
 
     async adapterRead(file: TFile | string) {
         const path = file instanceof TFile ? file.path : file;
-        return await serialized(getFileLockKey(path), () => this.app.vault.adapter.read(path));
+        return await processReadFile(file, () => this.app.vault.adapter.read(path));
     }
     async adapterReadBinary(file: TFile | string) {
         const path = file instanceof TFile ? file.path : file;
-        return await serialized(getFileLockKey(path), () => this.app.vault.adapter.readBinary(path));
+        return await processReadFile(file, () => this.app.vault.adapter.readBinary(path));
     }
 
     async adapterWrite(file: TFile | string, data: string | ArrayBuffer | Uint8Array, options?: DataWriteOptions) {
         const path = file instanceof TFile ? file.path : file;
         if (typeof (data) === "string") {
-            return await serialized(getFileLockKey(path), () => this.app.vault.adapter.write(path, data, options));
+            return await processWriteFile(file, () => this.app.vault.adapter.write(path, data, options));
         } else {
-            return await serialized(getFileLockKey(path), () => this.app.vault.adapter.writeBinary(path, toArrayBuffer(data), options));
+            return await processWriteFile(file, () => this.app.vault.adapter.writeBinary(path, toArrayBuffer(data), options));
         }
     }
 
     async vaultCacheRead(file: TFile) {
-        return await serialized(getFileLockKey(file), () => this.app.vault.cachedRead(file));
+        return await processReadFile(file, () => this.app.vault.cachedRead(file));
     }
 
     async vaultRead(file: TFile) {
-        return await serialized(getFileLockKey(file), () => this.app.vault.read(file));
+        return await processReadFile(file, () => this.app.vault.read(file));
     }
 
     async vaultReadBinary(file: TFile) {
-        return await serialized(getFileLockKey(file), () => this.app.vault.readBinary(file));
+        return await processReadFile(file, () => this.app.vault.readBinary(file));
     }
 
     async vaultModify(file: TFile, data: string | ArrayBuffer | Uint8Array, options?: DataWriteOptions) {
         if (typeof (data) === "string") {
-            return await serialized(getFileLockKey(file), async () => {
+            return await processWriteFile(file, async () => {
                 const oldData = await this.app.vault.read(file);
-                if (data === oldData) return false
+                if (data === oldData) {
+                    markChangesAreSame(file, file.stat.mtime, options.mtime);
+                    return false
+                }
                 await this.app.vault.modify(file, data, options)
                 return true;
             }
             );
         } else {
-            return await serialized(getFileLockKey(file), async () => {
+            return await processWriteFile(file, async () => {
                 const oldData = await this.app.vault.readBinary(file);
                 if (await isDocContentSame(createBinaryBlob(oldData), createBinaryBlob(data))) {
+                    markChangesAreSame(file, file.stat.mtime, options.mtime);
                     return false;
                 }
                 await this.app.vault.modifyBinary(file, toArrayBuffer(data), options)
@@ -87,16 +102,16 @@ export class SerializedFileAccess {
     }
     async vaultCreate(path: string, data: string | ArrayBuffer | Uint8Array, options?: DataWriteOptions): Promise<TFile> {
         if (typeof (data) === "string") {
-            return await serialized(getFileLockKey(path), () => this.app.vault.create(path, data, options));
+            return await processWriteFile(path, () => this.app.vault.create(path, data, options));
         } else {
-            return await serialized(getFileLockKey(path), () => this.app.vault.createBinary(path, toArrayBuffer(data), options));
+            return await processWriteFile(path, () => this.app.vault.createBinary(path, toArrayBuffer(data), options));
         }
     }
     async delete(file: TFile | TFolder, force = false) {
-        return await serialized(getFileLockKey(file), () => this.app.vault.delete(file, force));
+        return await processWriteFile(file, () => this.app.vault.delete(file, force));
     }
     async trash(file: TFile | TFolder, force = false) {
-        return await serialized(getFileLockKey(file), () => this.app.vault.trash(file, force));
+        return await processWriteFile(file, () => this.app.vault.trash(file, force));
     }
 
     getAbstractFileByPath(path: FilePath | string): TAbstractFile | null {

src/StorageEventManager.ts

@@ -4,6 +4,7 @@ import { Logger } from "./lib/src/logger";
 import { isPlainText, shouldBeIgnored } from "./lib/src/path";
 import type { KeyedQueueProcessor } from "./lib/src/processor";
 import { LOG_LEVEL_NOTICE, type FilePath, type ObsidianLiveSyncSettings } from "./lib/src/types";
+import { delay } from "./lib/src/utils";
 import { type FileEventItem, type FileEventType, type FileInfo, type InternalFileInfo } from "./types";
 
 
@@ -110,6 +111,8 @@ export class StorageEventManagerObsidian extends StorageEventManager {
             let cache: null | string | ArrayBuffer;
             // new file or something changed, cache the changes.
             if (file instanceof TFile && (type == "CREATE" || type == "CHANGED")) {
+                // Wait for a bit while to let the writer has marked `touched` at the file.
+                await delay(10);
                 if (this.plugin.vaultAccess.recentlyTouched(file)) {
                     continue;
                 }

src/main.ts

@@ -1,16 +1,16 @@
 const isDebug = false;
 
 import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch, stringifyYaml, parseYaml } from "./deps";
-import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
+import { Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl, type MarkdownFileInfo } from "./deps";
 import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT, LOG_LEVEL_VERBOSE, type SavingEntry, MISSING_OR_ERROR, NOT_CONFLICTED, AUTO_MERGED, CANCELLED, LEAVE_TO_SUBSEQUENT, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, } from "./lib/src/types";
 import { type InternalFileInfo, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";
-import { createBinaryBlob, createTextBlob, fireAndForget, getDocData, isDocContentSame, isObjectDifferent, sendValue } from "./lib/src/utils";
+import { arrayToChunkedArray, createBinaryBlob, createTextBlob, fireAndForget, getDocData, isDocContentSame, isObjectDifferent, sendValue } from "./lib/src/utils";
 import { Logger, setGlobalLogFunction } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";
 import { ConflictResolveModal } from "./ConflictResolveModal";
 import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab";
 import { DocumentHistoryModal } from "./DocumentHistoryModal";
-import { applyPatch, cancelAllPeriodicTask, cancelAllTasks, cancelTask, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, flattenObject, path2id, scheduleTask, tryParseJSON, isValidPath, isInternalMetadata, isPluginMetadata, stripInternalMetadataPrefix, isChunk, askSelectString, askYesNo, askString, PeriodicProcessor, getPath, getPathWithoutPrefix, getPathFromTFile, performRebuildDB, memoIfNotExist, memoObject, retrieveMemoObject, disposeMemoObject, isCustomisationSyncMetadata } from "./utils";
+import { applyPatch, cancelAllPeriodicTask, cancelAllTasks, cancelTask, generatePatchObj, id2path, isObjectMargeApplicable, isSensibleMargeApplicable, flattenObject, path2id, scheduleTask, tryParseJSON, isValidPath, isInternalMetadata, isPluginMetadata, stripInternalMetadataPrefix, isChunk, askSelectString, askYesNo, askString, PeriodicProcessor, getPath, getPathWithoutPrefix, getPathFromTFile, performRebuildDB, memoIfNotExist, memoObject, retrieveMemoObject, disposeMemoObject, isCustomisationSyncMetadata, compareFileFreshness, BASE_IS_NEW, TARGET_IS_NEW, EVEN, compareMTime, markChangesAreSame } from "./utils";
 import { encrypt, tryDecrypt } from "./lib/src/e2ee_v2";
 import { balanceChunkPurgedDBs, enableEncryption, isCloudantURI, isErrorOfMissingDoc, isValidRemoteCouchDBURI, purgeUnreferencedChunks } from "./lib/src/utils_couchdb";
 import { logStore, type LogEntry, collectingChunks, pluginScanningCount, hiddenFilesProcessingCount, hiddenFilesEventCount, logMessages } from "./lib/src/stores";
@@ -33,6 +33,7 @@ import { LRUCache } from "./lib/src/LRUCache";
 import { SerializedFileAccess } from "./SerializedFileAccess.js";
 import { KeyedQueueProcessor, QueueProcessor, type QueueItemWithKey } from "./lib/src/processor.js";
 import { reactive, reactiveSource } from "./lib/src/reactive.js";
+import { initializeStores } from "./stores.js";
 
 setNoticeClass(Notice);
 
@@ -728,9 +729,9 @@ Note: We can always able to read V1 format. It will be progressively converted.
         this.packageVersion = packageVersion;
 
         Logger(`Self-hosted LiveSync v${manifestVersion} ${packageVersion} `);
+        await this.loadSettings();
         const lsKey = "obsidian-live-sync-ver" + this.getVaultName();
         const last_version = localStorage.getItem(lsKey);
-        await this.loadSettings();
         this.observeForLogs();
         this.statusBar = this.addStatusBarItem();
         this.statusBar.addClass("syncstatusbar");
@@ -757,9 +758,9 @@ Note: We can always able to read V1 format. It will be progressively converted.
         }
         localStorage.setItem(lsKey, `${VER}`);
         await this.openDatabase();
-        this.watchWorkspaceOpen = debounce(this.watchWorkspaceOpen.bind(this), 1000, false);
+        this.watchWorkspaceOpen = this.watchWorkspaceOpen.bind(this);
-        this.watchWindowVisibility = debounce(this.watchWindowVisibility.bind(this), 1000, false);
+        this.watchWindowVisibility = this.watchWindowVisibility.bind(this)
-        this.watchOnline = debounce(this.watchOnline.bind(this), 500, false);
+        this.watchOnline = this.watchOnline.bind(this);
         this.realizeSettingSyncMode = this.realizeSettingSyncMode.bind(this);
         this.parseReplicationResult = this.parseReplicationResult.bind(this);
 
@@ -821,6 +822,7 @@ Note: We can always able to read V1 format. It will be progressively converted.
         //@ts-ignore
         this.isMobile = this.app.isMobile;
         this.localDatabase = new LiveSyncLocalDB(vaultName, this);
+        initializeStores(vaultName);
         return await this.localDatabase.initializeDatabase();
     }
 
@@ -1010,8 +1012,8 @@ Note: We can always able to read V1 format. It will be progressively converted.
 
     async checkAndApplySettingFromMarkdown(filename: string, automated?: boolean) {
         if (automated && !this.settings.notifyAllSettingSyncFile) {
-            if (this.settings.settingSyncFile != filename) {
+            if (!this.settings.settingSyncFile || this.settings.settingSyncFile != filename) {
-                Logger(`Setting file (${filename}) is not matched to the current configuration. skipped.`, LOG_LEVEL_INFO);
+                Logger(`Setting file (${filename}) is not matched to the current configuration. skipped.`, LOG_LEVEL_VERBOSE);
                 return;
             }
         }
@@ -1175,7 +1177,7 @@ We can perform a command in this file.
 
 
     watchOnline() {
-        this.watchOnlineAsync();
+        scheduleTask("watch-online", 500, () => fireAndForget(() => this.watchOnlineAsync()));
     }
     async watchOnlineAsync() {
         // If some files were failed to retrieve, scan files again.
@@ -1186,7 +1188,7 @@ We can perform a command in this file.
         }
     }
     watchWindowVisibility() {
-        this.watchWindowVisibilityAsync();
+        scheduleTask("watch-window-visibility", 500, () => fireAndForget(() => this.watchWindowVisibilityAsync()));
     }
 
     async watchWindowVisibilityAsync() {
@@ -1291,10 +1293,11 @@ We can perform a command in this file.
     }
 
     pendingFileEventCount = reactiveSource(0);
+    processingFileEventCount = reactiveSource(0);
     fileEventQueue =
         new KeyedQueueProcessor(
             (items: FileEventItem[]) => this.handleFileEvent(items[0]),
-            { suspended: true, batchSize: 1, concurrentLimit: 5, delay: 100, yieldThreshold: FileWatchEventQueueMax, totalRemainingReactiveSource: this.pendingFileEventCount }
+            { suspended: true, batchSize: 1, concurrentLimit: 5, delay: 100, yieldThreshold: FileWatchEventQueueMax, totalRemainingReactiveSource: this.pendingFileEventCount, processingEntitiesReactiveSource: this.processingFileEventCount }
         ).replaceEnqueueProcessor((items, newItem) => this.queueNextFileEvent(items, newItem));
 
 
@@ -1306,7 +1309,7 @@ We can perform a command in this file.
         if (this.settings.suspendFileWatching) return;
         if (!this.isReady) return;
         if (!file) return;
-        this.watchWorkspaceOpenAsync(file);
+        scheduleTask("watch-workspace-open", 500, () => fireAndForget(() => this.watchWorkspaceOpenAsync(file)));
     }
 
     async watchWorkspaceOpenAsync(file: TFile) {
@@ -1468,47 +1471,45 @@ We can perform a command in this file.
         // let performPullFileAgain = false;
         if (existDoc && existDoc._conflicts) {
             if (this.settings.writeDocumentsIfConflicted) {
-                Logger(`Processing: ${file.path}: Conflicted revision has been deleted, but there were more conflicts. `, LOG_LEVEL_INFO);
+                Logger(`Processing: ${path}: Conflicted revision has been deleted, but there were more conflicts. `, LOG_LEVEL_INFO);
                 await this.processEntryDoc(docEntry, file, true);
                 return;
             } else if (force != true) {
-                Logger(`Processing: ${file.path}: Conflicted revision has been deleted, but there were more conflicts...`);
+                Logger(`Processing: ${path}: Conflicted revision has been deleted, but there were more conflicts...`);
-                this.queueConflictCheck(file);
+                this.queueConflictCheck(path);
                 return;
             }
         }
         // If there are no conflicts, or forced to overwrite.
 
         if (docEntry._deleted || docEntry.deleted || existDoc === false) {
-            if (path != file.path) {
+            if (!file) {
-                Logger(`delete skipped: ${file.path} :Not exactly matched`, LOG_LEVEL_VERBOSE);
+                Logger(`delete skipped: ${path} :Already not exist on storage`, LOG_LEVEL_VERBOSE);
+                return;
+            }
+            if (file.path != path) {
+                Logger(`delete skipped: ${path} :Not exactly matched`, LOG_LEVEL_VERBOSE);
+                return;
             }
             if (existDoc === false) {
                 await this.deleteVaultItem(file);
             } else {
                 // Conflict has been resolved at this time, 
-                await this.pullFile(path, null, force);
+                await this.pullFile(path, null, true);
             }
             return;
         }
-        const localMtime = ~~((file?.stat?.mtime || 0) / 1000);
-        const docMtime = ~~(docEntry.mtime / 1000);
 
-        // const doc = await this.localDatabase.getDBEntry(path, { rev: docEntry._rev });
+        const compareResult = compareFileFreshness(file, docEntry);
-        // if (doc === false) return;
+
         const doc = existDoc;
-        // if (doc === false) {
-        //     // The latest file 
-        //     await this.pullFile(path, null, force);
-        //     // Logger(`delete skipped: ${file.path} :Not exactly matched`, LOG_LEVEL_VERBOSE);
-        //     return;
-        // }
 
         if (doc.datatype != "newnote" && doc.datatype != "plain") {
             Logger(msg + "ERROR, Invalid datatype: " + path + "(" + doc.datatype + ")", LOG_LEVEL_NOTICE);
             return;
         }
-        if (!force && localMtime >= docMtime) return;
+        // if (!force && localMtime >= docMtime) return;
+        if (!force && (compareResult == BASE_IS_NEW || compareResult == EVEN)) return;
         if (!isValidPath(path)) {
             Logger(msg + "ERROR, invalid path: " + path, LOG_LEVEL_NOTICE);
             return;
@@ -1518,7 +1519,7 @@ We can perform a command in this file.
         try {
             let outFile;
             let isChanged = true;
-            if (mode == "create") {
+            if (!file) {
                 const normalizedPath = normalizePath(path);
                 await this.vaultAccess.vaultCreate(normalizedPath, writeData, { ctime: doc.ctime, mtime: doc.mtime, });
                 outFile = this.vaultAccess.getAbstractFileByPath(normalizedPath) as TFile;
@@ -1581,9 +1582,12 @@ We can perform a command in this file.
         if (!this.settings.suspendParseReplicationResult) {
             const lsKey = "obsidian-livesync-queuefiles-" + this.getVaultName();
             const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[];
-            const ret = await this.localDatabase.allDocsRaw<EntryDoc>({ keys: ids, include_docs: true });
+            const batchSize = 100;
-            for (const doc of ret.rows) {
+            const chunkedIds = arrayToChunkedArray(ids, batchSize);
-                this.replicationResultProcessor.enqueue(doc.doc);
+            for await (const idsBatch of chunkedIds) {
+                const ret = await this.localDatabase.allDocsRaw<EntryDoc>({ keys: idsBatch, include_docs: true, limit: 100 });
+                this.replicationResultProcessor.enqueueAll(ret.rows.map(doc => doc.doc));
+                await this.replicationResultProcessor.waitForPipeline();
             }
         }
     }
@@ -1669,17 +1673,18 @@ We can perform a command in this file.
             this.databaseQueuedProcessor.enqueueWithKey(change.path, change);
         }
         return;
-    }, { batchSize: 1, suspended: true, concurrentLimit: 1, delay: 0, totalRemainingReactiveSource: this.replicationResultCount }).startPipeline().onUpdateProgress(() => {
+    }, { batchSize: 1, suspended: true, concurrentLimit: 100, delay: 0, totalRemainingReactiveSource: this.replicationResultCount }).startPipeline().onUpdateProgress(() => {
         this.saveQueuedFiles();
     });
     //---> Sync
     parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>) {
         if (this.settings.suspendParseReplicationResult) {
             this.replicationResultProcessor.suspend()
-        } else {
-            this.replicationResultProcessor.resume()
         }
         this.replicationResultProcessor.enqueueAll(docs);
+        if (!this.settings.suspendParseReplicationResult) {
+            this.replicationResultProcessor.resume()
+        }
     }
 
 
@@ -1764,7 +1769,7 @@ We can perform a command in this file.
         })
         const waitingLabel = reactive(() => {
             const e = this.pendingFileEventCount.value;
-            const proc = this.fileEventQueue.processingEntities;
+            const proc = this.processingFileEventCount.value;
             const pend = e - proc;
             const labelProc = proc != 0 ? `⏳${proc} ` : "";
             const labelPend = pend != 0 ? ` 🛫${pend}` : "";
@@ -1805,13 +1810,19 @@ We can perform a command in this file.
         const newLog = log;
         // scheduleTask("update-display", 50, () => {
         this.statusBar?.setText(newMsg.split("\n")[0]);
+        const selector = `.CodeMirror-wrap,` +
+            `.markdown-preview-view.cm-s-obsidian,` +
+            `.markdown-source-view.cm-s-obsidian,` +
+            `.canvas-wrapper,` +
+            `.empty-state`
+            ;
         if (this.settings.showStatusOnEditor) {
             const root = activeDocument.documentElement;
-            const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
+            const q = root.querySelectorAll(selector);
             q.forEach(e => e.setAttr("data-log", '' + (newMsg + "\n" + newLog) + ''))
         } else {
             const root = activeDocument.documentElement;
-            const q = root.querySelectorAll(`.CodeMirror-wrap,.cm-s-obsidian>.cm-editor,.canvas-wrapper`);
+            const q = root.querySelectorAll(selector);
             q.forEach(e => e.setAttr("data-log", ''))
         }
         // }, true);
@@ -1987,7 +1998,6 @@ Or if you are sure know what had been happened, we can unlock the database from
 
         const syncFiles = filesStorage.filter((e) => onlyInStorageNames.indexOf(e.path) == -1);
         Logger("Updating database by new files");
-        // this.setStatusBarText(`UPDATE DATABASE`);
 
         const initProcess = [];
         const logLevel = showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
@@ -2046,8 +2056,8 @@ Or if you are sure know what had been happened, we can unlock the database from
             }));
         }
         if (!initialScan) {
-            let caches: { [key: string]: { storageMtime: number; docMtime: number } } = {};
+            // let caches: { [key: string]: { storageMtime: number; docMtime: number } } = {};
-            caches = await this.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number } }>("diff-caches") || {};
+            // caches = await this.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number } }>("diff-caches") || {};
             type FileDocPair = { file: TFile, id: DocumentID };
 
             const processPrepareSyncFile = new QueueProcessor(
@@ -2073,7 +2083,7 @@ Or if you are sure know what had been happened, we can unlock the database from
                     new QueueProcessor(
                         async (loadedPairs) => {
                             const e = loadedPairs[0];
-                            await this.syncFileBetweenDBandStorage(e.file, e.doc, initialScan, caches);
+                            await this.syncFileBetweenDBandStorage(e.file, e.doc, initialScan);
                             return;
                         }, { batchSize: 1, concurrentLimit: 5, delay: 10, suspended: false }
                     ))
@@ -2081,7 +2091,7 @@ Or if you are sure know what had been happened, we can unlock the database from
             processPrepareSyncFile.startPipeline();
             initProcess.push(async () => {
                 await processPrepareSyncFile.waitForPipeline();
-                await this.kvDB.set("diff-caches", caches);
+                // await this.kvDB.set("diff-caches", caches);
             })
         }
         await Promise.all(initProcess);
@@ -2130,6 +2140,10 @@ Or if you are sure know what had been happened, we can unlock the database from
         if (baseLeaf == false || leftLeaf == false || rightLeaf == false) {
             return false;
         }
+        if (leftLeaf.deleted && rightLeaf.deleted) {
+            // Both are deleted
+            return false;
+        }
         // diff between base and each revision
         const dmp = new diff_match_patch();
         const mapLeft = dmp.diff_linesToChars_(baseLeaf.data, leftLeaf.data);
@@ -2290,6 +2304,9 @@ Or if you are sure know what had been happened, we can unlock the database from
             if (baseLeaf == false || leftLeaf == false || rightLeaf == false) {
                 return false;
             }
+            if (leftLeaf.deleted && rightLeaf.deleted) {
+                return false;
+            }
             const baseObj = { data: tryParseJSON(baseLeaf.data, {}) } as Record<string | number | symbol, any>;
             const leftObj = { data: tryParseJSON(leftLeaf.data, {}) } as Record<string | number | symbol, any>;
             const rightObj = { data: tryParseJSON(rightLeaf.data, {}) } as Record<string | number | symbol, any>;
@@ -2375,7 +2392,6 @@ Or if you are sure know what had been happened, we can unlock the database from
                 if (p != undefined) {
                     // remove conflicted revision.
                     await this.localDatabase.deleteDBEntry(path, { rev: conflictedRev });
-
                     const file = this.vaultAccess.getAbstractFileByPath(stripAllPrefixes(path)) as TFile;
                     if (file) {
                         if (await this.vaultAccess.vaultModify(file, p)) {
@@ -2412,10 +2428,10 @@ Or if you are sure know what had been happened, we can unlock the database from
         const isBinary = !isPlainText(path);
         const alwaysNewer = this.settings.resolveConflictsByNewerFile;
         if (isSame || isBinary || alwaysNewer) {
-            const lMtime = ~~(leftLeaf.mtime / 1000);
+            const result = compareMTime(leftLeaf.mtime, rightLeaf.mtime)
-            const rMtime = ~~(rightLeaf.mtime / 1000);
             let loser = leftLeaf;
-            if (lMtime > rMtime) {
+            // if (lMtime > rMtime) {
+            if (result != TARGET_IS_NEW) {
                 loser = rightLeaf;
             }
             await this.localDatabase.deleteDBEntry(path, { rev: loser.rev });
@@ -2437,7 +2453,7 @@ Or if you are sure know what had been happened, we can unlock the database from
 
     conflictProcessQueueCount = reactiveSource(0);
     conflictResolveQueue =
-        new KeyedQueueProcessor(async (entries: { filename: FilePathWithPrefix, file: TFile }[]) => {
+        new KeyedQueueProcessor(async (entries: { filename: FilePathWithPrefix }[]) => {
             const entry = entries[0];
             const filename = entry.filename;
             const conflictCheckResult = await this.checkConflictAndPerformAutoMerge(filename);
@@ -2478,11 +2494,12 @@ Or if you are sure know what had been happened, we can unlock the database from
         new QueueProcessor((files: FilePathWithPrefix[]) => {
             const filename = files[0];
             const file = this.vaultAccess.getAbstractFileByPath(filename);
-            if (!file) return;
+            // if (!file) return;
-            if (!(file instanceof TFile)) return;
+            // if (!(file instanceof TFile)) return;
+            if ((file instanceof TFolder)) return;
             // Check again?
 
-            return [{ key: filename, entity: { filename, file } }];
+            return [{ key: filename, entity: { filename } }];
             // this.conflictResolveQueue.enqueueWithKey(filename, { filename, file });
         }, {
             suspended: false, batchSize: 1, concurrentLimit: 5, delay: 10, keepResultUntilDownstreamConnected: true, pipeTo: this.conflictResolveQueue, totalRemainingReactiveSource: this.conflictProcessQueueCount
@@ -2569,7 +2586,7 @@ Or if you are sure know what had been happened, we can unlock the database from
         //when to opened file;
     }
 
-    async syncFileBetweenDBandStorage(file: TFile, doc: LoadedEntry, initialScan: boolean, caches: { [key: string]: { storageMtime: number; docMtime: number } }) {
+    async syncFileBetweenDBandStorage(file: TFile, doc: LoadedEntry, initialScan: boolean) {
         if (!doc) {
             throw new Error(`Missing doc:${(file as any).path}`)
         }
@@ -2582,47 +2599,37 @@ Or if you are sure know what had been happened, we can unlock the database from
             }
         }
 
-        const storageMtime = ~~(file.stat.mtime / 1000);
+        const compareResult = compareFileFreshness(file, doc);
-        const docMtime = ~~(doc.mtime / 1000);
+        switch (compareResult) {
-        const dK = `${file.path}-diff`;
+            case BASE_IS_NEW:
-        const isLastDiff = dK in caches ? caches[dK] : { storageMtime: 0, docMtime: 0 };
+                if (!this.isFileSizeExceeded(file.stat.size)) {
-        if (isLastDiff.docMtime == docMtime && isLastDiff.storageMtime == storageMtime) {
+                    Logger("STORAGE -> DB :" + file.path);
-            // Logger("STORAGE .. DB :" + file.path, LOG_LEVEL_VERBOSE);
+                    await this.updateIntoDB(file, initialScan);
-            caches[dK] = { storageMtime, docMtime };
+                    fireAndForget(() => this.checkAndApplySettingFromMarkdown(file.path, true));
-            return caches;
+                } else {
-        }
+                    Logger(`STORAGE -> DB : ${file.path} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
-        if (storageMtime > docMtime) {
+                }
-            //newer local file.
+                break;
-            if (!this.isFileSizeExceeded(file.stat.size)) {
+            case TARGET_IS_NEW:
-                Logger("STORAGE -> DB :" + file.path);
+                if (!this.isFileSizeExceeded(doc.size)) {
-                Logger(`${storageMtime} > ${docMtime}`);
+                    Logger("STORAGE <- DB :" + file.path);
-                await this.updateIntoDB(file, initialScan);
+                    const docx = await this.localDatabase.getDBEntry(getPathFromTFile(file), null, false, false, true);
-                fireAndForget(() => this.checkAndApplySettingFromMarkdown(file.path, true));
+                    if (docx != false) {
-                caches[dK] = { storageMtime, docMtime };
+                        await this.processEntryDoc(docx, file);
-                return caches;
+                    } else {
-            } else {
+                        Logger(`STORAGE <- DB : Cloud not read ${file.path}, possibly deleted`, LOG_LEVEL_NOTICE);
-                Logger(`STORAGE -> DB : ${file.path} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
+                    }
-            }
+                    return caches;
-        } else if (storageMtime < docMtime) {
-            //newer database file.
-            if (!this.isFileSizeExceeded(doc.size)) {
-                Logger("STORAGE <- DB :" + file.path);
-                Logger(`${storageMtime} < ${docMtime}`);
-                const docx = await this.localDatabase.getDBEntry(getPathFromTFile(file), null, false, false);
-                if (docx != false) {
-                    await this.processEntryDoc(docx, file);
                 } else {
                     Logger(`STORAGE <- DB : ${file.path} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
                 }
-                caches[dK] = { storageMtime, docMtime };
+                break;
-                return caches;
+            case EVEN:
-            } else {
+                Logger("STORAGE == DB :" + file.path + "", LOG_LEVEL_VERBOSE);
-                Logger("STORAGE <- DB :" + file.path + " Skipped (size)");
+                break;
-            }
+            default:
+                Logger("STORAGE ?? DB :" + file.path + " Something got weird");
         }
-        Logger("STORAGE == DB :" + file.path + "", LOG_LEVEL_VERBOSE);
-        caches[dK] = { storageMtime, docMtime };
-        return caches;
 
     }
 
@@ -2693,6 +2700,7 @@ Or if you are sure know what had been happened, we can unlock the database from
                     if (oldData.deleted != newData.deleted) return false;
                     if (!await isDocContentSame(old.data, newData.data)) return false;
                     Logger(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
+                    markChangesAreSame(old, d.mtime, old.mtime);
                     return true;
                     // d._rev = old._rev;
                 }
@@ -2711,10 +2719,11 @@ Or if you are sure know what had been happened, we can unlock the database from
             return true;
         }
         const ret = await this.localDatabase.putDBEntry(d, initialScan);
-
+        if (ret !== false) {
-        Logger(msg + fullPath);
+            Logger(msg + fullPath);
-        if (this.settings.syncOnSave && !this.suspended) {
+            if (this.settings.syncOnSave && !this.suspended) {
-            await this.replicate();
+                scheduleTask("perform-replicate-after-save", 250, () => this.replicate());
+            }
         }
         return ret != false;
     }

src/stores.ts

@@ -0,0 +1,7 @@
+import { PersistentMap } from "./lib/src/PersistentMap";
+
+export let sameChangePairs: PersistentMap<number[]>;
+
+export function initializeStores(vaultName: string) {
+    sameChangePairs = new PersistentMap<number[]>(`ls-persist-same-changes-${vaultName}`);
+}

src/utils.ts

@@ -1,4 +1,4 @@
-import { normalizePath, Platform, TAbstractFile, App, type RequestUrlParam, requestUrl } from "./deps";
+import { normalizePath, Platform, TAbstractFile, App, type RequestUrlParam, requestUrl, TFile } from "./deps";
 import { path2id_base, id2path_base, isValidFilenameInLinux, isValidFilenameInDarwin, isValidFilenameInWidows, isValidFilenameInAndroid, stripAllPrefixes } from "./lib/src/path";
 
 import { Logger } from "./lib/src/logger";
@@ -8,6 +8,7 @@ import { InputStringDialog, PopoverSelectString } from "./dialogs";
 import type ObsidianLiveSyncPlugin from "./main";
 import { writeString } from "./lib/src/strbin";
 import { fireAndForget } from "./lib/src/utils";
+import { sameChangePairs } from "./stores";
 
 export { scheduleTask, setPeriodicTask, cancelTask, cancelAllTasks, cancelPeriodicTask, cancelAllPeriodicTask, } from "./lib/src/task";
 
@@ -415,3 +416,48 @@ export async function performRebuildDB(plugin: ObsidianLiveSyncPlugin, method: "
         await plugin.addOnSetup.rebuildEverything();
     }
 }
+
+export const BASE_IS_NEW = Symbol("base");
+export const TARGET_IS_NEW = Symbol("target");
+export const EVEN = Symbol("even");
+
+
+// Why 2000? : ZIP FILE Does not have enough resolution.
+const resolution = 2000;
+export function compareMTime(baseMTime: number, targetMTime: number): typeof BASE_IS_NEW | typeof TARGET_IS_NEW | typeof EVEN {
+    const truncatedBaseMTime = (~~(baseMTime / resolution)) * resolution;
+    const truncatedTargetMTime = (~~(targetMTime / resolution)) * resolution;
+    // Logger(`Resolution MTime ${truncatedBaseMTime} and ${truncatedTargetMTime} `, LOG_LEVEL_VERBOSE);
+    if (truncatedBaseMTime == truncatedTargetMTime) return EVEN;
+    if (truncatedBaseMTime > truncatedTargetMTime) return BASE_IS_NEW;
+    if (truncatedBaseMTime < truncatedTargetMTime) return TARGET_IS_NEW;
+    throw new Error("Unexpected error");
+}
+
+export function markChangesAreSame(file: TFile | AnyEntry | string, mtime1: number, mtime2: number) {
+    if (mtime1 === mtime2) return true;
+    const key = typeof file == "string" ? file : file instanceof TFile ? file.path : file.path ?? file._id;
+    const pairs = sameChangePairs.get(key, []);
+    if (pairs.some(e => e == mtime1 || e == mtime2)) {
+        sameChangePairs.set(key, [...new Set([...pairs, mtime1, mtime2])]);
+    } else {
+        sameChangePairs.set(key, [mtime1, mtime2]);
+    }
+}
+export function isMarkedAsSameChanges(file: TFile | AnyEntry | string, mtimes: number[]) {
+    const key = typeof file == "string" ? file : file instanceof TFile ? file.path : file.path ?? file._id;
+    const pairs = sameChangePairs.get(key, []);
+    if (mtimes.every(e => pairs.indexOf(e) !== -1)) {
+        return EVEN;
+    }
+}
+export function compareFileFreshness(baseFile: TFile | AnyEntry, checkTarget: TFile | AnyEntry): typeof BASE_IS_NEW | typeof TARGET_IS_NEW | typeof EVEN {
+    const modifiedBase = baseFile instanceof TFile ? baseFile?.stat?.mtime ?? 0 : baseFile?.mtime ?? 0;
+    const modifiedTarget = checkTarget instanceof TFile ? checkTarget?.stat?.mtime ?? 0 : checkTarget?.mtime ?? 0;
+
+    if (modifiedBase && modifiedTarget && isMarkedAsSameChanges(baseFile, [modifiedBase, modifiedTarget])) {
+        return EVEN;
+    }
+    return compareMTime(modifiedBase, modifiedTarget);
+}
+

styles.css

@@ -98,8 +98,10 @@
 }
 
 .CodeMirror-wrap::before,
-.cm-s-obsidian > .cm-editor::before,
+.markdown-preview-view.cm-s-obsidian::before,
-.canvas-wrapper::before {
+.markdown-source-view.cm-s-obsidian::before,
+.canvas-wrapper::before,
+.empty-state::before {
     content: attr(data-log);
     text-align: right;
     white-space: pre-wrap;
@@ -115,6 +117,19 @@
     filter: grayscale(100%);
 }
 
+.empty-state::before,
+.markdown-preview-view.cm-s-obsidian::before,
+.markdown-source-view.cm-s-obsidian::before {
+    top: var(--header-height);
+    right: 1em;
+}
+
+.is-mobile .empty-state::before,
+.is-mobile .markdown-preview-view.cm-s-obsidian::before,
+.is-mobile .markdown-source-view.cm-s-obsidian::before {
+    top: var(--view-header-height);
+    right: 1em;
+}
 .canvas-wrapper::before {
     right: 48px;
 }
@@ -292,7 +307,7 @@ span.ls-mark-cr::after {
     position: absolute;
     top: 0;
     left: 0;
-    animation: ls-blink-diff 0.5s cubic-bezier(0.4, 0, 1, 1)  infinite alternate;
+    animation: ls-blink-diff 0.5s cubic-bezier(0.4, 0, 1, 1) infinite alternate;
 }
 @keyframes ls-blink-diff {
     0% {

updates.md

@@ -10,6 +10,31 @@ Note: we got a very performance improvement.
 Note at 0.22.2: **Now, to rescue mobile devices, Maximum file size is set to 50 by default**. Please configure the limit as you need. If you do not want to limit the sizes, set zero manually, please.
 
 #### Version history
+- 0.23.4
+  - Fixed:
+    - Now the result of conflict resolution could be surely written into the storage.
+    - Deleted files can be handled correctly again in the history dialogue and conflict dialogue.
+    - Some wrong log messages were fixed.
+    - Change handling now has become more stable.
+    - Some event handling became to be safer.
+  - Improved:
+    - Dumping document information shows conflicts and revisions.
+    - The timestamp-only differences can be surely cached.
+    - Timestamp difference detection can be rounded by two seconds.
+  - Refactored:
+    - A bit of organisation to write the test.
+- 0.22.3
+  - Fixed:
+    - No longer detects storage changes which have been caused by Self-hosted LiveSync itself.
+    - Setting sync file will be detected only if it has been configured now.
+      - And its log will be shown only while the verbose log is enabled.
+    - Customisation file enumeration has got less blingy.
+    - Deletion of files is now reliably synchronised.
+  - Fixed and improved:
+    - In-editor-status is now shown in the following areas:
+      - Note editing pane (Source mode and live-preview mode).
+      - New tab pane.
+      - Canvas pane.
 - 0.22.2
   - Fixed:
     - Now the results of resolving conflicts are surely synchronised.