bump

- `Verify and repair all files` is no longer broken.
New feature::
- Now `Verify and repair all files` can restore or show history
Improved:
- Performance improved

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.22.10",
+    "version": "0.22.11",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,12 +1,12 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.22.10",
+    "version": "0.22.11",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.22.10",
+            "version": "0.22.11",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.22.10",
+    "version": "0.22.11",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",

src/ObsidianLiveSyncSettingTab.ts

@@ -1,13 +1,14 @@
 import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "./deps";
-import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO } from "./lib/src/types";
+import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, LOG_LEVEL_INFO, type LoadedEntry } from "./lib/src/types";
 import { createBinaryBlob, createTextBlob, delay, isDocContentSame } from "./lib/src/utils";
-import { versionNumberString2Number } from "./lib/src/strbin";
+import { decodeBinary, versionNumberString2Number } from "./lib/src/strbin";
 import { Logger } from "./lib/src/logger";
 import { checkSyncInfo, isCloudantURI } from "./lib/src/utils_couchdb";
 import { testCrypt } from "./lib/src/e2ee_v2";
 import ObsidianLiveSyncPlugin from "./main";
 import { askYesNo, performRebuildDB, requestToCouchDB, scheduleTask } from "./utils";
-import { request, type ButtonComponent } from "obsidian";
+import { request, type ButtonComponent, TFile } from "obsidian";
+import { shouldBeIgnored } from "./lib/src/path";
 
 
 export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
@@ -1698,6 +1699,59 @@ ${stringifyYaml(pluginConfig)}`;
         const hatchWarn = containerHatchEl.createEl("div", { text: `To stop the boot up sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` });
         hatchWarn.addClass("op-warn-info");
 
+
+
+        const addResult = (path: string, file: TFile | false, fileOnDB: LoadedEntry | false) => {
+            resultArea.appendChild(resultArea.createEl("div", {}, el => {
+                el.appendChild(el.createEl("h6", { text: path }));
+                el.appendChild(el.createEl("div", {}, infoGroupEl => {
+                    infoGroupEl.appendChild(infoGroupEl.createEl("div", { text: `Storage : Modified: ${!file ? `Missing:` : `${new Date(file.stat.mtime).toLocaleString()}, Size:${file.stat.size}`}` }))
+                    infoGroupEl.appendChild(infoGroupEl.createEl("div", { text: `Database: Modified: ${!fileOnDB ? `Missing:` : `${new Date(fileOnDB.mtime).toLocaleString()}, Size:${fileOnDB.size}`}` }))
+                }));
+                if (fileOnDB && file) {
+                    el.appendChild(el.createEl("button", { text: "Show history" }, buttonEl => {
+                        buttonEl.onClickEvent(() => {
+                            this.plugin.showHistory(file, fileOnDB._id);
+                        })
+                    }))
+                }
+                if (file) {
+                    el.appendChild(el.createEl("button", { text: "Storage -> Database" }, buttonEl => {
+                        buttonEl.onClickEvent(() => {
+                            this.plugin.updateIntoDB(file, undefined, true);
+                            el.remove();
+                        })
+                    }))
+                }
+                if (fileOnDB) {
+                    el.appendChild(el.createEl("button", { text: "Database -> Storage" }, buttonEl => {
+                        buttonEl.onClickEvent(() => {
+                            this.plugin.pullFile(this.plugin.getPath(fileOnDB), [], true, undefined, false);
+                            el.remove();
+                        })
+                    }))
+                }
+                return el;
+            }))
+        }
+
+        const checkBetweenStorageAndDatabase = async (file: TFile, fileOnDB: LoadedEntry) => {
+            let content: Blob;
+            let dataContent: Blob;
+            if (fileOnDB.type == "newnote") {
+                dataContent = createBinaryBlob(decodeBinary(fileOnDB.data));
+                content = createBinaryBlob(await this.plugin.vaultAccess.vaultReadBinary(file));
+            } else {
+                dataContent = createTextBlob(fileOnDB.data);
+                content = createTextBlob(await this.plugin.vaultAccess.vaultRead(file));
+            }
+            if (await isDocContentSame(content, dataContent)) {
+                Logger(`Compare: SAME: ${file.path}`)
+            } else {
+                Logger(`Compare: CONTENT IS NOT MATCHED! ${file.path}`, LOG_LEVEL_NOTICE);
+                addResult(file.path, file, fileOnDB)
+            }
+        }
         new Setting(containerHatchEl)
             .setName("Verify and repair all files")
             .setDesc("Compare the content of files between on local database and storage. If not matched, you will asked which one want to keep.")
@@ -1708,47 +1762,36 @@ ${stringifyYaml(pluginConfig)}`;
                     .setWarning()
                     .onClick(async () => {
                         const files = this.app.vault.getFiles();
+                        const documents = [] as FilePathWithPrefix[];
+
+                        const adn = this.plugin.localDatabase.findAllNormalDocs()
+                        for await (const i of adn) documents.push(this.plugin.getPath(i));
+                        const allPaths = [...new Set([...documents, ...files.map(e => e.path as FilePathWithPrefix)])];
                         let i = 0;
-                        for (const file of files) {
+                        for (const path of allPaths) {
                             i++;
-                            Logger(`${i}/${files.length}\n${file.path}`, LOG_LEVEL_NOTICE, "verify");
+                            Logger(`${i}/${files.length}\n${path}`, LOG_LEVEL_NOTICE, "verify");
-                            if (!await this.plugin.isTargetFile(file)) continue;
+                            if (shouldBeIgnored(path)) continue;
-                            const fileOnDB = await this.plugin.localDatabase.getDBEntry(file.path as FilePathWithPrefix);
+                            const abstractFile = this.plugin.vaultAccess.getAbstractFileByPath(path);
-                            if (!fileOnDB) {
+                            const fileOnStorage = abstractFile instanceof TFile ? abstractFile : false;
-                                Logger(`Compare: Not found on local database: ${file.path}`, LOG_LEVEL_NOTICE);
+                            if (!await this.plugin.isTargetFile(path)) continue;
+
+                            if (fileOnStorage && this.plugin.isFileSizeExceeded(fileOnStorage.stat.size)) continue;
+                            const fileOnDB = await this.plugin.localDatabase.getDBEntry(path);
+                            if (fileOnDB && this.plugin.isFileSizeExceeded(fileOnDB.size)) continue;
+
+                            if (!fileOnDB && fileOnStorage) {
+                                Logger(`Compare: Not found on the local database: ${path}`, LOG_LEVEL_NOTICE);
+                                addResult(path, fileOnStorage, false)
                                 continue;
                             }
-                            let content: Blob;
+                            if (fileOnDB && !fileOnStorage) {
-                            if (fileOnDB.type == "newnote") {
+                                Logger(`Compare: Not found on the storage: ${path}`, LOG_LEVEL_NOTICE);
-                                content = createBinaryBlob(await this.plugin.vaultAccess.vaultReadBinary(file));
+                                addResult(path, false, fileOnDB)
-                            } else {
+                                continue;
-                                content = createTextBlob(await this.plugin.vaultAccess.vaultRead(file));
                             }
-                            if (isDocContentSame(content, fileOnDB.data)) {
+                            if (fileOnStorage && fileOnDB) {
-                                Logger(`Compare: SAME: ${file.path}`)
+                                await checkBetweenStorageAndDatabase(fileOnStorage, fileOnDB)
-                            } else {
-                                Logger(`Compare: CONTENT IS NOT MATCHED! ${file.path}`, LOG_LEVEL_NOTICE);
-                                resultArea.appendChild(resultArea.createEl("div", {}, el => {
-                                    el.appendChild(el.createEl("h6", { text: file.path }));
-                                    el.appendChild(el.createEl("div", {}, infoGroupEl => {
-                                        infoGroupEl.appendChild(infoGroupEl.createEl("div", { text: `Storage : Modified: ${new Date(file.stat.mtime).toLocaleString()}, Size:${file.stat.size}` }))
-                                        infoGroupEl.appendChild(infoGroupEl.createEl("div", { text: `Database: Modified: ${new Date(fileOnDB.mtime).toLocaleString()}, Size:${content.size}` }))
-                                    }));
-
-                                    el.appendChild(el.createEl("button", { text: "Storage -> Database" }, buttonEl => {
-                                        buttonEl.onClickEvent(() => {
-                                            this.plugin.updateIntoDB(file, undefined, true);
-                                            el.remove();
-                                        })
-                                    }))
-                                    el.appendChild(el.createEl("button", { text: "Database -> Storage" }, buttonEl => {
-                                        buttonEl.onClickEvent(() => {
-                                            this.plugin.pullFile(file.path as FilePathWithPrefix, [], true, undefined, false);
-                                            el.remove();
-                                        })
-                                    }))
-                                    return el;
-                                }))
                             }
                         }
                         Logger("done", LOG_LEVEL_NOTICE, "verify");

updates.md

@@ -10,6 +10,15 @@ Note: we got a very performance improvement.
 Note at 0.22.2: **Now, to rescue mobile devices, Maximum file size is set to 50 by default**. Please configure the limit as you need. If you do not want to limit the sizes, set zero manually, please.
 
 #### Version history
+- 0.22.11:
+  - Fixed:
+    - `Verify and repair all files` is no longer broken.
+  - New feature:
+    - Now `Verify and repair all files` is able to...
+      - Restore if the file only in the local database.
+      - Show the history.
+  - Improved:
+    - Performance improved.
 - 0.22.10
   - Fixed:
     - No longer unchanged hidden files and customisations are saved and transferred now.
@@ -64,80 +73,5 @@ Note at 0.22.2: **Now, to rescue mobile devices, Maximum file size is set to 50
     - `Copy setup URI` is now `Copy current settings as a new setup URI`
     - `Setup Wizard` is now `Minimal Setup`
     - `Check database configuration` is now `Check and Fix database configuration`
-- 0.22.5
-  - Fixed:
-    - Some description of settings have been refined
-  - New feature:
-    - TroubleShooting is now shown in the setting dialogue.
-- 0.22.4
-  - Fixed:
-    - Now the result of conflict resolution could be surely written into the storage.
-    - Deleted files can be handled correctly again in the history dialogue and conflict dialogue.
-    - Some wrong log messages were fixed.
-    - Change handling now has become more stable.
-    - Some event handling became to be safer.
-  - Improved:
-    - Dumping document information shows conflicts and revisions.
-    - The timestamp-only differences can be surely cached.
-    - Timestamp difference detection can be rounded by two seconds.
-  - Refactored:
-    - A bit of organisation to write the test.
-- 0.22.3
-  - Fixed:
-    - No longer detects storage changes which have been caused by Self-hosted LiveSync itself.
-    - Setting sync file will be detected only if it has been configured now.
-      - And its log will be shown only while the verbose log is enabled.
-    - Customisation file enumeration has got less blingy.
-    - Deletion of files is now reliably synchronised.
-  - Fixed and improved:
-    - In-editor-status is now shown in the following areas:
-      - Note editing pane (Source mode and live-preview mode).
-      - New tab pane.
-      - Canvas pane.
-- 0.22.2
-  - Fixed:
-    - Now the results of resolving conflicts are surely synchronised.
-  - Modified:
-    - Some setting items got new clear names. (`Sync Settings` -> `Targets`).
-  - New feature:
-    - We can limit the synchronising files by their size. (`Sync Settings` -> `Targets` -> `Maximum file size`).
-      - It depends on the size of the newer one.
-      - At Obsidian 1.5.3 on mobile, we should set this to around 50MB to avoid restarting Obsidian.
-    - Now the settings could be stored in a specific markdown file to synchronise or switch it (`General Setting` -> `Share settings via markdown`).
-      - [Screwdriver](https://github.com/vrtmrz/obsidian-screwdriver) is quite good, but mostly we only need this.
-    - Customisation of the obsoleted device is now able to be deleted at once.
-      - We have to put the maintenance mode in at the Customisation sync dialogue.
-- 0.22.1
-  - New feature:
-    - We can perform automatic conflict resolution for inactive files, and postpone only manual ones by `Postpone manual resolution of inactive files`.
-    - Now we can see the image in the document history dialogue.
-      - We can see the difference of the image, in the document history dialogue.
-        - And also we can highlight differences.
-  - Improved:
-    - Hidden file sync has been stabilised.
-    - Now automatically reloads the conflict-resolution dialogue when new conflicted revisions have arrived.
-  - Fixed:
-    - No longer periodic process runs after unloading the plug-in.
-    - Now the modification of binary files is surely stored in the storage.
-- 0.22.0
-  - Refined:
-    - Task scheduling logics has been rewritten.
-    - Screen updates are also now efficient.
-    - Possibly many bugs and fragile behaviour has been fixed.
-    - Status updates and logging have been thinned out to display.
-  - Fixed:
-    - Remote-chunk-fetching now works with keeping request intervals
-  - New feature:
-    - We can show only the icons in the editor.
-    - Progress indicators have been more meaningful:
-      -   📥 Unprocessed transferred items
-      -   📄 Working database operation
-      -   💾 Working write storage processes
-      -   ⏳ Working read storage processes
-      -   🛫 Pending read storage processes
-      -   ⚙️ Working or pending storage processes of hidden files
-      -   🧩 Waiting chunks
-      -   🔌 Working Customisation items (Configuration, snippets and plug-ins)
-
 
 ... To continue on to `updates_old.md`.

updates_old.md

@@ -1,3 +1,93 @@
+### 0.22.0
+A few years passed since Self-hosted LiveSync was born, and our codebase had been very complicated. This could be patient now, but it should be a tremendous hurt.
+Therefore at v0.22.0, for future maintainability, I refined task scheduling logic totally.
+
+Of course, I think this would be our suffering in some cases. However, I would love to ask you for your cooperation and contribution.
+
+Sorry for being absent so much long. And thank you for your patience!
+
+Note: we got a very performance improvement.
+Note at 0.22.2: **Now, to rescue mobile devices, Maximum file size is set to 50 by default**. Please configure the limit as you need. If you do not want to limit the sizes, set zero manually, please.
+
+#### Version history
+- 0.22.5
+  - Fixed:
+    - Some description of settings have been refined
+  - New feature:
+    - TroubleShooting is now shown in the setting dialogue.
+- 0.22.4
+  - Fixed:
+    - Now the result of conflict resolution could be surely written into the storage.
+    - Deleted files can be handled correctly again in the history dialogue and conflict dialogue.
+    - Some wrong log messages were fixed.
+    - Change handling now has become more stable.
+    - Some event handling became to be safer.
+  - Improved:
+    - Dumping document information shows conflicts and revisions.
+    - The timestamp-only differences can be surely cached.
+    - Timestamp difference detection can be rounded by two seconds.
+  - Refactored:
+    - A bit of organisation to write the test.
+- 0.22.3
+  - Fixed:
+    - No longer detects storage changes which have been caused by Self-hosted LiveSync itself.
+    - Setting sync file will be detected only if it has been configured now.
+      - And its log will be shown only while the verbose log is enabled.
+    - Customisation file enumeration has got less blingy.
+    - Deletion of files is now reliably synchronised.
+  - Fixed and improved:
+    - In-editor-status is now shown in the following areas:
+      - Note editing pane (Source mode and live-preview mode).
+      - New tab pane.
+      - Canvas pane.
+- 0.22.2
+  - Fixed:
+    - Now the results of resolving conflicts are surely synchronised.
+  - Modified:
+    - Some setting items got new clear names. (`Sync Settings` -> `Targets`).
+  - New feature:
+    - We can limit the synchronising files by their size. (`Sync Settings` -> `Targets` -> `Maximum file size`).
+      - It depends on the size of the newer one.
+      - At Obsidian 1.5.3 on mobile, we should set this to around 50MB to avoid restarting Obsidian.
+    - Now the settings could be stored in a specific markdown file to synchronise or switch it (`General Setting` -> `Share settings via markdown`).
+      - [Screwdriver](https://github.com/vrtmrz/obsidian-screwdriver) is quite good, but mostly we only need this.
+    - Customisation of the obsoleted device is now able to be deleted at once.
+      - We have to put the maintenance mode in at the Customisation sync dialogue.
+- 0.22.1
+  - New feature:
+    - We can perform automatic conflict resolution for inactive files, and postpone only manual ones by `Postpone manual resolution of inactive files`.
+    - Now we can see the image in the document history dialogue.
+      - We can see the difference of the image, in the document history dialogue.
+        - And also we can highlight differences.
+  - Improved:
+    - Hidden file sync has been stabilised.
+    - Now automatically reloads the conflict-resolution dialogue when new conflicted revisions have arrived.
+  - Fixed:
+    - No longer periodic process runs after unloading the plug-in.
+    - Now the modification of binary files is surely stored in the storage.
+- 0.22.0
+  - Refined:
+    - Task scheduling logics has been rewritten.
+    - Screen updates are also now efficient.
+    - Possibly many bugs and fragile behaviour has been fixed.
+    - Status updates and logging have been thinned out to display.
+  - Fixed:
+    - Remote-chunk-fetching now works with keeping request intervals
+  - New feature:
+    - We can show only the icons in the editor.
+    - Progress indicators have been more meaningful:
+      -   📥 Unprocessed transferred items
+      -   📄 Working database operation
+      -   💾 Working write storage processes
+      -   ⏳ Working read storage processes
+      -   🛫 Pending read storage processes
+      -   ⚙️ Working or pending storage processes of hidden files
+      -   🧩 Waiting chunks
+      -   🔌 Working Customisation items (Configuration, snippets and plug-ins)
+
+
+... To continue on to `updates_old.md`.
+
 ### 0.21.0
 The E2EE encryption V2 format has been reverted. That was probably the cause of the glitch.
 Instead, to maintain efficiency, files are treated with Blob until just before saving. Along with this, the old-fashioned encryption format has also been discontinued.