bump

- Hashing ChunkID has been improved.
- Logging keeps 400 lines now.

Refactored:
- Import statement has been fixed about types.

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.19.9",
+    "version": "0.19.11",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package-lock.json

@@ -1,17 +1,18 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.19.9",
+    "version": "0.19.11",
     "lockfileVersion": 2,
     "requires": true,
     "packages": {
         "": {
             "name": "obsidian-livesync",
-            "version": "0.19.9",
+            "version": "0.19.11",
             "license": "MIT",
             "dependencies": {
                 "diff-match-patch": "^1.0.5",
                 "idb": "^7.1.1",
-                "xxhash-wasm": "^0.4.2"
+                "xxhash-wasm": "^0.4.2",
+                "xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
             },
             "devDependencies": {
                 "@tsconfig/svelte": "^4.0.1",
@@ -4076,6 +4077,12 @@
             "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz",
             "integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA=="
         },
+        "node_modules/xxhash-wasm-102": {
+            "name": "xxhash-wasm",
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.0.2.tgz",
+            "integrity": "sha512-ibF0Or+FivM9lNrg+HGJfVX8WJqgo+kCLDc4vx6xMeTce7Aj+DLttKbxxRR/gNLSAelRc1omAPlJ77N/Jem07A=="
+        },
         "node_modules/yallist": {
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -7032,6 +7039,11 @@
             "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz",
             "integrity": "sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA=="
         },
+        "xxhash-wasm-102": {
+            "version": "npm:xxhash-wasm@1.0.2",
+            "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.0.2.tgz",
+            "integrity": "sha512-ibF0Or+FivM9lNrg+HGJfVX8WJqgo+kCLDc4vx6xMeTce7Aj+DLttKbxxRR/gNLSAelRc1omAPlJ77N/Jem07A=="
+        },
         "yallist": {
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.19.9",
+    "version": "0.19.11",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",
@@ -47,6 +47,7 @@
     "dependencies": {
         "diff-match-patch": "^1.0.5",
         "idb": "^7.1.1",
-        "xxhash-wasm": "^0.4.2"
+        "xxhash-wasm": "^0.4.2",
+        "xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
     }
 }

src/CmdConfigSync.ts

@@ -7,7 +7,7 @@ import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "./types";
 import { Parallels, delay, getDocData } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { WrappedNotice } from "./lib/src/wrapper";
-import { base64ToArrayBuffer, arrayBufferToBase64, readString, uint8ArrayToHexString } from "./lib/src/strbin";
+import { base64ToArrayBuffer, arrayBufferToBase64, readString, crc32CKHash } from "./lib/src/strbin";
 import { runWithLock } from "./lib/src/lock";
 import { LiveSyncCommands } from "./LiveSyncCommands";
 import { stripAllPrefixes } from "./lib/src/path";
@@ -35,14 +35,6 @@ function deserialize<T>(str: string, def: T) {
 export const pluginList = writable([] as PluginDataExDisplay[]);
 export const pluginIsEnumerating = writable(false);
 
-const encoder = new TextEncoder();
-const hashString = (async (key: string) => {
-    // const buff = writeString(key);
-    const buff = encoder.encode(key);
-    const digest = await crypto.subtle.digest('SHA-256', buff);
-    return uint8ArrayToHexString(new Uint8Array(digest));
-})
-
 export type PluginDataExFile = {
     filename: string,
     data?: string[],
@@ -209,7 +201,7 @@ export class ConfigSync extends LiveSyncCommands {
                                     for (const file of data.files) {
                                         const work = { ...file };
                                         const tempStr = getDocData(work.data);
-                                        work.data = [await hashString(tempStr)];
+                                        work.data = [crc32CKHash(tempStr)];
                                         xFiles.push(work);
                                     }
                                     entries.push({

src/CmdHiddenFileSync.ts

@@ -1,6 +1,6 @@
-import { Notice, normalizePath, PluginManifest } from "./deps";
-import { EntryDoc, LoadedEntry, LOG_LEVEL, InternalFileEntry, FilePathWithPrefix, FilePath } from "./lib/src/types";
-import { InternalFileInfo, ICHeader, ICHeaderEnd } from "./types";
+import { Notice, normalizePath, type PluginManifest } from "./deps";
+import { type EntryDoc, type LoadedEntry, LOG_LEVEL, type InternalFileEntry, type FilePathWithPrefix, type FilePath } from "./lib/src/types";
+import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "./types";
 import { Parallels, delay, isDocContentSame } from "./lib/src/utils";
 import { Logger } from "./lib/src/logger";
 import { PouchDB } from "./lib/src/pouchdb-browser.js";

src/DocumentHistoryModal.ts

@@ -1,8 +1,7 @@
-import { TFile, Modal, App } from "./deps";
+import { TFile, Modal, App, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps";
 import { getPathFromTFile, isValidPath } from "./utils";
 import { base64ToArrayBuffer, base64ToString, escapeStringToHTML } from "./lib/src/strbin";
 import ObsidianLiveSyncPlugin from "./main";
-import { DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
 import { type DocumentID, type FilePathWithPrefix, type LoadedEntry, LOG_LEVEL } from "./lib/src/types";
 import { Logger } from "./lib/src/logger";
 import { isErrorOfMissingDoc } from "./lib/src/utils_couchdb";

src/GlobalHistory.svelte

@@ -3,7 +3,7 @@
     import { onDestroy, onMount } from "svelte";
     import type { AnyEntry, FilePathWithPrefix } from "./lib/src/types";
     import { getDocData, isDocContentSame } from "./lib/src/utils";
-    import { diff_match_patch } from "diff-match-patch";
+    import { diff_match_patch } from "./deps";
     import { DocumentHistoryModal } from "./DocumentHistoryModal";
     import { isPlainText, stripAllPrefixes } from "./lib/src/path";
     import { TFile } from "./deps";

src/JsonResolvePane.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-    import { type Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
+    import { type Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "./deps";
     import type { FilePath, LoadedEntry } from "./lib/src/types";
     import { base64ToString } from "./lib/src/strbin";
     import { getDocData } from "./lib/src/utils";

src/ObsidianLiveSyncSettingTab.ts

@@ -1,5 +1,5 @@
 import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, TextAreaComponent, MarkdownRenderer, stringifyYaml } from "./deps";
-import { DEFAULT_SETTINGS, LOG_LEVEL, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix } from "./lib/src/types";
+import { DEFAULT_SETTINGS, LOG_LEVEL, type ObsidianLiveSyncSettings, type ConfigPassphraseStore, type RemoteDBSettings, type FilePathWithPrefix, type HashAlgorithm, type DocumentID } from "./lib/src/types";
 import { delay } from "./lib/src/utils";
 import { Semaphore } from "./lib/src/semaphore";
 import { versionNumberString2Number } from "./lib/src/strbin";
@@ -1548,22 +1548,29 @@ ${stringifyYaml(pluginConfig)}`;
                         for await (const docName of this.plugin.localDatabase.findAllDocNames()) {
                             if (!docName.startsWith("f:")) {
                                 const idEncoded = await this.plugin.path2id(docName as FilePathWithPrefix);
-                                const doc = await this.plugin.localDatabase.getDBEntry(docName as FilePathWithPrefix);
+                                const doc = await this.plugin.localDatabase.getRaw(docName as DocumentID);
                                 if (!doc) continue;
                                 if (doc.type != "newnote" && doc.type != "plain") {
                                     continue;
                                 }
+                                if (doc?.deleted ?? false) continue;
                                 const newDoc = { ...doc };
                                 //Prepare converted data
                                 newDoc._id = idEncoded;
-                                newDoc.path = this.plugin.getPath(newDoc);
+                                newDoc.path = docName as FilePathWithPrefix;
+                                delete newDoc._rev;
                                 try {
                                     const obfuscatedDoc = await this.plugin.localDatabase.getRaw(idEncoded, { revs_info: true });
                                     // Unfortunately we have to delete one of them.
                                     // Just now, save it as a conflicted document.
                                     obfuscatedDoc._revs_info?.shift(); // Drop latest revision.
                                     const previousRev = obfuscatedDoc._revs_info?.shift(); // Use second revision.
-                                    newDoc._rev = previousRev.rev;
+                                    if (previousRev) {
+                                        newDoc._rev = previousRev.rev;
+                                    } else {
+                                        //If there are no revisions, set the possibly unique one
+                                        newDoc._rev = "1-" + (`00000000000000000000000000000000${~~(Math.random() * 1e9)}${~~(Math.random() * 1e9)}${~~(Math.random() * 1e9)}${~~(Math.random() * 1e9)}`.slice(-32));
+                                    }
                                     const ret = await this.plugin.localDatabase.putRaw(newDoc, { force: true });
                                     if (ret.ok) {
                                         Logger(`${docName} has been converted as conflicted document`, LOG_LEVEL.NOTICE);
@@ -1594,6 +1601,7 @@ ${stringifyYaml(pluginConfig)}`;
                                 }
                             }
                         }
+                        Logger(`Converting finished`, LOG_LEVEL.NOTICE);
                     }));
         new Setting(containerHatchEl)
             .setName("Suspend file watching")
@@ -1707,6 +1715,22 @@ ${stringifyYaml(pluginConfig)}`;
                         await this.plugin.initializeDatabase();
                     })
             })
+
+        new Setting(containerHatchEl)
+            .setName("The Hash algorithm for chunk IDs")
+            .setDesc("xxhash64 is the current default.")
+            .setClass("wizardHidden")
+            .addDropdown((dropdown) =>
+                dropdown
+                    .addOptions({ "": "Old Algorithm", "xxhash32": "xxhash32 (Fast)", "xxhash64": "xxhash64 (Fastest)" } as Record<HashAlgorithm, string>)
+                    .setValue(this.plugin.settings.hashAlg)
+                    .onChange(async (value: HashAlgorithm) => {
+                        this.plugin.settings.hashAlg = value;
+                        await this.plugin.saveSettings();
+                    })
+            )
+            .setClass("wizardHidden");
+
         addScreenElement("50", containerHatchEl);
 
 

src/deps.ts

@@ -10,3 +10,4 @@ import {
 } from "obsidian";
 const normalizePath = normalizePath_ as <T extends string | FilePath>(from: T) => T;
 export { normalizePath }
+export { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";

src/main.ts

@@ -1,6 +1,6 @@
 const isDebug = false;
 
-import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "diff-match-patch";
+import { type Diff, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "./deps";
 import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, type RequestUrlParam, type RequestUrlResponse, requestUrl } from "./deps";
 import { type EntryDoc, type LoadedEntry, type ObsidianLiveSyncSettings, type diff_check_result, type diff_result_leaf, type EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, type diff_result, FLAGMD_REDFLAG, SYNCINFO_ID, SALT_OF_PASSPHRASE, type ConfigPassphraseStore, type CouchDBConnection, FLAGMD_REDFLAG2, FLAGMD_REDFLAG3, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type EntryHasPath, type DocumentID, type FilePathWithPrefix, type FilePath, type AnyEntry } from "./lib/src/types";
 import { type InternalFileInfo, type queueItem, type CacheData, type FileEventItem, FileWatchEventQueueMax } from "./types";

tsconfig.json

@@ -21,7 +21,8 @@
             "ES5",
             "ES6",
             "ES7",
-            "es2019.array"
+            "es2019.array",
+            "ES2020.BigInt",
         ]
     },
     "include": [

updates.md

@@ -14,50 +14,7 @@ I hope you will give it a try.
 
 #### Minors
 
-- 0.19.1
-  - Fixed: Fixed hidden file handling on Linux
-  - Improved: Now customization sync works more smoothly.
-- 0.19.2
-  - Fixed:
-    - Fixed garbage collection error while unreferenced chunks exist many.
-    - Fixed filename validation on Linux.
-  - Improved:
-    - Showing status is now thinned for performance.
-    - Enhance caching while collecting chunks.
-- 0.19.3
-  - Improved:
-    - Now replication will be paced by collecting chunks. If synchronisation has been deadlocked, please enable `Do not pace synchronization` once.
-- 0.19.4
-  - Improved:
-    - Reduced remote database checking to improve speed and reduce bandwidth.
-  - Fixed:
-    - Chunks which previously misinterpreted are now interpreted correctly.
-      - No more missing chunks which not be found forever, except if it has been actually missing.
-    - Deleted file detection on hidden file synchronising now works fine.
-    - Now the Customisation sync is surely quiet while it has been disabled.
-- 0.19.5
-  - Fixed:
-    - Now hidden file synchronisation would not be hanged, even if so many files exist.
-  - Improved:
-    - Customisation sync works more smoothly.
-  - Note: Concurrent processing has been rollbacked into the original implementation. As a result, the total number of processes is no longer shown next to the hourglass icon. However, only the processes that are running concurrently are shown.
-- 0.19.6
-  - Fixed:
-    - Logging has been tweaked.
-    - No more too many planes and rockets.
-    - The batch database update now surely only works in non-live mode.
-  - Internal things:
-    - Some frameworks has been upgraded.
-    - Import declaration has been fixed.
-  - Improved:
-    - The plug-in now asks to enable a new adaptor, when rebuilding, if it is not enabled yet.
-    - The setting dialogue refined.
-      - Configurations for compatibilities have been moved under the hatch.
-      - Made it clear that disabled is the default.
-      - Ambiguous names configuration have been renamed.
-      - Items that have no meaning in the settings are no longer displayed.
-      - Some items have been reordered for clarity.
-      - Each configuration has been grouped.
+- 0.19.1 to 0.19.6 has been moved into the updates_old.md
 - 0.19.7
   - Fixed:
     - The initial pane of Setting dialogue is now changed to General Settings.
@@ -75,5 +32,14 @@ I hope you will give it a try.
     - We can fix the database obfuscated and plain paths that have been mixed up.
   - Improvements
     - Customisation Sync performance has been improved.
+- 0.19.10
+  - Fixed
+    - Fixed the issue about fixing the database.
+- 0.19.11
+  - Improvements:
+    - Hashing ChunkID has been improved.
+    - Logging keeps 400 lines now.
+  - Refactored:
+    - Import statement has been fixed about types.
 
 ... To continue on to `updates_old.md`.

updates_old.md

@@ -1,3 +1,61 @@
+### 0.19.0
+
+#### Customization sync
+
+Since `Plugin and their settings` have been broken, so I tried to fix it, not just fix it, but fix it the way it should be.
+
+Now, we have `Customization sync`.
+
+It is a real shame that the compatibility between these features has been broken. However, this new feature is surely useful and I believe that worth getting over the pain.
+We can use the new feature with the same configuration. Only the menu on the command palette has been changed. The dialog can be opened by `Show customization sync dialog`.
+
+I hope you will give it a try.
+
+#### Minors
+- 0.19.1
+  - Fixed: Fixed hidden file handling on Linux
+  - Improved: Now customization sync works more smoothly.
+- 0.19.2
+  - Fixed:
+    - Fixed garbage collection error while unreferenced chunks exist many.
+    - Fixed filename validation on Linux.
+  - Improved:
+    - Showing status is now thinned for performance.
+    - Enhance caching while collecting chunks.
+- 0.19.3
+  - Improved:
+    - Now replication will be paced by collecting chunks. If synchronisation has been deadlocked, please enable `Do not pace synchronization` once.
+- 0.19.4
+  - Improved:
+    - Reduced remote database checking to improve speed and reduce bandwidth.
+  - Fixed:
+    - Chunks which previously misinterpreted are now interpreted correctly.
+      - No more missing chunks which not be found forever, except if it has been actually missing.
+    - Deleted file detection on hidden file synchronising now works fine.
+    - Now the Customisation sync is surely quiet while it has been disabled.
+- 0.19.5
+  - Fixed:
+    - Now hidden file synchronisation would not be hanged, even if so many files exist.
+  - Improved:
+    - Customisation sync works more smoothly.
+  - Note: Concurrent processing has been rollbacked into the original implementation. As a result, the total number of processes is no longer shown next to the hourglass icon. However, only the processes that are running concurrently are shown.
+- 0.19.6
+  - Fixed:
+    - Logging has been tweaked.
+    - No more too many planes and rockets.
+    - The batch database update now surely only works in non-live mode.
+  - Internal things:
+    - Some frameworks has been upgraded.
+    - Import declaration has been fixed.
+  - Improved:
+    - The plug-in now asks to enable a new adaptor, when rebuilding, if it is not enabled yet.
+    - The setting dialogue refined.
+      - Configurations for compatibilities have been moved under the hatch.
+      - Made it clear that disabled is the default.
+      - Ambiguous names configuration have been renamed.
+      - Items that have no meaning in the settings are no longer displayed.
+      - Some items have been reordered for clarity.
+      - Each configuration has been grouped.
 
 ### 0.18.0