WIP

Fix check warning for max_document_size, max_http_request_size

docs/setup_own_server.md

@@ -10,9 +10,11 @@ But some additional configurations are required in `local.ini` to use from Self-
 ```
 [couchdb]
 single_node=true
+max_document_size = 50000000
 
 [chttpd]
 require_valid_user = true
+max_http_request_size = 4294967296
 
 [chttpd_auth]
 require_valid_user = true
@@ -94,4 +96,4 @@ Using Caddy is a handy way to serve the server with SSL automatically.
 
 I have published [docker-compose.yml and ini files](https://github.com/vrtmrz/self-hosted-livesync-server) that launches Caddy and CouchDB at once. Please try it out.
 
-And, be sure to check the server log and be careful of malicious access.
+And, be sure to check the server log and be careful of malicious access.

docs/setup_own_server_cn.md

@@ -11,9 +11,11 @@
 ```
 [couchdb]
 single_node=true
+max_document_size = 50000000
 
 [chttpd]
 require_valid_user = true
+max_http_request_size = 4294967296
 
 [chttpd_auth]
 require_valid_user = true
@@ -92,4 +94,4 @@ Note: 不推荐将 CouchDB 挂载到根目录
 
 提供了 [docker-compose.yml 和 ini 文件](https://github.com/vrtmrz/self-hosted-livesync-server) 可以同时启动 Caddy 和 CouchDB。
 
-注意检查服务器日志，当心恶意访问。
+注意检查服务器日志，当心恶意访问。

docs/setup_own_server_ja.md

@@ -8,12 +8,14 @@ CouchDBを構築するには、[Dockerのイメージ](https://hub.docker.com/_/
 ```
 [couchdb]
 single_node=true
+max_document_size = 50000000
 
 [chttpd]
 require_valid_user = true
 
 [chttpd_auth]
 require_valid_user = true
+max_http_request_size = 4294967296
 authentication_redirect = /_utils/session.html
 
 [httpd]

esbuild.config.mjs

@@ -13,7 +13,7 @@ if you want to view the source, please visit the github repository of this plugi
 const prod = process.argv[2] === "production";
 const manifestJson = JSON.parse(fs.readFileSync("./manifest.json"));
 const packageJson = JSON.parse(fs.readFileSync("./package.json"));
-const updateInfo = JSON.stringify(fs.readFileSync("./updates.md") + "");
+const updateInfo = JSON.stringify("PATCHED-"+fs.readFileSync("./updates.md") + "");
 esbuild
     .build({
         banner: {

manifest.json

@@ -1,7 +1,7 @@
 {
     "id": "obsidian-livesync",
     "name": "Self-hosted LiveSync",
-    "version": "0.16.5",
+    "version": "0.16.8",
     "minAppVersion": "0.9.12",
     "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "author": "vorotamoroz",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "obsidian-livesync",
-    "version": "0.16.5",
+    "version": "0.16.8",
     "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
     "main": "main.js",
     "type": "module",
@@ -13,34 +13,32 @@
     "author": "vorotamoroz",
     "license": "MIT",
     "devDependencies": {
-        "@rollup/plugin-commonjs": "^18.0.0",
-        "@rollup/plugin-node-resolve": "^11.2.1",
-        "@rollup/plugin-typescript": "^8.2.1",
         "@types/diff-match-patch": "^1.0.32",
         "@types/pouchdb": "^6.4.0",
         "@types/pouchdb-browser": "^6.1.3",
-        "@typescript-eslint/eslint-plugin": "^5.7.0",
-        "@typescript-eslint/parser": "^5.0.0",
-        "builtin-modules": "^3.2.0",
-        "esbuild": "0.13.12",
-        "esbuild-svelte": "^0.7.0",
-        "eslint": "^7.32.0",
-        "eslint-config-airbnb-base": "^14.2.1",
-        "eslint-plugin-import": "^2.25.2",
+        "@types/xxhashjs": "^0.2.2",
+        "@typescript-eslint/eslint-plugin": "^5.44.0",
+        "@typescript-eslint/parser": "^5.44.0",
+        "builtin-modules": "^3.3.0",
+        "esbuild": "0.15.15",
+        "esbuild-svelte": "^0.7.3",
+        "eslint": "^8.28.0",
+        "eslint-config-airbnb-base": "^15.0.0",
+        "eslint-plugin-import": "^2.26.0",
         "obsidian": "^0.16.3",
-        "postcss": "^8.4.14",
-        "postcss-load-config": "^3.1.4",
-        "rollup": "^2.32.1",
-        "svelte": "^3.49.0",
+        "postcss": "^8.4.19",
+        "postcss-load-config": "^4.0.1",
+        "svelte": "^3.53.1",
         "svelte-preprocess": "^4.10.7",
-        "tslib": "^2.2.0",
-        "typescript": "^4.2.4"
+        "tslib": "^2.4.1",
+        "typescript": "^4.9.3"
     },
     "dependencies": {
         "diff-match-patch": "^1.0.5",
-        "esbuild": "0.13.12",
-        "esbuild-svelte": "^0.7.0",
-        "idb": "^7.0.2",
-        "xxhash-wasm": "^0.4.2"
+        "esbuild": "0.15.15",
+        "esbuild-svelte": "^0.7.3",
+        "idb": "^7.1.1",
+        "xxhash-wasm": "^0.4.2",
+        "xxhashjs": "^0.2.2"
     }
-}
+}

rollup.config.js

@@ -1,31 +0,0 @@
-import typescript from "@rollup/plugin-typescript";
-import { nodeResolve } from "@rollup/plugin-node-resolve";
-import commonjs from "@rollup/plugin-commonjs";
-
-const isProd = process.env.BUILD === "production";
-
-const banner = `/*
-THIS IS A GENERATED/BUNDLED FILE BY ROLLUP
-if you want to view the source visit the plugins github repository
-*/
-`;
-
-export default {
-    input: "./src/main.ts",
-    output: {
-        dir: ".",
-        sourcemap: "inline",
-        sourcemapExcludeSources: isProd,
-        format: "cjs",
-        exports: "default",
-        banner,
-    },
-    external: ["obsidian"],
-    plugins: [
-        typescript({ exclude: ["pouchdb-browser.js", "pouchdb-browser-webpack"] }),
-        nodeResolve({
-            browser: true,
-        }),
-        commonjs(),
-    ],
-};

src/LocalPouchDB.ts

@@ -7,6 +7,7 @@ import { EntryDoc, LOG_LEVEL } from "./lib/src/types.js";
 import { enableEncryption } from "./lib/src/utils.js";
 import { isCloudantURI, isValidRemoteCouchDBURI } from "./lib/src/utils_couchdb.js";
 import { id2path, path2id } from "./utils.js";
+import XXH from "xxhashjs";
 
 export class LocalPouchDB extends LocalPouchDBBase {
 
@@ -33,9 +34,16 @@ export class LocalPouchDB extends LocalPouchDBBase {
         await this.kvDB.destroy();
     }
 
+    async prepareHashFunctions() {
+        if (this.h32 != null) return;
+        // const { h32, h32Raw } = await xxhash();
+        this.h32 = (input: string, seed: number) => (XXH.h32(input, seed).toString(16))// h32;
+        this.h32Raw = (input: Uint8Array, seed: number) => (XXH.h32(input.buffer, seed).toNumber())// h32;
+    }
+
     last_successful_post = false;
     getLastPostFailedBySize() {
-        return this.last_successful_post;
+        return !this.last_successful_post;
     }
     async fetchByAPI(request: RequestUrlParam): Promise<RequestUrlResponse> {
         const ret = await requestUrl(request);
@@ -75,7 +83,7 @@ export class LocalPouchDB extends LocalPouchDBBase {
                 const method = opts.method ?? "GET";
                 if (opts.body) {
                     const opts_length = opts.body.toString().length;
-                    if (opts_length > 1024 * 1024 * 10) {
+                    if (opts_length > 1000 * 1000 * 10) {
                         // over 10MB
                         if (isCloudantURI(uri)) {
                             this.last_successful_post = false;

src/main.ts

@@ -44,6 +44,16 @@ const ICHeaderEnd = "i;";
 const ICHeaderLength = ICHeader.length;
 const FileWatchEventQueueMax = 10;
 
+function getAbstractFileByPath(path: string): TAbstractFile | null {
+    // Hidden API but so useful.
+    if ("getAbstractFileByPathInsensitive" in app.vault && (app.vault.adapter?.insensitive ?? false)) {
+        // @ts-ignore
+        return app.vault.getAbstractFileByPathInsensitive(path);
+    } else {
+        return app.vault.getAbstractFileByPath(path);
+    }
+}
+
 /**
  * returns is internal chunk of file
  * @param str ID
@@ -97,7 +107,7 @@ const askString = (app: App, title: string, key: string, placeholder: string): P
 };
 let touchedFiles: string[] = [];
 function touch(file: TFile | string) {
-    const f = file instanceof TFile ? file : app.vault.getAbstractFileByPath(file) as TFile;
+    const f = file instanceof TFile ? file : getAbstractFileByPath(file) as TFile;
     const key = `${f.path}-${f.stat.mtime}-${f.stat.size}`;
     touchedFiles.unshift(key);
     touchedFiles = touchedFiles.slice(0, 100);
@@ -147,7 +157,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     isRedFlagRaised(): boolean {
-        const redflag = this.app.vault.getAbstractFileByPath(normalizePath(FLAGMD_REDFLAG));
+        const redflag = getAbstractFileByPath(normalizePath(FLAGMD_REDFLAG));
         if (redflag != null) {
             return true;
         }
@@ -1050,7 +1060,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             for (const i of newFiles) {
                 try {
                     const newFilePath = normalizePath(this.getFilePath(i));
-                    const newFile = this.app.vault.getAbstractFileByPath(newFilePath);
+                    const newFile = getAbstractFileByPath(newFilePath);
                     if (newFile instanceof TFile) {
                         Logger(`save ${newFile.path} into db`);
                         await this.updateIntoDB(newFile);
@@ -1278,7 +1288,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                         await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
                         // this.batchFileChange = this.batchFileChange.filter((e) => e != file.path);
                         Logger(msg + path);
-                        const xf = this.app.vault.getAbstractFileByPath(file.path) as TFile;
+                        const xf = getAbstractFileByPath(file.path) as TFile;
                         touch(xf);
                         this.app.vault.trigger("modify", xf);
                     } catch (ex) {
@@ -1295,7 +1305,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                     await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
                     Logger(msg + path);
                     // this.batchFileChange = this.batchFileChange.filter((e) => e != file.path);
-                    const xf = this.app.vault.getAbstractFileByPath(file.path) as TFile;
+                    const xf = getAbstractFileByPath(file.path) as TFile;
                     touch(xf);
                     this.app.vault.trigger("modify", xf);
                 } catch (ex) {
@@ -1345,7 +1355,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
     async handleDBChangedAsync(change: EntryBody) {
 
-        const targetFile = this.app.vault.getAbstractFileByPath(id2path(change._id));
+        const targetFile = getAbstractFileByPath(id2path(change._id));
         if (targetFile == null) {
             if (change._deleted || change.deleted) {
                 return;
@@ -1461,7 +1471,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
         if (!this.isTargetFile(id2path(doc._id))) return;
         const skipOldFile = this.settings.skipOlderFilesOnSync && false; //patched temporary.
         if ((!isInternalChunk(doc._id)) && skipOldFile) {
-            const info = this.app.vault.getAbstractFileByPath(id2path(doc._id));
+            const info = getAbstractFileByPath(id2path(doc._id));
 
             if (info && info instanceof TFile) {
                 const localMtime = ~~((info as TFile).stat.mtime / 1000);
@@ -2027,7 +2037,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
                     const p = conflictCheckResult.diff.map((e) => e[1]).join("");
                     await this.localDatabase.deleteDBEntry(filename, { rev: conflictCheckResult.left.rev });
                     await this.localDatabase.deleteDBEntry(filename, { rev: conflictCheckResult.right.rev });
-                    const file = this.app.vault.getAbstractFileByPath(filename) as TFile;
+                    const file = getAbstractFileByPath(filename) as TFile;
                     if (file) {
                         await this.app.vault.modify(file, p);
                         await this.updateIntoDB(file);
@@ -2074,7 +2084,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             const checkFiles = JSON.parse(JSON.stringify(this.conflictedCheckFiles)) as string[];
             for (const filename of checkFiles) {
                 try {
-                    const file = this.app.vault.getAbstractFileByPath(filename);
+                    const file = getAbstractFileByPath(filename);
                     if (file != null && file instanceof TFile) {
                         await this.showIfConflicted(file.path);
                     }
@@ -2106,7 +2116,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
     }
 
     async pullFile(filename: string, fileList?: TFile[], force?: boolean, rev?: string, waitForReady = true) {
-        const targetFile = this.app.vault.getAbstractFileByPath(id2path(filename));
+        const targetFile = getAbstractFileByPath(id2path(filename));
         if (!this.isTargetFile(id2path(filename))) return;
         if (targetFile == null) {
             //have to create;
@@ -2137,7 +2147,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
             throw new Error(`Missing doc:${(file as any).path}`)
         }
         if (!(file instanceof TFile) && "path" in file) {
-            const w = this.app.vault.getAbstractFileByPath((file as any).path);
+            const w = getAbstractFileByPath((file as any).path);
             if (w instanceof TFile) {
                 file = w;
             } else {

updates.md

@@ -19,6 +19,18 @@
   - New Feature
     - The feature of automatically deleting old expired metadata has been implemented.
     We can configure it in `Delete old metadata of deleted files on start-up` in the `General Settings` pane.
+- 0.16.6
+  - Fixed
+    - Automatic (temporary) batch size adjustment has been restored to work correctly.
+    - Chunk splitting has been backed to the previous behaviour for saving them correctly.
+  - Improved
+    - Corrupted chunks will be detected automatically.
+    - Now on the case-insensitive system, `aaa.md` and `AAA.md` will be treated as the same file or path at applying changesets.
+- 0.16.7 Nothing has been changed except toolsets, framework library, and as like them. Please inform me if something had been getting strange!
+- 0.16.8 Now we can synchronise without `bad_request:invalid UTF-8 JSON` even while end-to-end encryption has been disabled.
+
+Note:
+Before 0.16.5, LiveSync had some issues making chunks. In this case, synchronisation had became been always failing after a corrupted one should be made. After 0.16.6, the corrupted chunk is automatically detected. Sorry for troubling you but please do `rebuild everything` when this plug-in notified so.
 
 ### 0.15.0
 - Outdated configuration items have been removed.