public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2025-12-13 09:45:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag '0.24.0.dev-rc8'

Browse Source
This commit is contained in:
vorotamoroz
2024-10-28 10:24:38 +09:00
parent 8f5bc387b4 5afe24c460
commit 25e30fa09d
87 changed files with 10200 additions and 6744 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.eslintignore
View File

@@ -2,4 +2,8 @@ node_modules
build
.eslintrc.js.bak
src/lib/src/patches/pouchdb-utils
esbuild.config.mjs
esbuild.config.mjs
rollup.config.js
src/lib/test
src/lib/src/cli
main.js

29
.eslintrc
View File

@@ -1,20 +1,13 @@
{
"root": true,
"parser": "@typescript-eslint/parser",
"plugins": [
"@typescript-eslint"
],
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/eslint-recommended",
"plugin:@typescript-eslint/recommended"
],
"plugins": ["@typescript-eslint"],
"extends": ["eslint:recommended", "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended"],
"parserOptions": {
"sourceType": "module",
"project": [
"tsconfig.json"
]
"project": ["tsconfig.json"]
},
"ignorePatterns": [],
"rules": {
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": [
@@ -28,9 +21,17 @@
"no-prototype-builtins": "off",
"@typescript-eslint/no-empty-function": "off",
"require-await": "warn",
"no-async-promise-executor": "off",
"@typescript-eslint/require-await": "warn",
"@typescript-eslint/no-misused-promises": "warn",
"@typescript-eslint/no-floating-promises": "warn",
"no-async-promise-executor": "warn",
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/no-unnecessary-type-assertion": "error",
"no-constant-condition": ["error", { "checkLoops": false }]
}
"no-constant-condition": [
"error",
{
"checkLoops": false
}
]
}
}

2
esbuild.config.mjs
View File

@@ -12,7 +12,7 @@ import inlineWorkerPlugin from "esbuild-plugin-inline-worker";
import { terserOption } from "./terser.config.mjs";
const prod = process.argv[2] === "production";
const keepTest = !prod;
const keepTest = true; //!prod;
const manifestJson = JSON.parse(fs.readFileSync("./manifest.json") + "");
const packageJson = JSON.parse(fs.readFileSync("./package.json") + "");

229
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "obsidian-livesync",
"version": "0.23.23",
"version": "0.24.0.dev-rc8",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "obsidian-livesync",
"version": "0.23.23",
"version": "0.24.0.dev-rc8",
"license": "MIT",
"dependencies": {
"@aws-sdk/client-s3": "^3.645.0",
@@ -19,6 +19,7 @@
"idb": "^8.0.0",
"minimatch": "^10.0.1",
"octagonal-wheels": "^0.1.15",
"svelte-check": "^4.0.4",
"xxhash-wasm": "0.4.2",
"xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
},
@@ -78,7 +79,6 @@
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz",
"integrity": "sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==",
"dev": true,
"dependencies": {
"@jridgewell/gen-mapping": "^0.3.0",
"@jridgewell/trace-mapping": "^0.3.9"
@@ -1578,7 +1578,6 @@
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
"integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
"dev": true,
"dependencies": {
"@jridgewell/set-array": "^1.0.1",
"@jridgewell/sourcemap-codec": "^1.4.10",
@@ -1592,7 +1591,6 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz",
"integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==",
"dev": true,
"engines": {
"node": ">=6.0.0"
}
@@ -1601,7 +1599,6 @@
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz",
"integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==",
"dev": true,
"engines": {
"node": ">=6.0.0"
}
@@ -1619,14 +1616,12 @@
"node_modules/@jridgewell/sourcemap-codec": {
"version": "1.4.15",
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
"integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==",
"dev": true
"integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
},
"node_modules/@jridgewell/trace-mapping": {
"version": "0.3.25",
"resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz",
"integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==",
"dev": true,
"dependencies": {
"@jridgewell/resolve-uri": "^3.1.0",
"@jridgewell/sourcemap-codec": "^1.4.14"
@@ -2408,8 +2403,7 @@
"node_modules/@types/estree": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz",
"integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==",
"dev": true
"integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA=="
},
"node_modules/@types/json5": {
"version": "0.0.29",
@@ -2848,7 +2842,6 @@
"version": "8.10.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz",
"integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==",
"dev": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -2915,7 +2908,6 @@
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz",
"integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==",
"dev": true,
"dependencies": {
"dequal": "^2.0.3"
}
@@ -3055,7 +3047,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.0.0.tgz",
"integrity": "sha512-+60uv1hiVFhHZeO+Lz0RYzsVHy5Wr1ayX0mwda9KPDVLNJgZ1T9Ny7VmFbLDzxsH0D87I86vgj3gFrjTJUYznw==",
"dev": true,
"dependencies": {
"dequal": "^2.0.3"
}
@@ -3154,11 +3145,24 @@
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/chokidar": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.1.tgz",
"integrity": "sha512-n8enUVCED/KVRQlab1hr3MVpcVMvxtZjmEa956u+4YijlmQED223XMSYj2tLuKvr4jcCTzNNMpQDUer72MMmzA==",
"dependencies": {
"readdirp": "^4.0.1"
},
"engines": {
"node": ">= 14.16.0"
},
"funding": {
"url": "https://paulmillr.com/funding/"
}
},
"node_modules/code-red": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/code-red/-/code-red-1.0.3.tgz",
"integrity": "sha512-kVwJELqiILQyG5aeuyKFbdsI1fmQy1Cmf7dQ8eGmVuJoaRVdwey7WaMknr2ZFeVSYSKT0rExsa8EGw0aoI/1QQ==",
"dev": true,
"dependencies": {
"@jridgewell/sourcemap-codec": "^1.4.14",
"@types/estree": "^1.0.0",
@@ -3226,7 +3230,6 @@
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz",
"integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==",
"dev": true,
"dependencies": {
"mdn-data": "2.0.30",
"source-map-js": "^1.0.1"
@@ -3350,7 +3353,6 @@
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
"dev": true,
"engines": {
"node": ">=6"
}
@@ -3916,7 +3918,6 @@
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz",
"integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==",
"dev": true,
"dependencies": {
"@types/estree": "^1.0.0"
}
@@ -4622,7 +4623,6 @@
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.1.tgz",
"integrity": "sha512-baJJdQLiYaJdvFbJqXrcGv3WU3QCzBlUcI5QhbesIm6/xPsvmO+2CDoi/GMOFBQEQm+PXkwOPrp9KK5ozZsp2w==",
"dev": true,
"dependencies": {
"@types/estree": "*"
}
@@ -4791,8 +4791,7 @@
"node_modules/locate-character": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz",
"integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==",
"dev": true
"integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA=="
},
"node_modules/locate-path": {
"version": "6.0.0",
@@ -4819,7 +4818,6 @@
"version": "0.30.8",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.8.tgz",
"integrity": "sha512-ISQTe55T2ao7XtlAStud6qwYPZjE4GK1S/BeVPus4jrq6JuOnQ00YKQC581RWhR122W7msZV263KzVeLoqidyQ==",
"dev": true,
"dependencies": {
"@jridgewell/sourcemap-codec": "^1.4.15"
},
@@ -4852,8 +4850,7 @@
"node_modules/mdn-data": {
"version": "2.0.30",
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz",
"integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==",
"dev": true
"integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA=="
},
"node_modules/merge2": {
"version": "1.4.1",
@@ -4931,6 +4928,14 @@
"node": "*"
}
},
"node_modules/mri": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
"integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==",
"engines": {
"node": ">=4"
}
},
"node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -5220,7 +5225,6 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.1.0.tgz",
"integrity": "sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==",
"dev": true,
"dependencies": {
"@types/estree": "^1.0.0",
"estree-walker": "^3.0.0",
@@ -5230,8 +5234,7 @@
"node_modules/picocolors": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz",
"integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==",
"dev": true
"integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew=="
},
"node_modules/picomatch": {
"version": "2.3.1",
@@ -5728,6 +5731,18 @@
}
]
},
"node_modules/readdirp": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.0.2.tgz",
"integrity": "sha512-yDMz9g+VaZkqBYS/ozoBJwaBhTbZo3UNYQHNRw1D3UFQB8oHB4uS/tAODO+ZLjGWmUbKnIlOWO+aaIiAxrUWHA==",
"engines": {
"node": ">= 14.16.0"
},
"funding": {
"type": "individual",
"url": "https://paulmillr.com/funding/"
}
},
"node_modules/regexp.prototype.flags": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz",
@@ -5826,6 +5841,17 @@
"queue-microtask": "^1.2.2"
}
},
"node_modules/sade": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz",
"integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==",
"dependencies": {
"mri": "^1.1.0"
},
"engines": {
"node": ">=6"
}
},
"node_modules/safe-array-concat": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz",
@@ -5964,7 +5990,6 @@
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz",
"integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==",
"dev": true,
"engines": {
"node": ">=0.10.0"
}
@@ -6111,7 +6136,6 @@
"version": "4.2.19",
"resolved": "https://registry.npmjs.org/svelte/-/svelte-4.2.19.tgz",
"integrity": "sha512-IY1rnGr6izd10B0A8LqsBfmlT5OILVuZ7XsI0vdGPEvuonFV7NYEUK4dAkm9Zg2q0Um92kYjTpS1CAP3Nh/KWw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@ampproject/remapping": "^2.2.1",
@@ -6133,6 +6157,54 @@
"node": ">=16"
}
},
"node_modules/svelte-check": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.0.5.tgz",
"integrity": "sha512-icBTBZ3ibBaywbXUat3cK6hB5Du+Kq9Z8CRuyLmm64XIe2/r+lQcbuBx/IQgsbrC+kT2jQ0weVpZSSRIPwB6jQ==",
"dependencies": {
"@jridgewell/trace-mapping": "^0.3.25",
"chokidar": "^4.0.1",
"fdir": "^6.2.0",
"picocolors": "^1.0.0",
"sade": "^1.7.4"
},
"bin": {
"svelte-check": "bin/svelte-check"
},
"engines": {
"node": ">= 18.0.0"
},
"peerDependencies": {
"svelte": "^4.0.0 || ^5.0.0-next.0",
"typescript": ">=5.0.0"
}
},
"node_modules/svelte-check/node_modules/fdir": {
"version": "6.4.0",
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.0.tgz",
"integrity": "sha512-3oB133prH1o4j/L5lLW7uOCF1PlD+/It2L0eL/iAqWMB91RBbqTewABqxhj0ibBd90EEmWZq7ntIWzVaWcXTGQ==",
"peerDependencies": {
"picomatch": "^3 || ^4"
},
"peerDependenciesMeta": {
"picomatch": {
"optional": true
}
}
},
"node_modules/svelte-check/node_modules/picomatch": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz",
"integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",
"optional": true,
"peer": true,
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/svelte-preprocess": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/svelte-preprocess/-/svelte-preprocess-6.0.2.tgz",
@@ -6389,7 +6461,6 @@
"version": "5.5.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz",
"integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
@@ -6591,7 +6662,6 @@
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz",
"integrity": "sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==",
"dev": true,
"requires": {
"@jridgewell/gen-mapping": "^0.3.0",
"@jridgewell/trace-mapping": "^0.3.9"
@@ -7624,7 +7694,6 @@
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz",
"integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==",
"dev": true,
"requires": {
"@jridgewell/set-array": "^1.0.1",
"@jridgewell/sourcemap-codec": "^1.4.10",
@@ -7634,14 +7703,12 @@
"@jridgewell/resolve-uri": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz",
"integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==",
"dev": true
"integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w=="
},
"@jridgewell/set-array": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz",
"integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==",
"dev": true
"integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw=="
},
"@jridgewell/source-map": {
"version": "0.3.5",
@@ -7656,14 +7723,12 @@
"@jridgewell/sourcemap-codec": {
"version": "1.4.15",
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
"integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==",
"dev": true
"integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
},
"@jridgewell/trace-mapping": {
"version": "0.3.25",
"resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz",
"integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==",
"dev": true,
"requires": {
"@jridgewell/resolve-uri": "^3.1.0",
"@jridgewell/sourcemap-codec": "^1.4.14"
@@ -8267,8 +8332,7 @@
"@types/estree": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz",
"integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==",
"dev": true
"integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA=="
},
"@types/json5": {
"version": "0.0.29",
@@ -8605,8 +8669,7 @@
"acorn": {
"version": "8.10.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz",
"integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==",
"dev": true
"integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw=="
},
"acorn-jsx": {
"version": "5.3.2",
@@ -8652,7 +8715,6 @@
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz",
"integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==",
"dev": true,
"requires": {
"dequal": "^2.0.3"
}
@@ -8748,7 +8810,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.0.0.tgz",
"integrity": "sha512-+60uv1hiVFhHZeO+Lz0RYzsVHy5Wr1ayX0mwda9KPDVLNJgZ1T9Ny7VmFbLDzxsH0D87I86vgj3gFrjTJUYznw==",
"dev": true,
"requires": {
"dequal": "^2.0.3"
}
@@ -8821,11 +8882,18 @@
"supports-color": "^7.1.0"
}
},
"chokidar": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.1.tgz",
"integrity": "sha512-n8enUVCED/KVRQlab1hr3MVpcVMvxtZjmEa956u+4YijlmQED223XMSYj2tLuKvr4jcCTzNNMpQDUer72MMmzA==",
"requires": {
"readdirp": "^4.0.1"
}
},
"code-red": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/code-red/-/code-red-1.0.3.tgz",
"integrity": "sha512-kVwJELqiILQyG5aeuyKFbdsI1fmQy1Cmf7dQ8eGmVuJoaRVdwey7WaMknr2ZFeVSYSKT0rExsa8EGw0aoI/1QQ==",
"dev": true,
"requires": {
"@jridgewell/sourcemap-codec": "^1.4.14",
"@types/estree": "^1.0.0",
@@ -8887,7 +8955,6 @@
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz",
"integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==",
"dev": true,
"requires": {
"mdn-data": "2.0.30",
"source-map-js": "^1.0.1"
@@ -8966,8 +9033,7 @@
"dequal": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
"dev": true
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="
},
"detect-libc": {
"version": "1.0.3",
@@ -9408,7 +9474,6 @@
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz",
"integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==",
"dev": true,
"requires": {
"@types/estree": "^1.0.0"
}
@@ -9906,7 +9971,6 @@
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.1.tgz",
"integrity": "sha512-baJJdQLiYaJdvFbJqXrcGv3WU3QCzBlUcI5QhbesIm6/xPsvmO+2CDoi/GMOFBQEQm+PXkwOPrp9KK5ozZsp2w==",
"dev": true,
"requires": {
"@types/estree": "*"
}
@@ -10027,8 +10091,7 @@
"locate-character": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz",
"integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==",
"dev": true
"integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA=="
},
"locate-path": {
"version": "6.0.0",
@@ -10049,7 +10112,6 @@
"version": "0.30.8",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.8.tgz",
"integrity": "sha512-ISQTe55T2ao7XtlAStud6qwYPZjE4GK1S/BeVPus4jrq6JuOnQ00YKQC581RWhR122W7msZV263KzVeLoqidyQ==",
"dev": true,
"requires": {
"@jridgewell/sourcemap-codec": "^1.4.15"
}
@@ -10072,8 +10134,7 @@
"mdn-data": {
"version": "2.0.30",
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz",
"integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==",
"dev": true
"integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA=="
},
"merge2": {
"version": "1.4.1",
@@ -10126,6 +10187,11 @@
"integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==",
"dev": true
},
"mri": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
"integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA=="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -10324,7 +10390,6 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.1.0.tgz",
"integrity": "sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==",
"dev": true,
"requires": {
"@types/estree": "^1.0.0",
"estree-walker": "^3.0.0",
@@ -10334,8 +10399,7 @@
"picocolors": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz",
"integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==",
"dev": true
"integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew=="
},
"picomatch": {
"version": "2.3.1",
@@ -10747,6 +10811,11 @@
"integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
"dev": true
},
"readdirp": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.0.2.tgz",
"integrity": "sha512-yDMz9g+VaZkqBYS/ozoBJwaBhTbZo3UNYQHNRw1D3UFQB8oHB4uS/tAODO+ZLjGWmUbKnIlOWO+aaIiAxrUWHA=="
},
"regexp.prototype.flags": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz",
@@ -10806,6 +10875,14 @@
"queue-microtask": "^1.2.2"
}
},
"sade": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz",
"integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==",
"requires": {
"mri": "^1.1.0"
}
},
"safe-array-concat": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz",
@@ -10903,8 +10980,7 @@
"source-map-js": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz",
"integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==",
"dev": true
"integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg=="
},
"source-map-support": {
"version": "0.5.21",
@@ -11008,7 +11084,6 @@
"version": "4.2.19",
"resolved": "https://registry.npmjs.org/svelte/-/svelte-4.2.19.tgz",
"integrity": "sha512-IY1rnGr6izd10B0A8LqsBfmlT5OILVuZ7XsI0vdGPEvuonFV7NYEUK4dAkm9Zg2q0Um92kYjTpS1CAP3Nh/KWw==",
"dev": true,
"requires": {
"@ampproject/remapping": "^2.2.1",
"@jridgewell/sourcemap-codec": "^1.4.15",
@@ -11026,6 +11101,33 @@
"periscopic": "^3.1.0"
}
},
"svelte-check": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.0.5.tgz",
"integrity": "sha512-icBTBZ3ibBaywbXUat3cK6hB5Du+Kq9Z8CRuyLmm64XIe2/r+lQcbuBx/IQgsbrC+kT2jQ0weVpZSSRIPwB6jQ==",
"requires": {
"@jridgewell/trace-mapping": "^0.3.25",
"chokidar": "^4.0.1",
"fdir": "^6.2.0",
"picocolors": "^1.0.0",
"sade": "^1.7.4"
},
"dependencies": {
"fdir": {
"version": "6.4.0",
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.0.tgz",
"integrity": "sha512-3oB133prH1o4j/L5lLW7uOCF1PlD+/It2L0eL/iAqWMB91RBbqTewABqxhj0ibBd90EEmWZq7ntIWzVaWcXTGQ==",
"requires": {}
},
"picomatch": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz",
"integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",
"optional": true,
"peer": true
}
}
},
"svelte-preprocess": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/svelte-preprocess/-/svelte-preprocess-6.0.2.tgz",
@@ -11181,8 +11283,7 @@
"typescript": {
"version": "5.5.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz",
"integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==",
"dev": true
"integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q=="
},
"unbox-primitive": {
"version": "1.0.2",

10
package.json
View File

@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.23.23",
"version": "0.24.0.dev-rc8",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"type": "module",
@@ -8,7 +8,10 @@
"dev": "node esbuild.config.mjs",
"build": "node esbuild.config.mjs production",
"buildDev": "node esbuild.config.mjs dev",
"lint": "eslint src"
"lint": "eslint src",
"svelte-check": "svelte-check --tsconfig ./tsconfig.json",
"tsc-check": "tsc --noEmit",
"check": "npm run lint && npm run svelte-check && npm run tsc-check"
},
"keywords": [],
"author": "vorotamoroz",
@@ -66,7 +69,8 @@
"idb": "^8.0.0",
"minimatch": "^10.0.1",
"octagonal-wheels": "^0.1.15",
"svelte-check": "^4.0.4",
"xxhash-wasm": "0.4.2",
"xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2"
}
}
}

15
src/common/events.ts
View File

@@ -3,9 +3,22 @@ export const EVENT_PLUGIN_LOADED = "plugin-loaded";
export const EVENT_PLUGIN_UNLOADED = "plugin-unloaded";
export const EVENT_SETTING_SAVED = "setting-saved";
export const EVENT_FILE_RENAMED = "file-renamed";
export const EVENT_FILE_SAVED = "file-saved";
export const EVENT_LEAF_ACTIVE_CHANGED = "leaf-active-changed";
export const EVENT_LOG_ADDED = "log-added";
export const EVENT_REQUEST_OPEN_SETTINGS = "request-open-settings";
export const EVENT_REQUEST_OPEN_SETTING_WIZARD = "request-open-setting-wizard";
export const EVENT_REQUEST_OPEN_SETUP_URI = "request-open-setup-uri";
export const EVENT_REQUEST_COPY_SETUP_URI = "request-copy-setup-uri";
export const EVENT_REQUEST_SHOW_HISTORY = "show-history";
export const EVENT_REQUEST_RELOAD_SETTING_TAB = "reload-setting-tab";
export const EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG = "request-open-plugin-sync-dialog";
// export const EVENT_FILE_CHANGED = "file-changed";

6
src/common/types.ts
View File

@@ -1,5 +1,5 @@
import { type PluginManifest, TFile } from "../deps.ts";
import { type DatabaseEntry, type EntryBody, type FilePath } from "../lib/src/common/types.ts";
import { type DatabaseEntry, type EntryBody, type FilePath, type UXFileInfoStub, type UXInternalFileInfoStub } from "../lib/src/common/types.ts";
export interface PluginDataEntry extends DatabaseEntry {
deviceVaultName: string;
@@ -49,9 +49,9 @@ export type queueItem = {
};
export type CacheData = string | ArrayBuffer;
export type FileEventType = "CREATE" | "DELETE" | "CHANGED" | "RENAME" | "INTERNAL";
export type FileEventType = "CREATE" | "DELETE" | "CHANGED" | "INTERNAL";
export type FileEventArgs = {
file: FileInfo | InternalFileInfo;
file: UXFileInfoStub | UXInternalFileInfoStub;
cache?: CacheData;
oldPath?: string;
ctx?: any;

277
src/common/utils.ts
View File

@@ -1,10 +1,9 @@
import { normalizePath, Platform, TAbstractFile, App, type RequestUrlParam, requestUrl, TFile } from "../deps.ts";
import { normalizePath, Platform, TAbstractFile, type RequestUrlParam, requestUrl } from "../deps.ts";
import { path2id_base, id2path_base, isValidFilenameInLinux, isValidFilenameInDarwin, isValidFilenameInWidows, isValidFilenameInAndroid, stripAllPrefixes } from "../lib/src/string_and_binary/path.ts";
import { Logger } from "../lib/src/common/logger.ts";
import { LOG_LEVEL_VERBOSE, type AnyEntry, type DocumentID, type EntryHasPath, type FilePath, type FilePathWithPrefix } from "../lib/src/common/types.ts";
import { LOG_LEVEL_VERBOSE, type AnyEntry, type DocumentID, type EntryHasPath, type FilePath, type FilePathWithPrefix, type UXFileInfo, type UXFileInfoStub } from "../lib/src/common/types.ts";
import { CHeader, ICHeader, ICHeaderLength, ICXHeader, PSCHeader } from "./types.ts";
import { InputStringDialog, PopoverSelectString } from "./dialogs.ts";
import type ObsidianLiveSyncPlugin from "../main.ts";
import { writeString } from "../lib/src/string_and_binary/convert.ts";
import { fireAndForget } from "../lib/src/common/utils.ts";
@@ -46,6 +45,10 @@ export function getPathWithoutPrefix(entry: AnyEntry) {
export function getPathFromTFile(file: TAbstractFile) {
return file.path as FilePath;
}
export function getPathFromUXFileInfo(file: UXFileInfoStub | string | FilePathWithPrefix) {
if (typeof file == "string") return file as FilePathWithPrefix;
return file.path;
}
const memos: { [key: string]: any } = {};
@@ -72,206 +75,6 @@ export function disposeMemoObject(key: string) {
delete memos[key];
}
export function isSensibleMargeApplicable(path: string) {
if (path.endsWith(".md")) return true;
return false;
}
export function isObjectMargeApplicable(path: string) {
if (path.endsWith(".canvas")) return true;
if (path.endsWith(".json")) return true;
return false;
}
export function tryParseJSON(str: string, fallbackValue?: any) {
try {
return JSON.parse(str);
} catch (ex) {
return fallbackValue;
}
}
const MARK_OPERATOR = `\u{0001}`;
const MARK_DELETED = `${MARK_OPERATOR}__DELETED`;
const MARK_ISARRAY = `${MARK_OPERATOR}__ARRAY`;
const MARK_SWAPPED = `${MARK_OPERATOR}__SWAP`;
function unorderedArrayToObject(obj: Array<any>) {
return obj.map(e => ({ [e.id as string]: e })).reduce((p, c) => ({ ...p, ...c }), {})
}
function objectToUnorderedArray(obj: object) {
const entries = Object.entries(obj);
if (entries.some(e => e[0] != e[1]?.id)) throw new Error("Item looks like not unordered array")
return entries.map(e => e[1]);
}
function generatePatchUnorderedArray(from: Array<any>, to: Array<any>) {
if (from.every(e => typeof (e) == "object" && ("id" in e)) && to.every(e => typeof (e) == "object" && ("id" in e))) {
const fObj = unorderedArrayToObject(from);
const tObj = unorderedArrayToObject(to);
const diff = generatePatchObj(fObj, tObj);
if (Object.keys(diff).length > 0) {
return { [MARK_ISARRAY]: diff };
} else {
return {};
}
}
return { [MARK_SWAPPED]: to };
}
export function generatePatchObj(from: Record<string | number | symbol, any>, to: Record<string | number | symbol, any>) {
const entries = Object.entries(from);
const tempMap = new Map<string | number | symbol, any>(entries);
const ret = {} as Record<string | number | symbol, any>;
const newEntries = Object.entries(to);
for (const [key, value] of newEntries) {
if (!tempMap.has(key)) {
//New
ret[key] = value;
tempMap.delete(key);
} else {
//Exists
const v = tempMap.get(key);
if (typeof (v) !== typeof (value) || (Array.isArray(v) !== Array.isArray(value))) {
//if type is not match, replace completely.
ret[key] = { [MARK_SWAPPED]: value };
} else {
if (v === null && value === null) {
// NO OP.
} else if (v === null && value !== null) {
ret[key] = { [MARK_SWAPPED]: value };
} else if (v !== null && value === null) {
ret[key] = { [MARK_SWAPPED]: value };
} else if (typeof (v) == "object" && typeof (value) == "object" && !Array.isArray(v) && !Array.isArray(value)) {
const wk = generatePatchObj(v, value);
if (Object.keys(wk).length > 0) ret[key] = wk;
} else if (typeof (v) == "object" && typeof (value) == "object" && Array.isArray(v) && Array.isArray(value)) {
const wk = generatePatchUnorderedArray(v, value);
if (Object.keys(wk).length > 0) ret[key] = wk;
} else if (typeof (v) != "object" && typeof (value) != "object") {
if (JSON.stringify(tempMap.get(key)) !== JSON.stringify(value)) {
ret[key] = value;
}
} else {
if (JSON.stringify(tempMap.get(key)) !== JSON.stringify(value)) {
ret[key] = { [MARK_SWAPPED]: value };
}
}
}
tempMap.delete(key);
}
}
//Not used item, means deleted one
for (const [key,] of tempMap) {
ret[key] = MARK_DELETED
}
return ret;
}
export function applyPatch(from: Record<string | number | symbol, any>, patch: Record<string | number | symbol, any>) {
const ret = from;
const patches = Object.entries(patch);
for (const [key, value] of patches) {
if (value == MARK_DELETED) {
delete ret[key];
continue;
}
if (value === null) {
ret[key] = null;
continue;
}
if (typeof (value) == "object") {
if (MARK_SWAPPED in value) {
ret[key] = value[MARK_SWAPPED];
continue;
}
if (MARK_ISARRAY in value) {
if (!(key in ret)) ret[key] = [];
if (!Array.isArray(ret[key])) {
throw new Error("Patch target type is mismatched (array to something)");
}
const orgArrayObject = unorderedArrayToObject(ret[key]);
const appliedObject = applyPatch(orgArrayObject, value[MARK_ISARRAY]);
const appliedArray = objectToUnorderedArray(appliedObject);
ret[key] = [...appliedArray];
} else {
if (!(key in ret)) {
ret[key] = value;
continue;
}
ret[key] = applyPatch(ret[key], value);
}
} else {
ret[key] = value;
}
}
return ret;
}
export function mergeObject(
objA: Record<string | number | symbol, any> | [any],
objB: Record<string | number | symbol, any> | [any]
) {
const newEntries = Object.entries(objB);
const ret: any = { ...objA };
if (
typeof objA !== typeof objB ||
Array.isArray(objA) !== Array.isArray(objB)
) {
return objB;
}
for (const [key, v] of newEntries) {
if (key in ret) {
const value = ret[key];
if (
typeof v !== typeof value ||
Array.isArray(v) !== Array.isArray(value)
) {
//if type is not match, replace completely.
ret[key] = v;
} else {
if (
typeof v == "object" &&
typeof value == "object" &&
!Array.isArray(v) &&
!Array.isArray(value)
) {
ret[key] = mergeObject(v, value);
} else if (
typeof v == "object" &&
typeof value == "object" &&
Array.isArray(v) &&
Array.isArray(value)
) {
ret[key] = [...new Set([...v, ...value])];
} else {
ret[key] = v;
}
}
} else {
ret[key] = v;
}
}
const retSorted = Object.fromEntries(Object.entries(ret).sort((a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
if (Array.isArray(objA) && Array.isArray(objB)) {
return Object.values(retSorted);
}
return retSorted;
}
export function flattenObject(obj: Record<string | number | symbol, any>, path: string[] = []): [string, any][] {
if (typeof (obj) != "object") return [[path.join("."), obj]];
if (obj === null) return [[path.join("."), null]];
if (Array.isArray(obj)) return [[path.join("."), JSON.stringify(obj)]];
const e = Object.entries(obj);
const ret = []
for (const [key, value] of e) {
const p = flattenObject(value, [...path, key]);
ret.push(...p);
}
return ret;
}
export function isValidPath(filename: string) {
if (Platform.isDesktop) {
@@ -319,29 +122,6 @@ export function isCustomisationSyncMetadata(str: string): boolean {
return str.startsWith(ICXHeader);
}
export const askYesNo = (app: App, message: string): Promise<"yes" | "no"> => {
return new Promise((res) => {
const popover = new PopoverSelectString(app, message, undefined, undefined, (result) => res(result as "yes" | "no"));
popover.open();
});
};
export const askSelectString = (app: App, message: string, items: string[]): Promise<string> => {
const getItemsFun = () => items;
return new Promise((res) => {
const popover = new PopoverSelectString(app, message, "", getItemsFun, (result) => res(result));
popover.open();
});
};
export const askString = (app: App, title: string, key: string, placeholder: string, isPassword: boolean = false): Promise<string | false> => {
return new Promise((res) => {
const dialog = new InputStringDialog(app, title, key, placeholder, isPassword, (result) => res(result));
dialog.open();
});
};
export class PeriodicProcessor {
_process: () => Promise<any>;
@@ -413,20 +193,6 @@ export const requestToCouchDB = async (baseUri: string, username: string, passwo
return await _requestToCouchDB(baseUri, username, password, origin, uri, body, method);
};
export async function performRebuildDB(plugin: ObsidianLiveSyncPlugin, method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice" | "localOnlyWithChunks") {
if (method == "localOnly") {
await plugin.addOnSetup.fetchLocal();
}
if (method == "localOnlyWithChunks") {
await plugin.addOnSetup.fetchLocal(true);
}
if (method == "remoteOnly") {
await plugin.addOnSetup.rebuildRemote();
}
if (method == "rebuildBothByThisDevice") {
await plugin.addOnSetup.rebuildEverything();
}
}
export const BASE_IS_NEW = Symbol("base");
export const TARGET_IS_NEW = Symbol("target");
@@ -445,9 +211,9 @@ export function compareMTime(baseMTime: number, targetMTime: number): typeof BAS
throw new Error("Unexpected error");
}
export function markChangesAreSame(file: TFile | AnyEntry | string, mtime1: number, mtime2: number) {
export function markChangesAreSame(file: AnyEntry | string | UXFileInfoStub, mtime1: number, mtime2: number) {
if (mtime1 === mtime2) return true;
const key = typeof file == "string" ? file : file instanceof TFile ? file.path : file.path ?? file._id;
const key = typeof file == "string" ? file : "_id" in file ? file._id : file.path;
const pairs = sameChangePairs.get(key, []) || [];
if (pairs.some(e => e == mtime1 || e == mtime2)) {
sameChangePairs.set(key, [...new Set([...pairs, mtime1, mtime2])]);
@@ -455,20 +221,20 @@ export function markChangesAreSame(file: TFile | AnyEntry | string, mtime1: numb
sameChangePairs.set(key, [mtime1, mtime2]);
}
}
export function isMarkedAsSameChanges(file: TFile | AnyEntry | string, mtimes: number[]) {
const key = typeof file == "string" ? file : file instanceof TFile ? file.path : file.path ?? file._id;
export function isMarkedAsSameChanges(file: UXFileInfoStub | AnyEntry | string, mtimes: number[]) {
const key = typeof file == "string" ? file : "_id" in file ? file._id : file.path;
const pairs = sameChangePairs.get(key, []) || [];
if (mtimes.every(e => pairs.indexOf(e) !== -1)) {
return EVEN;
}
}
export function compareFileFreshness(baseFile: TFile | AnyEntry | undefined, checkTarget: TFile | AnyEntry | undefined): typeof BASE_IS_NEW | typeof TARGET_IS_NEW | typeof EVEN {
export function compareFileFreshness(baseFile: UXFileInfoStub | AnyEntry | undefined, checkTarget: UXFileInfo | AnyEntry | undefined): typeof BASE_IS_NEW | typeof TARGET_IS_NEW | typeof EVEN {
if (baseFile === undefined && checkTarget == undefined) return EVEN;
if (baseFile == undefined) return TARGET_IS_NEW;
if (checkTarget == undefined) return BASE_IS_NEW;
const modifiedBase = baseFile instanceof TFile ? baseFile?.stat?.mtime ?? 0 : baseFile?.mtime ?? 0;
const modifiedTarget = checkTarget instanceof TFile ? checkTarget?.stat?.mtime ?? 0 : checkTarget?.mtime ?? 0;
const modifiedBase = "stat" in baseFile ? baseFile?.stat?.mtime ?? 0 : baseFile?.mtime ?? 0;
const modifiedTarget = "stat" in checkTarget ? checkTarget?.stat?.mtime ?? 0 : checkTarget?.mtime ?? 0;
if (modifiedBase && modifiedTarget && isMarkedAsSameChanges(baseFile, [modifiedBase, modifiedTarget])) {
return EVEN;
@@ -484,15 +250,15 @@ const _cached = new Map<string, {
export type MemoOption = {
key: string;
forceUpdate?: boolean;
validator?: () => boolean;
validator?: (context: Map<string, any>) => boolean;
}
export function useMemo<T>({ key, forceUpdate, validator }: MemoOption, updateFunc: (context: Map<string, any>, prev: T) => T): T {
const cached = _cached.get(key);
if (cached && !forceUpdate && (!validator || validator && !validator())) {
const context = cached?.context || new Map<string, any>();
if (cached && !forceUpdate && (!validator || validator && !validator(context))) {
return cached.value;
}
const context = cached?.context || new Map<string, any>();
const value = updateFunc(context, cached?.value);
if (value !== cached?.value) {
_cached.set(key, { value, context });
@@ -535,4 +301,13 @@ export function disposeMemo(key: string) {
export function disposeAllMemo() {
_cached.clear();
}
}
export function displayRev(rev: string) {
const [number, hash] = rev.split("-");
return `${number}-${hash.substring(0, 6)}`;
}
// export function getPathFromUXFileInfo(file: UXFileInfoStub | UXFileInfo | string) {
// return (typeof file == "string" ? file : file.path) as FilePathWithPrefix;
// }

776
src/features/CmdHiddenFileSync.ts
View File

@@ -1,776 +0,0 @@
import { normalizePath, type PluginManifest, type ListedFiles } from "../deps.ts";
import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED, type SavingEntry, type DocumentID } from "../lib/src/common/types.ts";
import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "../common/types.ts";
import { readAsBlob, isDocContentSame, sendSignal, readContent, createBlob } from "../lib/src/common/utils.ts";
import { Logger } from "../lib/src/common/logger.ts";
import { PouchDB } from "../lib/src/pouchdb/pouchdb-browser.js";
import { isInternalMetadata, PeriodicProcessor } from "../common/utils.ts";
import { serialized } from "../lib/src/concurrency/lock.ts";
import { JsonResolveModal } from "../ui/JsonResolveModal.ts";
import { LiveSyncCommands } from "./LiveSyncCommands.ts";
import { addPrefix, stripAllPrefixes } from "../lib/src/string_and_binary/path.ts";
import { QueueProcessor } from "../lib/src/concurrency/processor.ts";
import { hiddenFilesEventCount, hiddenFilesProcessingCount } from "../lib/src/mock_and_interop/stores.ts";
export class HiddenFileSync extends LiveSyncCommands {
periodicInternalFileScanProcessor: PeriodicProcessor = new PeriodicProcessor(this.plugin, async () => this.settings.syncInternalFiles && this.localDatabase.isReady && await this.syncInternalFilesAndDatabase("push", false));
get kvDB() {
return this.plugin.kvDB;
}
getConflictedDoc(path: FilePathWithPrefix, rev: string) {
return this.plugin.getConflictedDoc(path, rev);
}
onunload() {
this.periodicInternalFileScanProcessor?.disable();
}
onload() {
this.plugin.addCommand({
id: "livesync-scaninternal",
name: "Sync hidden files",
callback: () => {
this.syncInternalFilesAndDatabase("safe", true);
},
});
}
async onInitializeDatabase(showNotice: boolean) {
if (this.settings.syncInternalFiles) {
try {
Logger("Synchronizing hidden files...");
await this.syncInternalFilesAndDatabase("push", showNotice);
Logger("Synchronizing hidden files done");
} catch (ex) {
Logger("Synchronizing hidden files failed");
Logger(ex, LOG_LEVEL_VERBOSE);
}
}
}
async beforeReplicate(showNotice: boolean) {
if (this.localDatabase.isReady && this.settings.syncInternalFiles && this.settings.syncInternalFilesBeforeReplication && !this.settings.watchInternalFileChanges) {
await this.syncInternalFilesAndDatabase("push", showNotice);
}
}
async onResume() {
this.periodicInternalFileScanProcessor?.disable();
if (this.plugin.suspended)
return;
if (this.settings.syncInternalFiles) {
await this.syncInternalFilesAndDatabase("safe", false);
}
this.periodicInternalFileScanProcessor.enable(this.settings.syncInternalFiles && this.settings.syncInternalFilesInterval ? (this.settings.syncInternalFilesInterval * 1000) : 0);
}
parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>) {
return false;
}
realizeSettingSyncMode(): Promise<void> {
this.periodicInternalFileScanProcessor?.disable();
if (this.plugin.suspended)
return Promise.resolve();
if (!this.plugin.isReady)
return Promise.resolve();
this.periodicInternalFileScanProcessor.enable(this.settings.syncInternalFiles && this.settings.syncInternalFilesInterval ? (this.settings.syncInternalFilesInterval * 1000) : 0);
return Promise.resolve();
}
procInternalFile(filename: string) {
this.internalFileProcessor.enqueue(filename);
}
internalFileProcessor = new QueueProcessor<string, any>(
async (filenames) => {
Logger(`START :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE);
await this.syncInternalFilesAndDatabase("pull", false, false, filenames);
Logger(`DONE :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE);
return;
}, { batchSize: 100, concurrentLimit: 1, delay: 10, yieldThreshold: 100, suspended: false, totalRemainingReactiveSource: hiddenFilesEventCount }
);
recentProcessedInternalFiles = [] as string[];
async watchVaultRawEventsAsync(path: FilePath) {
if (!this.settings.syncInternalFiles) return;
// Exclude files handled by customization sync
const configDir = normalizePath(this.app.vault.configDir);
const synchronisedInConfigSync = !this.settings.usePluginSync ? [] : Object.values(this.settings.pluginSyncExtendedSetting).filter(e => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase());
if (synchronisedInConfigSync.some(e => e.startsWith(path.toLowerCase()))) {
Logger(`Hidden file skipped: ${path} is synchronized in customization sync.`, LOG_LEVEL_VERBOSE);
return;
}
const stat = await this.vaultAccess.adapterStat(path);
// sometimes folder is coming.
if (stat != null && stat.type != "file") {
return;
}
const mtime = stat == null ? 0 : stat?.mtime ?? 0;
const storageMTime = ~~((mtime) / 1000);
const key = `${path}-${storageMTime}`;
if (mtime != 0 && this.recentProcessedInternalFiles.contains(key)) {
//If recently processed, it may caused by self.
return;
}
this.recentProcessedInternalFiles = [key, ...this.recentProcessedInternalFiles].slice(0, 100);
// const id = await this.path2id(path, ICHeader);
const prefixedFileName = addPrefix(path, ICHeader);
const filesOnDB = await this.localDatabase.getDBEntryMeta(prefixedFileName);
const dbMTime = ~~((filesOnDB && filesOnDB.mtime || 0) / 1000);
// Skip unchanged file.
if (dbMTime == storageMTime) {
// Logger(`STORAGE --> DB:${path}: (hidden) Nothing changed`);
return;
}
// Do not compare timestamp. Always local data should be preferred except this plugin wrote one.
if (storageMTime == 0) {
await this.deleteInternalFileOnDatabase(path);
} else {
await this.storeInternalFileToDatabase({ path: path, mtime, ctime: stat?.ctime ?? mtime, size: stat?.size ?? 0 });
}
}
async resolveConflictOnInternalFiles() {
// Scan all conflicted internal files
const conflicted = this.localDatabase.findEntries(ICHeader, ICHeaderEnd, { conflicts: true });
this.conflictResolutionProcessor.suspend();
try {
for await (const doc of conflicted) {
if (!("_conflicts" in doc))
continue;
if (isInternalMetadata(doc._id)) {
this.conflictResolutionProcessor.enqueue(doc.path);
}
}
} catch (ex) {
Logger("something went wrong on resolving all conflicted internal files");
Logger(ex, LOG_LEVEL_VERBOSE);
}
await this.conflictResolutionProcessor.startPipeline().waitForAllProcessed();
}
async resolveByNewerEntry(id: DocumentID, path: FilePathWithPrefix, currentDoc: EntryDoc, currentRev: string, conflictedRev: string) {
const conflictedDoc = await this.localDatabase.getRaw(id, { rev: conflictedRev });
// determine which revision should been deleted.
// simply check modified time
const mtimeCurrent = ("mtime" in currentDoc && currentDoc.mtime) || 0;
const mtimeConflicted = ("mtime" in conflictedDoc && conflictedDoc.mtime) || 0;
// Logger(`Revisions:${new Date(mtimeA).toLocaleString} and ${new Date(mtimeB).toLocaleString}`);
// console.log(`mtime:${mtimeA} - ${mtimeB}`);
const delRev = mtimeCurrent < mtimeConflicted ? currentRev : conflictedRev;
// delete older one.
await this.localDatabase.removeRevision(id, delRev);
Logger(`Older one has been deleted:${path}`);
const cc = await this.localDatabase.getRaw(id, { conflicts: true });
if (cc._conflicts?.length === 0) {
await this.extractInternalFileFromDatabase(stripAllPrefixes(path))
} else {
this.conflictResolutionProcessor.enqueue(path);
}
// check the file again
}
conflictResolutionProcessor = new QueueProcessor(async (paths: FilePathWithPrefix[]) => {
const path = paths[0];
sendSignal(`cancel-internal-conflict:${path}`);
try {
// Retrieve data
const id = await this.path2id(path, ICHeader);
const doc = await this.localDatabase.getRaw(id, { conflicts: true });
// if (!("_conflicts" in doc)){
// return [];
// }
if (doc._conflicts === undefined) return [];
if (doc._conflicts.length == 0)
return [];
Logger(`Hidden file conflicted:${path}`);
const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
const revA = doc._rev;
const revB = conflicts[0];
if (path.endsWith(".json")) {
const conflictedRev = conflicts[0];
const conflictedRevNo = Number(conflictedRev.split("-")[0]);
//Search
const revFrom = (await this.localDatabase.getRaw<EntryDoc>(id, { revs_info: true }));
const commonBase = revFrom._revs_info?.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first()?.rev ?? "";
const result = await this.plugin.mergeObject(path, commonBase, doc._rev, conflictedRev);
if (result) {
Logger(`Object merge:${path}`, LOG_LEVEL_INFO);
const filename = stripAllPrefixes(path);
const isExists = await this.plugin.vaultAccess.adapterExists(filename);
if (!isExists) {
await this.vaultAccess.ensureDirectory(filename);
}
await this.plugin.vaultAccess.adapterWrite(filename, result);
const stat = await this.vaultAccess.adapterStat(filename);
if (!stat) {
throw new Error(`conflictResolutionProcessor: Failed to stat file ${filename}`);
}
await this.storeInternalFileToDatabase({ path: filename, ...stat });
await this.extractInternalFileFromDatabase(filename);
await this.localDatabase.removeRevision(id, revB);
this.conflictResolutionProcessor.enqueue(path);
return [];
} else {
Logger(`Object merge is not applicable.`, LOG_LEVEL_VERBOSE);
}
return [{ path, revA, revB, id, doc }];
}
// When not JSON file, resolve conflicts by choosing a newer one.
await this.resolveByNewerEntry(id, path, doc, revA, revB);
return [];
} catch (ex) {
Logger(`Failed to resolve conflict (Hidden): ${path}`);
Logger(ex, LOG_LEVEL_VERBOSE);
return [];
}
}, {
suspended: false, batchSize: 1, concurrentLimit: 5, delay: 10, keepResultUntilDownstreamConnected: true, yieldThreshold: 10,
pipeTo: new QueueProcessor(async (results) => {
const { id, doc, path, revA, revB } = results[0];
const docAMerge = await this.localDatabase.getDBEntry(path, { rev: revA });
const docBMerge = await this.localDatabase.getDBEntry(path, { rev: revB });
if (docAMerge != false && docBMerge != false) {
if (await this.showJSONMergeDialogAndMerge(docAMerge, docBMerge)) {
// Again for other conflicted revisions.
this.conflictResolutionProcessor.enqueue(path);
}
return;
} else {
// If either revision could not read, force resolving by the newer one.
await this.resolveByNewerEntry(id, path, doc, revA, revB);
}
}, { suspended: false, batchSize: 1, concurrentLimit: 1, delay: 10, keepResultUntilDownstreamConnected: false, yieldThreshold: 10 })
})
queueConflictCheck(path: FilePathWithPrefix) {
this.conflictResolutionProcessor.enqueue(path);
}
//TODO: Tidy up. Even though it is experimental feature, So dirty...
async syncInternalFilesAndDatabase(direction: "push" | "pull" | "safe" | "pullForce" | "pushForce", showMessage: boolean, filesAll: InternalFileInfo[] | false = false, targetFiles: string[] | false = false) {
await this.resolveConflictOnInternalFiles();
const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
Logger("Scanning hidden files.", logLevel, "sync_internal");
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
const configDir = normalizePath(this.app.vault.configDir);
let files: InternalFileInfo[] =
filesAll ? filesAll : (await this.scanInternalFiles())
const synchronisedInConfigSync = !this.settings.usePluginSync ? [] : Object.values(this.settings.pluginSyncExtendedSetting).filter(e => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase());
files = files.filter(file => synchronisedInConfigSync.every(filterFile => !file.path.toLowerCase().startsWith(filterFile)))
const filesOnDB = ((await this.localDatabase.allDocsRaw({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted);
const allFileNamesSrc = [...new Set([...files.map(e => normalizePath(e.path)), ...filesOnDB.map(e => stripAllPrefixes(this.getPath(e)))])];
const allFileNames = allFileNamesSrc.filter(filename => !targetFiles || (targetFiles && targetFiles.indexOf(filename) !== -1)).filter(path => synchronisedInConfigSync.every(filterFile => !path.toLowerCase().startsWith(filterFile)))
function compareMTime(a: number, b: number) {
const wa = ~~(a / 1000);
const wb = ~~(b / 1000);
const diff = wa - wb;
return diff;
}
const fileCount = allFileNames.length;
let processed = 0;
let filesChanged = 0;
// count updated files up as like this below:
// .obsidian: 2
// .obsidian/workspace: 1
// .obsidian/plugins: 1
// .obsidian/plugins/recent-files-obsidian: 1
// .obsidian/plugins/recent-files-obsidian/data.json: 1
const updatedFolders: { [key: string]: number; } = {};
const countUpdatedFolder = (path: string) => {
const pieces = path.split("/");
let c = pieces.shift();
let pathPieces = "";
filesChanged++;
while (c) {
pathPieces += (pathPieces != "" ? "/" : "") + c;
pathPieces = normalizePath(pathPieces);
if (!(pathPieces in updatedFolders)) {
updatedFolders[pathPieces] = 0;
}
updatedFolders[pathPieces]++;
c = pieces.shift();
}
};
// Cache update time information for files which have already been processed (mainly for files that were skipped due to the same content)
let caches: { [key: string]: { storageMtime: number; docMtime: number; }; } = {};
caches = await this.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number; }; }>("diff-caches-internal") || {};
const filesMap = files.reduce((acc, cur) => {
acc[cur.path] = cur;
return acc;
}, {} as { [key: string]: InternalFileInfo; });
const filesOnDBMap = filesOnDB.reduce((acc, cur) => {
acc[stripAllPrefixes(this.getPath(cur))] = cur;
return acc;
}, {} as { [key: string]: InternalFileEntry; });
await new QueueProcessor(async (filenames: FilePath[]) => {
const filename = filenames[0];
processed++;
if (processed % 100 == 0) {
Logger(`Hidden file: ${processed}/${fileCount}`, logLevel, "sync_internal");
}
if (!filename) return [];
if (ignorePatterns.some(e => filename.match(e)))
return [];
if (await this.plugin.isIgnoredByIgnoreFiles(filename)) {
return [];
}
const fileOnStorage = filename in filesMap ? filesMap[filename] : undefined;
const fileOnDatabase = filename in filesOnDBMap ? filesOnDBMap[filename] : undefined;
return [{
filename,
fileOnStorage,
fileOnDatabase,
}]
}, { suspended: true, batchSize: 1, concurrentLimit: 10, delay: 0, totalRemainingReactiveSource: hiddenFilesProcessingCount })
.pipeTo(new QueueProcessor(async (params) => {
const
{
filename,
fileOnStorage: xFileOnStorage,
fileOnDatabase: xFileOnDatabase
} = params[0];
if (xFileOnStorage && xFileOnDatabase) {
const cache = filename in caches ? caches[filename] : { storageMtime: 0, docMtime: 0 };
// Both => Synchronize
if ((direction != "pullForce" && direction != "pushForce") && xFileOnDatabase.mtime == cache.docMtime && xFileOnStorage.mtime == cache.storageMtime) {
return;
}
const nw = compareMTime(xFileOnStorage.mtime, xFileOnDatabase.mtime);
if (nw > 0 || direction == "pushForce") {
await this.storeInternalFileToDatabase(xFileOnStorage);
}
if (nw < 0 || direction == "pullForce") {
// skip if not extraction performed.
if (!await this.extractInternalFileFromDatabase(filename))
return;
}
// If process successfully updated or file contents are same, update cache.
cache.docMtime = xFileOnDatabase.mtime;
cache.storageMtime = xFileOnStorage.mtime;
caches[filename] = cache;
countUpdatedFolder(filename);
} else if (!xFileOnStorage && xFileOnDatabase) {
if (direction == "push" || direction == "pushForce") {
if (xFileOnDatabase.deleted)
return;
await this.deleteInternalFileOnDatabase(filename, false);
} else if (direction == "pull" || direction == "pullForce") {
if (await this.extractInternalFileFromDatabase(filename)) {
countUpdatedFolder(filename);
}
} else if (direction == "safe") {
if (xFileOnDatabase.deleted)
return;
if (await this.extractInternalFileFromDatabase(filename)) {
countUpdatedFolder(filename);
}
}
} else if (xFileOnStorage && !xFileOnDatabase) {
if (direction == "push" || direction == "pushForce" || direction == "safe") {
await this.storeInternalFileToDatabase(xFileOnStorage);
} else {
await this.extractInternalFileFromDatabase(xFileOnStorage.path);
}
} else {
throw new Error("Invalid state on hidden file sync");
// Something corrupted?
}
return;
}, { suspended: true, batchSize: 1, concurrentLimit: 5, delay: 0 }))
.root
.enqueueAll(allFileNames)
.startPipeline().waitForAllDoneAndTerminate();
await this.kvDB.set("diff-caches-internal", caches);
// When files has been retrieved from the database. they must be reloaded.
if ((direction == "pull" || direction == "pullForce") && filesChanged != 0) {
// Show notification to restart obsidian when something has been changed in configDir.
if (configDir in updatedFolders) {
// Numbers of updated files that is below of configDir.
let updatedCount = updatedFolders[configDir];
try {
//@ts-ignore
const manifests = Object.values(this.app.plugins.manifests) as any as PluginManifest[];
//@ts-ignore
const enabledPlugins = this.app.plugins.enabledPlugins as Set<string>;
const enabledPluginManifests = manifests.filter(e => enabledPlugins.has(e.id));
for (const manifest of enabledPluginManifests) {
if (manifest.dir && manifest.dir in updatedFolders) {
// If notified about plug-ins, reloading Obsidian may not be necessary.
updatedCount -= updatedFolders[manifest.dir];
const updatePluginId = manifest.id;
const updatePluginName = manifest.name;
this.plugin.askInPopup(`updated-${updatePluginId}`, `Files in ${updatePluginName} has been updated, Press {HERE} to reload ${updatePluginName}, or press elsewhere to dismiss this message.`, (anchor) => {
anchor.text = "HERE";
anchor.addEventListener("click", async () => {
Logger(`Unloading plugin: ${updatePluginName}`, LOG_LEVEL_NOTICE, "plugin-reload-" + updatePluginId);
// @ts-ignore
await this.app.plugins.unloadPlugin(updatePluginId);
// @ts-ignore
await this.app.plugins.loadPlugin(updatePluginId);
Logger(`Plugin reloaded: ${updatePluginName}`, LOG_LEVEL_NOTICE, "plugin-reload-" + updatePluginId);
});
}
);
}
}
} catch (ex) {
Logger("Error on checking plugin status.");
Logger(ex, LOG_LEVEL_VERBOSE);
}
// If something changes left, notify for reloading Obsidian.
if (updatedCount != 0) {
if (!this.plugin.isReloadingScheduled) {
this.plugin.askInPopup(`updated-any-hidden`, `Hidden files have been synchronised, Press {HERE} to schedule a reload of Obsidian, or press elsewhere to dismiss this message.`, (anchor) => {
anchor.text = "HERE";
anchor.addEventListener("click", () => {
this.plugin.scheduleAppReload();
});
});
}
}
}
}
Logger(`Hidden files scanned: ${filesChanged} files had been modified`, logLevel, "sync_internal");
}
async storeInternalFileToDatabase(file: InternalFileInfo, forceWrite = false) {
if (await this.plugin.isIgnoredByIgnoreFiles(file.path)) {
return
}
const id = await this.path2id(file.path, ICHeader);
const prefixedFileName = addPrefix(file.path, ICHeader);
const content = createBlob(await this.plugin.vaultAccess.adapterReadAuto(file.path));
const mtime = file.mtime;
return await serialized("file-" + prefixedFileName, async () => {
try {
const old = await this.localDatabase.getDBEntry(prefixedFileName, undefined, false, false);
let saveData: SavingEntry;
if (old === false) {
saveData = {
_id: id,
path: prefixedFileName,
data: content,
mtime,
ctime: mtime,
datatype: "newnote",
size: file.size,
children: [],
deleted: false,
type: "newnote",
eden: {},
};
} else {
if (await isDocContentSame(readAsBlob(old), content) && !forceWrite) {
// Logger(`STORAGE --> DB:${file.path}: (hidden) Not changed`, LOG_LEVEL_VERBOSE);
return;
}
saveData =
{
...old,
data: content,
mtime,
size: file.size,
datatype: old.datatype,
children: [],
deleted: false,
type: old.datatype,
};
}
const ret = await this.localDatabase.putDBEntry(saveData);
Logger(`STORAGE --> DB:${file.path}: (hidden) Done`);
return ret;
} catch (ex) {
Logger(`STORAGE --> DB:${file.path}: (hidden) Failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
return false;
}
});
}
async deleteInternalFileOnDatabase(filename: FilePath, forceWrite = false) {
const id = await this.path2id(filename, ICHeader);
const prefixedFileName = addPrefix(filename, ICHeader);
const mtime = new Date().getTime();
if (await this.plugin.isIgnoredByIgnoreFiles(filename)) {
return
}
await serialized("file-" + prefixedFileName, async () => {
try {
const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, undefined, true) as InternalFileEntry | false;
let saveData: InternalFileEntry;
if (old === false) {
saveData = {
_id: id,
path: prefixedFileName,
mtime,
ctime: mtime,
size: 0,
children: [],
deleted: true,
type: "newnote",
eden: {}
};
} else {
// Remove all conflicted before deleting.
const conflicts = await this.localDatabase.getRaw(old._id, { conflicts: true });
if (conflicts._conflicts !== undefined) {
for (const conflictRev of conflicts._conflicts) {
await this.localDatabase.removeRevision(old._id, conflictRev);
Logger(`STORAGE -x> DB:${filename}: (hidden) conflict removed ${old._rev} => ${conflictRev}`, LOG_LEVEL_VERBOSE);
}
}
if (old.deleted) {
Logger(`STORAGE -x> DB:${filename}: (hidden) already deleted`);
return;
}
saveData =
{
...old,
mtime,
size: 0,
children: [],
deleted: true,
type: "newnote",
};
}
await this.localDatabase.putRaw(saveData);
Logger(`STORAGE -x> DB:${filename}: (hidden) Done`);
} catch (ex) {
Logger(`STORAGE -x> DB:${filename}: (hidden) Failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
return false;
}
});
}
async extractInternalFileFromDatabase(filename: FilePath, force = false) {
const isExists = await this.plugin.vaultAccess.adapterExists(filename);
const prefixedFileName = addPrefix(filename, ICHeader);
if (await this.plugin.isIgnoredByIgnoreFiles(filename)) {
return;
}
return await serialized("file-" + prefixedFileName, async () => {
try {
// Check conflicted status
const fileOnDB = await this.localDatabase.getDBEntry(prefixedFileName, { conflicts: true }, false, true, true);
if (fileOnDB === false)
throw new Error(`File not found on database.:${filename}`);
// Prevent overwrite for Prevent overwriting while some conflicted revision exists.
if (fileOnDB?._conflicts?.length) {
Logger(`Hidden file ${filename} has conflicted revisions, to keep in safe, writing to storage has been prevented`, LOG_LEVEL_INFO);
return;
}
const deleted = fileOnDB.deleted || fileOnDB._deleted || false;
if (deleted) {
if (!isExists) {
Logger(`STORAGE <x- DB:${filename}: deleted (hidden) Deleted on DB, but the file is already not found on storage.`);
} else {
Logger(`STORAGE <x- DB:${filename}: deleted (hidden).`);
await this.plugin.vaultAccess.adapterRemove(filename);
try {
//@ts-ignore internalAPI
await this.app.vault.adapter.reconcileInternalFile(filename);
} catch (ex) {
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE);
Logger(ex, LOG_LEVEL_VERBOSE);
}
}
return true;
}
if (!isExists) {
await this.vaultAccess.ensureDirectory(filename);
await this.plugin.vaultAccess.adapterWrite(filename, readContent(fileOnDB), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
try {
//@ts-ignore internalAPI
await this.app.vault.adapter.reconcileInternalFile(filename);
} catch (ex) {
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE);
Logger(ex, LOG_LEVEL_VERBOSE);
}
Logger(`STORAGE <-- DB:${filename}: written (hidden,new${force ? ", force" : ""})`);
return true;
} else {
const content = await this.plugin.vaultAccess.adapterReadAuto(filename);
const docContent = readContent(fileOnDB);
if (await isDocContentSame(content, docContent) && !force) {
// Logger(`STORAGE <-- DB:${filename}: skipped (hidden) Not changed`, LOG_LEVEL_VERBOSE);
return true;
}
await this.plugin.vaultAccess.adapterWrite(filename, docContent, { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
try {
//@ts-ignore internalAPI
await this.app.vault.adapter.reconcileInternalFile(filename);
} catch (ex) {
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE);
Logger(ex, LOG_LEVEL_VERBOSE);
}
Logger(`STORAGE <-- DB:${filename}: written (hidden, overwrite${force ? ", force" : ""})`);
return true;
}
} catch (ex) {
Logger(`STORAGE <-- DB:${filename}: written (hidden, overwrite${force ? ", force" : ""}) Failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
return false;
}
});
}
showJSONMergeDialogAndMerge(docA: LoadedEntry, docB: LoadedEntry): Promise<boolean> {
return new Promise((res) => {
Logger("Opening data-merging dialog", LOG_LEVEL_VERBOSE);
const docs = [docA, docB];
const path = stripAllPrefixes(docA.path);
const modal = new JsonResolveModal(this.app, path, [docA, docB], async (keep, result) => {
// modal.close();
try {
const filename = path;
let needFlush = false;
if (!result && !keep) {
Logger(`Skipped merging: ${filename}`);
res(false);
return;
}
//Delete old revisions
if (result || keep) {
for (const doc of docs) {
if (doc._rev != keep) {
if (await this.localDatabase.deleteDBEntry(this.getPath(doc), { rev: doc._rev })) {
Logger(`Conflicted revision has been deleted: ${filename}`);
needFlush = true;
}
}
}
}
if (!keep && result) {
const isExists = await this.plugin.vaultAccess.adapterExists(filename);
if (!isExists) {
await this.vaultAccess.ensureDirectory(filename);
}
await this.plugin.vaultAccess.adapterWrite(filename, result);
const stat = await this.plugin.vaultAccess.adapterStat(filename);
if (!stat) {
throw new Error("Stat failed");
}
const mtime = stat?.mtime ?? 0;
await this.storeInternalFileToDatabase({ path: filename, mtime, ctime: stat?.ctime ?? mtime, size: stat?.size ?? 0 }, true);
try {
//@ts-ignore internalAPI
await this.app.vault.adapter.reconcileInternalFile(filename);
} catch (ex) {
Logger("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE);
Logger(ex, LOG_LEVEL_VERBOSE);
}
Logger(`STORAGE <-- DB:${filename}: written (hidden,merged)`);
}
if (needFlush) {
await this.extractInternalFileFromDatabase(filename, false);
Logger(`STORAGE --> DB:${filename}: extracted (hidden,merged)`);
}
res(true);
} catch (ex) {
Logger("Could not merge conflicted json");
Logger(ex, LOG_LEVEL_VERBOSE);
res(false);
}
});
modal.open();
});
}
async scanInternalFiles(): Promise<InternalFileInfo[]> {
const configDir = normalizePath(this.app.vault.configDir);
const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
const synchronisedInConfigSync = !this.settings.usePluginSync ? [] : Object.values(this.settings.pluginSyncExtendedSetting).filter(e => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase());
const root = this.app.vault.getRoot();
const findRoot = root.path;
const filenames = (await this.getFiles(findRoot, [], undefined, ignoreFilter)).filter(e => e.startsWith(".")).filter(e => !e.startsWith(".trash"));
const files = filenames.filter(path => synchronisedInConfigSync.every(filterFile => !path.toLowerCase().startsWith(filterFile))).map(async (e) => {
return {
path: e as FilePath,
stat: await this.plugin.vaultAccess.adapterStat(e)
};
});
const result: InternalFileInfo[] = [];
for (const f of files) {
const w = await f;
if (await this.plugin.isIgnoredByIgnoreFiles(w.path)) {
continue
}
const mtime = w.stat?.mtime ?? 0
const ctime = w.stat?.ctime ?? mtime;
const size = w.stat?.size ?? 0;
result.push({
...w,
mtime, ctime, size
});
}
return result;
}
async getFiles(
path: string,
ignoreList: string[],
filter?: RegExp[],
ignoreFilter?: RegExp[]
) {
let w: ListedFiles;
try {
w = await this.app.vault.adapter.list(path);
} catch (ex) {
Logger(`Could not traverse(HiddenSync):${path}`, LOG_LEVEL_INFO);
Logger(ex, LOG_LEVEL_VERBOSE);
return [];
}
const filesSrc = [
...w.files
.filter((e) => !ignoreList.some((ee) => e.endsWith(ee)))
.filter((e) => !filter || filter.some((ee) => e.match(ee)))
.filter((e) => !ignoreFilter || ignoreFilter.every((ee) => !e.match(ee)))
];
let files = [] as string[];
for (const file of filesSrc) {
if (!await this.plugin.isIgnoredByIgnoreFiles(file)) {
files.push(file);
}
}
L1: for (const v of w.folders) {
for (const ignore of ignoreList) {
if (v.endsWith(ignore)) {
continue L1;
}
}
if (ignoreFilter && ignoreFilter.some(e => v.match(e))) {
continue L1;
}
if (await this.plugin.isIgnoredByIgnoreFiles(v)) {
continue L1;
}
files = files.concat(await this.getFiles(v, ignoreList, filter, ignoreFilter));
}
return files;
}
}

424
src/features/CmdSetupLiveSync.ts
View File

@@ -1,424 +0,0 @@
import { type EntryDoc, type ObsidianLiveSyncSettings, DEFAULT_SETTINGS, LOG_LEVEL_NOTICE, REMOTE_COUCHDB, REMOTE_MINIO } from "../lib/src/common/types.ts";
import { configURIBase } from "../common/types.ts";
import { Logger } from "../lib/src/common/logger.ts";
import { PouchDB } from "../lib/src/pouchdb/pouchdb-browser.js";
import { askSelectString, askYesNo, askString } from "../common/utils.ts";
import { decrypt, encrypt } from "../lib/src/encryption/e2ee_v2.ts";
import { LiveSyncCommands } from "./LiveSyncCommands.ts";
import { delay, fireAndForget } from "../lib/src/common/utils.ts";
import { confirmWithMessage } from "../common/dialogs.ts";
import { Platform } from "../deps.ts";
import { fetchAllUsedChunks } from "../lib/src/pouchdb/utils_couchdb.ts";
import type { LiveSyncCouchDBReplicator } from "../lib/src/replication/couchdb/LiveSyncReplicator.js";
export class SetupLiveSync extends LiveSyncCommands {
onunload() { }
onload(): void | Promise<void> {
this.plugin.registerObsidianProtocolHandler("setuplivesync", async (conf: any) => await this.setupWizard(conf.settings));
this.plugin.addCommand({
id: "livesync-copysetupuri",
name: "Copy settings as a new setup URI",
callback: () => fireAndForget(this.command_copySetupURI()),
});
this.plugin.addCommand({
id: "livesync-copysetupuri-short",
name: "Copy settings as a new setup URI (With customization sync)",
callback: () => fireAndForget(this.command_copySetupURIWithSync()),
});
this.plugin.addCommand({
id: "livesync-copysetupurifull",
name: "Copy settings as a new setup URI (Full)",
callback: () => fireAndForget(this.command_copySetupURIFull()),
});
this.plugin.addCommand({
id: "livesync-opensetupuri",
name: "Use the copied setup URI (Formerly Open setup URI)",
callback: () => fireAndForget(this.command_openSetupURI()),
});
}
onInitializeDatabase(showNotice: boolean) { }
beforeReplicate(showNotice: boolean) { }
onResume() { }
parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>): boolean | Promise<boolean> {
return false;
}
async realizeSettingSyncMode() { }
async command_copySetupURI(stripExtra = true) {
const encryptingPassphrase = await askString(this.app, "Encrypt your settings", "The passphrase to encrypt the setup URI", "", true);
if (encryptingPassphrase === false)
return;
const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" } as Partial<ObsidianLiveSyncSettings>;
if (stripExtra) {
delete setting.pluginSyncExtendedSetting;
}
const keys = Object.keys(setting) as (keyof ObsidianLiveSyncSettings)[];
for (const k of keys) {
if (JSON.stringify(k in setting ? setting[k] : "") == JSON.stringify(k in DEFAULT_SETTINGS ? DEFAULT_SETTINGS[k] : "*")) {
delete setting[k];
}
}
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri);
Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);
}
async command_copySetupURIFull() {
const encryptingPassphrase = await askString(this.app, "Encrypt your settings", "The passphrase to encrypt the setup URI", "", true);
if (encryptingPassphrase === false)
return;
const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" };
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri);
Logger("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);
}
async command_copySetupURIWithSync() {
await this.command_copySetupURI(false);
}
async command_openSetupURI() {
const setupURI = await askString(this.app, "Easy setup", "Set up URI", `${configURIBase}aaaaa`);
if (setupURI === false)
return;
if (!setupURI.startsWith(`${configURIBase}`)) {
Logger("Set up URI looks wrong.", LOG_LEVEL_NOTICE);
return;
}
const config = decodeURIComponent(setupURI.substring(configURIBase.length));
console.dir(config);
await this.setupWizard(config);
}
async setupWizard(confString: string) {
try {
const oldConf = JSON.parse(JSON.stringify(this.settings));
const encryptingPassphrase = await askString(this.app, "Passphrase", "The passphrase to decrypt your setup URI", "", true);
if (encryptingPassphrase === false)
return;
const newConf = await JSON.parse(await decrypt(confString, encryptingPassphrase, false));
if (newConf) {
const result = await askYesNo(this.app, "Importing LiveSync's conf, OK?");
if (result == "yes") {
const newSettingW = Object.assign({}, DEFAULT_SETTINGS, newConf) as ObsidianLiveSyncSettings;
this.plugin.replicator.closeReplication();
this.settings.suspendFileWatching = true;
console.dir(newSettingW);
// Back into the default method once.
newSettingW.configPassphraseStore = "";
newSettingW.encryptedPassphrase = "";
newSettingW.encryptedCouchDBConnection = "";
newSettingW.additionalSuffixOfDatabaseName = `${("appId" in this.app ? this.app.appId : "")}`
const setupJustImport = "Just import setting";
const setupAsNew = "Set it up as secondary or subsequent device";
const setupAsMerge = "Secondary device but try keeping local changes";
const setupAgain = "Reconfigure and reconstitute the data";
const setupManually = "Leave everything to me";
newSettingW.syncInternalFiles = false;
newSettingW.usePluginSync = false;
newSettingW.isConfigured = true;
// Migrate completely obsoleted configuration.
if (!newSettingW.useIndexedDBAdapter) {
newSettingW.useIndexedDBAdapter = true;
}
const setupType = await askSelectString(this.app, "How would you like to set it up?", [setupAsNew, setupAgain, setupAsMerge, setupJustImport, setupManually]);
if (setupType == setupJustImport) {
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
await this.plugin.saveSettings();
} else if (setupType == setupAsNew) {
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
await this.fetchLocal();
} else if (setupType == setupAsMerge) {
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
await this.fetchLocalWithRebuild();
} else if (setupType == setupAgain) {
const confirm = "I know this operation will rebuild all my databases with files on this device, and files that are on the remote database and I didn't synchronize to any other devices will be lost and want to proceed indeed.";
if (await askSelectString(this.app, "Do you really want to do this?", ["Cancel", confirm]) != confirm) {
return;
}
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
await this.rebuildEverything();
} else if (setupType == setupManually) {
const keepLocalDB = await askYesNo(this.app, "Keep local DB?");
const keepRemoteDB = await askYesNo(this.app, "Keep remote DB?");
if (keepLocalDB == "yes" && keepRemoteDB == "yes") {
// nothing to do. so peaceful.
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
this.suspendAllSync();
this.suspendExtraSync();
await this.plugin.saveSettings();
const replicate = await askYesNo(this.app, "Unlock and replicate?");
if (replicate == "yes") {
await this.plugin.replicate(true);
await this.plugin.markRemoteUnlocked();
}
Logger("Configuration loaded.", LOG_LEVEL_NOTICE);
return;
}
if (keepLocalDB == "no" && keepRemoteDB == "no") {
const reset = await askYesNo(this.app, "Drop everything?");
if (reset != "yes") {
Logger("Cancelled", LOG_LEVEL_NOTICE);
this.plugin.settings = oldConf;
return;
}
}
let initDB;
this.plugin.settings = newSettingW;
this.plugin.usedPassphrase = "";
await this.plugin.saveSettings();
if (keepLocalDB == "no") {
await this.plugin.resetLocalDatabase();
await this.plugin.localDatabase.initializeDatabase();
const rebuild = await askYesNo(this.app, "Rebuild the database?");
if (rebuild == "yes") {
initDB = this.plugin.initializeDatabase(true);
} else {
await this.plugin.markRemoteResolved();
}
}
if (keepRemoteDB == "no") {
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
}
if (keepLocalDB == "no" || keepRemoteDB == "no") {
const replicate = await askYesNo(this.app, "Replicate once?");
if (replicate == "yes") {
if (initDB != null) {
await initDB;
}
await this.plugin.replicate(true);
}
}
}
}
Logger("Configuration loaded.", LOG_LEVEL_NOTICE);
} else {
Logger("Cancelled.", LOG_LEVEL_NOTICE);
}
} catch (ex) {
Logger("Couldn't parse or decrypt configuration uri.", LOG_LEVEL_NOTICE);
}
}
suspendExtraSync() {
Logger("Hidden files and plugin synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL_NOTICE)
this.plugin.settings.syncInternalFiles = false;
this.plugin.settings.usePluginSync = false;
this.plugin.settings.autoSweepPlugins = false;
}
async askHiddenFileConfiguration(opt: { enableFetch?: boolean, enableOverwrite?: boolean }) {
this.plugin.addOnSetup.suspendExtraSync();
const message = `Would you like to enable \`Hidden File Synchronization\` or \`Customization sync\`?
${opt.enableFetch ? " - Fetch: Use files stored from other devices. \n" : ""}${opt.enableOverwrite ? "- Overwrite: Use files from this device. \n" : ""}- Custom: Synchronize only customization files with a dedicated interface.
- Keep them disabled: Do not use hidden file synchronization.
Of course, we are able to disable these features.`
const CHOICE_FETCH = "Fetch";
const CHOICE_OVERWRITE = "Overwrite";
const CHOICE_CUSTOMIZE = "Custom";
const CHOICE_DISMISS = "keep them disabled";
const choices = [];
if (opt?.enableFetch) {
choices.push(CHOICE_FETCH);
}
if (opt?.enableOverwrite) {
choices.push(CHOICE_OVERWRITE);
}
choices.push(CHOICE_CUSTOMIZE);
choices.push(CHOICE_DISMISS);
const ret = await confirmWithMessage(this.plugin, "Hidden file sync", message, choices, CHOICE_DISMISS, 40);
if (ret == CHOICE_FETCH) {
await this.configureHiddenFileSync("FETCH");
} else if (ret == CHOICE_OVERWRITE) {
await this.configureHiddenFileSync("OVERWRITE");
} else if (ret == CHOICE_DISMISS) {
await this.configureHiddenFileSync("DISABLE");
} else if (ret == CHOICE_CUSTOMIZE) {
await this.configureHiddenFileSync("CUSTOMIZE");
}
}
async configureHiddenFileSync(mode: "FETCH" | "OVERWRITE" | "MERGE" | "DISABLE" | "CUSTOMIZE") {
this.plugin.addOnSetup.suspendExtraSync();
if (mode == "DISABLE") {
this.plugin.settings.syncInternalFiles = false;
this.plugin.settings.usePluginSync = false;
await this.plugin.saveSettings();
return;
}
if (mode != "CUSTOMIZE") {
Logger("Gathering files for enabling Hidden File Sync", LOG_LEVEL_NOTICE);
if (mode == "FETCH") {
await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("pullForce", true);
} else if (mode == "OVERWRITE") {
await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("pushForce", true);
} else if (mode == "MERGE") {
await this.plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase("safe", true);
}
this.plugin.settings.syncInternalFiles = true;
await this.plugin.saveSettings();
Logger(`Done! Restarting the app is strongly recommended!`, LOG_LEVEL_NOTICE);
} else if (mode == "CUSTOMIZE") {
if (!this.plugin.deviceAndVaultName) {
let name = await askString(this.app, "Device name", "Please set this device name", `desktop`);
if (!name) {
if (Platform.isAndroidApp) {
name = "android-app"
} else if (Platform.isIosApp) {
name = "ios"
} else if (Platform.isMacOS) {
name = "macos"
} else if (Platform.isMobileApp) {
name = "mobile-app"
} else if (Platform.isMobile) {
name = "mobile"
} else if (Platform.isSafari) {
name = "safari"
} else if (Platform.isDesktop) {
name = "desktop"
} else if (Platform.isDesktopApp) {
name = "desktop-app"
} else {
name = "unknown"
}
name = name + Math.random().toString(36).slice(-4);
}
this.plugin.deviceAndVaultName = name;
}
this.plugin.settings.usePluginSync = true;
await this.plugin.saveSettings();
await this.plugin.addOnConfigSync.scanAllConfigFiles(true);
}
}
suspendAllSync() {
this.plugin.settings.liveSync = false;
this.plugin.settings.periodicReplication = false;
this.plugin.settings.syncOnSave = false;
this.plugin.settings.syncOnEditorSave = false;
this.plugin.settings.syncOnStart = false;
this.plugin.settings.syncOnFileOpen = false;
this.plugin.settings.syncAfterMerge = false;
//this.suspendExtraSync();
}
async suspendReflectingDatabase() {
if (this.plugin.settings.doNotSuspendOnFetching) return;
if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
Logger(`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`, LOG_LEVEL_NOTICE);
this.plugin.settings.suspendParseReplicationResult = true;
this.plugin.settings.suspendFileWatching = true;
await this.plugin.saveSettings();
}
async resumeReflectingDatabase() {
if (this.plugin.settings.doNotSuspendOnFetching) return;
if (this.plugin.settings.remoteType == REMOTE_MINIO) return;
Logger(`Database and storage reflection has been resumed!`, LOG_LEVEL_NOTICE);
this.plugin.settings.suspendParseReplicationResult = false;
this.plugin.settings.suspendFileWatching = false;
await this.plugin.syncAllFiles(true);
await this.plugin.loadQueuedFiles();
await this.plugin.saveSettings();
}
async askUseNewAdapter() {
if (!this.plugin.settings.useIndexedDBAdapter) {
const message = `Now this plugin has been configured to use the old database adapter for keeping compatibility. Do you want to deactivate it?`;
const CHOICE_YES = "Yes, disable and use latest";
const CHOICE_NO = "No, keep compatibility";
const choices = [CHOICE_YES, CHOICE_NO];
const ret = await confirmWithMessage(this.plugin, "Database adapter", message, choices, CHOICE_YES, 10);
if (ret == CHOICE_YES) {
this.plugin.settings.useIndexedDBAdapter = true;
}
}
}
async resetLocalDatabase() {
if (this.plugin.settings.isConfigured && this.plugin.settings.additionalSuffixOfDatabaseName == "") {
// Discard the non-suffixed database
await this.plugin.resetLocalDatabase();
}
this.plugin.settings.additionalSuffixOfDatabaseName = `${("appId" in this.app ? this.app.appId : "")}`
await this.plugin.resetLocalDatabase();
}
async fetchRemoteChunks() {
if (!this.plugin.settings.doNotSuspendOnFetching && this.plugin.settings.readChunksOnline && this.plugin.settings.remoteType == REMOTE_COUCHDB) {
Logger(`Fetching chunks`, LOG_LEVEL_NOTICE);
const replicator = this.plugin.getReplicator() as LiveSyncCouchDBReplicator;
const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.plugin.getIsMobile(), true);
if (typeof remoteDB == "string") {
Logger(remoteDB, LOG_LEVEL_NOTICE);
} else {
await fetchAllUsedChunks(this.localDatabase.localDatabase, remoteDB.db);
}
Logger(`Fetching chunks done`, LOG_LEVEL_NOTICE);
}
}
async fetchLocal(makeLocalChunkBeforeSync?: boolean) {
this.suspendExtraSync();
await this.askUseNewAdapter();
this.plugin.settings.isConfigured = true;
await this.suspendReflectingDatabase();
await this.plugin.realizeSettingSyncMode();
await this.resetLocalDatabase();
await delay(1000);
await this.plugin.openDatabase();
this.plugin.isReady = true;
if (makeLocalChunkBeforeSync) {
await this.plugin.createAllChunks(true);
}
await this.plugin.markRemoteResolved();
await delay(500);
await this.plugin.replicateAllFromServer(true);
await delay(1000);
await this.plugin.replicateAllFromServer(true);
await this.resumeReflectingDatabase();
await this.askHiddenFileConfiguration({ enableFetch: true });
}
async fetchLocalWithRebuild() {
return await this.fetchLocal(true);
}
async rebuildRemote() {
this.suspendExtraSync();
this.plugin.settings.isConfigured = true;
await this.plugin.realizeSettingSyncMode();
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await delay(500);
await this.askHiddenFileConfiguration({ enableOverwrite: true });
await delay(1000);
await this.plugin.replicateAllToServer(true);
await delay(1000);
await this.plugin.replicateAllToServer(true);
}
async rebuildEverything() {
this.suspendExtraSync();
await this.askUseNewAdapter();
this.plugin.settings.isConfigured = true;
await this.plugin.realizeSettingSyncMode();
await this.resetLocalDatabase();
await delay(1000);
await this.plugin.initializeDatabase(true);
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await delay(500);
await this.askHiddenFileConfiguration({ enableOverwrite: true });
await delay(1000);
await this.plugin.replicateAllToServer(true);
await delay(1000);
await this.plugin.replicateAllToServer(true);
}
}

505
src/features/CmdConfigSync.ts → src/features/ConfigSync/CmdConfigSync.ts
View File

@@ -1,25 +1,26 @@
import { writable } from 'svelte/store';
import { Notice, type PluginManifest, parseYaml, normalizePath, type ListedFiles, diff_match_patch } from "../deps.ts";
import { Notice, type PluginManifest, parseYaml, normalizePath, type ListedFiles, diff_match_patch, Platform, addIcon } from "../../deps.ts";
import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, AnyEntry, SavingEntry, diff_result } from "../lib/src/common/types.ts";
import { CANCELLED, LEAVE_TO_SUBSEQUENT, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_SHINY } from "../lib/src/common/types.ts";
import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "../common/types.ts";
import { createBlob, createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocData, getDocDataAsArray, isDocContentSame, isLoadedEntry, isObjectDifferent } from "../lib/src/common/utils.ts";
import { Logger } from "../lib/src/common/logger.ts";
import { digestHash } from "../lib/src/string_and_binary/hash.ts";
import { arrayBufferToBase64, decodeBinary, readString } from 'src/lib/src/string_and_binary/convert.ts';
import { serialized, shareRunningResult } from "../lib/src/concurrency/lock.ts";
import { LiveSyncCommands } from "./LiveSyncCommands.ts";
import { stripAllPrefixes } from "../lib/src/string_and_binary/path.ts";
import { EVEN, PeriodicProcessor, disposeMemoObject, isMarkedAsSameChanges, markChangesAreSame, memoIfNotExist, memoObject, retrieveMemoObject, scheduleTask } from "../common/utils.ts";
import { PluginDialogModal } from "../common/dialogs.ts";
import { JsonResolveModal } from "../ui/JsonResolveModal.ts";
import { QueueProcessor } from '../lib/src/concurrency/processor.ts';
import { pluginScanningCount } from '../lib/src/mock_and_interop/stores.ts';
import type ObsidianLiveSyncPlugin from '../main.ts';
import type { EntryDoc, LoadedEntry, InternalFileEntry, FilePathWithPrefix, FilePath, AnyEntry, SavingEntry, diff_result } from "../../lib/src/common/types.ts";
import { CANCELLED, LEAVE_TO_SUBSEQUENT, LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_SHINY } from "../../lib/src/common/types.ts";
import { ICXHeader, PERIODIC_PLUGIN_SWEEP, } from "../../common/types.ts";
import { createBlob, createSavingEntryFromLoadedEntry, createTextBlob, delay, fireAndForget, getDocData, getDocDataAsArray, isDocContentSame, isLoadedEntry, isObjectDifferent } from "../../lib/src/common/utils.ts";
import { digestHash } from "../../lib/src/string_and_binary/hash.ts";
import { arrayBufferToBase64, decodeBinary, readString } from '../../lib/src/string_and_binary/convert.ts';
import { serialized, shareRunningResult } from "../../lib/src/concurrency/lock.ts";
import { LiveSyncCommands } from "../LiveSyncCommands.ts";
import { stripAllPrefixes } from "../../lib/src/string_and_binary/path.ts";
import { EVEN, PeriodicProcessor, disposeMemoObject, isCustomisationSyncMetadata, isMarkedAsSameChanges, isPluginMetadata, markChangesAreSame, memoIfNotExist, memoObject, retrieveMemoObject, scheduleTask } from "../../common/utils.ts";
import { JsonResolveModal } from "../HiddenFileCommon/JsonResolveModal.ts";
import { QueueProcessor } from '../../lib/src/concurrency/processor.ts';
import { pluginScanningCount } from '../../lib/src/mock_and_interop/stores.ts';
import type ObsidianLiveSyncPlugin from '../../main.ts';
import { base64ToArrayBuffer, base64ToString } from 'octagonal-wheels/binary/base64';
import { ConflictResolveModal } from '../ui/ConflictResolveModal.ts';
import { ConflictResolveModal } from '../../modules/features/InteractiveConflictResolving/ConflictResolveModal.ts';
import { Semaphore } from 'octagonal-wheels/concurrency/semaphore';
import type { IObsidianModule } from '../../modules/AbstractObsidianModule.ts';
import { EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG, eventHub } from '../../common/events.ts';
import { PluginDialogModal } from "./PluginDialogModal.ts";
const d = "\u200b";
const d2 = "\n";
@@ -184,10 +185,10 @@ function deserialize<T>(str: string[], def: T) {
return o;
}
return JSON.parse(str.join("")) as T;
} catch (ex) {
} catch {
try {
return parseYaml(str.join(""));
} catch (ex) {
} catch {
return def;
}
}
@@ -322,13 +323,14 @@ export type PluginDataEx = {
mtime: number,
};
export class ConfigSync extends LiveSyncCommands {
export class ConfigSync extends LiveSyncCommands implements IObsidianModule {
constructor(plugin: ObsidianLiveSyncPlugin) {
super(plugin);
pluginScanningCount.onChanged((e) => {
const total = e.value;
pluginIsEnumerating.set(total != 0);
})
}
get kvDB() {
return this.plugin.kvDB;
@@ -340,13 +342,16 @@ export class ConfigSync extends LiveSyncCommands {
get useSyncPluginEtc() {
return this.plugin.settings.usePluginEtc;
}
_isThisModuleEnabled() {
return this.plugin.settings.usePluginSync;
}
pluginDialog?: PluginDialogModal = undefined;
periodicPluginSweepProcessor = new PeriodicProcessor(this.plugin, async () => await this.scanAllConfigFiles(false));
pluginList: IPluginDataExDisplay[] = [];
showPluginSyncModal() {
if (!this.settings.usePluginSync) {
if (!this._isThisModuleEnabled()) {
return;
}
if (this.pluginDialog) {
@@ -367,7 +372,14 @@ export class ConfigSync extends LiveSyncCommands {
this.hidePluginSyncModal();
this.periodicPluginSweepProcessor?.disable();
}
addRibbonIcon = this.plugin.addRibbonIcon.bind(this.plugin);
onload() {
addIcon(
"custom-sync",
`<g transform="rotate(-90 75 218)" fill="currentColor" fill-rule="evenodd">
<path d="m272 166-9.38 9.38 9.38 9.38 9.38-9.38c1.96-1.93 5.11-1.9 7.03 0.058 1.91 1.94 1.91 5.04 0 6.98l-9.38 9.38 5.86 5.86-11.7 11.7c-8.34 8.35-21.4 9.68-31.3 3.19l-3.84 3.98c-8.45 8.7-20.1 13.6-32.2 13.6h-5.55v-9.95h5.55c9.43-0.0182 18.5-3.84 25-10.6l3.95-4.09c-6.54-9.86-5.23-23 3.14-31.3l11.7-11.7 5.86 5.86 9.38-9.38c1.96-1.93 5.11-1.9 7.03 0.0564 1.91 1.93 1.91 5.04 2e-3 6.98z"/>
</g>`
);
this.plugin.addCommand({
id: "livesync-plugin-dialog-ex",
name: "Show customization sync dialog",
@@ -375,6 +387,10 @@ export class ConfigSync extends LiveSyncCommands {
this.showPluginSyncModal();
},
});
this.addRibbonIcon("custom-sync", "Show Customization sync", () => {
this.showPluginSyncModal();
}).addClass("livesync-ribbon-showcustom");
eventHub.onEvent(EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG, () => this.showPluginSyncModal());
}
getFileCategory(filePath: string): "CONFIG" | "THEME" | "SNIPPET" | "PLUGIN_MAIN" | "PLUGIN_ETC" | "PLUGIN_DATA" | "" {
@@ -399,33 +415,41 @@ export class ConfigSync extends LiveSyncCommands {
// Idea non-filter option?
return this.getFileCategory(filePath) != "";
}
async onInitializeDatabase(showNotice: boolean) {
if (this.settings.usePluginSync) {
try {
Logger("Scanning customizations...");
await this.scanAllConfigFiles(showNotice);
Logger("Scanning customizations : done");
} catch (ex) {
Logger("Scanning customizations : failed");
Logger(ex, LOG_LEVEL_VERBOSE);
}
}
}
async beforeReplicate(showNotice: boolean) {
if (this.settings.autoSweepPlugins && this.settings.usePluginSync) {
async $everyOnDatabaseInitialized(showNotice: boolean) {
if (!this._isThisModuleEnabled()) return true;
try {
this._log("Scanning customizations...");
await this.scanAllConfigFiles(showNotice);
this._log("Scanning customizations : done");
} catch (ex) {
this._log("Scanning customizations : failed");
this._log(ex, LOG_LEVEL_VERBOSE);
}
return true;
}
async onResume() {
if (this.plugin.suspended) {
return;
async $everyBeforeReplicate(showNotice: boolean) {
if (!this._isThisModuleEnabled()) return true;
if (this.settings.autoSweepPlugins) {
await this.scanAllConfigFiles(showNotice);
return true;
}
if (this.settings.autoSweepPlugins && this.settings.usePluginSync) {
return true;
}
async $everyOnResumeProcess(): Promise<boolean> {
if (!this._isThisModuleEnabled()) return true;
if (this._isMainSuspended()) {
return true;
}
if (this.settings.autoSweepPlugins) {
await this.scanAllConfigFiles(false);
}
this.periodicPluginSweepProcessor.enable(this.settings.autoSweepPluginsPeriodic && !this.settings.watchInternalFileChanges ? (PERIODIC_PLUGIN_SWEEP * 1000) : 0);
return true;
}
$everyAfterResumeProcess(): Promise<boolean> {
const q = activeDocument.querySelector(`.livesync-ribbon-showcustom`);
q?.toggleClass("sls-hidden", !this._isThisModuleEnabled());
return Promise.resolve(true);
}
async reloadPluginList(showMessage: boolean) {
this.pluginList = [];
@@ -452,7 +476,7 @@ export class ConfigSync extends LiveSyncCommands {
xFiles.push(work);
}
if (missingHash) {
Logger(`Digest created for ${path} to improve checking`, LOG_LEVEL_VERBOSE);
this._log(`Digest created for ${path} to improve checking`, LOG_LEVEL_VERBOSE);
wx.data = serialize(data);
fireAndForget(() => this.localDatabase.putDBEntry(createSavingEntryFromLoadedEntry(wx)));
}
@@ -487,8 +511,8 @@ export class ConfigSync extends LiveSyncCommands {
return [];
} catch (ex) {
Logger(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
return [];
}, { suspended: false, batchSize: 1, concurrentLimit: 10, delay: 100, yieldThreshold: 10, maintainDelay: false, totalRemainingReactiveSource: pluginScanningCount }).startPipeline();
@@ -511,8 +535,8 @@ export class ConfigSync extends LiveSyncCommands {
return [];
} catch (ex) {
Logger(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
return [];
}, { suspended: false, batchSize: 1, concurrentLimit: 10, delay: 100, yieldThreshold: 10, maintainDelay: false, totalRemainingReactiveSource: pluginScanningCount }).startPipeline();
@@ -558,11 +582,11 @@ export class ConfigSync extends LiveSyncCommands {
if (!loaded) {
const d = await this.localDatabase.getDBEntry(unifiedPathV2);
if (!d) {
Logger(`The file ${unifiedPathV2} is not found`, LOG_LEVEL_VERBOSE);
this._log(`The file ${unifiedPathV2} is not found`, LOG_LEVEL_VERBOSE);
return false;
}
if (!isLoadedEntry(d)) {
Logger(`The file ${unifiedPathV2} is not a note`, LOG_LEVEL_VERBOSE);
this._log(`The file ${unifiedPathV2} is not a note`, LOG_LEVEL_VERBOSE);
return false;
}
loaded = d;
@@ -588,8 +612,8 @@ export class ConfigSync extends LiveSyncCommands {
this.pluginList.filter(e => e instanceof PluginDataExDisplayV2 && e.confKey == confKey).forEach(e => (e as PluginDataExDisplayV2).applyLoadedManifest());
pluginList.set(this.pluginList);
} catch (ex) {
Logger(`The file ${loaded.path} seems to manifest, but could not be decoded as JSON`, LOG_LEVEL_VERBOSE);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`The file ${loaded.path} seems to manifest, but could not be decoded as JSON`, LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
this.loadedManifest_mTime.set(confKey, file.mtime);
} else {
@@ -662,22 +686,22 @@ export class ConfigSync extends LiveSyncCommands {
async migrateV1ToV2(showMessage: boolean, entry: AnyEntry): Promise<void> {
const v1Path = entry.path;
Logger(`Migrating ${entry.path} to V2`, showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
this._log(`Migrating ${entry.path} to V2`, showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
if (entry.deleted) {
Logger(`The entry ${v1Path} is already deleted`, LOG_LEVEL_VERBOSE);
this._log(`The entry ${v1Path} is already deleted`, LOG_LEVEL_VERBOSE);
return;
}
if (!v1Path.endsWith(".md") && !v1Path.startsWith(ICXHeader)) {
Logger(`The entry ${v1Path} is not a customisation sync binder`, LOG_LEVEL_VERBOSE);
this._log(`The entry ${v1Path} is not a customisation sync binder`, LOG_LEVEL_VERBOSE);
return
}
if (v1Path.indexOf("%") !== -1) {
Logger(`The entry ${v1Path} is already migrated`, LOG_LEVEL_VERBOSE);
this._log(`The entry ${v1Path} is already migrated`, LOG_LEVEL_VERBOSE);
return;
}
const loadedEntry = await this.localDatabase.getDBEntry(v1Path);
if (!loadedEntry) {
Logger(`The entry ${v1Path} is not found`, LOG_LEVEL_VERBOSE);
this._log(`The entry ${v1Path} is not found`, LOG_LEVEL_VERBOSE);
return;
}
@@ -698,8 +722,8 @@ export class ConfigSync extends LiveSyncCommands {
const relativeFilename = f.filename.split("/").slice(deletePrefixCount).join("/");
const v2Path = (prefixPath + relativeFilename) as FilePathWithPrefix;
// console.warn(`Migrating ${v1Path} / ${relativeFilename} to ${v2Path}`);
Logger(`Migrating ${v1Path} / ${relativeFilename} to ${v2Path}`, LOG_LEVEL_VERBOSE);
const newId = await this.plugin.path2id(v2Path);
this._log(`Migrating ${v1Path} / ${relativeFilename} to ${v2Path}`, LOG_LEVEL_VERBOSE);
const newId = await this.plugin.$$path2id(v2Path);
// const buf =
const data = createBlob([DUMMY_HEAD, DUMMY_END, ...getDocDataAsArray(f.data)]);
@@ -717,19 +741,19 @@ export class ConfigSync extends LiveSyncCommands {
}
const r = await this.plugin.localDatabase.putDBEntry(saving);
if (r && r.ok) {
Logger(`Migrated ${v1Path} / ${f.filename} to ${v2Path}`, LOG_LEVEL_INFO);
this._log(`Migrated ${v1Path} / ${f.filename} to ${v2Path}`, LOG_LEVEL_INFO);
const delR = await this.deleteConfigOnDatabase(v1Path);
if (delR) {
Logger(`Deleted ${v1Path} successfully`, LOG_LEVEL_INFO);
this._log(`Deleted ${v1Path} successfully`, LOG_LEVEL_INFO);
} else {
Logger(`Failed to delete ${v1Path}`, LOG_LEVEL_NOTICE);
this._log(`Failed to delete ${v1Path}`, LOG_LEVEL_NOTICE);
}
}
}
}
async updatePluginList(showMessage: boolean, updatedDocumentPath?: FilePathWithPrefix): Promise<void> {
if (!this.settings.usePluginSync) {
if (!this._isThisModuleEnabled()) {
this.pluginScanProcessor.clearQueue();
this.pluginList = [];
pluginList.set(this.pluginList)
@@ -777,9 +801,9 @@ export class ConfigSync extends LiveSyncCommands {
}
const fileA = await loadFile(dataA);
const fileB = await loadFile(dataB);
Logger(`Comparing: ${dataA.documentPath} <-> ${dataB.documentPath}`, LOG_LEVEL_VERBOSE);
this._log(`Comparing: ${dataA.documentPath} <-> ${dataB.documentPath}`, LOG_LEVEL_VERBOSE);
if (!fileA || !fileB) {
Logger(`Could not load ${dataA.name} for comparison: ${!fileA ? dataA.term : ""}${!fileB ? dataB.term : ""}`, LOG_LEVEL_NOTICE);
this._log(`Could not load ${dataA.name} for comparison: ${!fileA ? dataA.term : ""}${!fileB ? dataB.term : ""}`, LOG_LEVEL_NOTICE);
return false;
}
let path = stripAllPrefixes(fileA.path.split("/").slice(-1).join("/") as FilePath); // TODO:adjust
@@ -788,15 +812,15 @@ export class ConfigSync extends LiveSyncCommands {
}
if (fileA.path.endsWith(".json")) {
return serialized("config:merge-data", () => new Promise<boolean>((res) => {
Logger("Opening data-merging dialog", LOG_LEVEL_VERBOSE);
this._log("Opening data-merging dialog", LOG_LEVEL_VERBOSE);
// const docs = [docA, docB];
const modal = new JsonResolveModal(this.app, path, [fileA, fileB], async (keep, result) => {
if (result == null) return res(false);
try {
res(await this.applyData(dataA, result));
} catch (ex) {
Logger("Could not apply merged file");
Logger(ex, LOG_LEVEL_VERBOSE);
this._log("Could not apply merged file");
this._log(ex, LOG_LEVEL_VERBOSE);
res(false);
}
}, "Local", `${dataB.term}`, "B", true, true, "Difference between local and remote");
@@ -822,7 +846,7 @@ export class ConfigSync extends LiveSyncCommands {
right: { rev: "B", ...fileB, data: docBData },
diff: diff
}
console.dir(diffResult);
// console.dir(diffResult);
const d = new ConflictResolveModal(this.app, path, diffResult, true, dataB.term);
d.open();
const ret = await d.waitForResult();
@@ -841,11 +865,11 @@ export class ConfigSync extends LiveSyncCommands {
if (content) {
// const dt = createBlob(content);
const filename = data.files[0].filename;
Logger(`Applying ${filename} of ${data.displayName || data.name}..`);
this._log(`Applying ${filename} of ${data.displayName || data.name}..`);
const path = `${baseDir}/${filename}` as FilePath;
await this.vaultAccess.ensureDirectory(path);
await this.plugin.storageAccess.ensureDir(path);
// If the content has applied, modified time will be updated to the current time.
await this.vaultAccess.adapterWrite(path, content);
await this.plugin.storageAccess.writeHiddenFileAuto(path, content);
await this.storeCustomisationFileV2(path, this.plugin.deviceAndVaultName);
} else {
@@ -854,50 +878,54 @@ export class ConfigSync extends LiveSyncCommands {
// If files have applied, modified time will be updated to the current time.
const stat = { mtime: f.mtime, ctime: f.ctime };
const path = `${baseDir}/${f.filename}` as FilePath;
Logger(`Applying ${f.filename} of ${data.displayName || data.name}..`);
this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`);
// const contentEach = createBlob(f.data);
this.vaultAccess.ensureDirectory(path);
await this.plugin.storageAccess.ensureDir(path);
if (f.datatype == "newnote") {
let oldData;
try {
oldData = await this.vaultAccess.adapterReadBinary(path);
oldData = await this.plugin.storageAccess.readHiddenFileBinary(path);
} catch (ex) {
this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
oldData = new ArrayBuffer(0);
}
const content = base64ToArrayBuffer(f.data);
if (await isDocContentSame(oldData, content)) {
Logger(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE);
this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE);
continue;
}
await this.vaultAccess.adapterWrite(path, content, stat);
await this.plugin.storageAccess.writeHiddenFileAuto(path, content, stat);
} else {
let oldData;
try {
oldData = await this.vaultAccess.adapterRead(path);
oldData = await this.plugin.storageAccess.readHiddenFileText(path);
} catch (ex) {
this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
oldData = "";
}
const content = getDocData(f.data);
if (await isDocContentSame(oldData, content)) {
Logger(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE);
this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE);
continue;
}
await this.vaultAccess.adapterWrite(path, content, stat);
await this.plugin.storageAccess.writeHiddenFileAuto(path, content, stat);
}
Logger(`Applied ${f.filename} of ${data.displayName || data.name}..`);
this._log(`Applied ${f.filename} of ${data.displayName || data.name}..`);
await this.storeCustomisationFileV2(path, this.plugin.deviceAndVaultName);
}
}
} catch (ex) {
Logger(`Applying ${data.displayName || data.name}.. Failed`, LOG_LEVEL_NOTICE);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`Applying ${data.displayName || data.name}.. Failed`, LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
return true;
}
async applyData(data: IPluginDataExDisplay, content?: string): Promise<boolean> {
Logger(`Applying ${data.displayName || data.name
this._log(`Applying ${data.displayName || data.name
}..`);
if (data instanceof PluginDataExDisplayV2) {
@@ -912,22 +940,22 @@ export class ConfigSync extends LiveSyncCommands {
}
const loadedData = deserialize(getDocDataAsArray(dx.data), {}) as PluginDataEx;
for (const f of loadedData.files) {
Logger(`Applying ${f.filename} of ${data.displayName || data.name}..`);
this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`);
try {
// console.dir(f);
const path = `${baseDir}/${f.filename}`;
await this.vaultAccess.ensureDirectory(path);
await this.plugin.storageAccess.ensureDir(path);
if (!content) {
const dt = decodeBinary(f.data);
await this.vaultAccess.adapterWrite(path, dt);
await this.plugin.storageAccess.writeHiddenFileAuto(path, dt);
} else {
await this.vaultAccess.adapterWrite(path, content);
await this.plugin.storageAccess.writeHiddenFileAuto(path, content);
}
Logger(`Applying ${f.filename} of ${data.displayName || data.name}.. Done`);
this._log(`Applying ${f.filename} of ${data.displayName || data.name}.. Done`);
} catch (ex) {
Logger(`Applying ${f.filename} of ${data.displayName || data.name}.. Failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`Applying ${f.filename} of ${data.displayName || data.name}.. Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
}
}
@@ -935,7 +963,7 @@ export class ConfigSync extends LiveSyncCommands {
await this.storeCustomizationFiles(uPath);
await this.updatePluginList(true, uPath);
await delay(100);
Logger(`Config ${data.displayName || data.name} has been applied`, LOG_LEVEL_NOTICE);
this._log(`Config ${data.displayName || data.name} has been applied`, LOG_LEVEL_NOTICE);
if (data.category == "PLUGIN_DATA" || data.category == "PLUGIN_MAIN") {
//@ts-ignore
const manifests = Object.values(this.app.plugins.manifests) as any as PluginManifest[];
@@ -943,20 +971,20 @@ export class ConfigSync extends LiveSyncCommands {
const enabledPlugins = this.app.plugins.enabledPlugins as Set<string>;
const pluginManifest = manifests.find((manifest) => enabledPlugins.has(manifest.id) && manifest.dir == `${baseDir}/plugins/${data.name}`);
if (pluginManifest) {
Logger(`Unloading plugin: ${pluginManifest.name}`, LOG_LEVEL_NOTICE, "plugin-reload-" + pluginManifest.id);
this._log(`Unloading plugin: ${pluginManifest.name}`, LOG_LEVEL_NOTICE, "plugin-reload-" + pluginManifest.id);
// @ts-ignore
await this.app.plugins.unloadPlugin(pluginManifest.id);
// @ts-ignore
await this.app.plugins.loadPlugin(pluginManifest.id);
Logger(`Plugin reloaded: ${pluginManifest.name}`, LOG_LEVEL_NOTICE, "plugin-reload-" + pluginManifest.id);
this._log(`Plugin reloaded: ${pluginManifest.name}`, LOG_LEVEL_NOTICE, "plugin-reload-" + pluginManifest.id);
}
} else if (data.category == "CONFIG") {
this.plugin.askReload();
this.plugin.$$askReload();
}
return true;
} catch (ex) {
Logger(`Applying ${data.displayName || data.name}.. Failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`Applying ${data.displayName || data.name}.. Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
}
@@ -978,84 +1006,81 @@ export class ConfigSync extends LiveSyncCommands {
await Promise.allSettled(p);
// await this.deleteConfigOnDatabase(data.documentPath);
// await this.updatePluginList(false, data.documentPath);
Logger(`Deleted: ${data.category}/${data.name} of ${data.category} (${delList.length} items)`, LOG_LEVEL_NOTICE);
this._log(`Deleted: ${data.category}/${data.name} of ${data.category} (${delList.length} items)`, LOG_LEVEL_NOTICE);
}
return true;
} catch (ex) {
Logger(`Failed to delete: ${data.documentPath}`, LOG_LEVEL_NOTICE);
this._log(`Failed to delete: ${data.documentPath}`, LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
}
async parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>) {
if (docs._id.startsWith(ICXHeader)) {
if (this.plugin.settings.usePluginSync) {
await this.updatePluginList(false, (docs as AnyEntry).path ? (docs as AnyEntry).path : this.getPath((docs as AnyEntry)));
}
if (this.plugin.settings.usePluginSync && this.plugin.settings.notifyPluginOrSettingUpdated) {
if (!this.pluginDialog || (this.pluginDialog && !this.pluginDialog.isOpened())) {
const fragment = createFragment((doc) => {
doc.createEl("span", undefined, (a) => {
a.appendText(`Some configuration has been arrived, Press `);
a.appendChild(a.createEl("a", undefined, (anchor) => {
anchor.text = "HERE";
anchor.addEventListener("click", () => {
this.showPluginSyncModal();
});
}));
async $anyModuleParsedReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>) {
if (!docs._id.startsWith(ICXHeader)) return undefined;
if (this._isThisModuleEnabled()) {
await this.updatePluginList(false, (docs as AnyEntry).path ? (docs as AnyEntry).path : this.getPath((docs as AnyEntry)));
}
if (this._isThisModuleEnabled() && this.plugin.settings.notifyPluginOrSettingUpdated) {
if (!this.pluginDialog || (this.pluginDialog && !this.pluginDialog.isOpened())) {
const fragment = createFragment((doc) => {
doc.createEl("span", undefined, (a) => {
a.appendText(`Some configuration has been arrived, Press `);
a.appendChild(a.createEl("a", undefined, (anchor) => {
anchor.text = "HERE";
anchor.addEventListener("click", () => {
this.showPluginSyncModal();
});
}));
a.appendText(` to open the config sync dialog , or press elsewhere to dismiss this message.`);
});
a.appendText(` to open the config sync dialog , or press elsewhere to dismiss this message.`);
});
});
const updatedPluginKey = "popupUpdated-plugins";
scheduleTask(updatedPluginKey, 1000, async () => {
const popup = await memoIfNotExist(updatedPluginKey, () => new Notice(fragment, 0));
const updatedPluginKey = "popupUpdated-plugins";
scheduleTask(updatedPluginKey, 1000, async () => {
const popup = await memoIfNotExist(updatedPluginKey, () => new Notice(fragment, 0));
//@ts-ignore
const isShown = popup?.noticeEl?.isShown();
if (!isShown) {
memoObject(updatedPluginKey, new Notice(fragment, 0));
}
scheduleTask(updatedPluginKey + "-close", 20000, () => {
const popup = retrieveMemoObject<Notice>(updatedPluginKey);
if (!popup)
return;
//@ts-ignore
const isShown = popup?.noticeEl?.isShown();
if (!isShown) {
memoObject(updatedPluginKey, new Notice(fragment, 0));
if (popup?.noticeEl?.isShown()) {
popup.hide();
}
scheduleTask(updatedPluginKey + "-close", 20000, () => {
const popup = retrieveMemoObject<Notice>(updatedPluginKey);
if (!popup)
return;
//@ts-ignore
if (popup?.noticeEl?.isShown()) {
popup.hide();
}
disposeMemoObject(updatedPluginKey);
});
disposeMemoObject(updatedPluginKey);
});
}
});
}
return true;
}
return false;
return true;
}
async realizeSettingSyncMode(): Promise<void> {
async $everyRealizeSettingSyncMode(): Promise<boolean> {
this.periodicPluginSweepProcessor?.disable();
if (this.plugin.suspended)
return;
if (!this.settings.usePluginSync) {
return;
}
if (!this._isMainReady) return true;
if (!this._isMainSuspended()) return true;
if (!this._isThisModuleEnabled()) return true;
if (this.settings.autoSweepPlugins) {
await this.scanAllConfigFiles(false);
}
this.periodicPluginSweepProcessor.enable(this.settings.autoSweepPluginsPeriodic && !this.settings.watchInternalFileChanges ? (PERIODIC_PLUGIN_SWEEP * 1000) : 0);
return;
return true;
}
recentProcessedInternalFiles = [] as string[];
async makeEntryFromFile(path: FilePath): Promise<false | PluginDataExFile> {
const stat = await this.vaultAccess.adapterStat(path);
const stat = await this.plugin.storageAccess.statHidden(path);
let version: string | undefined;
let displayName: string | undefined;
if (!stat) {
return false;
}
const contentBin = await this.vaultAccess.adapterReadBinary(path);
const contentBin = await this.plugin.storageAccess.readHiddenFileBinary(path);
let content: string[];
try {
content = await arrayBufferToBase64(contentBin);
@@ -1070,12 +1095,13 @@ export class ConfigSync extends LiveSyncCommands {
displayName = `${json.name}`;
}
} catch (ex) {
Logger(`Configuration sync data: ${path} looks like manifest, but could not read the version`, LOG_LEVEL_INFO);
this._log(`Configuration sync data: ${path} looks like manifest, but could not read the version`, LOG_LEVEL_INFO);
this._log(ex, LOG_LEVEL_VERBOSE);
}
}
} catch (ex) {
Logger(`The file ${path} could not be encoded`);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`The file ${path} could not be encoded`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
const mtime = stat.mtime;
@@ -1096,12 +1122,12 @@ export class ConfigSync extends LiveSyncCommands {
const prefixedFileName = vf;
const id = await this.path2id(prefixedFileName);
const stat = await this.vaultAccess.adapterStat(path);
const stat = await this.plugin.storageAccess.statHidden(path);
if (!stat) {
return false;
}
const mtime = stat.mtime;
const content = await this.vaultAccess.adapterReadBinary(path);
const content = await this.plugin.storageAccess.readHiddenFileBinary(path);
const contentBlob = createBlob([DUMMY_HEAD, DUMMY_END, ...await arrayBufferToBase64(content)]);
// const contentBlob = createBlob(content);
try {
@@ -1123,7 +1149,7 @@ export class ConfigSync extends LiveSyncCommands {
};
} else {
if (isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN) {
Logger(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Already checked the same)`, LOG_LEVEL_DEBUG);
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Already checked the same)`, LOG_LEVEL_DEBUG);
return;
}
const docXDoc = await this.localDatabase.getDBEntryFromMeta(old, {}, false, false);
@@ -1135,7 +1161,7 @@ export class ConfigSync extends LiveSyncCommands {
const oldContent = dataSrc.substring(dataStart + DUMMY_END.length);
const oldContentArray = base64ToArrayBuffer(oldContent);
if (await isDocContentSame(oldContentArray, content)) {
Logger(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (the same content)`, LOG_LEVEL_VERBOSE);
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (the same content)`, LOG_LEVEL_VERBOSE);
markChangesAreSame(prefixedFileName, old.mtime, mtime + 1);
return true;
}
@@ -1152,12 +1178,12 @@ export class ConfigSync extends LiveSyncCommands {
};
}
const ret = await this.localDatabase.putDBEntry(saveData);
Logger(`STORAGE --> DB:${prefixedFileName}: (config) Done`);
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`);
fireAndForget(() => this.updatePluginListV2(false, this.filenameWithUnifiedKey(path)));
return ret;
} catch (ex) {
Logger(`STORAGE --> DB:${prefixedFileName}: (config) Failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
})
@@ -1165,7 +1191,7 @@ export class ConfigSync extends LiveSyncCommands {
async storeCustomizationFiles(path: FilePath, termOverRide?: string) {
const term = termOverRide || this.plugin.deviceAndVaultName;
if (term == "") {
Logger("We have to configure the device name", LOG_LEVEL_NOTICE);
this._log("We have to configure the device name", LOG_LEVEL_NOTICE);
return;
}
if (this.useV2) {
@@ -1207,7 +1233,7 @@ export class ConfigSync extends LiveSyncCommands {
for (const target of fileTargets) {
const data = await this.makeEntryFromFile(target);
if (data == false) {
Logger(`Config: skipped (Possibly is not exist): ${target} `, LOG_LEVEL_VERBOSE);
this._log(`Config: skipped (Possibly is not exist): ${target} `, LOG_LEVEL_VERBOSE);
continue;
}
if (data.version) {
@@ -1222,9 +1248,9 @@ export class ConfigSync extends LiveSyncCommands {
}
dt.mtime = mtime;
// Logger(`Configuration saving: ${prefixedFileName}`);
// this._log(`Configuration saving: ${prefixedFileName}`);
if (dt.files.length == 0) {
Logger(`Nothing left: deleting.. ${path}`);
this._log(`Nothing left: deleting.. ${path}`);
await this.deleteConfigOnDatabase(prefixedFileName);
await this.updatePluginList(false, prefixedFileName);
return
@@ -1250,7 +1276,7 @@ export class ConfigSync extends LiveSyncCommands {
};
} else {
if (old.mtime == mtime) {
// Logger(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same time)`, LOG_LEVEL_VERBOSE);
// this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same time)`, LOG_LEVEL_VERBOSE);
return true;
}
const oldC = await this.localDatabase.getDBEntryFromMeta(old, {}, false, false);
@@ -1258,11 +1284,11 @@ export class ConfigSync extends LiveSyncCommands {
const d = await deserialize(getDocDataAsArray(oldC.data), {}) as PluginDataEx;
if (d.files.length == dt.files.length) {
const diffs = (d.files.map(previous => ({ prev: previous, curr: dt.files.find(e => e.filename == previous.filename) })).map(async e => {
try { return await isDocContentSame(e.curr?.data ?? [], e.prev.data) } catch (_) { return false }
try { return await isDocContentSame(e.curr?.data ?? [], e.prev.data) } catch { return false }
}))
const isSame = (await Promise.all(diffs)).every(e => e == true);
if (isSame) {
Logger(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same content)`, LOG_LEVEL_VERBOSE);
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same content)`, LOG_LEVEL_VERBOSE);
return true;
}
}
@@ -1281,20 +1307,25 @@ export class ConfigSync extends LiveSyncCommands {
}
const ret = await this.localDatabase.putDBEntry(saveData);
await this.updatePluginList(false, saveData.path);
Logger(`STORAGE --> DB:${prefixedFileName}: (config) Done`);
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`);
return ret;
} catch (ex) {
Logger(`STORAGE --> DB:${prefixedFileName}: (config) Failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
})
}
async $anyProcessOptionalFileEvent(path: FilePath): Promise<boolean | undefined> {
return await this.watchVaultRawEventsAsync(path);
}
async watchVaultRawEventsAsync(path: FilePath) {
if (!this.settings.usePluginSync) return false;
if (!this.isTargetPath(path)) return false;
const stat = await this.vaultAccess.adapterStat(path);
if (!this._isMainReady) return false;
if (this._isMainSuspended()) return false;
if (!this._isThisModuleEnabled()) return false;
// if (!this.isTargetPath(path)) return false;
const stat = await this.plugin.storageAccess.statHidden(path);
// Make sure that target is a file.
if (stat && stat.type != "file")
return false;
@@ -1304,13 +1335,16 @@ export class ConfigSync extends LiveSyncCommands {
e.mode != MODE_SELECTIVE && e.mode != MODE_SHINY
).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase());
if (synchronisedInConfigSync.some(e => e.startsWith(path.toLowerCase()))) {
Logger(`Customization file skipped: ${path}`, LOG_LEVEL_VERBOSE);
return;
this._log(`Customization file skipped: ${path}`, LOG_LEVEL_VERBOSE);
// This file could be handled by the other module.
return false;
}
// this._log(`Customization file detected: ${path}`, LOG_LEVEL_VERBOSE);
const storageMTime = ~~((stat && stat.mtime || 0) / 1000);
const key = `${path}-${storageMTime}`;
if (this.recentProcessedInternalFiles.contains(key)) {
// If recently processed, it may caused by self.
// return true to prevent pass the event to the next.
return true;
}
this.recentProcessedInternalFiles = [key, ...this.recentProcessedInternalFiles].slice(0, 100);
@@ -1319,18 +1353,18 @@ export class ConfigSync extends LiveSyncCommands {
scheduleTask(keySchedule, 100, async () => {
await this.storeCustomizationFiles(path);
})
// Okay, it may handled after 100ms.
// This was my own job.
return true;
}
async scanAllConfigFiles(showMessage: boolean) {
await shareRunningResult("scanAllConfigFiles", async () => {
const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
Logger("Scanning customizing files.", logLevel, "scan-all-config");
this._log("Scanning customizing files.", logLevel, "scan-all-config");
const term = this.plugin.deviceAndVaultName;
if (term == "") {
Logger("We have to configure the device name", LOG_LEVEL_NOTICE);
this._log("We have to configure the device name", LOG_LEVEL_NOTICE);
return;
}
const filesAll = await this.scanInternalFiles();
@@ -1358,8 +1392,8 @@ export class ConfigSync extends LiveSyncCommands {
await this.deleteConfigOnDatabase(unifiedFilenameWithKey);
}
} catch (ex) {
Logger(`scanAllConfigFiles - Error: ${item._id}`, LOG_LEVEL_VERBOSE);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`scanAllConfigFiles - Error: ${item._id}`, LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
} finally {
releaser();
}
@@ -1374,8 +1408,8 @@ export class ConfigSync extends LiveSyncCommands {
try {
await this.storeCustomisationFileV2(filePath, term);
} catch (ex) {
Logger(`scanAllConfigFiles - Error: ${filePath}`, LOG_LEVEL_VERBOSE);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`scanAllConfigFiles - Error: ${filePath}`, LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
finally {
releaser();
@@ -1383,8 +1417,7 @@ export class ConfigSync extends LiveSyncCommands {
})
}
await Promise.all(taskExtra.map(e => e()));
this.updatePluginList(false).then(/* fire and forget */);
fireAndForget(() => this.updatePluginList(false));
} else {
const files = filesAll.filter(e => this.isTargetPath(e)).map(e => ({ key: this.filenameToUnifiedKey(e), file: e }));
const virtualPathsOfLocalFiles = [...new Set(files.map(e => e.key))];
@@ -1393,7 +1426,7 @@ export class ConfigSync extends LiveSyncCommands {
for (const vp of virtualPathsOfLocalFiles) {
const p = files.find(e => e.key == vp)?.file;
if (!p) {
Logger(`scanAllConfigFiles - File not found: ${vp}`, LOG_LEVEL_VERBOSE);
this._log(`scanAllConfigFiles - File not found: ${vp}`, LOG_LEVEL_VERBOSE);
continue;
}
await this.storeCustomizationFiles(p);
@@ -1402,7 +1435,7 @@ export class ConfigSync extends LiveSyncCommands {
for (const vp of deleteCandidate) {
await this.deleteConfigOnDatabase(vp);
}
this.updatePluginList(false).then(/* fire and forget */);
fireAndForget(() => this.updatePluginList(false))
}
});
}
@@ -1416,11 +1449,11 @@ export class ConfigSync extends LiveSyncCommands {
const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, undefined, false) as InternalFileEntry | false;
let saveData: InternalFileEntry;
if (old === false) {
Logger(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted (Not found on database)`);
this._log(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted (Not found on database)`);
return true;
} else {
if (old.deleted) {
Logger(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted`);
this._log(`STORAGE -x> DB:${prefixedFileName}: (config) already deleted`);
return true;
}
saveData =
@@ -1435,11 +1468,11 @@ export class ConfigSync extends LiveSyncCommands {
}
await this.localDatabase.putRaw(saveData);
await this.updatePluginList(false, prefixedFileName);
Logger(`STORAGE -x> DB:${prefixedFileName}: (config) Done`);
this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Done`);
return true;
} catch (ex) {
Logger(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
});
@@ -1450,7 +1483,99 @@ export class ConfigSync extends LiveSyncCommands {
return filenames as FilePath[];
}
async $allAskUsingOptionalSyncFeature(opt: { enableFetch?: boolean, enableOverwrite?: boolean }): Promise<boolean> {
await this._askHiddenFileConfiguration(opt);
return true;
}
async _askHiddenFileConfiguration(opt: { enableFetch?: boolean, enableOverwrite?: boolean }) {
const message = `Would you like to enable **Customization sync**?
> [!DETAILS]-
> This feature allows you to sync your customisations -- such as configurations, themes, snippets, and plugins -- across your devices in a fully controlled manner, unlike the fully automatic behaviour of hidden file synchronisation.
>
> You may use this feature alongside hidden file synchronisation. When both features are enabled, items configured as \`Automatic\` in this feature will be managed by **hidden file synchronisation**.
> Do not worry, you will be prompted to enable or keep disabled **hidden file synchronisation** after this dialogue.
`
const CHOICE_CUSTOMIZE = "Yes, Enable it";
const CHOICE_DISABLE = "No, Disable it";
const CHOICE_DISMISS = "Later";
const choices = [];
choices.push(CHOICE_CUSTOMIZE);
choices.push(CHOICE_DISABLE);
choices.push(CHOICE_DISMISS);
const ret = await this.plugin.confirm.confirmWithMessage("Customisation sync", message, choices, CHOICE_DISMISS, 40);
if (ret == CHOICE_CUSTOMIZE) {
await this.configureHiddenFileSync("CUSTOMIZE");
} else if (ret == CHOICE_DISABLE) {
await this.configureHiddenFileSync("DISABLE_CUSTOM");
}
}
$anyGetOptionalConflictCheckMethod(path: FilePathWithPrefix): Promise<boolean | "newer"> {
if (isPluginMetadata(path)) {
return Promise.resolve("newer")
}
if (isCustomisationSyncMetadata(path)) {
return Promise.resolve("newer");
}
return Promise.resolve(false);
}
$allSuspendExtraSync(): Promise<boolean> {
if (this.plugin.settings.usePluginSync || this.plugin.settings.autoSweepPlugins) {
this._log("Customisation sync have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL_NOTICE)
this.plugin.settings.usePluginSync = false;
this.plugin.settings.autoSweepPlugins = false;
}
return Promise.resolve(true);
}
async $anyConfigureOptionalSyncFeature(mode: "CUSTOMIZE" | "DISABLE" | "DISABLE_CUSTOM") {
await this.configureHiddenFileSync(mode);
}
async configureHiddenFileSync(mode: "CUSTOMIZE" | "DISABLE" | "DISABLE_CUSTOM") {
if (mode == "DISABLE") {
this.plugin.settings.usePluginSync = false;
await this.plugin.saveSettings();
return;
}
if (mode == "CUSTOMIZE") {
if (!this.plugin.deviceAndVaultName) {
let name = await this.plugin.confirm.askString("Device name", "Please set this device name", `desktop`);
if (!name) {
if (Platform.isAndroidApp) {
name = "android-app"
} else if (Platform.isIosApp) {
name = "ios"
} else if (Platform.isMacOS) {
name = "macos"
} else if (Platform.isMobileApp) {
name = "mobile-app"
} else if (Platform.isMobile) {
name = "mobile"
} else if (Platform.isSafari) {
name = "safari"
} else if (Platform.isDesktop) {
name = "desktop"
} else if (Platform.isDesktopApp) {
name = "desktop-app"
} else {
name = "unknown"
}
name = name + Math.random().toString(36).slice(-4);
}
this.plugin.deviceAndVaultName = name;
}
this.plugin.settings.usePluginSync = true;
this.plugin.settings.useAdvancedMode = true;
await this.plugin.saveSettings();
await this.scanAllConfigFiles(true);
}
}
async getFiles(
path: string,
@@ -1461,8 +1586,8 @@ export class ConfigSync extends LiveSyncCommands {
try {
w = await this.app.vault.adapter.list(path);
} catch (ex) {
Logger(`Could not traverse(ConfigSync):${path}`, LOG_LEVEL_INFO);
Logger(ex, LOG_LEVEL_VERBOSE);
this._log(`Could not traverse(ConfigSync):${path}`, LOG_LEVEL_INFO);
this._log(ex, LOG_LEVEL_VERBOSE);
return [];
}
let files = [

12
src/ui/components/PluginCombo.svelte → src/features/ConfigSync/PluginCombo.svelte
View File

@@ -1,10 +1,10 @@
<script lang="ts">
import { PluginDataExDisplayV2, type IPluginDataExDisplay } from "../../features/CmdConfigSync";
import { ConfigSync, PluginDataExDisplayV2, type IPluginDataExDisplay } from "./CmdConfigSync.ts";
import { Logger } from "../../lib/src/common/logger";
import { type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "../../lib/src/common/types";
import { getDocData, timeDeltaToHumanReadable, unique } from "../../lib/src/common/utils";
import type ObsidianLiveSyncPlugin from "../../main";
import { askString } from "../../common/utils";
// import { askString } from "../../common/utils";
import { Menu } from "obsidian";
export let list: IPluginDataExDisplay[] = [];
@@ -21,7 +21,11 @@
export let plugin: ObsidianLiveSyncPlugin;
export let isMaintenanceMode: boolean = false;
export let isFlagged: boolean = false;
const addOn = plugin.addOnConfigSync;
const addOn = plugin.getAddOn<ConfigSync>(ConfigSync.name)!;
if (!addOn) {
Logger(`Could not load the add-on ${ConfigSync.name}`, LOG_LEVEL_INFO);
throw new Error(`Could not load the add-on ${ConfigSync.name}`);
}
export let selected = "";
let freshness = "";
@@ -303,7 +307,7 @@
Logger(`Could not find local item`, LOG_LEVEL_VERBOSE);
return;
}
const duplicateTermName = await askString(plugin.app, "Duplicate", "device name", "");
const duplicateTermName = await plugin.confirm.askString("Duplicate", "device name", "");
if (duplicateTermName) {
if (duplicateTermName.contains("/")) {
Logger(`We can not use "/" to the device name`, LOG_LEVEL_NOTICE);

35
src/features/ConfigSync/PluginDialogModal.ts Normal file
View File

@@ -0,0 +1,35 @@
import { App, Modal } from "../../deps.ts";
import ObsidianLiveSyncPlugin from "../../main.ts";
import PluginPane from "./PluginPane.svelte";
export class PluginDialogModal extends Modal {
plugin: ObsidianLiveSyncPlugin;
component: PluginPane | undefined;
isOpened() {
return this.component != undefined;
}
constructor(app: App, plugin: ObsidianLiveSyncPlugin) {
super(app);
this.plugin = plugin;
}
onOpen() {
const { contentEl } = this;
this.contentEl.style.overflow = "auto";
this.contentEl.style.display = "flex";
this.contentEl.style.flexDirection = "column";
this.titleEl.setText("Customization Sync (Beta3)")
if (!this.component) {
this.component = new PluginPane({
target: contentEl, props: { plugin: this.plugin },
});
}
}
onClose() {
if (this.component) {
this.component.$destroy();
this.component = undefined;
}
}
}

33
src/ui/PluginPane.svelte → src/features/ConfigSync/PluginPane.svelte
View File

@@ -1,18 +1,31 @@
<script lang="ts">
import { onMount } from "svelte";
import ObsidianLiveSyncPlugin from "../main";
import { type IPluginDataExDisplay, pluginIsEnumerating, pluginList, pluginManifestStore, pluginV2Progress } from "../features/CmdConfigSync";
import PluginCombo from "./components/PluginCombo.svelte";
import ObsidianLiveSyncPlugin from "../../main";
import { ConfigSync, type IPluginDataExDisplay, pluginIsEnumerating, pluginList, pluginManifestStore, pluginV2Progress } from "./CmdConfigSync.ts";
import PluginCombo from "./PluginCombo.svelte";
import { Menu, type PluginManifest } from "obsidian";
import { unique } from "../lib/src/common/utils";
import { MODE_SELECTIVE, MODE_AUTOMATIC, MODE_PAUSED, type SYNC_MODE, type PluginSyncSettingEntry, MODE_SHINY } from "../lib/src/common/types";
import { normalizePath } from "../deps";
import { unique } from "../../lib/src/common/utils";
import { MODE_SELECTIVE, MODE_AUTOMATIC, MODE_PAUSED, type SYNC_MODE, type PluginSyncSettingEntry, MODE_SHINY } from "../../lib/src/common/types";
import { normalizePath } from "../../deps";
import { HiddenFileSync } from "../HiddenFileSync/CmdHiddenFileSync.ts";
import { LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger";
export let plugin: ObsidianLiveSyncPlugin;
$: hideNotApplicable = false;
$: thisTerm = plugin.deviceAndVaultName;
const addOn = plugin.addOnConfigSync;
const addOn = plugin.getAddOn(ConfigSync.name) as ConfigSync;
if (!addOn) {
const msg = "AddOn Module (ConfigSync) has not been loaded. This is very unexpected situation. Please report this issue.";
Logger(msg, LOG_LEVEL_NOTICE);
throw new Error(msg);
}
const addOnHiddenFileSync = plugin.getAddOn(HiddenFileSync.name) as HiddenFileSync;
if (!addOnHiddenFileSync) {
const msg = "AddOn Module (HiddenFileSync) has not been loaded. This is very unexpected situation. Please report this issue.";
Logger(msg, LOG_LEVEL_NOTICE);
throw new Error(msg);
}
let list: IPluginDataExDisplay[] = [];
@@ -70,7 +83,7 @@
await requestUpdate();
}
async function replicate() {
await plugin.replicate(true);
await plugin.$$replicate(true);
}
function selectAllNewest(selectMode: boolean) {
selectNewestPulse++;
@@ -117,7 +130,7 @@
setMode(key, MODE_AUTOMATIC);
const configDir = normalizePath(plugin.app.vault.configDir);
const files = (plugin.settings.pluginSyncExtendedSetting[key]?.files ?? []).map((e) => `${configDir}/${e}`);
plugin.addOnHiddenFileSync.syncInternalFilesAndDatabase(direction, true, false, files);
addOnHiddenFileSync.syncInternalFilesAndDatabase(direction, true, false, files);
}
function askOverwriteModeForAutomatic(evt: MouseEvent, key: string) {
const menu = new Menu();
@@ -205,7 +218,7 @@
plugin.settings.pluginSyncExtendedSetting[key].files = files;
plugin.settings.pluginSyncExtendedSetting[key].mode = mode;
}
plugin.saveSettingData();
plugin.$$saveSettingData();
}
function getIcon(mode: SYNC_MODE) {
if (mode in ICONS) {

15
src/ui/JsonResolveModal.ts → src/features/HiddenFileCommon/JsonResolveModal.ts
View File

@@ -1,7 +1,7 @@
import { App, Modal } from "../deps.ts";
import { type FilePath, type LoadedEntry } from "../lib/src/common/types.ts";
import { App, Modal } from "../../deps.ts";
import { type FilePath, type LoadedEntry } from "../../lib/src/common/types.ts";
import JsonResolvePane from "./JsonResolvePane.svelte";
import { waitForSignal } from "../lib/src/common/utils.ts";
import { waitForSignal } from "../../lib/src/common/utils.ts";
export class JsonResolveModal extends Modal {
// result: Array<[number, string]>;
@@ -30,11 +30,14 @@ export class JsonResolveModal extends Modal {
this.defaultSelect = defaultSelect || "";
this.title = title;
this.hideLocal = hideLocal ?? false;
waitForSignal(`cancel-internal-conflict:${filename}`).then(() => this.close());
void waitForSignal(`cancel-internal-conflict:${filename}`).then(() => this.close());
}
async UICallback(keepRev?: string, mergedStr?: string) {
if (this.callback) {
await this.callback(keepRev, mergedStr);
}
this.close();
await this.callback?.(keepRev, mergedStr);
this.callback = undefined;
}
@@ -67,7 +70,7 @@ export class JsonResolveModal extends Modal {
contentEl.empty();
// contentEl.empty();
if (this.callback != undefined) {
this.callback(undefined);
void this.callback(undefined);
}
if (this.component != undefined) {
this.component.$destroy();

9
src/ui/JsonResolvePane.svelte → src/features/HiddenFileCommon/JsonResolvePane.svelte
View File

@@ -1,9 +1,8 @@
<script lang="ts">
import { type Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "../deps";
import type { FilePath, LoadedEntry } from "../lib/src/common/types";
import { decodeBinary, readString } from "../lib/src/string_and_binary/convert";
import { getDocData } from "../lib/src/common/utils";
import { mergeObject } from "../common/utils";
import { type Diff, DIFF_DELETE, DIFF_INSERT, diff_match_patch } from "../../deps.ts";
import type { FilePath, LoadedEntry } from "../../lib/src/common/types.ts";
import { decodeBinary, readString } from "../../lib/src/string_and_binary/convert.ts";
import { getDocData, mergeObject } from "../../lib/src/common/utils.ts";
export let docs: LoadedEntry[] = [];
export let callback: (keepRev?: string, mergedStr?: string) => Promise<void> = async (_, __) => {

967
src/features/HiddenFileSync/CmdHiddenFileSync.ts Normal file
View File

@@ -0,0 +1,967 @@
import { normalizePath, type PluginManifest, type ListedFiles } from "../../deps.ts";
import { type EntryDoc, type LoadedEntry, type InternalFileEntry, type FilePathWithPrefix, type FilePath, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MODE_SELECTIVE, MODE_PAUSED, type SavingEntry, type DocumentID, type UXStat, MODE_AUTOMATIC } from "../../lib/src/common/types.ts";
import { type InternalFileInfo, ICHeader, ICHeaderEnd } from "../../common/types.ts";
import { readAsBlob, isDocContentSame, sendSignal, readContent, createBlob, fireAndForget } from "../../lib/src/common/utils.ts";
import { BASE_IS_NEW, compareMTime, EVEN, getPath, isInternalMetadata, isMarkedAsSameChanges, markChangesAreSame, PeriodicProcessor, TARGET_IS_NEW } from "../../common/utils.ts";
import { serialized } from "../../lib/src/concurrency/lock.ts";
import { JsonResolveModal } from "../HiddenFileCommon/JsonResolveModal.ts";
import { LiveSyncCommands } from "../LiveSyncCommands.ts";
import { addPrefix, stripAllPrefixes } from "../../lib/src/string_and_binary/path.ts";
import { QueueProcessor } from "../../lib/src/concurrency/processor.ts";
import { hiddenFilesEventCount, hiddenFilesProcessingCount } from "../../lib/src/mock_and_interop/stores.ts";
import type { IObsidianModule } from "../../modules/AbstractObsidianModule.ts";
export class HiddenFileSync extends LiveSyncCommands implements IObsidianModule {
_isThisModuleEnabled() {
return this.plugin.settings.syncInternalFiles;
}
periodicInternalFileScanProcessor: PeriodicProcessor = new PeriodicProcessor(this.plugin, async () => this._isThisModuleEnabled() && this._isDatabaseReady() && await this.syncInternalFilesAndDatabase("push", false));
get kvDB() {
return this.plugin.kvDB;
}
getConflictedDoc(path: FilePathWithPrefix, rev: string) {
return this.plugin.localDatabase.getConflictedDoc(path, rev);
}
onunload() {
this.periodicInternalFileScanProcessor?.disable();
}
onload() {
this.plugin.addCommand({
id: "livesync-scaninternal",
name: "Sync hidden files",
callback: () => {
void this.syncInternalFilesAndDatabase("safe", true);
},
});
}
async $everyOnDatabaseInitialized(showNotice: boolean) {
if (this._isThisModuleEnabled()) {
try {
this._log("Synchronizing hidden files...");
await this.syncInternalFilesAndDatabase("push", showNotice);
this._log("Synchronizing hidden files done");
} catch (ex) {
this._log("Synchronizing hidden files failed");
this._log(ex, LOG_LEVEL_VERBOSE);
}
}
return true;
}
async $everyBeforeReplicate(showNotice: boolean) {
if (this._isThisModuleEnabled() && this._isDatabaseReady() && this.settings.syncInternalFilesBeforeReplication && !this.settings.watchInternalFileChanges) {
await this.syncInternalFilesAndDatabase("push", showNotice);
}
return true;
}
$everyOnloadAfterLoadSettings(): Promise<boolean> {
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
this.ignorePatterns = ignorePatterns;
return Promise.resolve(true);
}
async $everyOnResumeProcess(): Promise<boolean> {
this.periodicInternalFileScanProcessor?.disable();
if (this._isMainSuspended())
return true;
if (this._isThisModuleEnabled()) {
await this.syncInternalFilesAndDatabase("safe", false);
}
this.periodicInternalFileScanProcessor.enable(this._isThisModuleEnabled() && this.settings.syncInternalFilesInterval ? (this.settings.syncInternalFilesInterval * 1000) : 0);
return true
}
$everyRealizeSettingSyncMode(): Promise<boolean> {
this.periodicInternalFileScanProcessor?.disable();
if (this._isMainSuspended())
return Promise.resolve(true);
if (!this.plugin.isReady)
return Promise.resolve(true);
this.periodicInternalFileScanProcessor.enable(this._isThisModuleEnabled() && this.settings.syncInternalFilesInterval ? (this.settings.syncInternalFilesInterval * 1000) : 0);
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
this.ignorePatterns = ignorePatterns;
return Promise.resolve(true);
}
procInternalFile(filename: string) {
this.internalFileProcessor.enqueue(filename);
}
internalFileProcessor = new QueueProcessor<string, any>(
async (filenames) => {
this._log(`START :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE);
await this.syncInternalFilesAndDatabase("pull", false, false, filenames);
this._log(`DONE :Applying hidden ${filenames.length} files change`, LOG_LEVEL_VERBOSE);
return;
}, { batchSize: 100, concurrentLimit: 1, delay: 10, yieldThreshold: 100, suspended: false, totalRemainingReactiveSource: hiddenFilesEventCount }
);
async $anyProcessOptionalFileEvent(path: FilePath): Promise<boolean | undefined> {
return await this.watchVaultRawEventsAsync(path);
}
async watchVaultRawEventsAsync(path: FilePath): Promise<boolean | undefined> {
if (!this._isMainReady) return false;
if (this._isMainSuspended()) return false;
if (!this._isThisModuleEnabled()) return false;
// Exclude files handled by customization sync
const configDir = normalizePath(this.app.vault.configDir);
const synchronisedInConfigSync = !this.settings.usePluginSync ? [] :
Object.values(this.settings.pluginSyncExtendedSetting).
filter(e => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED).
map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase());
if (synchronisedInConfigSync.some(e => e.startsWith(path.toLowerCase()))) {
this._log(`Hidden file skipped: ${path} is synchronized in customization sync.`, LOG_LEVEL_VERBOSE);
return false;
}
const stat = await this.plugin.storageAccess.statHidden(path);
// sometimes folder is coming.
if (stat != null && stat.type != "file") {
return false;
}
if (this.isKnownChange(path, stat?.mtime ?? 0)) {
// This could be caused by self. so return true to prevent further processing.
return true;
}
const mtime = stat == null ? 0 : stat?.mtime ?? 0;
const storageMTime = ~~((mtime) / 1000);
const prefixedFileName = addPrefix(path, ICHeader);
const filesOnDB = await this.localDatabase.getDBEntryMeta(prefixedFileName);
const dbMTime = ~~((filesOnDB && filesOnDB.mtime || 0) / 1000);
// Skip unchanged file.
if (dbMTime == storageMTime) {
// this._log(`STORAGE --> DB:${path}: (hidden) Nothing changed`);
// Handled, but nothing changed. also return true to prevent further processing.
return true;
}
try {
if (storageMTime == 0) {
await this.deleteInternalFileOnDatabase(path);
} else {
await this.storeInternalFileToDatabase({ path: path, mtime, ctime: stat?.ctime ?? mtime, size: stat?.size ?? 0 });
}
// Surely processed.
return true;
} catch (ex) {
this._log(`Failed to process hidden file:${path}`);
this._log(ex, LOG_LEVEL_VERBOSE);
}
// Could not be processed. but it was own task. so return true to prevent further processing.
return true;
}
async resolveConflictOnInternalFiles() {
// Scan all conflicted internal files
const conflicted = this.localDatabase.findEntries(ICHeader, ICHeaderEnd, { conflicts: true });
this.conflictResolutionProcessor.suspend();
try {
for await (const doc of conflicted) {
if (!("_conflicts" in doc))
continue;
if (isInternalMetadata(doc._id)) {
this.conflictResolutionProcessor.enqueue(doc.path);
}
}
} catch (ex) {
this._log("something went wrong on resolving all conflicted internal files");
this._log(ex, LOG_LEVEL_VERBOSE);
}
await this.conflictResolutionProcessor.startPipeline().waitForAllProcessed();
}
async resolveByNewerEntry(id: DocumentID, path: FilePathWithPrefix, currentDoc: EntryDoc, currentRev: string, conflictedRev: string) {
const conflictedDoc = await this.localDatabase.getRaw(id, { rev: conflictedRev });
// determine which revision should been deleted.
// simply check modified time
const mtimeCurrent = ("mtime" in currentDoc && currentDoc.mtime) || 0;
const mtimeConflicted = ("mtime" in conflictedDoc && conflictedDoc.mtime) || 0;
// this._log(`Revisions:${new Date(mtimeA).toLocaleString} and ${new Date(mtimeB).toLocaleString}`);
// console.log(`mtime:${mtimeA} - ${mtimeB}`);
const delRev = mtimeCurrent < mtimeConflicted ? currentRev : conflictedRev;
// delete older one.
await this.localDatabase.removeRevision(id, delRev);
this._log(`Older one has been deleted:${path}`);
const cc = await this.localDatabase.getRaw(id, { conflicts: true });
if (cc._conflicts?.length === 0) {
await this.extractInternalFileFromDatabase(stripAllPrefixes(path))
} else {
this.conflictResolutionProcessor.enqueue(path);
}
// check the file again
}
conflictResolutionProcessor = new QueueProcessor(async (paths: FilePathWithPrefix[]) => {
const path = paths[0];
sendSignal(`cancel-internal-conflict:${path}`);
try {
// Retrieve data
const id = await this.path2id(path, ICHeader);
const doc = await this.localDatabase.getRaw(id, { conflicts: true });
// if (!("_conflicts" in doc)){
// return [];
// }
if (doc._conflicts === undefined) return [];
if (doc._conflicts.length == 0)
return [];
this._log(`Hidden file conflicted:${path}`);
const conflicts = doc._conflicts.sort((a, b) => Number(a.split("-")[0]) - Number(b.split("-")[0]));
const revA = doc._rev;
const revB = conflicts[0];
if (path.endsWith(".json")) {
const conflictedRev = conflicts[0];
const conflictedRevNo = Number(conflictedRev.split("-")[0]);
//Search
const revFrom = (await this.localDatabase.getRaw<EntryDoc>(id, { revs_info: true }));
const commonBase = revFrom._revs_info?.filter(e => e.status == "available" && Number(e.rev.split("-")[0]) < conflictedRevNo).first()?.rev ?? "";
const result = await this.plugin.localDatabase.mergeObject(path, commonBase, doc._rev, conflictedRev);
if (result) {
this._log(`Object merge:${path}`, LOG_LEVEL_INFO);
const filename = stripAllPrefixes(path);
const isExists = await this.plugin.storageAccess.isExistsIncludeHidden(filename);
if (!isExists) {
await this.plugin.storageAccess.ensureDir(filename);
}
await this.plugin.storageAccess.writeHiddenFileAuto(filename, result);
const stat = await this.plugin.storageAccess.statHidden(filename);
if (!stat) {
throw new Error(`conflictResolutionProcessor: Failed to stat file ${filename}`);
}
await this.storeInternalFileToDatabase({ path: filename, ...stat });
await this.extractInternalFileFromDatabase(filename);
await this.localDatabase.removeRevision(id, revB);
this.conflictResolutionProcessor.enqueue(path);
return [];
} else {
this._log(`Object merge is not applicable.`, LOG_LEVEL_VERBOSE);
}
return [{ path, revA, revB, id, doc }];
}
// When not JSON file, resolve conflicts by choosing a newer one.
await this.resolveByNewerEntry(id, path, doc, revA, revB);
return [];
} catch (ex) {
this._log(`Failed to resolve conflict (Hidden): ${path}`);
this._log(ex, LOG_LEVEL_VERBOSE);
return [];
}
}, {
suspended: false, batchSize: 1, concurrentLimit: 5, delay: 10, keepResultUntilDownstreamConnected: true, yieldThreshold: 10,
pipeTo: new QueueProcessor(async (results) => {
const { id, doc, path, revA, revB } = results[0];
const docAMerge = await this.localDatabase.getDBEntry(path, { rev: revA });
const docBMerge = await this.localDatabase.getDBEntry(path, { rev: revB });
if (docAMerge != false && docBMerge != false) {
if (await this.showJSONMergeDialogAndMerge(docAMerge, docBMerge)) {
// Again for other conflicted revisions.
this.conflictResolutionProcessor.enqueue(path);
}
return;
} else {
// If either revision could not read, force resolving by the newer one.
await this.resolveByNewerEntry(id, path, doc, revA, revB);
}
}, { suspended: false, batchSize: 1, concurrentLimit: 1, delay: 10, keepResultUntilDownstreamConnected: false, yieldThreshold: 10 })
})
$anyGetOptionalConflictCheckMethod(path: FilePathWithPrefix): Promise<boolean | "newer"> {
if (isInternalMetadata(path)) {
this.queueConflictCheck(path);
return Promise.resolve(true);
}
return Promise.resolve(false);
}
async $anyProcessOptionalSyncFiles(doc: LoadedEntry): Promise<boolean | undefined> {
if (isInternalMetadata(doc._id) && this._isThisModuleEnabled()) {
//system file
const filename = getPath(doc);
if (await this.plugin.$$isTargetFile(filename)) {
this.procInternalFile(filename);
return true;
} else {
this._log(`Skipped (Not target:${filename})`, LOG_LEVEL_VERBOSE);
return false;
}
}
return false;
}
queueConflictCheck(path: FilePathWithPrefix) {
this.conflictResolutionProcessor.enqueue(path);
}
knownChanges: { [key: string]: number; } = {};
markAsKnownChange(path: string, mtime: number) {
this.knownChanges[path] = mtime;
}
isKnownChange(path: string, mtime: number) {
return this.knownChanges[path] == mtime;
}
ignorePatterns: RegExp[] = [];
//TODO: Tidy up. Even though it is experimental feature, So dirty...
async syncInternalFilesAndDatabase(direction: "push" | "pull" | "safe" | "pullForce" | "pushForce", showMessage: boolean, filesAll: InternalFileInfo[] | false = false, targetFilesSrc: string[] | false = false) {
const targetFiles = targetFilesSrc ? targetFilesSrc.map(e => stripAllPrefixes(e as FilePathWithPrefix)) : false;
// debugger;
await this.resolveConflictOnInternalFiles();
const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
this._log("Scanning hidden files.", logLevel, "sync_internal");
const configDir = normalizePath(this.app.vault.configDir);
let files: InternalFileInfo[] =
filesAll ? filesAll : (await this.scanInternalFiles())
const allowedInHiddenFileSync = this.settings.usePluginSync ? Object.values(this.settings.pluginSyncExtendedSetting).filter(e => e.mode == MODE_AUTOMATIC).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase()) : undefined;
if (allowedInHiddenFileSync) {
const systemOrNot = files.reduce((acc, cur) => {
if (cur.path.startsWith(configDir)) {
acc.system.push(cur);
} else {
acc.user.push(cur);
}
return acc;
}, { system: [] as InternalFileInfo[], user: [] as InternalFileInfo[] });
files =
[...systemOrNot.user,
...systemOrNot.system.filter(file => allowedInHiddenFileSync.some(filterFile => file.path.toLowerCase().startsWith(filterFile)))];
}
const filesOnDB = ((await this.localDatabase.allDocsRaw({ startkey: ICHeader, endkey: ICHeaderEnd, include_docs: true })).rows.map(e => e.doc) as InternalFileEntry[]).filter(e => !e.deleted);
const allFileNamesSrc = [...new Set([...files.map(e => normalizePath(e.path)), ...filesOnDB.map(e => stripAllPrefixes(this.getPath(e)))])];
let allFileNames = allFileNamesSrc.filter(filename => !targetFiles || (targetFiles && targetFiles.indexOf(filename) !== -1));
if (allowedInHiddenFileSync) {
allFileNames = allFileNames.filter(file => allowedInHiddenFileSync.some(filterFile => file.toLowerCase().startsWith(filterFile)));
}
const fileCount = allFileNames.length;
let processed = 0;
let filesChanged = 0;
// count updated files up as like this below:
// .obsidian: 2
// .obsidian/workspace: 1
// .obsidian/plugins: 1
// .obsidian/plugins/recent-files-obsidian: 1
// .obsidian/plugins/recent-files-obsidian/data.json: 1
const updatedFolders: { [key: string]: number; } = {};
const countUpdatedFolder = (path: string) => {
const pieces = path.split("/");
let c = pieces.shift();
let pathPieces = "";
filesChanged++;
while (c) {
pathPieces += (pathPieces != "" ? "/" : "") + c;
pathPieces = normalizePath(pathPieces);
if (!(pathPieces in updatedFolders)) {
updatedFolders[pathPieces] = 0;
}
updatedFolders[pathPieces]++;
c = pieces.shift();
}
};
const filesMap = files.reduce((acc, cur) => {
acc[cur.path] = cur;
return acc;
}, {} as { [key: string]: InternalFileInfo; });
const filesOnDBMap = filesOnDB.reduce((acc, cur) => {
acc[stripAllPrefixes(this.getPath(cur))] = cur;
return acc;
}, {} as { [key: string]: InternalFileEntry; });
await new QueueProcessor(async (filenames: FilePath[]) => {
const filename = filenames[0];
processed++;
if (processed % 100 == 0) {
this._log(`Hidden file: ${processed}/${fileCount}`, logLevel, "sync_internal");
}
if (!filename) return [];
if (this.ignorePatterns.some(e => filename.match(e)))
return [];
if (await this.plugin.$$isIgnoredByIgnoreFiles(filename)) {
return [];
}
const fileOnStorage = filename in filesMap ? filesMap[filename] : undefined;
const fileOnDatabase = filename in filesOnDBMap ? filesOnDBMap[filename] : undefined;
return [{
filename,
fileOnStorage,
fileOnDatabase,
}]
}, { suspended: true, batchSize: 1, concurrentLimit: 10, delay: 0, totalRemainingReactiveSource: hiddenFilesProcessingCount })
.pipeTo(new QueueProcessor(async (params) => {
const
{
filename,
fileOnStorage: xFileOnStorage,
fileOnDatabase: xFileOnDatabase
} = params[0];
const xFileOnDatabaseExists = xFileOnDatabase !== undefined && !(xFileOnDatabase.deleted || xFileOnDatabase._deleted);
if (xFileOnStorage && xFileOnDatabaseExists) {
// Both => Synchronize
if ((direction != "pullForce" && direction != "pushForce") && isMarkedAsSameChanges(filename, [xFileOnDatabase.mtime, xFileOnStorage.mtime]) == EVEN) {
this._log(`Hidden file skipped: ${filename} is marked as same`, LOG_LEVEL_VERBOSE);
return;
}
const nw = compareMTime(xFileOnStorage.mtime, xFileOnDatabase.mtime);
if (nw == BASE_IS_NEW || direction == "pushForce") {
if (await this.storeInternalFileToDatabase(xFileOnStorage) !== false) {
// countUpdatedFolder(filename);
}
} else if (nw == TARGET_IS_NEW || direction == "pullForce") {
// skip if not extraction performed.
if (await this.extractInternalFileFromDatabase(filename))
countUpdatedFolder(filename);
} else {
// Even, or not forced. skip.
}
} else if (!xFileOnStorage && xFileOnDatabaseExists) {
if (direction == "push" || direction == "pushForce") {
if (xFileOnDatabase.deleted)
return;
await this.deleteInternalFileOnDatabase(filename, false);
} else if (direction == "pull" || direction == "pullForce") {
if (await this.extractInternalFileFromDatabase(filename)) {
countUpdatedFolder(filename);
}
} else if (direction == "safe") {
if (xFileOnDatabase.deleted)
return;
if (await this.extractInternalFileFromDatabase(filename)) {
countUpdatedFolder(filename);
}
}
} else if (xFileOnStorage && !xFileOnDatabaseExists) {
if (direction == "push" || direction == "pushForce" || direction == "safe") {
await this.storeInternalFileToDatabase(xFileOnStorage);
} else {
// Apply the deletion
if (await this.extractInternalFileFromDatabase(xFileOnStorage.path)) {
countUpdatedFolder(xFileOnStorage.path);
}
}
} else {
throw new Error("Invalid state on hidden file sync");
// Something corrupted?
}
return;
}, { suspended: true, batchSize: 1, concurrentLimit: 5, delay: 0 }))
.root
.enqueueAll(allFileNames)
.startPipeline().waitForAllDoneAndTerminate();
// When files has been retrieved from the database. they must be reloaded.
if ((direction == "pull" || direction == "pullForce") && filesChanged != 0) {
// Show notification to restart obsidian when something has been changed in configDir.
if (configDir in updatedFolders) {
// Numbers of updated files that is below of configDir.
let updatedCount = updatedFolders[configDir];
try {
//@ts-ignore
const manifests = Object.values(this.app.plugins.manifests) as any as PluginManifest[];
//@ts-ignore
const enabledPlugins = this.app.plugins.enabledPlugins as Set<string>;
const enabledPluginManifests = manifests.filter(e => enabledPlugins.has(e.id));
for (const manifest of enabledPluginManifests) {
if (manifest.dir && manifest.dir in updatedFolders) {
// If notified about plug-ins, reloading Obsidian may not be necessary.
updatedCount -= updatedFolders[manifest.dir];
const updatePluginId = manifest.id;
const updatePluginName = manifest.name;
this.plugin.confirm.askInPopup(`updated-${updatePluginId}`, `Files in ${updatePluginName} has been updated, Press {HERE} to reload ${updatePluginName}, or press elsewhere to dismiss this message.`, (anchor) => {
anchor.text = "HERE";
anchor.addEventListener("click", () => {
fireAndForget(async () => {
this._log(`Unloading plugin: ${updatePluginName}`, LOG_LEVEL_NOTICE, "plugin-reload-" + updatePluginId);
// @ts-ignore
await this.app.plugins.unloadPlugin(updatePluginId);
// @ts-ignore
await this.app.plugins.loadPlugin(updatePluginId);
this._log(`Plugin reloaded: ${updatePluginName}`, LOG_LEVEL_NOTICE, "plugin-reload-" + updatePluginId);
});
});
}
);
}
}
} catch (ex) {
this._log("Error on checking plugin status.");
this._log(ex, LOG_LEVEL_VERBOSE);
}
// If something changes left, notify for reloading Obsidian.
if (updatedCount != 0) {
if (!this.plugin.isReloadingScheduled) {
this.plugin.confirm.askInPopup(`updated-any-hidden`, `Hidden files have been synchronised, Press {HERE} to schedule a reload of Obsidian, or press elsewhere to dismiss this message.`, (anchor) => {
anchor.text = "HERE";
anchor.addEventListener("click", () => {
this.plugin.$$scheduleAppReload();
});
});
}
}
}
}
this._log(`Hidden files scanned: ${filesChanged} files had been modified`, logLevel, "sync_internal");
}
async storeInternalFileToDatabase(file: InternalFileInfo, forceWrite = false) {
const storeFilePath = file.path;
const storageFilePath = file.path;
if (await this.plugin.$$isIgnoredByIgnoreFiles(storageFilePath)) {
return undefined;
}
const id = await this.path2id(storeFilePath, ICHeader);
const prefixedFileName = addPrefix(storeFilePath, ICHeader);
const content = createBlob(await this.plugin.storageAccess.readHiddenFileAuto(storageFilePath));
const mtime = file.mtime;
return await serialized("file-" + prefixedFileName, async () => {
try {
const old = await this.localDatabase.getDBEntry(prefixedFileName, undefined, false, false);
let saveData: SavingEntry;
if (old === false) {
saveData = {
_id: id,
path: prefixedFileName,
data: content,
mtime,
ctime: mtime,
datatype: "newnote",
size: file.size,
children: [],
deleted: false,
type: "newnote",
eden: {},
};
} else {
if (await isDocContentSame(readAsBlob(old), content) && !forceWrite) {
// this._log(`STORAGE --> DB:${file.path}: (hidden) Not changed`, LOG_LEVEL_VERBOSE);
const stat = await this.plugin.storageAccess.statHidden(storageFilePath);
if (stat) {
markChangesAreSame(storageFilePath, old.mtime, stat.mtime);
}
return undefined;
}
saveData =
{
...old,
data: content,
mtime,
size: file.size,
datatype: old.datatype,
children: [],
deleted: false,
type: old.datatype,
};
}
const ret = await this.localDatabase.putDBEntry(saveData);
if (ret !== false) {
this._log(`STORAGE --> DB:${storageFilePath}: (hidden) Done`);
return true;
} else {
this._log(`STORAGE --> DB:${storageFilePath}: (hidden) Failed`);
return false;
}
} catch (ex) {
this._log(`STORAGE --> DB:${storageFilePath}: (hidden) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
});
}
async deleteInternalFileOnDatabase(filenameSrc: FilePath, forceWrite = false) {
const storeFilePath = filenameSrc;
const storageFilePath = filenameSrc;
const displayFileName = filenameSrc;
const id = await this.path2id(storeFilePath, ICHeader);
const prefixedFileName = addPrefix(storeFilePath, ICHeader);
const mtime = new Date().getTime();
if (await this.plugin.$$isIgnoredByIgnoreFiles(storageFilePath)) {
return undefined
}
return await serialized("file-" + prefixedFileName, async () => {
try {
const old = await this.localDatabase.getDBEntryMeta(prefixedFileName, undefined, true) as InternalFileEntry | false;
let saveData: InternalFileEntry;
if (old === false) {
saveData = {
_id: id,
path: prefixedFileName,
mtime,
ctime: mtime,
size: 0,
children: [],
deleted: true,
type: "newnote",
eden: {}
};
} else {
// Remove all conflicted before deleting.
const conflicts = await this.localDatabase.getRaw(old._id, { conflicts: true });
if (conflicts._conflicts !== undefined) {
for (const conflictRev of conflicts._conflicts) {
await this.localDatabase.removeRevision(old._id, conflictRev);
this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) conflict removed ${old._rev} => ${conflictRev}`, LOG_LEVEL_VERBOSE);
}
}
if (old.deleted) {
this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) already deleted`);
return undefined;
}
saveData =
{
...old,
mtime,
size: 0,
children: [],
deleted: true,
type: "newnote",
};
}
const ret = await this.localDatabase.putRaw(saveData);
if (ret && ret.ok) {
this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Done`);
return true;
} else {
this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`);
return false;
}
} catch (ex) {
this._log(`STORAGE -x> DB: ${displayFileName}: (hidden) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
});
}
async extractInternalFileFromDatabase(filenameSrc: FilePath, force = false) {
const storeFilePath = filenameSrc;
const storageFilePath = filenameSrc;
const isExists = await this.plugin.storageAccess.isExistsIncludeHidden(storageFilePath);
const prefixedFileName = addPrefix(storeFilePath, ICHeader);
const displayFileName = `${storeFilePath}`;
if (await this.plugin.$$isIgnoredByIgnoreFiles(storageFilePath)) {
return undefined;
}
return await serialized("file-" + prefixedFileName, async () => {
try {
// Check conflicted status
const fileOnDB = await this.localDatabase.getDBEntry(prefixedFileName, { conflicts: true }, false, true, true);
if (fileOnDB === false)
throw new Error(`File not found on database.:${displayFileName}`);
// Prevent overwrite for Prevent overwriting while some conflicted revision exists.
if (fileOnDB?._conflicts?.length) {
this._log(`Hidden file ${displayFileName} has conflicted revisions, to keep in safe, writing to storage has been prevented`, LOG_LEVEL_INFO);
return false;
}
const deleted = fileOnDB.deleted || fileOnDB._deleted || false;
if (deleted) {
if (!isExists) {
this._log(`STORAGE <x- DB: ${displayFileName}: deleted (hidden) Deleted on DB, but the file is already not found on storage.`);
} else {
this._log(`STORAGE <x- DB: ${displayFileName}: deleted (hidden).`);
await this.plugin.storageAccess.removeHidden(storageFilePath);
try {
await this.plugin.storageAccess.triggerHiddenFile(storageFilePath);
} catch (ex) {
this._log("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
}
return true;
}
if (!isExists) {
await this.plugin.storageAccess.ensureDir(storageFilePath);
await this.plugin.storageAccess.writeHiddenFileAuto(storageFilePath, readContent(fileOnDB), { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime });
try {
await this.plugin.storageAccess.triggerHiddenFile(storageFilePath);
} catch (ex) {
this._log("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
this._log(`STORAGE <-- DB: ${displayFileName}: written (hidden,new${force ? ", force" : ""})`);
return true;
} else {
const content = await this.plugin.storageAccess.readHiddenFileAuto(storageFilePath);
const docContent = readContent(fileOnDB);
if (await isDocContentSame(content, docContent) && !force) {
// this._log(`STORAGE <-- DB:${filename}: skipped (hidden) Not changed`, LOG_LEVEL_VERBOSE);
const stat = await this.plugin.storageAccess.statHidden(storageFilePath);
if (stat) {
markChangesAreSame(storageFilePath, fileOnDB.mtime, stat.mtime);
}
return undefined;
}
if (await this.plugin.storageAccess.writeHiddenFileAuto(storageFilePath, docContent, { mtime: fileOnDB.mtime, ctime: fileOnDB.ctime })) {
const stat = await this.plugin.storageAccess.statHidden(storageFilePath) as UXStat;
this.markAsKnownChange(storageFilePath, stat.mtime);
try {
// await this.app.vault.adapter.reconcileInternalFile(filename);
await this.plugin.storageAccess.triggerHiddenFile(storageFilePath);
} catch (ex) {
this._log("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
this._log(`STORAGE <-- DB: ${displayFileName}: written (hidden, overwrite${force ? ", force" : ""})`);
return true;
} else {
this._log(`STORAGE <-- DB: ${displayFileName}: written (hidden, overwrite${force ? ", force" : ""}) Failed`);
return false;
}
}
} catch (ex) {
this._log(`STORAGE <-- DB: ${displayFileName}: written (hidden, overwrite${force ? ", force" : ""}) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
});
}
showJSONMergeDialogAndMerge(docA: LoadedEntry, docB: LoadedEntry): Promise<boolean> {
return new Promise((res) => {
this._log("Opening data-merging dialog", LOG_LEVEL_VERBOSE);
const docs = [docA, docB];
const strippedPath = stripAllPrefixes(docA.path);
const storageFilePath = strippedPath;
const storeFilePath = strippedPath;
const displayFilename = `${storeFilePath}`;
// const path = this.prefixedConfigDir2configDir(stripAllPrefixes(docA.path)) || docA.path;
const modal = new JsonResolveModal(this.app, storageFilePath, [docA, docB], async (keep, result) => {
// modal.close();
try {
// const filename = storeFilePath;
let needFlush = false;
if (!result && !keep) {
this._log(`Skipped merging: ${displayFilename}`);
res(false);
return;
}
//Delete old revisions
if (result || keep) {
for (const doc of docs) {
if (doc._rev != keep) {
if (await this.localDatabase.deleteDBEntry(this.getPath(doc), { rev: doc._rev })) {
this._log(`Conflicted revision has been deleted: ${displayFilename}`);
needFlush = true;
}
}
}
}
if (!keep && result) {
const isExists = await this.plugin.storageAccess.isExistsIncludeHidden(storageFilePath);
if (!isExists) {
await this.plugin.storageAccess.ensureDir(storageFilePath);
}
await this.plugin.storageAccess.writeHiddenFileAuto(storageFilePath, result);
const stat = await this.plugin.storageAccess.statHidden(storageFilePath);
if (!stat) {
throw new Error("Stat failed");
}
const mtime = stat?.mtime ?? 0;
await this.storeInternalFileToDatabase({ path: storageFilePath, mtime, ctime: stat?.ctime ?? mtime, size: stat?.size ?? 0 }, true);
try {
//@ts-ignore internalAPI
await this.app.vault.adapter.reconcileInternalFile(storageFilePath);
} catch (ex) {
this._log("Failed to call internal API(reconcileInternalFile)", LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
this._log(`STORAGE <-- DB:${displayFilename}: written (hidden,merged)`);
}
if (needFlush) {
if (await this.extractInternalFileFromDatabase(storeFilePath, false)) {
this._log(`STORAGE --> DB:${displayFilename}: extracted (hidden,merged)`);
} else {
this._log(`STORAGE --> DB:${displayFilename}: extracted (hidden,merged) Failed`);
}
}
res(true);
} catch (ex) {
this._log("Could not merge conflicted json");
this._log(ex, LOG_LEVEL_VERBOSE);
res(false);
}
});
modal.open();
});
}
async $allAskUsingOptionalSyncFeature(opt: { enableFetch?: boolean, enableOverwrite?: boolean }) {
await this._askHiddenFileConfiguration(opt);
return true;
}
async _askHiddenFileConfiguration(opt: { enableFetch?: boolean, enableOverwrite?: boolean }) {
const messageFetch = `${opt.enableFetch ? `> - Fetch: Use the files stored from other devices. Choose this option if you have already configured hidden file synchronization on those devices and wish to accept their files.\n` : ""}`;
const messageOverwrite = `${opt.enableOverwrite ? `> - Overwrite: Use the files from this device. Select this option if you want to overwrite the files stored on other devices.\n` : ""}`;
const messageMerge = `> - Merge: Merge the files from this device with those on other devices. Choose this option if you wish to combine files from multiple sources.
> However, please be reminded that merging may cause conflicts if the files are not identical. Additionally, this process may occur within the same folder, potentially breaking your plug-in or theme settings that comprise multiple files.\n`;
const message = `Would you like to enable **Hidden File Synchronization**?
> [!DETAILS]-
> This feature allows you to synchronize all hidden files without any user interaction.
> To enable this feature, you should choose one of the following options:
${messageFetch}${messageOverwrite}${messageMerge}
> [!IMPORTANT]
> Please keep in mind that enabling this feature alongside customisation sync may override certain behaviors.`
const CHOICE_FETCH = "Fetch";
const CHOICE_OVERWRITE = "Overwrite";
const CHOICE_MERGE = "Merge";
const CHOICE_DISABLE = "Disable";
const choices = [];
if (opt?.enableFetch) {
choices.push(CHOICE_FETCH);
}
if (opt?.enableOverwrite) {
choices.push(CHOICE_OVERWRITE);
}
choices.push(CHOICE_MERGE);
choices.push(CHOICE_DISABLE);
const ret = await this.plugin.confirm.confirmWithMessage("Hidden file sync", message, choices, CHOICE_DISABLE, 40);
if (ret == CHOICE_FETCH) {
await this.configureHiddenFileSync("FETCH");
} else if (ret == CHOICE_OVERWRITE) {
await this.configureHiddenFileSync("OVERWRITE");
} else if (ret == CHOICE_MERGE) {
await this.configureHiddenFileSync("MERGE");
} else if (ret == CHOICE_DISABLE) {
await this.configureHiddenFileSync("DISABLE_HIDDEN");
}
}
$allSuspendExtraSync(): Promise<boolean> {
if (this.plugin.settings.syncInternalFiles) {
this._log("Hidden file synchronization have been temporarily disabled. Please enable them after the fetching, if you need them.", LOG_LEVEL_NOTICE)
this.plugin.settings.syncInternalFiles = false;
}
return Promise.resolve(true);
}
async $anyConfigureOptionalSyncFeature(mode: "FETCH" | "OVERWRITE" | "MERGE" | "DISABLE" | "DISABLE_HIDDEN") {
await this.configureHiddenFileSync(mode);
}
async configureHiddenFileSync(mode: "FETCH" | "OVERWRITE" | "MERGE" | "DISABLE" | "DISABLE_HIDDEN") {
if (mode != "FETCH" && mode != "OVERWRITE" && mode != "MERGE" && mode != "DISABLE" && mode != "DISABLE_HIDDEN") {
return;
}
if (mode == "DISABLE" || mode == "DISABLE_HIDDEN") {
// await this.plugin.$allSuspendExtraSync();
this.plugin.settings.syncInternalFiles = false;
await this.plugin.saveSettings();
return;
}
this._log("Gathering files for enabling Hidden File Sync", LOG_LEVEL_NOTICE);
if (mode == "FETCH") {
await this.syncInternalFilesAndDatabase("pullForce", true);
} else if (mode == "OVERWRITE") {
await this.syncInternalFilesAndDatabase("pushForce", true);
} else if (mode == "MERGE") {
await this.syncInternalFilesAndDatabase("safe", true);
}
this.plugin.settings.useAdvancedMode = true;
this.plugin.settings.syncInternalFiles = true;
await this.plugin.saveSettings();
this._log(`Done! Restarting the app is strongly recommended!`, LOG_LEVEL_NOTICE);
}
async scanInternalFiles(): Promise<InternalFileInfo[]> {
const configDir = normalizePath(this.app.vault.configDir);
const ignoreFilter = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
const synchronisedInConfigSync = !this.settings.usePluginSync ? [] : Object.values(this.settings.pluginSyncExtendedSetting).filter(e => e.mode == MODE_SELECTIVE || e.mode == MODE_PAUSED).map(e => e.files).flat().map(e => `${configDir}/${e}`.toLowerCase());
const root = this.app.vault.getRoot();
const findRoot = root.path;
const filenames = (await this.getFiles(findRoot, [], undefined, ignoreFilter)).filter(e => e.startsWith(".")).filter(e => !e.startsWith(".trash"));
const files = filenames.filter(path => synchronisedInConfigSync.every(filterFile => !path.toLowerCase().startsWith(filterFile))).map(async (e) => {
return {
path: e as FilePath,
stat: await this.plugin.storageAccess.statHidden(e) // this.plugin.vaultAccess.adapterStat(e)
};
});
const result: InternalFileInfo[] = [];
for (const f of files) {
const w = await f;
if (await this.plugin.$$isIgnoredByIgnoreFiles(w.path)) {
continue
}
const mtime = w.stat?.mtime ?? 0
const ctime = w.stat?.ctime ?? mtime;
const size = w.stat?.size ?? 0;
result.push({
...w,
mtime, ctime, size
});
}
return result;
}
async getFiles(
path: string,
ignoreList: string[],
filter?: RegExp[],
ignoreFilter?: RegExp[]
) {
let w: ListedFiles;
try {
w = await this.app.vault.adapter.list(path);
} catch (ex) {
this._log(`Could not traverse(HiddenSync):${path}`, LOG_LEVEL_INFO);
this._log(ex, LOG_LEVEL_VERBOSE);
return [];
}
const filesSrc = [
...w.files
.filter((e) => !ignoreList.some((ee) => e.endsWith(ee)))
.filter((e) => !filter || filter.some((ee) => e.match(ee)))
.filter((e) => !ignoreFilter || ignoreFilter.every((ee) => !e.match(ee)))
];
let files = [] as string[];
for (const file of filesSrc) {
if (!await this.plugin.$$isIgnoredByIgnoreFiles(file)) {
files.push(file);
}
}
L1: for (const v of w.folders) {
for (const ignore of ignoreList) {
if (v.endsWith(ignore)) {
continue L1;
}
}
if (ignoreFilter && ignoreFilter.some(e => v.match(e))) {
continue L1;
}
if (await this.plugin.$$isIgnoredByIgnoreFiles(v)) {
continue L1;
}
files = files.concat(await this.getFiles(v, ignoreList, filter, ignoreFilter));
}
return files;
}
}

38
src/features/LiveSyncCommands.ts
View File

@@ -1,5 +1,6 @@
import { type AnyEntry, type DocumentID, type EntryDoc, type EntryHasPath, type FilePath, type FilePathWithPrefix } from "../lib/src/common/types.ts";
import { PouchDB } from "../lib/src/pouchdb/pouchdb-browser.js";
import { Logger } from "octagonal-wheels/common/logger";
import { getPath } from "../common/utils.ts";
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, type AnyEntry, type DocumentID, type EntryHasPath, type FilePath, type FilePathWithPrefix, type LOG_LEVEL } from "../lib/src/common/types.ts";
import type ObsidianLiveSyncPlugin from "../main.ts";
@@ -14,17 +15,15 @@ export abstract class LiveSyncCommands {
get localDatabase() {
return this.plugin.localDatabase;
}
get vaultAccess() {
return this.plugin.vaultAccess;
}
id2path(id: DocumentID, entry?: EntryHasPath, stripPrefix?: boolean): FilePathWithPrefix {
return this.plugin.id2path(id, entry, stripPrefix);
return this.plugin.$$id2path(id, entry, stripPrefix);
}
async path2id(filename: FilePathWithPrefix | FilePath, prefix?: string): Promise<DocumentID> {
return await this.plugin.path2id(filename, prefix);
return await this.plugin.$$path2id(filename, prefix);
}
getPath(entry: AnyEntry): FilePathWithPrefix {
return this.plugin.getPath(entry);
return getPath(entry);
}
constructor(plugin: ObsidianLiveSyncPlugin) {
@@ -32,9 +31,22 @@ export abstract class LiveSyncCommands {
}
abstract onunload(): void;
abstract onload(): void | Promise<void>;
abstract onInitializeDatabase(showNotice: boolean): void | Promise<void>;
abstract beforeReplicate(showNotice: boolean): void | Promise<void>;
abstract onResume(): void | Promise<void>;
abstract parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>): Promise<boolean> | boolean;
abstract realizeSettingSyncMode(): Promise<void>;
_isMainReady() {
return this.plugin._isMainReady();
}
_isMainSuspended() {
return this.plugin._isMainSuspended();
}
_isDatabaseReady() {
return this.plugin._isDatabaseReady();
}
_log = (msg: any, level: LOG_LEVEL = LOG_LEVEL_INFO, key?: string) => {
if (typeof msg === "string" && level !== LOG_LEVEL_NOTICE) {
msg = `[${this.constructor.name}]\u{200A} ${msg}`;
}
// console.log(msg);
Logger(msg, level, key);
};
}

2
src/lib

Submodule src/lib updated: 3108e3e3db...5079b0bf79

3989
src/main.ts
View File

File diff suppressed because it is too large Load Diff

183
src/modules/AbstractModule.ts Normal file
View File

@@ -0,0 +1,183 @@
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
import type { LOG_LEVEL } from "../lib/src/common/types";
import type { LiveSyncCore } from "../main";
import { unique } from "octagonal-wheels/collection";
import type { IObsidianModule } from "./AbstractObsidianModule.ts";
import type { ICoreModuleBase, AllInjectableProps, AllExecuteProps, EveryExecuteProps, AnyExecuteProps, ICoreModule } from "./ModuleTypes";
function isOverridableKey(key: string): key is keyof ICoreModuleBase {
return key.startsWith("$");
}
function isInjectableKey(key: string): key is keyof AllInjectableProps {
return key.startsWith("$$");
}
function isAllExecuteKey(key: string): key is keyof AllExecuteProps {
return key.startsWith("$all");
}
function isEveryExecuteKey(key: string): key is keyof EveryExecuteProps {
return key.startsWith("$every");
}
function isAnyExecuteKey(key: string): key is keyof AnyExecuteProps {
return key.startsWith("$any");
}
/**
* All $prefixed functions are hooked by the modules. Be careful to call them directly.
* Please refer to the module's source code to understand the function.
* $$ : Completely overridden functions.
* $all : Process all modules and return all results.
* $every : Process all modules until the first failure.
* $any : Process all modules until the first success.
* $ : Other interceptive points. You should manually assign the module
* All of above performed on injectModules function.
*/
export function injectModules<T extends ICoreModule>(target: T, modules: ICoreModule[]) {
const allKeys = unique([...Object.keys(Object.getOwnPropertyDescriptors(target)),
...Object.keys(Object.getOwnPropertyDescriptors(Object.getPrototypeOf(target)))]).filter(e => e.startsWith("$")) as (keyof ICoreModule)[];
const moduleMap = new Map<string, IObsidianModule[]>();
for (const module of modules) {
for (const key of allKeys) {
if (isOverridableKey(key)) {
if (key in module) {
const list = moduleMap.get(key) || [];
if (typeof module[key] === 'function') {
module[key] = module[key].bind(module) as any;
}
list.push(module);
moduleMap.set(key, list);
}
}
}
}
Logger(`Injecting modules for ${target.constructor.name}`, LOG_LEVEL_VERBOSE);
for (const key of allKeys) {
const modules = moduleMap.get(key) || [];
if (isInjectableKey(key)) {
if (modules.length == 0) {
throw new Error(`No module injected for ${key}. This is a fatal error.`);
}
target[key] = modules[0][key]! as any;
Logger(`[${modules[0].constructor.name}]: Injected ${key} `, LOG_LEVEL_VERBOSE);
} else if (isAllExecuteKey(key)) {
const modules = moduleMap.get(key) || [];
target[key] = async (...args: any) => {
for (const module of modules) {
try {
//@ts-ignore
await module[key]!(...args);
} catch (ex) {
Logger(`[${module.constructor.name}]: All handler for ${key} failed`, LOG_LEVEL_VERBOSE);
Logger(ex, LOG_LEVEL_VERBOSE);
}
}
return true;
}
for (const module of modules) {
Logger(`[${module.constructor.name}]: Injected (All) ${key} `, LOG_LEVEL_VERBOSE);
}
} else if (isEveryExecuteKey(key)) {
target[key] = async (...args: any) => {
for (const module of modules) {
try {
//@ts-ignore:2556
const ret = await module[key]!(...args);
if (ret !== undefined && !ret) {
// Failed then return that falsy value.
return ret;
}
} catch (ex) {
Logger(`[${module.constructor.name}]: Every handler for ${key} failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
}
}
return true;
}
for (const module of modules) {
Logger(`[${module.constructor.name}]: Injected (Every) ${key} `, LOG_LEVEL_VERBOSE);
}
} else if (isAnyExecuteKey(key)) {
//@ts-ignore
target[key] = async (...args: any[]) => {
for (const module of modules) {
try {
//@ts-ignore:2556
const ret = await module[key](...args);
// If truly value returned, then return that value.
if (ret) {
return ret;
}
} catch (ex) {
Logger(`[${module.constructor.name}]: Any handler for ${key} failed`);
Logger(ex, LOG_LEVEL_VERBOSE);
}
}
return false;
}
for (const module of modules) {
Logger(`[${module.constructor.name}]: Injected (Any) ${key} `, LOG_LEVEL_VERBOSE);
}
} else {
Logger(`No injected handler for ${key} `, LOG_LEVEL_VERBOSE);
}
}
Logger(`Injected modules for ${target.constructor.name}`, LOG_LEVEL_VERBOSE);
return true;
}
export abstract class AbstractModule {
_log = (msg: any, level: LOG_LEVEL = LOG_LEVEL_INFO, key?: string) => {
if (typeof msg === "string" && level !== LOG_LEVEL_NOTICE) {
msg = `[${this.constructor.name}]\u{200A} ${msg}`;
}
// console.log(msg);
Logger(msg, level, key);
};
get localDatabase() {
return this.core.localDatabase;
}
get settings() {
return this.core.settings;
}
set settings(value) {
this.core.settings = value;
}
constructor(public core: LiveSyncCore) {
Logger(`[${this.constructor.name}] Loaded`, LOG_LEVEL_VERBOSE);
}
saveSettings = this.core.saveSettings.bind(this.core);
// abstract $everyTest(): Promise<boolean>;
addTestResult(key: string, value: boolean, summary?: string, message?: string) {
this.core.$$addTestResult(`${this.constructor.name}`, key, value, summary, message);
}
testDone(result: boolean = true) {
return Promise.resolve(result);
}
testFail(message: string) {
this._log(message, LOG_LEVEL_NOTICE);
return this.testDone(false);
}
async _test(key: string, process: () => Promise<any>) {
this._log(`Testing ${key}`, LOG_LEVEL_VERBOSE);
try {
const ret = await process();
if (ret !== true) {
this.addTestResult(key, false, ret.toString());
return this.testFail(`${key} failed: ${ret}`);
}
this.addTestResult(key, true, "");
}
catch (ex: any) {
this.addTestResult(key, false, "Failed by Exception", ex.toString());
return this.testFail(`${key} failed: ${ex}`);
}
return this.testDone();
}
}

55
src/modules/AbstractObsidianModule.ts Normal file
View File

@@ -0,0 +1,55 @@
import { type Prettify } from "../lib/src/common/types";
import type { LiveSyncCore } from "../main";
import type ObsidianLiveSyncPlugin from "../main";
import { AbstractModule } from "./AbstractModule.ts";
import type { ChainableExecuteFunction, OverridableFunctionsKeys } from "./ModuleTypes";
export type IObsidianModuleBase = OverridableFunctionsKeys<ObsidianLiveSyncPlugin>;
export type IObsidianModule = Prettify<Partial<IObsidianModuleBase>>
export type ModuleKeys = keyof IObsidianModule;
export type ChainableModuleProps = ChainableExecuteFunction<ObsidianLiveSyncPlugin>;
export abstract class AbstractObsidianModule extends AbstractModule {
addCommand = this.plugin.addCommand.bind(this.plugin);
registerView = this.plugin.registerView.bind(this.plugin);
addRibbonIcon = this.plugin.addRibbonIcon.bind(this.plugin);
registerObsidianProtocolHandler = this.plugin.registerObsidianProtocolHandler.bind(this.plugin);
get localDatabase() {
return this.plugin.localDatabase;
}
get settings() {
return this.plugin.settings;
}
set settings(value) {
this.plugin.settings = value;
}
get app() {
return this.plugin.app;
}
constructor(public plugin: ObsidianLiveSyncPlugin, public core: LiveSyncCore) {
super(core);
}
saveSettings = this.plugin.saveSettings.bind(this.plugin);
_isMainReady() {
return this.core._isMainReady();
}
_isMainSuspended() {
return this.core._isMainSuspended();
}
_isDatabaseReady() {
return this.core._isDatabaseReady();
}
//should be overridden
_isThisModuleEnabled() {
return true
}
}

40
src/modules/ModuleTypes.ts Normal file
View File

@@ -0,0 +1,40 @@
import type { Prettify } from "../lib/src/common/types";
import type { LiveSyncCore } from "../main";
export type OverridableFunctionsKeys<T> = {
[K in keyof T as K extends `$${string}` ? K : never]: T[K];
}
export type ChainableExecuteFunction<T> = {
[K in keyof T as K extends `$${string}` ? (T[K] extends (...args: any) => ChainableFunctionResult ? K : never) : never]: T[K];
}
export type ICoreModuleBase = OverridableFunctionsKeys<LiveSyncCore>;
export type ICoreModule = Prettify<Partial<ICoreModuleBase>>
export type CoreModuleKeys = keyof ICoreModule;
export type ChainableFunctionResult =
Promise<boolean | undefined | string>
| Promise<boolean | undefined>
| Promise<boolean>
| Promise<void>;
export type ChainableFunctionResultOrAll = Promise<boolean | undefined | string | void>;
type AllExecuteFunction<T> = {
[K in keyof T as K extends `$all${string}` ? (T[K] extends (...args: any[]) => ChainableFunctionResultOrAll ? K : never) : never]: T[K];
}
type EveryExecuteFunction<T> = {
[K in keyof T as K extends `$every${string}` ? (T[K] extends (...args: any[]) => ChainableFunctionResult ? K : never) : never]: T[K];
}
type AnyExecuteFunction<T> = {
[K in keyof T as K extends `$any${string}` ? (T[K] extends (...args: any[]) => ChainableFunctionResult ? K : never) : never]: T[K];
}
type InjectableFunction<T> = {
[K in keyof T as K extends `$$${string}` ? (T[K] extends (...args: any[]) => any ? K : never) : never]: T[K];
}
export type AllExecuteProps = AllExecuteFunction<LiveSyncCore>;
export type EveryExecuteProps = EveryExecuteFunction<LiveSyncCore>;
export type AnyExecuteProps = AnyExecuteFunction<LiveSyncCore>;
export type AllInjectableProps = InjectableFunction<LiveSyncCore>;

262
src/modules/core/ModuleDatabaseFileAccess.ts Normal file
View File

@@ -0,0 +1,262 @@
import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { EVENT_FILE_SAVED, eventHub } from "../../common/events";
import { getPathFromUXFileInfo, isInternalMetadata, markChangesAreSame } from "../../common/utils";
import type { UXFileInfoStub, FilePathWithPrefix, UXFileInfo, MetaEntry, LoadedEntry, FilePath, SavingEntry } from "../../lib/src/common/types";
import type { DatabaseFileAccess } from "../interfaces/DatabaseFileAccess";
import { type IObsidianModule } from "../AbstractObsidianModule.ts";
import { isPlainText, shouldBeIgnored } from "../../lib/src/string_and_binary/path";
import { createBlob, createTextBlob, delay, determineTypeFromBlob, isDocContentSame, readContent } from "../../lib/src/common/utils";
import { serialized } from "octagonal-wheels/concurrency/lock";
import { AbstractModule } from "../AbstractModule.ts";
export class ModuleDatabaseFileAccess extends AbstractModule implements IObsidianModule, DatabaseFileAccess {
$everyOnload(): Promise<boolean> {
this.core.databaseFileAccess = this;
return Promise.resolve(true);
}
async $everyModuleTest(): Promise<boolean> {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
const testString = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam nec purus nec nunc"
// Before test, we need to delete completely.
const conflicts = await this.getConflictedRevs("autoTest.md" as FilePathWithPrefix);
for (const rev of conflicts) {
await this.delete("autoTest.md" as FilePathWithPrefix, rev);
}
await this.delete("autoTest.md" as FilePathWithPrefix);
// OK, begin!
await this._test("storeContent", async () => (await this.storeContent("autoTest.md" as FilePathWithPrefix, testString)));
// For test, we need to clear the caches.
await this.localDatabase.hashCaches.clear();
await this._test("readContent", async () => {
const content = await this.fetch("autoTest.md" as FilePathWithPrefix);
if (!content) return "File not found";
if (content.deleted) return "File is deleted";
return (await content.body.text() == testString) ? true : `Content is not same ${await content.body.text()}`;
});
await this._test("delete", async () => (await this.delete("autoTest.md" as FilePathWithPrefix)));
await this._test("read deleted content", async () => {
const content = await this.fetch("autoTest.md" as FilePathWithPrefix);
if (!content) return true;
if (content.deleted) return true;
return `Still exist !:${await content.body.text()},${JSON.stringify(content, undefined, 2)}`;
});
await delay(100);
return this.testDone();
}
async checkIsTargetFile(file: UXFileInfoStub | FilePathWithPrefix): Promise<boolean> {
const path = getPathFromUXFileInfo(file);
if (!await this.core.$$isTargetFile(path)) {
this._log(`File is not target`, LOG_LEVEL_VERBOSE);
return false;
}
if (shouldBeIgnored(path)) {
this._log(`File should be ignored`, LOG_LEVEL_VERBOSE);
return false;
}
return true;
}
async delete(file: UXFileInfoStub | FilePathWithPrefix, rev?: string): Promise<boolean> {
if (!await this.checkIsTargetFile(file)) {
return true;
}
const fullPath = getPathFromUXFileInfo(file);
try {
this._log(`deleteDB By path:${fullPath}`);
return await this.deleteFromDBbyPath(fullPath, rev);
} catch (ex) {
this._log(`Failed to delete ${fullPath}`);
this._log(ex, LOG_LEVEL_VERBOSE);
return false;
}
}
async createChunks(file: UXFileInfo, force: boolean = false, skipCheck?: boolean): Promise<boolean> {
return await this._store(file, force, skipCheck, true);
}
async store(file: UXFileInfo, force: boolean = false, skipCheck?: boolean): Promise<boolean> {
return await this._store(file, force, skipCheck, false);
}
async storeContent(path: FilePathWithPrefix, content: string): Promise<boolean> {
const blob = createTextBlob(content);
const bytes = (await blob.arrayBuffer()).byteLength;
const dummyUXFileInfo: UXFileInfo = {
name: path.split("/").pop() as string,
path: path,
stat: {
size: bytes,
ctime: Date.now(),
mtime: Date.now(),
type: "file",
},
body: blob,
}
return await this._store(dummyUXFileInfo, true, false, false);
}
async _store(file: UXFileInfo, force: boolean = false, skipCheck?: boolean, onlyChunks?: boolean): Promise<boolean> {
if (!skipCheck) {
if (!await this.checkIsTargetFile(file)) {
return true;
}
}
if (!file) {
this._log("File seems bad", LOG_LEVEL_VERBOSE);
return false;
}
const path = getPathFromUXFileInfo(file);
const isPlain = isPlainText(file.name);
const possiblyLarge = !isPlain;
const content = file.body;
if (possiblyLarge) this._log(`Processing: ${path}`, LOG_LEVEL_VERBOSE);
const datatype = determineTypeFromBlob(content);
const fullPath = file.path;
const id = await this.core.$$path2id(fullPath);
const d: SavingEntry = {
_id: id,
path: file.path,
data: content,
ctime: file.stat.ctime,
mtime: file.stat.mtime,
size: file.stat.size,
children: [],
datatype: datatype,
type: datatype,
eden: {},
};
//upsert should locked
const msg = `STORAGE -> DB (${datatype}) `;
const isNotChanged = await serialized("file-" + fullPath, async () => {
// Commented out temporarily: this checks that the file was made ourself.
// if (this.core.storageAccess.recentlyTouched(file)) {
// return true;
// }
try {
const old = await this.localDatabase.getDBEntry(fullPath, undefined, false, true, false);
if (old !== false) {
const oldData = { data: old.data, deleted: old._deleted || old.deleted };
const newData = { data: d.data, deleted: d._deleted || d.deleted };
if (oldData.deleted != newData.deleted) return false;
if (!await isDocContentSame(old.data, newData.data)) return false;
this._log(msg + "Skipped (not changed) " + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
markChangesAreSame(old, d.mtime, old.mtime);
return true;
// d._rev = old._rev;
}
} catch (ex) {
if (force) {
this._log(msg + "Error, Could not check the diff for the old one." + (force ? "force writing." : "") + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
} else {
this._log(msg + "Error, Could not check the diff for the old one." + fullPath + ((d._deleted || d.deleted) ? " (deleted)" : ""), LOG_LEVEL_VERBOSE);
}
this._log(ex, LOG_LEVEL_VERBOSE);
return !force;
}
return false;
});
if (isNotChanged) {
this._log(msg + " Skip " + fullPath, LOG_LEVEL_VERBOSE);
return true;
}
const ret = await this.localDatabase.putDBEntry(d, onlyChunks);
if (ret !== false) {
this._log(msg + fullPath);
eventHub.emitEvent(EVENT_FILE_SAVED);
}
return ret != false;
}
async getConflictedRevs(file: UXFileInfoStub | FilePathWithPrefix): Promise<string[]> {
if (!await this.checkIsTargetFile(file)) {
return [];
}
const filename = getPathFromUXFileInfo(file);
const doc = await this.localDatabase.getDBEntryMeta(filename, { conflicts: true }, true);
if (doc === false) {
return [];
}
return doc._conflicts || [];
}
async fetch(file: UXFileInfoStub | FilePathWithPrefix,
rev?: string, waitForReady?: boolean, skipCheck = false): Promise<UXFileInfo | false> {
if (skipCheck && !await this.checkIsTargetFile(file)) {
return false;
}
const entry = await this.fetchEntry(file, rev, waitForReady, true);
if (entry === false) {
return false;
}
const data = createBlob(readContent(entry));
const fileInfo: UXFileInfo = {
name: entry.path.split("/").pop() as string,
path: entry.path,
stat: {
size: entry.size,
ctime: entry.ctime,
mtime: entry.mtime,
type: "file",
},
body: data,
deleted: entry.deleted || entry._deleted,
}
if (isInternalMetadata(entry.path)) {
fileInfo.isInternal = true;
}
return fileInfo;
}
async fetchEntryMeta(file: UXFileInfoStub | FilePathWithPrefix,
rev?: string, skipCheck = false): Promise<MetaEntry | false> {
const filename = getPathFromUXFileInfo(file);
if (skipCheck && !await this.checkIsTargetFile(file)) {
return false;
}
const doc = await this.localDatabase.getDBEntryMeta(
filename, rev ? { rev: rev } : undefined, true);
if (doc === false) {
return false;
}
return doc as MetaEntry;
}
async fetchEntryFromMeta(meta: MetaEntry, waitForReady: boolean = true, skipCheck = false): Promise<LoadedEntry | false> {
if (skipCheck && !await this.checkIsTargetFile(meta.path)) {
return false;
}
const doc = await this.localDatabase.getDBEntryFromMeta(meta as LoadedEntry, undefined, false, waitForReady, true);
if (doc === false) {
return false;
}
return doc;
}
async fetchEntry(file: UXFileInfoStub | FilePathWithPrefix,
rev?: string, waitForReady: boolean = true, skipCheck = false): Promise<LoadedEntry | false> {
if (skipCheck && !await this.checkIsTargetFile(file)) {
return false;
}
const entry = await this.fetchEntryMeta(file, rev, true);
if (entry === false) {
return false;
}
const doc = await this.fetchEntryFromMeta(entry, waitForReady, true);
return doc;
}
async deleteFromDBbyPath(fullPath: FilePath | FilePathWithPrefix, rev?: string): Promise<boolean> {
if (!await this.checkIsTargetFile(fullPath)) {
this._log(`storeFromStorage: File is not target: ${fullPath}`);
return true;
}
const opt = rev ? { rev: rev } : undefined;
const ret = await this.localDatabase.deleteDBEntry(fullPath, opt);
eventHub.emitEvent(EVENT_FILE_SAVED);
return ret;
}
}

358
src/modules/core/ModuleFileHandler.ts Normal file
View File

@@ -0,0 +1,358 @@
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { serialized } from "octagonal-wheels/concurrency/lock";
import type { FileEventItem } from "../../common/types";
import type { FilePath, FilePathWithPrefix, MetaEntry, UXFileInfo, UXFileInfoStub, UXInternalFileInfoStub } from "../../lib/src/common/types";
import { AbstractModule } from "../AbstractModule.ts";
import { compareFileFreshness, EVEN, getPath, getPathFromUXFileInfo, getPathWithoutPrefix, markChangesAreSame } from "../../common/utils";
import { getDocDataAsArray, isDocContentSame, readContent } from "../../lib/src/common/utils";
import { shouldBeIgnored } from "../../lib/src/string_and_binary/path";
import type { ICoreModule } from "../ModuleTypes";
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
export class ModuleFileHandler extends AbstractModule implements ICoreModule {
get db() {
return this.core.databaseFileAccess;
}
get storage() {
return this.core.storageAccess;
}
$everyOnloadStart(): Promise<boolean> {
this.core.fileHandler = this;
return Promise.resolve(true);
}
async readFileFromStub(file: UXFileInfoStub | UXFileInfo) {
if ("body" in file && file.body) {
return file;
}
const readFile = await this.storage.readStubContent(file);
if (!readFile) {
throw new Error(`File ${file.path} is not exist on the storage`);
}
return readFile
}
async storeFileToDB(info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix, force: boolean = false, onlyChunks: boolean = false): Promise<boolean | undefined> {
const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
if (file == null) {
this._log(`File ${info} is not exist on the storage`, LOG_LEVEL_VERBOSE);
return false;
}
// const file = item.args.file;
if (file.isInternal) {
this._log(`Internal file ${file.path} is not allowed to be processed on processFileEvent`, LOG_LEVEL_VERBOSE);
return false;
}
// First, check the file on the database
const entry = await this.db.fetchEntry(file, undefined, true, true);
if (!entry || entry.deleted || entry._deleted) {
// If the file is not exist on the database, then it should be created.
const readFile = await this.readFileFromStub(file);
if (!onlyChunks) {
return await this.db.store(readFile);
} else {
return await this.db.createChunks(readFile, false, true);
}
}
// entry is exist on the database, check the difference between the file and the entry.
let shouldApplied = false;
if (!force && !onlyChunks) {
// 1. if the time stamp is far different, then it should be updated.
// Note: This checks only the mtime with the resolution reduced to 2 seconds.
// 2 seconds it for the ZIP file's mtime. If not, we cannot backup the vault as the ZIP file.
// This is hardcoded on `compareMtime` of `src/common/utils.ts`.
if (compareFileFreshness(file, entry) !== EVEN) {
shouldApplied = true;
}
// 2. if not, the content should be checked.
let readFile: UXFileInfo | undefined = undefined;
if (!shouldApplied) {
readFile = await this.readFileFromStub(file);
if (await isDocContentSame(getDocDataAsArray(entry.data), readFile.body)) {
if (shouldApplied) {
// Timestamp is different but the content is same. therefore, two timestamps should be handled as same.
// So, mark the changes are same.
markChangesAreSame(file, file.stat.mtime, entry.mtime);
}
} else {
shouldApplied = true;
}
}
if (!shouldApplied) {
this._log(`File ${file.path} is not changed`, LOG_LEVEL_VERBOSE);
return true;
}
if (!readFile) readFile = await this.readFileFromStub(file);
// If the file is changed, then the file should be stored.
if (onlyChunks) {
return await this.db.createChunks(readFile, false, true);
} else {
return await this.db.store(readFile, false, true);
}
} else {
// If force is true, then it should be updated.
const readFile = await this.readFileFromStub(file);
if (onlyChunks) {
return await this.db.createChunks(readFile, true, true);
} else {
return await this.db.store(readFile, true, true);
}
}
}
async deleteFileFromDB(info: UXFileInfoStub | UXInternalFileInfoStub | FilePath): Promise<boolean | undefined> {
const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
if (file == null) {
this._log(`File ${info} is not exist on the storage`, LOG_LEVEL_VERBOSE);
return false;
}
// const file = item.args.file;
if (file.isInternal) {
this._log(`Internal file ${file.path} is not allowed to be processed on processFileEvent`, LOG_LEVEL_VERBOSE);
return false;
}
// First, check the file on the database
const entry = await this.db.fetchEntry(file, undefined, true, true);
if (!entry || entry.deleted || entry._deleted) {
this._log(`File ${file.path} is not exist or already deleted on the database`, LOG_LEVEL_VERBOSE);
return false;
}
// Check the file is already conflicted. if so, only the conflicted one should be deleted.
const conflictedRevs = await this.db.getConflictedRevs(file);
if (conflictedRevs.length > 0) {
// If conflicted, then it should be deleted. entry._rev should be own file's rev.
// TODO: I BELIEVED SO. BUT I NOTICED THAT I AN NOT SURE. I SHOULD CHECK THIS.
// ANYWAY, I SHOULD DELETE THE FILE. ACTUALLY WE SIMPLY DELETED THE FILE UNTIL PREVIOUS VERSIONS.
return await this.db.delete(file, entry._rev);
}
// Otherwise, the file should be deleted simply. This is the previous behaviour.
return await this.db.delete(file);
}
async deleteRevisionFromDB(info: UXFileInfoStub | FilePath | FilePathWithPrefix, rev: string): Promise<boolean | undefined> {
//TODO: Possibly check the conflicting.
return await this.db.delete(info, rev);
}
async resolveConflictedByDeletingRevision(info: UXFileInfoStub | FilePath, rev: string): Promise<boolean | undefined> {
if (!await this.deleteRevisionFromDB(info, rev)) {
this._log(`Failed to delete the conflicted revision ${rev} of ${getPathFromUXFileInfo(info)}`, LOG_LEVEL_VERBOSE);
return false;
}
if (!await this.dbToStorageWithSpecificRev(info, rev, true)) {
this._log(`Failed to apply the resolved revision ${rev} of ${getPathFromUXFileInfo(info)} to the storage`, LOG_LEVEL_VERBOSE);
return false;
}
}
async dbToStorageWithSpecificRev(info: UXFileInfoStub | UXFileInfo | FilePath | null, rev: string, force?: boolean): Promise<boolean> {
const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
if (file == null) {
this._log(`File ${info} is not exist on the storage`, LOG_LEVEL_VERBOSE);
return false;
}
const docEntry = await this.db.fetchEntryMeta(file, rev, true);
if (!docEntry) {
this._log(`File ${file.path} is not exist on the database`, LOG_LEVEL_VERBOSE);
return false;
}
return await this.dbToStorage(docEntry, file, force);
}
async dbToStorage(entryInfo: MetaEntry | FilePathWithPrefix, info: UXFileInfoStub | UXFileInfo | FilePath | null, force?: boolean): Promise<boolean> {
const file = typeof info === "string" ? this.storage.getFileStub(info) : info;
const mode = file == null ? "create" : "modify";
const docEntry = typeof entryInfo === "string" ?
await this.db.fetchEntryMeta(entryInfo, undefined, true) : await this.db.fetchEntryMeta(entryInfo.path, undefined, true);
if (!docEntry) {
this._log(`File ${entryInfo} is not exist on the database`, LOG_LEVEL_VERBOSE);
return false;
}
const path = getPath(docEntry);
// 1. Check if it already conflicted.
const revs = await this.db.getConflictedRevs(path);
if (revs.length > 0) {
// Some conflicts are exist.
if (this.settings.writeDocumentsIfConflicted) {
// If configured to write the document even if conflicted, then it should be written.
// NO OP
} else {
// If not, then it should be checked. and will be processed later (i.e., after the conflict is resolved).
await this.core.$$queueConflictCheckIfOpen(path);
return true;
}
}
// 2. Check if the file is already exist on the storage.
const existDoc = this.storage.getStub(path);
if (existDoc && existDoc.isFolder) {
this._log(`Folder ${path} is already exist on the storage as a folder`, LOG_LEVEL_VERBOSE);
// We can do nothing, and other modules should also nothing to do.
return true;
}
// Check existence of both file and docEntry.
const existOnDB = !(docEntry._deleted || docEntry.deleted || false);
const existOnStorage = existDoc != null;
if (!existOnDB && !existOnStorage) {
this._log(`File ${path} seems to be deleted, but already not on storage`, LOG_LEVEL_VERBOSE);
return true;
}
if (!existOnDB && existOnStorage) {
// Deletion has been Transferred. Storage files will be deleted.
// Note: If the folder becomes empty, the folder will be deleted if not configured to keep it.
// This behaviour is implemented on the `ModuleFileAccessObsidian`.
// And it does not care actually deleted.
await this.storage.deleteVaultItem(path);
return true;
}
// Okay, the file is exist on the database. Let's check the file is exist on the storage.
const docRead = await this.db.fetchEntryFromMeta(docEntry);
if (!docRead) {
this._log(`File ${path} is not exist on the database`, LOG_LEVEL_VERBOSE);
return false;
}
const docData = readContent(docRead);
if (!existOnStorage) {
// The file is not exist on the storage. We do not care about the differences.
await this.storage.ensureDir(path);
return await this.storage.writeFileAuto(path, docData, { ctime: docRead.ctime, mtime: docRead.mtime });
}
if (!force) {
// Ok, we have to compare.
let shouldApplied = false;
// 1. if the time stamp is far different, then it should be updated.
// Note: This checks only the mtime with the resolution reduced to 2 seconds.
// 2 seconds it for the ZIP file's mtime. If not, we cannot backup the vault as the ZIP file.
// This is hardcoded on `compareMtime` of `src/common/utils.ts`.
if (compareFileFreshness(existDoc, docEntry) !== EVEN) {
shouldApplied = true;
}
// 2. if not, the content should be checked.
if (shouldApplied) {
const readFile = await this.readFileFromStub(existDoc);
if (await isDocContentSame(docData, readFile.body)) {
// The content is same. So, we do not need to update the file.
shouldApplied = false;
// Timestamp is different but the content is same. therefore, two timestamps should be handled as same.
// So, mark the changes are same.
markChangesAreSame(docRead, docRead.mtime, existDoc.stat.mtime);
} else {
shouldApplied = true;
}
}
if (!shouldApplied) {
this._log(`File ${docRead.path} is not changed`, LOG_LEVEL_VERBOSE);
return true;
}
// Let's apply the changes.
}
await this.storage.ensureDir(path);
const ret = await this.storage.writeFileAuto(path, docData, { ctime: docRead.ctime, mtime: docRead.mtime });
this.storage.touched(path);
this.storage.triggerFileEvent(mode, path);
return ret;
}
async $anyHandlerProcessesFileEvent(item: FileEventItem): Promise<boolean | undefined> {
const eventItem = item.args;
const type = item.type;
const path = eventItem.file.path;
if (!await this.core.$$isTargetFile(path)) {
this._log(`File ${path} is not the target file`, LOG_LEVEL_VERBOSE);
return false;
}
if (shouldBeIgnored(path)) {
this._log(`File ${path} should be ignored`, LOG_LEVEL_VERBOSE);
return false;
}
const lockKey = `processFileEvent-${path}`;
return await serialized(lockKey, async () => {
switch (type) {
case "CREATE":
case "CHANGED":
return await this.storeFileToDB(item.args.file);
case "DELETE":
return await this.deleteFileFromDB(item.args.file);
case "INTERNAL":
// this should be handled on the other module.
return false;
default:
this._log(`Unsupported event type: ${type}`, LOG_LEVEL_VERBOSE);
return false;
}
});
}
async $anyProcessReplicatedDoc(entry: MetaEntry): Promise<boolean | undefined> {
return await serialized(entry.path, async () => {
if (!await this.core.$$isTargetFile(entry.path)) {
this._log(`File ${entry.path} is not the target file`, LOG_LEVEL_VERBOSE);
return false;
}
if (shouldBeIgnored(entry.path)) {
this._log(`File ${entry.path} should be ignored`, LOG_LEVEL_VERBOSE);
return false;
}
const path = getPath(entry);
const targetFile = this.storage.getStub(getPathWithoutPrefix(entry));
if (targetFile && targetFile.isFolder) {
this._log(`${getPath(entry)} is already exist as the folder`);
// Nothing to do and other modules should also nothing to do.
return true;
} else {
this._log(`Processing ${path} (${entry._id.substring(0, 8)}: ${entry._rev?.substring(0, 5)}) :Started...`, LOG_LEVEL_VERBOSE);
const ret = await this.dbToStorage(entry, targetFile);
this._log(`Processing ${path} (${entry._id.substring(0, 8)} :${entry._rev?.substring(0, 5)}) : Done`);
return ret;
}
});
}
async createAllChunks(showingNotice?: boolean): Promise<void> {
this._log("Collecting local files on the storage", LOG_LEVEL_VERBOSE);
const semaphore = Semaphore(10);
let processed = 0;
const filesStorageSrc = this.storage.getFiles();
const incProcessed = () => {
processed++;
if (processed % 25 == 0) this._log(`Creating missing chunks: ${processed} of ${total} files`, showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO, "chunkCreation");
}
const total = filesStorageSrc.length;
const procAllChunks = filesStorageSrc.map(async (file) => {
if (!await this.core.$$isTargetFile(file)) {
incProcessed();
return true;
}
if (shouldBeIgnored(file.path)) {
incProcessed();
return true;
}
const release = await semaphore.acquire();
incProcessed();
try {
await this.storeFileToDB(file, false, true);
} catch (ex) {
this._log(ex, LOG_LEVEL_VERBOSE);
} finally {
release();
}
});
await Promise.all(procAllChunks);
this._log(`Creating chunks Done: ${processed} of ${total} files`, showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO, "chunkCreation");
}
}

23
src/modules/core/ModuleLocalDatabaseObsidian.ts Normal file
View File

@@ -0,0 +1,23 @@
import { $f } from "../../lib/src/common/i18n";
import { LiveSyncLocalDB } from "../../lib/src/pouchdb/LiveSyncLocalDB.ts";
import { initializeStores } from "../../common/stores.ts";
import { AbstractModule } from "../AbstractModule.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleLocalDatabaseObsidian extends AbstractModule implements ICoreModule {
$everyOnloadStart(): Promise<boolean> {
return Promise.resolve(true);
}
async $$openDatabase(): Promise<boolean> {
if (this.localDatabase != null) {
await this.localDatabase.close();
}
const vaultName = this.core.$$getVaultName();
this._log($f`Waiting for ready...`);
this.core.localDatabase = new LiveSyncLocalDB(vaultName, this.core);
initializeStores(vaultName);
return await this.localDatabase.initializeDatabase();
}
}

35
src/modules/core/ModulePeriodicProcess.ts Normal file
View File

@@ -0,0 +1,35 @@
import { PeriodicProcessor } from "../../common/utils";
import { AbstractModule } from "../AbstractModule";
import type { ICoreModule } from "../ModuleTypes";
export class ModulePeriodicProcess extends AbstractModule implements ICoreModule {
periodicSyncProcessor = new PeriodicProcessor(this.core, async () => await this.core.$$replicate());
_disablePeriodic() {
this.periodicSyncProcessor?.disable();
return Promise.resolve(true);
}
_resumePeriodic() {
this.periodicSyncProcessor.enable(this.settings.periodicReplication ? this.settings.periodicReplicationInterval * 1000 : 0);
return Promise.resolve(true);
}
$allOnUnload() {
return this._disablePeriodic();
}
$everyBeforeRealizeSetting(): Promise<boolean> {
return this._disablePeriodic();
}
$everyBeforeSuspendProcess(): Promise<boolean> {
return this._disablePeriodic();
}
$everyAfterResumeProcess(): Promise<boolean> {
return this._resumePeriodic();
}
$everyAfterRealizeSetting(): Promise<boolean> {
return this._resumePeriodic();
}
}

16
src/modules/core/ModulePouchDB.ts Normal file
View File

@@ -0,0 +1,16 @@
import { AbstractModule } from "../AbstractModule";
import type { ICoreModule } from "../ModuleTypes";
import { PouchDB } from "../../lib/src/pouchdb/pouchdb-browser";
export class ModulePouchDB extends AbstractModule implements ICoreModule {
$$createPouchDBInstance<T extends object>(name?: string, options?: PouchDB.Configuration.DatabaseConfiguration): PouchDB.Database<T> {
const optionPass = options ?? {};
if (this.settings.useIndexedDBAdapter) {
optionPass.adapter = "indexeddb";
//@ts-ignore :missing def
optionPass.purged_infos_limit = 1;
return new PouchDB(name + "-indexeddb", optionPass);
}
return new PouchDB(name, optionPass);
}
}

225
src/modules/core/ModuleRebuilder.ts Normal file
View File

@@ -0,0 +1,225 @@
import { delay } from "octagonal-wheels/promises";
import { FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3_HR, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, REMOTE_COUCHDB, REMOTE_MINIO } from "../../lib/src/common/types.ts";
import { AbstractModule } from "../AbstractModule.ts";
import type { Rebuilder } from "../interfaces/DatabaseRebuilder.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
import type { LiveSyncCouchDBReplicator } from "../../lib/src/replication/couchdb/LiveSyncReplicator.ts";
import { fetchAllUsedChunks } from "../../lib/src/pouchdb/utils_couchdb.ts";
export class ModuleRebuilder extends AbstractModule implements ICoreModule, Rebuilder {
$everyOnload(): Promise<boolean> {
this.core.rebuilder = this;
return Promise.resolve(true);
}
async $performRebuildDB(method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice" | "localOnlyWithChunks"): Promise<void> {
if (method == "localOnly") {
await this.$fetchLocal();
}
if (method == "localOnlyWithChunks") {
await this.$fetchLocal(true);
}
if (method == "remoteOnly") {
await this.$rebuildRemote();
}
if (method == "rebuildBothByThisDevice") {
await this.$rebuildEverything();
}
}
async askUsingOptionalFeature(opt: {
enableFetch?: boolean;
enableOverwrite?: boolean;
}) {
if (await this.core.confirm.askYesNoDialog("Do you want to enable extra features? If you are new to Self-hosted LiveSync, try the core feature first!", { title: "Enable extra features", defaultOption: "No", timeout: 15 }) == "yes") {
await this.core.$allAskUsingOptionalSyncFeature(opt);
}
}
async rebuildRemote() {
await this.core.$allSuspendExtraSync();
this.core.settings.isConfigured = true;
await this.core.realizeSettingSyncMode();
await this.core.$$markRemoteLocked();
await this.core.$$tryResetRemoteDatabase();
await this.core.$$markRemoteLocked();
await delay(500);
await this.askUsingOptionalFeature({ enableOverwrite: true });
await delay(1000);
await this.core.$$replicateAllToServer(true);
await delay(1000);
await this.core.$$replicateAllToServer(true, true);
}
$rebuildRemote(): Promise<void> {
return this.rebuildRemote();
}
async rebuildEverything() {
await this.core.$allSuspendExtraSync();
await this.askUseNewAdapter();
this.core.settings.isConfigured = true;
await this.core.realizeSettingSyncMode();
await this.resetLocalDatabase();
await delay(1000);
await this.core.$$initializeDatabase(true);
await this.core.$$markRemoteLocked();
await this.core.$$tryResetRemoteDatabase();
await this.core.$$markRemoteLocked();
await delay(500);
await this.askUsingOptionalFeature({ enableOverwrite: true });
await delay(1000);
await this.core.$$replicateAllToServer(true);
await delay(1000);
await this.core.$$replicateAllToServer(true, true);
}
$rebuildEverything(): Promise<void> {
return this.rebuildEverything();
}
$fetchLocal(makeLocalChunkBeforeSync?: boolean): Promise<void> {
return this.fetchLocal(makeLocalChunkBeforeSync);
}
async scheduleRebuild(): Promise<void> {
try {
await this.core.storageAccess.writeFileAuto(FLAGMD_REDFLAG2_HR, "");
} catch (ex) {
this._log("Could not create red_flag_rebuild.md", LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
this.core.$$performRestart();
}
async scheduleFetch(): Promise<void> {
try {
await this.core.storageAccess.writeFileAuto(FLAGMD_REDFLAG3_HR, "");
} catch (ex) {
this._log("Could not create red_flag_fetch.md", LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
this.core.$$performRestart();
}
async $$tryResetRemoteDatabase(): Promise<void> {
await this.core.replicator.tryResetRemoteDatabase(this.settings);
}
async $$tryCreateRemoteDatabase(): Promise<void> {
await this.core.replicator.tryCreateRemoteDatabase(this.settings);
}
async $$resetLocalDatabase(): Promise<void> {
this.core.storageAccess.clearTouched();
await this.localDatabase.resetDatabase();
}
async suspendAllSync() {
this.core.settings.liveSync = false;
this.core.settings.periodicReplication = false;
this.core.settings.syncOnSave = false;
this.core.settings.syncOnEditorSave = false;
this.core.settings.syncOnStart = false;
this.core.settings.syncOnFileOpen = false;
this.core.settings.syncAfterMerge = false;
await this.core.$allSuspendExtraSync();
}
async suspendReflectingDatabase() {
if (this.core.settings.doNotSuspendOnFetching) return;
if (this.core.settings.remoteType == REMOTE_MINIO) return;
this._log(`Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`, LOG_LEVEL_NOTICE);
this.core.settings.suspendParseReplicationResult = true;
this.core.settings.suspendFileWatching = true;
await this.core.saveSettings();
}
async resumeReflectingDatabase() {
if (this.core.settings.doNotSuspendOnFetching) return;
if (this.core.settings.remoteType == REMOTE_MINIO) return;
this._log(`Database and storage reflection has been resumed!`, LOG_LEVEL_NOTICE);
this.core.settings.suspendParseReplicationResult = false;
this.core.settings.suspendFileWatching = false;
await this.core.$$performFullScan(true);
await this.core.$everyBeforeReplicate(false); //TODO: Check actual need of this.
await this.core.saveSettings();
}
async askUseNewAdapter() {
if (!this.core.settings.useIndexedDBAdapter) {
const message = `Now this core has been configured to use the old database adapter for keeping compatibility. Do you want to deactivate it?`;
const CHOICE_YES = "Yes, disable and use latest";
const CHOICE_NO = "No, keep compatibility";
const choices = [CHOICE_YES, CHOICE_NO];
const ret = await this.core.confirm.confirmWithMessage("Database adapter", message, choices, CHOICE_YES, 10);
if (ret == CHOICE_YES) {
this.core.settings.useIndexedDBAdapter = true;
}
}
}
async fetchLocal(makeLocalChunkBeforeSync?: boolean) {
await this.core.$allSuspendExtraSync();
await this.askUseNewAdapter();
this.core.settings.isConfigured = true;
await this.suspendReflectingDatabase();
await this.core.realizeSettingSyncMode();
await this.resetLocalDatabase();
await delay(1000);
await this.core.$$openDatabase();
this.core.isReady = true;
if (makeLocalChunkBeforeSync) {
await this.core.fileHandler.createAllChunks(true);
}
await this.core.$$markRemoteResolved();
await delay(500);
await this.core.$$replicateAllFromServer(true);
await delay(1000);
await this.core.$$replicateAllFromServer(true);
await this.resumeReflectingDatabase();
await this.askUsingOptionalFeature({ enableFetch: true });
}
async fetchLocalWithRebuild() {
return await this.fetchLocal(true);
}
async $allSuspendAllSync(): Promise<boolean> {
await this.suspendAllSync();
return true;
}
async resetLocalDatabase() {
if (this.core.settings.isConfigured && this.core.settings.additionalSuffixOfDatabaseName == "") {
// Discard the non-suffixed database
await this.core.$$resetLocalDatabase();
}
const suffix = (await this.core.$anyGetAppId()) || "";
this.core.settings.additionalSuffixOfDatabaseName = suffix;
await this.core.$$resetLocalDatabase();
}
async fetchRemoteChunks() {
if (!this.core.settings.doNotSuspendOnFetching && this.core.settings.readChunksOnline && this.core.settings.remoteType == REMOTE_COUCHDB) {
this._log(`Fetching chunks`, LOG_LEVEL_NOTICE);
const replicator = this.core.getReplicator() as LiveSyncCouchDBReplicator;
const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.core.getIsMobile(), true);
if (typeof remoteDB == "string") {
this._log(remoteDB, LOG_LEVEL_NOTICE);
} else {
await fetchAllUsedChunks(this.localDatabase.localDatabase, remoteDB.db);
}
this._log(`Fetching chunks done`, LOG_LEVEL_NOTICE);
}
}
async resolveAllConflictedFilesByNewerOnes() {
this._log(`Resolving conflicts by newer ones`, LOG_LEVEL_NOTICE);
const files = this.core.storageAccess.getFileNames();
let i = 0;
for (const file of files) {
if (i++ % 10) this._log(`Check and Processing ${i} / ${files.length}`, LOG_LEVEL_NOTICE, "resolveAllConflictedFilesByNewerOnes");
await this.core.$anyResolveConflictByNewest(file);
}
this._log(`Done!`, LOG_LEVEL_NOTICE, "resolveAllConflictedFilesByNewerOnes");
}
}

351
src/modules/core/ModuleReplicator.ts Normal file
View File

@@ -0,0 +1,351 @@
import { fireAndForget, yieldMicrotask } from "octagonal-wheels/promises";
import type { LiveSyncLocalDB } from "../../lib/src/pouchdb/LiveSyncLocalDB";
import { AbstractModule } from "../AbstractModule";
import type { ICoreModule } from "../ModuleTypes";
import { Logger, LOG_LEVEL_NOTICE, LOG_LEVEL_INFO, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { isLockAcquired, shareRunningResult, skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
import { purgeUnreferencedChunks, balanceChunkPurgedDBs } from "../../lib/src/pouchdb/utils_couchdb";
import { LiveSyncCouchDBReplicator } from "../../lib/src/replication/couchdb/LiveSyncReplicator";
import { throttle } from "octagonal-wheels/function";
import { arrayToChunkedArray } from "octagonal-wheels/collection";
import { SYNCINFO_ID, VER, type EntryBody, type EntryDoc, type LoadedEntry, type MetaEntry } from "../../lib/src/common/types";
import { QueueProcessor } from "octagonal-wheels/concurrency/processor";
import { getPath, isChunk, isValidPath, scheduleTask } from "../../common/utils";
import { sendValue } from "octagonal-wheels/messagepassing/signal";
import { isAnyNote } from "../../lib/src/common/utils";
import { EVENT_FILE_SAVED, eventHub } from "../../common/events";
export class ModuleReplicator extends AbstractModule implements ICoreModule {
$everyOnloadAfterLoadSettings(): Promise<boolean> {
eventHub.onEvent(EVENT_FILE_SAVED, () => {
if (this.settings.syncOnSave && !this.core.suspended) {
scheduleTask("perform-replicate-after-save", 250, () => this.core.$$waitForReplicationOnce());
}
})
return Promise.resolve(true);
}
async setReplicator() {
const replicator = await this.core.$anyNewReplicator();
if (!replicator) {
this._log("No replicator is available, this is the fatal error.", LOG_LEVEL_NOTICE);
return false
}
this.core.replicator = replicator;
await yieldMicrotask();
return true;
}
$everyOnInitializeDatabase(db: LiveSyncLocalDB): Promise<boolean> {
return this.setReplicator();
}
$everyOnResetDatabase(db: LiveSyncLocalDB): Promise<boolean> {
return this.setReplicator();
}
async $everyBeforeReplicate(showMessage: boolean): Promise<boolean> {
await this.loadQueuedFiles();
return true;
}
async $$replicate(showMessage: boolean = false): Promise<boolean | void> {
//--?
if (!this.core.isReady) return;
if (isLockAcquired("cleanup")) {
Logger("Database cleaning up is in process. replication has been cancelled", LOG_LEVEL_NOTICE);
return;
}
if (this.settings.versionUpFlash != "") {
Logger("Open settings and check message, please. replication has been cancelled.", LOG_LEVEL_NOTICE);
return;
}
if (!await this.core.$everyCommitPendingFileEvent()) {
Logger("Some file events are pending. Replication has been cancelled.", LOG_LEVEL_NOTICE);
return false;
}
if (!await this.core.$everyBeforeReplicate(showMessage)) {
Logger(`Replication has been cancelled by some module failure`, LOG_LEVEL_NOTICE);
return false;
}
//<-- Here could be an module.
const ret = await this.core.replicator.openReplication(this.settings, false, showMessage, false);
if (!ret) {
if (this.core.replicator.tweakSettingsMismatched) {
await this.core.$$askResolvingMismatchedTweaks();
} else {
if (this.core.replicator?.remoteLockedAndDeviceNotAccepted) {
if (this.core.replicator.remoteCleaned && this.settings.useIndexedDBAdapter) {
Logger(`The remote database has been cleaned.`, showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO);
await skipIfDuplicated("cleanup", async () => {
const count = await purgeUnreferencedChunks(this.localDatabase.localDatabase, true);
const message = `The remote database has been cleaned up.
To synchronize, this device must be also cleaned up. ${count} chunk(s) will be erased from this device.
However, If there are many chunks to be deleted, maybe fetching again is faster.
We will lose the history of this device if we fetch the remote database again.
Even if you choose to clean up, you will see this option again if you exit Obsidian and then synchronise again.`
const CHOICE_FETCH = "Fetch again";
const CHOICE_CLEAN = "Cleanup";
const CHOICE_DISMISS = "Dismiss";
const ret = await this.core.confirm.confirmWithMessage("Cleaned", message, [
CHOICE_FETCH,
CHOICE_CLEAN,
CHOICE_DISMISS], CHOICE_DISMISS, 30);
if (ret == CHOICE_FETCH) {
await this.core.rebuilder.$performRebuildDB("localOnly");
}
if (ret == CHOICE_CLEAN) {
const replicator = this.core.getReplicator();
if (!(replicator instanceof LiveSyncCouchDBReplicator)) return;
const remoteDB = await replicator.connectRemoteCouchDBWithSetting(this.settings, this.core.getIsMobile(), true);
if (typeof remoteDB == "string") {
Logger(remoteDB, LOG_LEVEL_NOTICE);
return false;
}
await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
this.localDatabase.hashCaches.clear();
// Perform the synchronisation once.
if (await this.core.replicator.openReplication(this.settings, false, showMessage, true)) {
await balanceChunkPurgedDBs(this.localDatabase.localDatabase, remoteDB.db);
await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
this.localDatabase.hashCaches.clear();
await this.core.getReplicator().markRemoteResolved(this.settings);
Logger("The local database has been cleaned up.", showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO)
} else {
Logger("Replication has been cancelled. Please try it again.", showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO)
}
}
});
} else {
const message = `
The remote database has been rebuilt.
To synchronize, this device must fetch everything again once.
Or if you are sure know what had been happened, we can unlock the database from the setting dialog.
`
const CHOICE_FETCH = "Fetch again";
const CHOICE_DISMISS = "Dismiss";
const ret = await this.core.confirm.confirmWithMessage("Locked", message, [
CHOICE_FETCH,
CHOICE_DISMISS], CHOICE_DISMISS, 10);
if (ret == CHOICE_FETCH) {
const CHOICE_RESTART = "Restart";
const CHOICE_WITHOUT_RESTART = "Without restart";
if (await this.core.confirm.askSelectStringDialogue(
"Self-hosted LiveSync restarts Obsidian to fetch everything safely. However, you can do it without restarting. Please choose one.",
[CHOICE_RESTART, CHOICE_WITHOUT_RESTART], {
title: "Fetch again",
defaultAction: CHOICE_RESTART,
timeout: 30,
}
) == CHOICE_RESTART) {
await this.core.rebuilder.scheduleFetch();
// await this.core.$$scheduleAppReload();
return;
} else {
await this.core.rebuilder.$performRebuildDB("localOnly");
}
}
}
}
}
}
return ret;
}
$$parseReplicationResult(docs: Array<PouchDB.Core.ExistingDocument<EntryDoc>>): void {
if (this.settings.suspendParseReplicationResult && !this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.suspend()
}
this.replicationResultProcessor.enqueueAll(docs);
if (!this.settings.suspendParseReplicationResult && this.replicationResultProcessor.isSuspended) {
this.replicationResultProcessor.resume()
}
}
_saveQueuedFiles = throttle(() => {
const saveData = this.replicationResultProcessor._queue.filter(e => e !== undefined && e !== null).map((e) => e?._id ?? "" as string) as string[];
const kvDBKey = "queued-files"
// localStorage.setItem(lsKey, saveData);
fireAndForget(() => this.core.kvDB.set(kvDBKey, saveData));
}, 100);
saveQueuedFiles() {
this._saveQueuedFiles();
}
async loadQueuedFiles() {
if (this.settings.suspendParseReplicationResult) return;
if (!this.settings.isConfigured) return;
const kvDBKey = "queued-files"
// const ids = [...new Set(JSON.parse(localStorage.getItem(lsKey) || "[]"))] as string[];
const ids = [...new Set(await this.core.kvDB.get<string[]>(kvDBKey) ?? [])];
const batchSize = 100;
const chunkedIds = arrayToChunkedArray(ids, batchSize);
for await (const idsBatch of chunkedIds) {
const ret = await this.localDatabase.allDocsRaw<EntryDoc>({
keys: idsBatch,
include_docs: true,
limit: 100
});
const docs = ret.rows.filter(e => e.doc).map(e => e.doc) as PouchDB.Core.ExistingDocument<EntryDoc>[];
const errors = ret.rows.filter(e => !e.doc && !e.value.deleted);
if (errors.length > 0) {
Logger("Some queued processes were not resurrected");
Logger(JSON.stringify(errors), LOG_LEVEL_VERBOSE);
}
this.replicationResultProcessor.enqueueAll(docs);
await this.replicationResultProcessor.waitForAllProcessed();
}
}
replicationResultProcessor = new QueueProcessor(async (docs: PouchDB.Core.ExistingDocument<EntryDoc>[]) => {
if (this.settings.suspendParseReplicationResult) return;
const change = docs[0];
if (!change) return;
if (isChunk(change._id)) {
// SendSignal?
// this.parseIncomingChunk(change);
sendValue(`leaf-${change._id}`, change);
return;
}
if (await this.core.$anyModuleParsedReplicationResultItem(change)) return;
// any addon needs this item?
// for (const proc of this.core.addOns) {
// if (await proc.parseReplicationResultItem(change)) {
// return;
// }
// }
if (change.type == "versioninfo") {
if (change.version > VER) {
this.core.replicator.closeReplication();
Logger(`Remote database updated to incompatible version. update your Self-hosted LiveSync plugin.`, LOG_LEVEL_NOTICE);
}
return;
}
if (change._id == SYNCINFO_ID || // Synchronisation information data
change._id.startsWith("_design") //design document
) {
return;
}
if (isAnyNote(change)) {
const docPath = getPath(change);
if (!await this.core.$$isTargetFile(docPath)) {
Logger(`Skipped: ${docPath}`, LOG_LEVEL_VERBOSE);
return;
}
if (this.databaseQueuedProcessor._isSuspended) {
Logger(`Processing scheduled: ${docPath}`, LOG_LEVEL_INFO);
}
const size = change.size;
if (this.core.$$isFileSizeExceeded(size)) {
Logger(`Processing ${docPath} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
return;
}
this.databaseQueuedProcessor.enqueue(change);
}
return;
}, {
batchSize: 1,
suspended: true,
concurrentLimit: 100,
delay: 0,
totalRemainingReactiveSource: this.core.replicationResultCount
}).replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter(e => e._id != newItem._id);
return [...q, newItem];
}).startPipeline().onUpdateProgress(() => {
this.saveQueuedFiles();
});
databaseQueuedProcessor = new QueueProcessor(async (docs: EntryBody[]) => {
const dbDoc = docs[0] as LoadedEntry; // It has no `data`
const path = getPath(dbDoc);
// If `Read chunks online` is disabled, chunks should be transferred before here.
// However, in some cases, chunks are after that. So, if missing chunks exist, we have to wait for them.
const doc = await this.localDatabase.getDBEntryFromMeta({ ...dbDoc }, {}, false, true, true);
if (!doc) {
Logger(`Something went wrong while gathering content of ${path} (${dbDoc._id.substring(0, 8)}, ${dbDoc._rev?.substring(0, 10)}) `, LOG_LEVEL_NOTICE)
return;
}
if (await this.core.$anyProcessOptionalSyncFiles(dbDoc)) {
// Already processed
} else if (isValidPath(getPath(doc))) {
this.storageApplyingProcessor.enqueue(doc as MetaEntry);
} else {
Logger(`Skipped: ${doc._id.substring(0, 8)}`, LOG_LEVEL_VERBOSE);
}
return;
}, {
suspended: true,
batchSize: 1,
concurrentLimit: 10,
yieldThreshold: 1,
delay: 0,
totalRemainingReactiveSource: this.core.databaseQueueCount
}).replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter(e => e._id != newItem._id);
return [...q, newItem];
}).startPipeline();
storageApplyingProcessor = new QueueProcessor(async (docs: MetaEntry[]) => {
const entry = docs[0];
await this.core.$anyProcessReplicatedDoc(entry);
return;
}, {
suspended: true,
batchSize: 1,
concurrentLimit: 6,
yieldThreshold: 1,
delay: 0,
totalRemainingReactiveSource: this.core.storageApplyingCount
}).replaceEnqueueProcessor((queue, newItem) => {
const q = queue.filter(e => e._id != newItem._id);
return [...q, newItem];
}).startPipeline()
$everyBeforeSuspendProcess(): Promise<boolean> {
this.core.replicator.closeReplication();
return Promise.resolve(true);
}
async $$replicateAllToServer(showingNotice: boolean = false, sendChunksInBulkDisabled: boolean = false): Promise<boolean> {
if (!this.core.isReady) return false;
if (!await this.core.$everyBeforeReplicate(showingNotice)) {
Logger(`Replication has been cancelled by some module failure`, LOG_LEVEL_NOTICE);
return false;
}
if (!sendChunksInBulkDisabled) {
if (this.core.replicator instanceof LiveSyncCouchDBReplicator) {
if (await this.core.confirm.askYesNoDialog("Do you want to send all chunks before replication?", { defaultOption: "No", timeout: 20 }) == "yes") {
await this.core.replicator.sendChunks(this.core.settings, undefined, true, 0);
}
}
}
const ret = await this.core.replicator.replicateAllToServer(this.settings, showingNotice);
if (ret) return true;
const checkResult = await this.core.$anyAfterConnectCheckFailed();
if (checkResult == "CHECKAGAIN") return await this.core.$$replicateAllToServer(showingNotice);
return !checkResult;
}
async $$replicateAllFromServer(showingNotice: boolean = false): Promise<boolean> {
if (!this.core.isReady) return false;
const ret = await this.core.replicator.replicateAllFromServer(this.settings, showingNotice);
if (ret) return true;
const checkResult = await this.core.$anyAfterConnectCheckFailed();
if (checkResult == "CHECKAGAIN") return await this.core.$$replicateAllFromServer(showingNotice);
return !checkResult;
}
async $$waitForReplicationOnce(): Promise<boolean | void> {
return await shareRunningResult(`replication`, () => this.core.$$replicate());
}
}

33
src/modules/core/ModuleReplicatorCouchDB.ts Normal file
View File

@@ -0,0 +1,33 @@
import { fireAndForget } from "octagonal-wheels/promises";
import { REMOTE_MINIO, type RemoteDBSettings } from "../../lib/src/common/types";
import { LiveSyncCouchDBReplicator } from "../../lib/src/replication/couchdb/LiveSyncReplicator";
import type { LiveSyncAbstractReplicator } from "../../lib/src/replication/LiveSyncAbstractReplicator";
import { AbstractModule } from "../AbstractModule";
import type { ICoreModule } from "../ModuleTypes";
export class ModuleReplicatorCouchDB extends AbstractModule implements ICoreModule {
$anyNewReplicator(settingOverride: Partial<RemoteDBSettings> = {}): Promise<LiveSyncAbstractReplicator> {
const settings = { ...this.settings, ...settingOverride };
// If new remote types were added, add them here. Do not use `REMOTE_COUCHDB` directly for the safety valve.
if (settings.remoteType == REMOTE_MINIO) {
return undefined!
}
return Promise.resolve(new LiveSyncCouchDBReplicator(this.core));
}
$everyAfterResumeProcess(): Promise<boolean> {
if (this.settings.remoteType != REMOTE_MINIO) {
// If LiveSync enabled, open replication
if (this.settings.liveSync) {
fireAndForget(() => this.core.replicator.openReplication(this.settings, true, false, false));
}
// If sync on start enabled, open replication
if (!this.settings.liveSync && this.settings.syncOnStart) {
// Possibly ok as if only share the result
fireAndForget(() => this.core.replicator.openReplication(this.settings, false, false, false));
}
}
return Promise.resolve(true);
}
}

17
src/modules/core/ModuleReplicatorMinIO.ts Normal file
View File

@@ -0,0 +1,17 @@
import { REMOTE_MINIO, type RemoteDBSettings } from "../../lib/src/common/types";
import { LiveSyncJournalReplicator } from "../../lib/src/replication/journal/LiveSyncJournalReplicator";
import type { LiveSyncAbstractReplicator } from "../../lib/src/replication/LiveSyncAbstractReplicator";
import { AbstractModule } from "../AbstractModule";
import type { ICoreModule } from "../ModuleTypes";
export class ModuleReplicatorMinIO extends AbstractModule implements ICoreModule {
$anyNewReplicator(settingOverride: Partial<RemoteDBSettings> = {}): Promise<LiveSyncAbstractReplicator> {
const settings = { ...this.settings, ...settingOverride };
if (settings.remoteType == REMOTE_MINIO) {
return Promise.resolve(new LiveSyncJournalReplicator(this.core));
}
return undefined!;
}
}

146
src/modules/core/ModuleTargetFilter.ts Normal file
View File

@@ -0,0 +1,146 @@
import { LRUCache } from "octagonal-wheels/memory/LRUCache";
import { getPathFromUXFileInfo, id2path, isInternalMetadata, path2id, stripInternalMetadataPrefix, useMemo } from "../../common/utils";
import { LOG_LEVEL_VERBOSE, type DocumentID, type EntryHasPath, type FilePath, type FilePathWithPrefix, type ObsidianLiveSyncSettings, type UXFileInfoStub } from "../../lib/src/common/types";
import { addPrefix, isAcceptedAll, stripAllPrefixes } from "../../lib/src/string_and_binary/path";
import { AbstractModule } from "../AbstractModule";
import type { ICoreModule } from "../ModuleTypes";
import { EVENT_REQUEST_RELOAD_SETTING_TAB, EVENT_SETTING_SAVED, eventHub } from "../../common/events";
import { isDirty } from "../../lib/src/common/utils";
export class ModuleTargetFilter extends AbstractModule implements ICoreModule {
reloadIgnoreFiles() {
this.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim());
}
$everyOnload(): Promise<boolean> {
eventHub.onEvent(EVENT_SETTING_SAVED, (evt: CustomEvent<ObsidianLiveSyncSettings>) => {
this.reloadIgnoreFiles();
});
eventHub.onEvent(EVENT_REQUEST_RELOAD_SETTING_TAB, (evt: CustomEvent<ObsidianLiveSyncSettings>) => {
this.reloadIgnoreFiles();
});
return Promise.resolve(true);
}
$$id2path(id: DocumentID, entry?: EntryHasPath, stripPrefix?: boolean): FilePathWithPrefix {
const tempId = id2path(id, entry);
if (stripPrefix && isInternalMetadata(tempId)) {
const out = stripInternalMetadataPrefix(tempId);
return out;
}
return tempId;
}
async $$path2id(filename: FilePathWithPrefix | FilePath, prefix?: string): Promise<DocumentID> {
const destPath = addPrefix(filename, prefix ?? "");
return await path2id(destPath, this.settings.usePathObfuscation ? this.settings.passphrase : "", !this.settings.handleFilenameCaseSensitive);
}
$$isFileSizeExceeded(size: number) {
if (this.settings.syncMaxSizeInMB > 0 && size > 0) {
if (this.settings.syncMaxSizeInMB * 1024 * 1024 < size) {
return true;
}
}
return false;
}
get fileListPossiblyChanged() {
if (isDirty("totalFileEventCount", this.core.totalFileEventCount)) {
return true;
}
return false;
}
async $$isTargetFile(file: string | UXFileInfoStub, keepFileCheckList = false) {
const fileCount = useMemo<Record<string, number>>({
key: "fileCount", // forceUpdate: !keepFileCheckList,
}, (ctx, prev) => {
if (keepFileCheckList && prev) return prev;
if (!keepFileCheckList && prev && !this.fileListPossiblyChanged) {
return prev;
}
const fileList = (ctx.get("fileList") ?? []) as FilePathWithPrefix[];
// const fileNameList = (ctx.get("fileNameList") ?? []) as FilePath[];
// const fileNames =
const vaultFiles = this.core.storageAccess.getFileNames().sort();
if (prev && vaultFiles.length == fileList.length) {
const fl3 = new Set([...fileList, ...vaultFiles]);
if (fileList.length == fl3.size && vaultFiles.length == fl3.size) {
return prev;
}
}
ctx.set("fileList", vaultFiles);
const fileCount: Record<string, number> = {};
for (const file of vaultFiles) {
const lc = file.toLowerCase();
if (!fileCount[lc]) {
fileCount[lc] = 1;
} else {
fileCount[lc]++;
}
}
return fileCount;
})
const filepath = getPathFromUXFileInfo(file);
const lc = filepath.toLowerCase();
if (this.core.shouldCheckCaseInsensitive) {
if (lc in fileCount && fileCount[lc] > 1) {
return false;
}
}
const fileNameLC = getPathFromUXFileInfo(file).split("/").pop()?.toLowerCase();
if (this.settings.useIgnoreFiles) {
if (this.ignoreFiles.some(e => e.toLowerCase() == fileNameLC)) {
// We must reload ignore files due to the its change.
await this.readIgnoreFile(filepath);
}
if (await this.core.$$isIgnoredByIgnoreFiles(file)) {
return false;
}
}
if (!this.localDatabase?.isTargetFile(filepath)) return false;
return true
}
ignoreFileCache = new LRUCache<string, string[] | false>(300, 250000, true);
ignoreFiles = [] as string[]
async readIgnoreFile(path: string) {
try {
const file = await this.core.storageAccess.readFileText(path);
const gitignore = file.split(/\r?\n/g);
this.ignoreFileCache.set(path, gitignore);
return gitignore;
} catch (ex) {
this._log(`Failed to read ignore file ${path}`);
this._log(ex, LOG_LEVEL_VERBOSE);
this.ignoreFileCache.set(path, false);
return false;
}
}
async getIgnoreFile(path: string) {
if (this.ignoreFileCache.has(path)) {
return this.ignoreFileCache.get(path) ?? false;
} else {
return await this.readIgnoreFile(path);
}
}
async $$isIgnoredByIgnoreFiles(file: string | UXFileInfoStub): Promise<boolean> {
if (!this.settings.useIgnoreFiles) {
return false;
}
const filepath = getPathFromUXFileInfo(file)
if (this.ignoreFileCache.has(filepath)) {
// Renew
await this.readIgnoreFile(filepath);
}
if (!await isAcceptedAll(stripAllPrefixes(filepath), this.ignoreFiles, (filename) => this.getIgnoreFile(filename))) {
return true;
}
return false;
}
}

101
src/modules/coreFeatures/ModuleCheckRemoteSize.ts Normal file
View File

@@ -0,0 +1,101 @@
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { AbstractModule } from "../AbstractModule.ts";
import { sizeToHumanReadable } from "octagonal-wheels/number";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleCheckRemoteSize extends AbstractModule implements ICoreModule {
async $allScanStat(): Promise<boolean> {
this._log(`Checking storage sizes`, LOG_LEVEL_VERBOSE);
if (this.settings.notifyThresholdOfRemoteStorageSize < 0) {
const message = `We can set a maximum database capacity warning, **to take action before running out of space on the remote storage**.
Do you want to enable this?
> [!MORE]-
> - 0: Do not warn about storage size.
> This is recommended if you have enough space on the remote storage especially you have self-hosted. And you can check the storage size and rebuild manually.
> - 800: Warn if the remote storage size exceeds 800MB.
> This is recommended if you are using fly.io with 1GB limit or IBM Cloudant.
> - 2000: Warn if the remote storage size exceeds 2GB.
If we have reached the limit, we will be asked to enlarge the limit step by step.
`
const ANSWER_0 = "No, never warn please";
const ANSWER_800 = "800MB (Cloudant, fly.io)";
const ANSWER_2000 = "2GB (Standard)";
const ASK_ME_NEXT_TIME = "Ask me later";
const ret = await this.core.confirm.askSelectStringDialogue(message, [ANSWER_0, ANSWER_800, ANSWER_2000, ASK_ME_NEXT_TIME], {
defaultAction: ASK_ME_NEXT_TIME,
title: "Setting up database size notification", timeout: 40
});
if (ret == ANSWER_0) {
this.settings.notifyThresholdOfRemoteStorageSize = 0;
await this.core.saveSettings();
} else if (ret == ANSWER_800) {
this.settings.notifyThresholdOfRemoteStorageSize = 800;
await this.core.saveSettings();
} else if (ret == ANSWER_2000) {
this.settings.notifyThresholdOfRemoteStorageSize = 2000;
await this.core.saveSettings();
}
}
if (this.settings.notifyThresholdOfRemoteStorageSize > 0) {
const remoteStat = await this.core.replicator?.getRemoteStatus(this.settings);
if (remoteStat) {
const estimatedSize = remoteStat.estimatedSize;
if (estimatedSize) {
const maxSize = this.settings.notifyThresholdOfRemoteStorageSize * 1024 * 1024;
if (estimatedSize > maxSize) {
const message = `**Your database is getting larger!** But do not worry, we can address it now. The time before running out of space on the remote storage.
| Measured size | Configured size |
| --- | --- |
| ${sizeToHumanReadable(estimatedSize)} | ${sizeToHumanReadable(maxSize)} |
> [!MORE]-
> If you have been using it for many years, there may be unreferenced chunks - that is, garbage - accumulating in the database. Therefore, we recommend rebuilding everything. It will probably become much smaller.
>
> If the volume of your vault is simply increasing, it is better to rebuild everything after organizing the files. Self-hosted LiveSync does not delete the actual data even if you delete it to speed up the process. It is roughly [documented](https://github.com/vrtmrz/obsidian-livesync/blob/main/docs/tech_info.md).
>
> If you don't mind the increase, you can increase the notification limit by 100MB. This is the case if you are running it on your own server. However, it is better to rebuild everything from time to time.
>
> [!WARNING]
> If you perform rebuild everything, make sure all devices are synchronised. The plug-in will merge as much as possible, though.
\n`;
const newMax = ~~(estimatedSize / 1024 / 1024) + 100;
const ANSWER_ENLARGE_LIMIT = `increase to ${newMax}MB`;
const ANSWER_REBUILD = "Rebuild Everything Now";
const ANSWER_IGNORE = "Dismiss";
const ret = await this.core.confirm.askSelectStringDialogue(message, [ANSWER_ENLARGE_LIMIT, ANSWER_REBUILD, ANSWER_IGNORE,], {
defaultAction: ANSWER_IGNORE,
title: "Remote storage size exceeded the limit", timeout: 60
});
if (ret == ANSWER_REBUILD) {
const ret = await this.core.confirm.askYesNoDialog("This may take a bit of a long time. Do you really want to rebuild everything now?", { defaultOption: "No" });
if (ret == "yes") {
this.core.settings.notifyThresholdOfRemoteStorageSize = -1;
await this.saveSettings();
await this.core.rebuilder.scheduleRebuild();
}
} else if (ret == ANSWER_ENLARGE_LIMIT) {
this.settings.notifyThresholdOfRemoteStorageSize = ~~(estimatedSize / 1024 / 1024) + 100;
this._log(`Threshold has been enlarged to ${this.settings.notifyThresholdOfRemoteStorageSize}MB`, LOG_LEVEL_NOTICE);
await this.core.saveSettings();
} else {
// Dismiss or Close the dialog
}
this._log(`Remote storage size: ${sizeToHumanReadable(estimatedSize)} exceeded ${sizeToHumanReadable(this.settings.notifyThresholdOfRemoteStorageSize * 1024 * 1024)} `, LOG_LEVEL_INFO);
} else {
this._log(`Remote storage size: ${sizeToHumanReadable(estimatedSize)}`, LOG_LEVEL_INFO);
}
}
}
}
return true;
}
}

75
src/modules/coreFeatures/ModuleConflictChecker.ts Normal file
View File

@@ -0,0 +1,75 @@
import { AbstractModule } from "../AbstractModule.ts";
import { LOG_LEVEL_NOTICE, type FilePathWithPrefix } from "../../lib/src/common/types";
import { QueueProcessor } from "octagonal-wheels/concurrency/processor";
import { sendValue } from "octagonal-wheels/messagepassing/signal";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleConflictChecker extends AbstractModule implements ICoreModule {
async $$queueConflictCheckIfOpen(file: FilePathWithPrefix): Promise<void> {
const path = file;
if (this.settings.checkConflictOnlyOnOpen) {
const af = this.core.$$getActiveFilePath();
if (af && af != path) {
this._log(`${file} is conflicted, merging process has been postponed.`, LOG_LEVEL_NOTICE);
return;
}
}
await this.core.$$queueConflictCheck(path);
}
async $$queueConflictCheck(file: FilePathWithPrefix): Promise<void> {
const optionalConflictResult = await this.core.$anyGetOptionalConflictCheckMethod(file);
if (optionalConflictResult == true) {
// The conflict has been resolved by another process.
return;
} else if (optionalConflictResult === "newer") {
// The conflict should be resolved by the newer entry.
await this.core.$anyResolveConflictByNewest(file);
} else {
this.conflictCheckQueue.enqueue(file);
}
}
$$waitForAllConflictProcessed(): Promise<boolean> {
return this.conflictResolveQueue.waitForAllProcessed();
}
// TODO-> Move to ModuleConflictResolver?
conflictResolveQueue = new QueueProcessor(async (filenames: FilePathWithPrefix[]) => {
await this.core.$$resolveConflict(filenames[0]);
}, {
suspended: false,
batchSize: 1,
concurrentLimit: 1,
delay: 10,
keepResultUntilDownstreamConnected: false
}).replaceEnqueueProcessor((queue, newEntity) => {
const filename = newEntity;
sendValue("cancel-resolve-conflict:" + filename, true);
const newQueue = [...queue].filter(e => e != newEntity);
return [...newQueue, newEntity];
});
conflictCheckQueue = // First process - Check is the file actually need resolve -
new QueueProcessor((files: FilePathWithPrefix[]) => {
const filename = files[0];
// const file = await this.core.storageAccess.isExists(filename);
// if (!file) return [];
// if (!(file instanceof TFile)) return;
// if ((file instanceof TFolder)) return [];
// Check again?
return Promise.resolve([filename]);
// this.conflictResolveQueue.enqueueWithKey(filename, { filename, file });
}, {
suspended: false,
batchSize: 1,
concurrentLimit: 5,
delay: 10,
keepResultUntilDownstreamConnected: true,
pipeTo: this.conflictResolveQueue,
totalRemainingReactiveSource: this.core.conflictProcessQueueCount
});
}

141
src/modules/coreFeatures/ModuleConflictResolver.ts Normal file
View File

@@ -0,0 +1,141 @@
import { serialized } from "octagonal-wheels/concurrency/lock";
import { AbstractModule } from "../AbstractModule.ts";
import { AUTO_MERGED, CANCELLED, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, MISSING_OR_ERROR, NOT_CONFLICTED, type diff_check_result, type FilePathWithPrefix } from "../../lib/src/common/types";
import { compareMTime, displayRev, TARGET_IS_NEW } from "../../common/utils";
import diff_match_patch from "diff-match-patch";
import { stripAllPrefixes, isPlainText } from "../../lib/src/string_and_binary/path";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleConflictResolver extends AbstractModule implements ICoreModule {
async $$resolveConflictByDeletingRev(path: FilePathWithPrefix, deleteRevision: string, subTitle = ""): Promise<typeof MISSING_OR_ERROR | typeof AUTO_MERGED> {
const title = `Resolving ${subTitle ? `[${subTitle}]` : ""}:`;
if (!await this.core.fileHandler.deleteRevisionFromDB(path, deleteRevision)) {
this._log(`${title} Could not delete conflicted revision ${displayRev(deleteRevision)} of ${path}`, LOG_LEVEL_NOTICE);
return MISSING_OR_ERROR;
}
this._log(`${title} Conflicted revision deleted ${displayRev(deleteRevision)} ${path}`, LOG_LEVEL_INFO);
if ((await this.core.databaseFileAccess.getConflictedRevs(path)).length != 0) {
this._log(`${title} some conflicts are left in ${path}`, LOG_LEVEL_INFO);
return AUTO_MERGED;
}
// If no conflicts were found, write the resolved content to the storage.
if (!await this.core.fileHandler.dbToStorage(path, stripAllPrefixes(path), true)) {
this._log(`Could not write the resolved content to the storage: ${path}`, LOG_LEVEL_NOTICE);
return MISSING_OR_ERROR;
}
this._log(`${path} Has been merged automatically`, LOG_LEVEL_NOTICE);
return AUTO_MERGED;
}
async checkConflictAndPerformAutoMerge(path: FilePathWithPrefix): Promise<diff_check_result> {
//
const ret = await this.localDatabase.tryAutoMerge(path, !this.settings.disableMarkdownAutoMerge);
if ("ok" in ret) {
return ret.ok;
}
if ("result" in ret) {
const p = ret.result;
// Merged content is coming.
// 1. Store the merged content to the storage
if (!await this.core.databaseFileAccess.storeContent(path, p)) {
this._log(`Merged content cannot be stored:${path}`, LOG_LEVEL_NOTICE);
return MISSING_OR_ERROR;
}
// 2. As usual, delete the conflicted revision and if there are no conflicts, write the resolved content to the storage.
return await this.core.$$resolveConflictByDeletingRev(path, ret.conflictedRev, "Sensible");
}
const { rightRev, leftLeaf, rightLeaf } = ret;
// should be one or more conflicts;
if (leftLeaf == false) {
// what's going on..
this._log(`could not get current revisions:${path}`, LOG_LEVEL_NOTICE);
return MISSING_OR_ERROR;
}
if (rightLeaf == false) {
// Conflicted item could not load, delete this.
return await this.core.$$resolveConflictByDeletingRev(path, rightRev, "MISSING OLD REV");
}
const isSame = leftLeaf.data == rightLeaf.data && leftLeaf.deleted == rightLeaf.deleted;
const isBinary = !isPlainText(path);
const alwaysNewer = this.settings.resolveConflictsByNewerFile;
if (isSame || isBinary || alwaysNewer) {
const result = compareMTime(leftLeaf.mtime, rightLeaf.mtime)
let loser = leftLeaf;
// if (lMtime > rMtime) {
if (result != TARGET_IS_NEW) {
loser = rightLeaf;
}
const subTitle = [`${isSame ? "same" : ""}`, `${isBinary ? "binary" : ""}`, `${alwaysNewer ? "alwaysNewer" : ""}`].join(",");
return await this.core.$$resolveConflictByDeletingRev(path, loser.rev, subTitle);
}
// make diff.
const dmp = new diff_match_patch();
const diff = dmp.diff_main(leftLeaf.data, rightLeaf.data);
dmp.diff_cleanupSemantic(diff);
this._log(`conflict(s) found:${path}`);
return {
left: leftLeaf,
right: rightLeaf,
diff: diff,
};
}
async $$resolveConflict(filename: FilePathWithPrefix): Promise<void> {
// const filename = filenames[0];
return await serialized(`conflict-resolve:${filename}`, async () => {
const conflictCheckResult = await this.checkConflictAndPerformAutoMerge(filename);
if (conflictCheckResult === MISSING_OR_ERROR || conflictCheckResult === NOT_CONFLICTED || conflictCheckResult === CANCELLED) {
// nothing to do.
this._log(`conflict:Nothing to do:${filename}`);
return;
}
if (conflictCheckResult === AUTO_MERGED) {
//auto resolved, but need check again;
if (this.settings.syncAfterMerge && !this.core.suspended) {
//Wait for the running replication, if not running replication, run it once.
await this.core.$$waitForReplicationOnce();
}
this._log("conflict:Automatically merged, but we have to check it again");
await this.core.$$queueConflictCheck(filename);
return;
}
if (this.settings.showMergeDialogOnlyOnActive) {
const af = this.core.$$getActiveFilePath();
if (af && af != filename) {
this._log(`${filename} is conflicted. Merging process has been postponed to the file have got opened.`, LOG_LEVEL_NOTICE);
return;
}
}
this._log("conflict:Manual merge required!");
await this.core.$anyResolveConflictByUI(filename, conflictCheckResult);
});
}
async $anyResolveConflictByNewest(filename: FilePathWithPrefix): Promise<boolean> {
const revs = await this.core.databaseFileAccess.getConflictedRevs(filename);
if (revs.length == 0) {
return Promise.resolve(true);
}
const mTimeAndRev = (await Promise.all(revs.map(async (rev) => {
const leaf = await this.core.databaseFileAccess.fetchEntryMeta(filename, rev);
if (leaf == false) {
return [0, rev] as [number, string];
}
return [leaf.mtime, rev] as [number, string];
}))).sort((a, b) => b[0] - a[0]);
this._log(`Resolving conflict by newest: ${filename} (Newest: ${new Date(mTimeAndRev[0][0]).toLocaleString()}) (${mTimeAndRev.length} revisions exists)`);
for (let i = 1; i < mTimeAndRev.length; i++) {
this._log(`conflict: Deleting the older revision ${mTimeAndRev[i][1]} (${new Date(mTimeAndRev[i][0]).toLocaleString()}) of ${filename}`);
await this.core.$$resolveConflictByDeletingRev(filename, mTimeAndRev[i][1], "NEWEST");
}
return true;
}
}

112
src/modules/coreFeatures/ModuleRedFlag.ts Normal file
View File

@@ -0,0 +1,112 @@
import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { normalizePath } from "../../deps.ts";
import { FLAGMD_REDFLAG, FLAGMD_REDFLAG2, FLAGMD_REDFLAG2_HR, FLAGMD_REDFLAG3, FLAGMD_REDFLAG3_HR } from "../../lib/src/common/types.ts";
import { AbstractModule } from "../AbstractModule.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleRedFlag extends AbstractModule implements ICoreModule {
async isFlagFileExist(path: string) {
const redflag = await this.core.storageAccess.isExists(normalizePath(path));
if (redflag) {
return true;
}
return false;
}
async deleteFlagFile(path: string) {
try {
const isFlagged = await this.core.storageAccess.isExists(normalizePath(path));
if (isFlagged) {
await this.core.storageAccess.delete(path, true);
}
} catch (ex) {
this._log(`Could not delete ${path}`);
this._log(ex, LOG_LEVEL_VERBOSE);
}
}
isRedFlagRaised = async () => await this.isFlagFileExist(FLAGMD_REDFLAG)
isRedFlag2Raised = async () => await this.isFlagFileExist(FLAGMD_REDFLAG2) || await this.isFlagFileExist(FLAGMD_REDFLAG2_HR)
isRedFlag3Raised = async () => await this.isFlagFileExist(FLAGMD_REDFLAG3) || await this.isFlagFileExist(FLAGMD_REDFLAG3_HR)
async deleteRedFlag2() {
await this.deleteFlagFile(FLAGMD_REDFLAG2);
await this.deleteFlagFile(FLAGMD_REDFLAG2_HR);
}
async deleteRedFlag3() {
await this.deleteFlagFile(FLAGMD_REDFLAG3);
await this.deleteFlagFile(FLAGMD_REDFLAG3_HR);
}
async $everyOnLayoutReady(): Promise<boolean> {
try {
const isRedFlagRaised = await this.isRedFlagRaised();
const isRedFlag2Raised = await this.isRedFlag2Raised();
const isRedFlag3Raised = await this.isRedFlag3Raised();
if (isRedFlagRaised || isRedFlag2Raised || isRedFlag3Raised) {
if (isRedFlag2Raised) {
if (await this.core.confirm.askYesNoDialog("Rebuild everything has been scheduled! Are you sure to rebuild everything?", { defaultOption: "Yes", timeout: 0 }) !== "yes") {
await this.deleteRedFlag2();
await this.core.$$performRestart();
return false;
}
}
if (isRedFlag3Raised) {
if (await this.core.confirm.askYesNoDialog("Fetch again has been scheduled! Are you sure?", { defaultOption: "Yes", timeout: 0 }) !== "yes") {
await this.deleteRedFlag3();
await this.core.$$performRestart();
return false;
}
}
this.settings.batchSave = false;
await this.core.$allSuspendAllSync();
await this.core.$allSuspendExtraSync();
this.settings.suspendFileWatching = true;
await this.saveSettings();
if (isRedFlag2Raised) {
this._log(`${FLAGMD_REDFLAG2} or ${FLAGMD_REDFLAG2_HR} has been detected! Self-hosted LiveSync suspends all sync and rebuild everything.`, LOG_LEVEL_NOTICE);
await this.core.rebuilder.$rebuildEverything();
await this.deleteRedFlag2();
if (await this.core.confirm.askYesNoDialog("Do you want to resume file and database processing, and restart obsidian now?", { defaultOption: "Yes", timeout: 15 }) == "yes") {
this.settings.suspendFileWatching = false;
await this.saveSettings();
this.core.$$performRestart();
return false;
}
} else if (isRedFlag3Raised) {
this._log(`${FLAGMD_REDFLAG3} or ${FLAGMD_REDFLAG3_HR} has been detected! Self-hosted LiveSync will discard the local database and fetch everything from the remote once again.`, LOG_LEVEL_NOTICE);
const makeLocalChunkBeforeSync = ((await this.core.confirm.askYesNoDialog(`Do you want to create local chunks before fetching?
> [!MORE]-
> If creating local chunks before fetching, only the difference between the local and remote will be fetched.
`, { defaultOption: "Yes", title: "Trick to transfer efficiently" })) == "yes");
await this.core.rebuilder.$fetchLocal(makeLocalChunkBeforeSync);
await this.deleteRedFlag3();
if (this.settings.suspendFileWatching) {
if (await this.core.confirm.askYesNoDialog("Do you want to resume file and database processing, and restart obsidian now?", { defaultOption: "Yes", timeout: 15 }) == "yes") {
this.settings.suspendFileWatching = false;
await this.saveSettings();
this.core.$$performRestart();
return false;
}
} else {
this._log("Your content of files will be synchronised gradually. Please wait for the completion.", LOG_LEVEL_NOTICE);
}
} else {
// Case of FLAGMD_REDFLAG.
this.settings.writeLogToTheFile = true;
// await this.plugin.openDatabase();
const warningMessage = "The red flag is raised! The whole initialize steps are skipped, and any file changes are not captured.";
this._log(warningMessage, LOG_LEVEL_NOTICE);
}
}
} catch (ex) {
this._log("Something went wrong on FlagFile Handling", LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
return true;
}
}

16
src/modules/coreFeatures/ModuleRemoteGovernor.ts Normal file
View File

@@ -0,0 +1,16 @@
import { AbstractModule } from "../AbstractModule.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleRemoteGovernor extends AbstractModule implements ICoreModule {
async $$markRemoteLocked(lockByClean: boolean = false): Promise<void> {
return await this.core.replicator.markRemoteLocked(this.settings, true, lockByClean);
}
async $$markRemoteUnlocked(): Promise<void> {
return await this.core.replicator.markRemoteLocked(this.settings, false, false);
}
async $$markRemoteResolved(): Promise<void> {
return await this.core.replicator.markRemoteResolved(this.settings);
}
}

97
src/modules/coreFeatures/ModuleResolveMismatchedTweaks.ts Normal file
View File

@@ -0,0 +1,97 @@
import { Logger, LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
import { extractObject } from "octagonal-wheels/object";
import { TweakValuesShouldMatchedTemplate, CompatibilityBreakingTweakValues, confName, type TweakValues } from "../../lib/src/common/types.ts";
import { escapeMarkdownValue } from "../../lib/src/common/utils.ts";
import { AbstractModule } from "../AbstractModule.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleResolvingMismatchedTweaks extends AbstractModule implements ICoreModule {
async $anyAfterConnectCheckFailed(): Promise<boolean | "CHECKAGAIN" | undefined> {
if (!this.core.replicator.tweakSettingsMismatched) return false;
const ret = await this.core.$$askResolvingMismatchedTweaks();
if (ret == "OK") return false;
if (ret == "CHECKAGAIN") return "CHECKAGAIN";
if (ret == "IGNORE") return true;
}
async $$askResolvingMismatchedTweaks(): Promise<"OK" | "CHECKAGAIN" | "IGNORE"> {
if (!this.core.replicator.tweakSettingsMismatched) {
return "OK";
}
const preferred = extractObject(TweakValuesShouldMatchedTemplate, this.core.replicator.preferredTweakValue!);
const mine = extractObject(TweakValuesShouldMatchedTemplate, this.settings);
const items = Object.entries(TweakValuesShouldMatchedTemplate);
let rebuildRequired = false;
// Making tables:
let table = `| Value name | This device | Configured | \n` + `|: --- |: --- :|: ---- :| \n`;
// const items = [mine,preferred]
for (const v of items) {
const key = v[0] as keyof typeof TweakValuesShouldMatchedTemplate;
const valueMine = escapeMarkdownValue(mine[key]);
const valuePreferred = escapeMarkdownValue(preferred[key]);
if (valueMine == valuePreferred) continue;
if (CompatibilityBreakingTweakValues.indexOf(key) !== -1) {
rebuildRequired = true;
}
table += `| ${confName(key)} | ${valueMine} | ${valuePreferred} | \n`;
}
const additionalMessage = rebuildRequired ? `
**Note**: We have detected that some of the values are different to make incompatible the local database with the remote database.
If you choose to use the configured values, the local database will be rebuilt, and if you choose to use the values of this device, the remote database will be rebuilt.
Both of them takes a few minutes. Please choose after considering the situation.` : "";
const message = `
Your configuration has not been matched with the one on the remote server.
(Which you had decided once before, or set by initially synchronised device).
Configured values:
${table}
Please select which one you want to use.
- Use configured: Update settings of this device by configured one on the remote server.
You should select this if you have changed the settings on ** another device **.
- Update with mine: Update settings on the remote server by the settings of this device.
You should select this if you have changed the settings on ** this device **.
- Dismiss: Ignore this message and keep the current settings.
You cannot synchronise until you resolve this issue without enabling \`Do not check configuration mismatch before replication\`.${additionalMessage}`;
const CHOICE_USE_REMOTE = "Use configured";
const CHOICE_USR_MINE = "Update with mine";
const CHOICE_DISMISS = "Dismiss";
const CHOICE_AND_VALUES = [
[CHOICE_USE_REMOTE, preferred],
[CHOICE_USR_MINE, true],
[CHOICE_DISMISS, false]]
const CHOICES = Object.fromEntries(CHOICE_AND_VALUES) as Record<string, TweakValues | boolean>;
const retKey = await this.core.confirm.confirmWithMessage("Tweaks Mismatched or Changed", message, Object.keys(CHOICES), CHOICE_DISMISS, 60);
if (!retKey) return "IGNORE";
const conf = CHOICES[retKey];
if (conf === true) {
await this.core.replicator.setPreferredRemoteTweakSettings(this.settings);
if (rebuildRequired) {
await this.core.rebuilder.$rebuildRemote();
}
Logger(`Tweak values on the remote server have been updated. Your other device will see this message.`, LOG_LEVEL_NOTICE);
return "CHECKAGAIN";
}
if (conf) {
this.settings = { ...this.settings, ...conf };
await this.core.replicator.setPreferredRemoteTweakSettings(this.settings);
await this.core.$$saveSettingData();
if (rebuildRequired) {
await this.core.rebuilder.$fetchLocal();
}
Logger(`Configuration has been updated as configured by the other device.`, LOG_LEVEL_NOTICE);
return "CHECKAGAIN";
}
return "IGNORE";
}
}

326
src/modules/coreObsidian/ModuleFileAccessObsidian.ts Normal file
View File

@@ -0,0 +1,326 @@
import { normalizePath, TFile, TFolder, type ListedFiles } from "obsidian";
import { SerializedFileAccess } from "./storageLib/SerializedFileAccess";
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
import { LOG_LEVEL_INFO, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import type { FilePath, FilePathWithPrefix, UXDataWriteOptions, UXFileInfo, UXFileInfoStub, UXFolderInfo, UXStat } from "../../lib/src/common/types";
import { TFileToUXFileInfoStub, TFolderToUXFileInfoStub } from "./storageLib/utilObsidian.ts";
import { StorageEventManagerObsidian, type StorageEventManager } from "./storageLib/StorageEventManager";
import type { StorageAccess } from "../interfaces/StorageAccess";
import { createBlob } from "../../lib/src/common/utils";
export class ModuleFileAccessObsidian extends AbstractObsidianModule implements IObsidianModule, StorageAccess {
vaultAccess!: SerializedFileAccess;
vaultManager: StorageEventManager = new StorageEventManagerObsidian(this.plugin, this.core);
$everyOnload(): Promise<boolean> {
this.core.storageAccess = this;
return Promise.resolve(true);
}
$everyOnFirstInitialize(): Promise<boolean> {
this.vaultManager.beginWatch();
return Promise.resolve(true);
}
$allOnUnload(): Promise<boolean> {
// this.vaultManager.
return Promise.resolve(true);
}
// $$flushFileEventQueue(): void {
// this.vaultManager.flushQueue();
// }
$everyCommitPendingFileEvent(): Promise<boolean> {
this.vaultManager.flushQueue();
return Promise.resolve(true);
}
$everyOnloadStart(): Promise<boolean> {
this.vaultAccess = new SerializedFileAccess(this.app, this.plugin);
return Promise.resolve(true);
}
$$isStorageInsensitive(): boolean {
return this.vaultAccess.isStorageInsensitive();
}
async writeFileAuto(path: string, data: string | ArrayBuffer, opt?: UXDataWriteOptions): Promise<boolean> {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return this.vaultAccess.vaultModify(file, data, opt);
} else if (file === null) {
return await this.vaultAccess.vaultCreate(path, data, opt) instanceof TFile;
} else {
this._log(`Could not write file (Possibly already exists as a folder): ${path}`, LOG_LEVEL_VERBOSE);
return false;
}
}
readFileAuto(path: string): Promise<string | ArrayBuffer> {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return this.vaultAccess.vaultRead(file);
} else {
throw new Error(`Could not read file (Possibly does not exist): ${path}`);
}
}
readFileText(path: string): Promise<string> {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return this.vaultAccess.vaultRead(file);
} else {
throw new Error(`Could not read file (Possibly does not exist): ${path}`);
}
}
isExists(path: string): Promise<boolean> {
return Promise.resolve(this.vaultAccess.getAbstractFileByPath(path) instanceof TFile);
}
async writeHiddenFileAuto(path: string, data: string | ArrayBuffer, opt?: UXDataWriteOptions): Promise<boolean> {
try {
await this.vaultAccess.adapterWrite(path, data, opt);
return true;
} catch (e) {
this._log(`Could not write hidden file: ${path}`, LOG_LEVEL_VERBOSE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
}
async appendHiddenFile(path: string, data: string, opt?: UXDataWriteOptions): Promise<boolean> {
try {
await this.vaultAccess.adapterAppend(path, data, opt);
return true;
} catch (e) {
this._log(`Could not append hidden file: ${path}`, LOG_LEVEL_VERBOSE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
}
stat(path: string): Promise<UXStat | null> {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file === null) return Promise.resolve(null);
if (file instanceof TFile) {
return Promise.resolve({
ctime: file.stat.ctime,
mtime: file.stat.mtime,
size: file.stat.size,
type: "file"
});
} else {
throw new Error(`Could not stat file (Possibly does not exist): ${path}`);
}
}
statHidden(path: string): Promise<UXStat | null> {
return this.vaultAccess.adapterStat(path);
}
async removeHidden(path: string): Promise<boolean> {
try {
await this.vaultAccess.adapterRemove(path);
return true;
} catch (e) {
this._log(`Could not remove hidden file: ${path}`, LOG_LEVEL_VERBOSE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
}
async readHiddenFileAuto(path: string): Promise<string | ArrayBuffer> {
return await this.vaultAccess.adapterReadAuto(path);
}
async readHiddenFileText(path: string): Promise<string> {
return await this.vaultAccess.adapterRead(path);
}
async readHiddenFileBinary(path: string): Promise<ArrayBuffer> {
return await this.vaultAccess.adapterReadBinary(path);
}
async isExistsIncludeHidden(path: string): Promise<boolean> {
return await this.vaultAccess.adapterStat(path) !== null;
}
async ensureDir(path: string): Promise<boolean> {
try {
await this.vaultAccess.ensureDirectory(path);
return true;
} catch (e) {
this._log(`Could not ensure directory: ${path}`, LOG_LEVEL_VERBOSE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
}
triggerFileEvent(event: string, path: string): void {
// this.app.vault.trigger("file-change", path);
this.vaultAccess.trigger(event, this.vaultAccess.getAbstractFileByPath(normalizePath(path)));
}
async triggerHiddenFile(path: string): Promise<void> {
//@ts-ignore internal function
await this.app.vault.adapter.reconcileInternalFile(path);
}
// getFileStub(file: TFile): UXFileInfoStub {
// return TFileToUXFileInfoStub(file);
// }
getFileStub(path: string): UXFileInfoStub | null {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return TFileToUXFileInfoStub(file);
} else {
return null;
}
}
async readStubContent(stub: UXFileInfoStub): Promise<UXFileInfo | false> {
const file = this.vaultAccess.getAbstractFileByPath(stub.path);
if (!(file instanceof TFile)) {
this._log(`Could not read file (Possibly does not exist or a folder): ${stub.path}`, LOG_LEVEL_VERBOSE);
return false;
}
const data = await this.vaultAccess.vaultReadAuto(file);
return {
...stub,
body: createBlob(data)
}
}
getStub(path: string): UXFileInfoStub | UXFolderInfo | null {
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file instanceof TFile) {
return TFileToUXFileInfoStub(file);
} else if (file instanceof TFolder) {
return TFolderToUXFileInfoStub(file);
}
return null;
}
getFiles(): UXFileInfoStub[] {
return this.vaultAccess.getFiles().map(f => TFileToUXFileInfoStub(f));
}
getFileNames(): FilePath[] {
return this.vaultAccess.getFiles().map(f => f.path as FilePath);
}
async getFilesIncludeHidden(
basePath: string,
includeFilter?: RegExp[],
excludeFilter?: RegExp[],
skipFolder: string[] = [".git", ".trash", "node_modules"]
): Promise<FilePath[]> {
let w: ListedFiles;
try {
w = await this.app.vault.adapter.list(basePath);
} catch (ex) {
this._log(`Could not traverse(getFilesIncludeHidden):${basePath}`, LOG_LEVEL_INFO);
this._log(ex, LOG_LEVEL_VERBOSE);
return [];
}
skipFolder = skipFolder.map(e => e.toLowerCase());
let files = [] as string[];
for (const file of w.files) {
if (excludeFilter && excludeFilter.some(ee => file.match(ee))) {
// If excludeFilter and includeFilter are both set, the file will be included in the list.
if (includeFilter) {
if (!includeFilter.some(e => file.match(e))) continue;
} else {
continue;
}
}
if (includeFilter) {
if (!includeFilter.some(e => file.match(e))) continue;
}
if (await this.plugin.$$isIgnoredByIgnoreFiles(file)) continue;
files.push(file);
}
for (const v of w.folders) {
const folderName = (v.split("/").pop() ?? "").toLowerCase();
if (skipFolder.some(e => folderName === e)) {
continue
}
if (excludeFilter && excludeFilter.some(e => v.match(e))) {
if (includeFilter) {
if (!includeFilter.some(e => v.match(e))) {
continue;
}
}
}
if (includeFilter) {
if (!includeFilter.some(e => v.match(e))) continue;
}
// OK, deep dive!
files = files.concat(await this.getFilesIncludeHidden(v, includeFilter, excludeFilter, skipFolder));
}
return files as FilePath[];
}
touched(file: UXFileInfoStub | FilePathWithPrefix): void {
const path = typeof file === "string" ? file : file.path;
this.vaultAccess.touch(path as FilePath);
}
recentlyTouched(file: UXFileInfoStub | FilePathWithPrefix): boolean {
const xFile = typeof file === "string" ? this.vaultAccess.getAbstractFileByPath(file) as TFile : file;
if (xFile === null) return false;
if (xFile instanceof TFolder) return false;
return this.vaultAccess.recentlyTouched(xFile);
}
clearTouched(): void {
this.vaultAccess.clearTouched();
}
delete(file: FilePathWithPrefix | UXFileInfoStub | string, force: boolean): Promise<void> {
const xPath = typeof file === "string" ? file : file.path;
const xFile = this.vaultAccess.getAbstractFileByPath(xPath);
if (xFile === null) return Promise.resolve();
if (!(xFile instanceof TFile) && !(xFile instanceof TFolder)) return Promise.resolve();
return this.vaultAccess.delete(xFile, force);
}
trash(file: FilePathWithPrefix | UXFileInfoStub | string, system: boolean): Promise<void> {
const xPath = typeof file === "string" ? file : file.path;
const xFile = this.vaultAccess.getAbstractFileByPath(xPath);
if (xFile === null) return Promise.resolve();
if (!(xFile instanceof TFile) && !(xFile instanceof TFolder)) return Promise.resolve();
return this.vaultAccess.trash(xFile, system);
}
// $readFileBinary(path: string): Promise<ArrayBuffer> {
// const file = this.vaultAccess.getAbstractFileByPath(path);
// if (file instanceof TFile) {
// return this.vaultAccess.vaultReadBinary(file);
// } else {
// throw new Error(`Could not read file (Possibly does not exist): ${path}`);
// }
// }
// async $appendFileAuto(path: string, data: string | ArrayBuffer, opt?: DataWriteOptions): Promise<boolean> {
// const file = this.vaultAccess.getAbstractFileByPath(path);
// if (file instanceof TFile) {
// return this.vaultAccess.a(file, data, opt);
// } else if (file !== null) {
// return await this.vaultAccess.vaultCreate(path, data, opt) instanceof TFile;
// } else {
// this._log(`Could not append file (Possibly already exists as a folder): ${path}`, LOG_LEVEL_VERBOSE);
// return false;
// }
// }
async _deleteVaultItem(file: TFile | TFolder) {
if (file instanceof TFile) {
if (!await this.core.$$isTargetFile(file.path)) return;
}
const dir = file.parent;
if (this.settings.trashInsteadDelete) {
await this.vaultAccess.trash(file, false);
} else {
await this.vaultAccess.delete(file, true);
}
this._log(`xxx <- STORAGE (deleted) ${file.path}`);
if (dir) {
this._log(`files: ${dir.children.length}`);
if (dir.children.length == 0) {
if (!this.settings.doNotDeleteFolder) {
this._log(`All files under the parent directory (${dir.path}) have been deleted, so delete this one.`);
await this._deleteVaultItem(dir);
}
}
}
}
async deleteVaultItem(fileSrc: FilePathWithPrefix | UXFileInfoStub | UXFolderInfo): Promise<void> {
const path = typeof fileSrc === "string" ? fileSrc : fileSrc.path;
const file = this.vaultAccess.getAbstractFileByPath(path);
if (file === null) return;
if (file instanceof TFile || file instanceof TFolder) {
return await this._deleteVaultItem(file);
}
}
}

71
src/modules/coreObsidian/ModuleInputUIObsidian.ts Normal file
View File

@@ -0,0 +1,71 @@
import { AbstractObsidianModule, type IObsidianModule } from '../AbstractObsidianModule.ts';
import { scheduleTask } from 'octagonal-wheels/concurrency/task';
import { disposeMemoObject, memoIfNotExist, memoObject, retrieveMemoObject } from '../../common/utils.ts';
import { askSelectString, askString, askYesNo, confirmWithMessage, confirmWithMessageWithWideButton } from './UILib/dialogs.ts';
import { Notice } from '../../deps.ts';
import type { Confirm } from '../interfaces/Confirm.ts';
// This module cannot be a common module because it depends on Obsidian's API.
// However, we have to make compatible one for other platform.
export class ModuleInputUIObsidian extends AbstractObsidianModule implements IObsidianModule, Confirm {
$everyOnload(): Promise<boolean> {
this.core.confirm = this;
return Promise.resolve(true);
}
askYesNo(message: string): Promise<"yes" | "no"> {
return askYesNo(this.app, message);
}
askString(title: string, key: string, placeholder: string, isPassword: boolean = false): Promise<string | false> {
return askString(this.app, title, key, placeholder, isPassword);
}
async askYesNoDialog(message: string, opt: { title?: string, defaultOption?: "Yes" | "No", timeout?: number } = { title: "Confirmation" }): Promise<"yes" | "no"> {
const ret = await confirmWithMessageWithWideButton(this.plugin, opt.title || "Confirmation", message, ["Yes", "No"], opt.defaultOption ?? "No", opt.timeout);
return ret == "Yes" ? "yes" : "no";
}
askSelectString(message: string, items: string[]): Promise<string> {
return askSelectString(this.app, message, items);
}
askSelectStringDialogue(message: string, buttons: string[], opt: { title?: string, defaultAction: (typeof buttons)[number], timeout?: number }): Promise<(typeof buttons)[number] | false> {
return confirmWithMessageWithWideButton(this.plugin, opt.title || "Select", message, buttons, opt.defaultAction, opt.timeout);
}
askInPopup(key: string, dialogText: string, anchorCallback: (anchor: HTMLAnchorElement) => void) {
const fragment = createFragment((doc) => {
const [beforeText, afterText] = dialogText.split("{HERE}", 2);
doc.createEl("span", undefined, (a) => {
a.appendText(beforeText);
a.appendChild(a.createEl("a", undefined, (anchor) => {
anchorCallback(anchor);
}));
a.appendText(afterText);
});
});
const popupKey = "popup-" + key;
scheduleTask(popupKey, 1000, async () => {
const popup = await memoIfNotExist(popupKey, () => new Notice(fragment, 0));
const isShown = popup?.noticeEl?.isShown();
if (!isShown) {
memoObject(popupKey, new Notice(fragment, 0));
}
scheduleTask(popupKey + "-close", 20000, () => {
const popup = retrieveMemoObject<Notice>(popupKey);
if (!popup)
return;
if (popup?.noticeEl?.isShown()) {
popup.hide();
}
disposeMemoObject(popupKey);
});
});
}
confirmWithMessage(title: string, contentMd: string, buttons: string[], defaultAction: (typeof buttons)[number], timeout?: number): Promise<(typeof buttons)[number] | false> {
return confirmWithMessage(this.plugin, title, contentMd, buttons, defaultAction, timeout);
}
}

124
src/common/dialogs.ts → src/modules/coreObsidian/UILib/dialogs.ts
View File

@@ -1,45 +1,29 @@
import { ButtonComponent } from "obsidian";
import { App, FuzzySuggestModal, MarkdownRenderer, Modal, Plugin, Setting } from "../deps.ts";
import ObsidianLiveSyncPlugin from "../main.ts";
import { App, FuzzySuggestModal, MarkdownRenderer, Modal, Plugin, Setting } from "../../../deps.ts";
import { EVENT_PLUGIN_UNLOADED, eventHub } from "../../../common/events.ts";
import { delay } from "octagonal-wheels/promises";
//@ts-ignore
import PluginPane from "../ui/PluginPane.svelte";
class AutoClosableModal extends Modal {
removeEvent: (() => void) | undefined;
export class PluginDialogModal extends Modal {
plugin: ObsidianLiveSyncPlugin;
component: PluginPane | undefined;
isOpened() {
return this.component != undefined;
}
constructor(app: App, plugin: ObsidianLiveSyncPlugin) {
constructor(app: App) {
super(app);
this.plugin = plugin;
this.removeEvent = eventHub.on(EVENT_PLUGIN_UNLOADED, async () => {
await delay(100);
if (!this.removeEvent) return;
this.close();
this.removeEvent = undefined;
});
}
onOpen() {
const { contentEl } = this;
this.contentEl.style.overflow = "auto";
this.contentEl.style.display = "flex";
this.contentEl.style.flexDirection = "column";
this.titleEl.setText("Customization Sync (Beta3)")
if (!this.component) {
this.component = new PluginPane({
target: contentEl,
props: { plugin: this.plugin },
});
}
}
onClose() {
if (this.component) {
this.component.$destroy();
this.component = undefined;
if (this.removeEvent) {
this.removeEvent();
this.removeEvent = undefined
}
}
}
export class InputStringDialog extends Modal {
export class InputStringDialog extends AutoClosableModal {
result: string | false = false;
onSubmit: (result: string | false) => void;
title: string;
@@ -85,6 +69,7 @@ export class InputStringDialog extends Modal {
}
onClose() {
super.onClose();
const { contentEl } = this;
contentEl.empty();
if (this.isManuallyClosed) {
@@ -133,7 +118,7 @@ export class PopoverSelectString extends FuzzySuggestModal<string> {
}
}
export class MessageBox extends Modal {
export class MessageBox extends AutoClosableModal {
plugin: Plugin;
title: string;
@@ -145,10 +130,11 @@ export class MessageBox extends Modal {
timeout: number | undefined;
timer: ReturnType<typeof setInterval> | undefined = undefined;
defaultButtonComponent: ButtonComponent | undefined;
wideButton: boolean;
onSubmit: (result: string | false) => void;
constructor(plugin: Plugin, title: string, contentMd: string, buttons: string[], defaultAction: (typeof buttons)[number], timeout: number | undefined, onSubmit: (result: (typeof buttons)[number] | false) => void) {
constructor(plugin: Plugin, title: string, contentMd: string, buttons: string[], defaultAction: (typeof buttons)[number], timeout: number | undefined, wideButton: boolean, onSubmit: (result: (typeof buttons)[number] | false) => void) {
super(plugin.app);
this.plugin = plugin;
this.title = title;
@@ -157,6 +143,7 @@ export class MessageBox extends Modal {
this.onSubmit = onSubmit;
this.defaultAction = defaultAction;
this.timeout = timeout;
this.wideButton = wideButton;
if (this.timeout) {
this.timer = setInterval(() => {
if (this.timeout === undefined) return;
@@ -164,6 +151,7 @@ export class MessageBox extends Modal {
if (this.timeout < 0) {
if (this.timer) {
clearInterval(this.timer);
this.defaultButtonComponent?.setButtonText(`${defaultAction}`);
this.timer = undefined;
}
this.result = defaultAction;
@@ -179,16 +167,36 @@ export class MessageBox extends Modal {
onOpen() {
const { contentEl } = this;
this.titleEl.setText(this.title);
const div = contentEl.createDiv();
div.style.userSelect = "text";
void MarkdownRenderer.render(this.plugin.app, this.contentMd, div, "/", this.plugin);
const buttonSetting = new Setting(contentEl);
const labelWrapper = contentEl.createDiv();
labelWrapper.addClass("sls-dialogue-note-wrapper");
const labelEl = labelWrapper.createEl("label", { text: "To stop the countdown, tap anywhere on the dialogue" });
labelEl.addClass("sls-dialogue-note-countdown");
if (!this.timeout || !this.timer) {
labelWrapper.empty();
labelWrapper.style.display = "none";
}
buttonSetting.infoEl.style.display = "none";
buttonSetting.controlEl.style.flexWrap = "wrap";
if (this.wideButton) {
buttonSetting.controlEl.style.flexDirection = "column";
buttonSetting.controlEl.style.alignItems = "center";
buttonSetting.controlEl.style.justifyContent = "center";
buttonSetting.controlEl.style.flexGrow = "1";
}
contentEl.addEventListener("click", () => {
if (this.timer) {
labelWrapper.empty();
labelWrapper.style.display = "none";
clearInterval(this.timer);
this.timer = undefined;
this.defaultButtonComponent?.setButtonText(`${this.defaultAction}`);
}
})
const div = contentEl.createDiv();
MarkdownRenderer.render(this.plugin.app, this.contentMd, div, "/", this.plugin);
const buttonSetting = new Setting(contentEl);
buttonSetting.controlEl.style.flexWrap = "wrap";
for (const button of this.buttons) {
buttonSetting.addButton((btn) => {
btn
@@ -204,6 +212,11 @@ export class MessageBox extends Modal {
})
if (button == this.defaultAction) {
this.defaultButtonComponent = btn;
btn.setCta();
}
if (this.wideButton) {
btn.buttonEl.style.flexGrow = "1";
btn.buttonEl.style.width = "100%";
}
return btn;
}
@@ -212,6 +225,7 @@ export class MessageBox extends Modal {
}
onClose() {
super.onClose();
const { contentEl } = this;
contentEl.empty();
if (this.timer) {
@@ -227,9 +241,41 @@ export class MessageBox extends Modal {
}
export function confirmWithMessage(plugin: Plugin, title: string, contentMd: string, buttons: string[], defaultAction: (typeof buttons)[number], timeout?: number): Promise<(typeof buttons)[number] | false> {
return new Promise((res) => {
const dialog = new MessageBox(plugin, title, contentMd, buttons, defaultAction, timeout, (result) => res(result));
const dialog = new MessageBox(plugin, title, contentMd, buttons, defaultAction, timeout, false, (result) => res(result));
dialog.open();
});
}
export function confirmWithMessageWithWideButton(plugin: Plugin, title: string, contentMd: string, buttons: string[], defaultAction: (typeof buttons)[number], timeout?: number): Promise<(typeof buttons)[number] | false> {
return new Promise((res) => {
const dialog = new MessageBox(plugin, title, contentMd, buttons, defaultAction, timeout, true, (result) => res(result));
dialog.open();
});
}
export const askYesNo = (app: App, message: string): Promise<"yes" | "no"> => {
return new Promise((res) => {
const popover = new PopoverSelectString(app, message, undefined, undefined, (result) => res(result as "yes" | "no"));
popover.open();
});
};
export const askSelectString = (app: App, message: string, items: string[]): Promise<string> => {
const getItemsFun = () => items;
return new Promise((res) => {
const popover = new PopoverSelectString(app, message, "", getItemsFun, (result) => res(result));
popover.open();
});
};
export const askString = (app: App, title: string, key: string, placeholder: string, isPassword: boolean = false): Promise<string | false> => {
return new Promise((res) => {
const dialog = new InputStringDialog(app, title, key, placeholder, isPassword, (result) => res(result));
dialog.open();
});
};

39
src/storages/SerializedFileAccess.ts → src/modules/coreObsidian/storageLib/SerializedFileAccess.ts
View File

@@ -1,13 +1,14 @@
import { type App, TFile, type DataWriteOptions, TFolder, TAbstractFile } from "../deps.ts";
import { serialized } from "../lib/src/concurrency/lock.ts";
import { Logger } from "../lib/src/common/logger.ts";
import { isPlainText } from "../lib/src/string_and_binary/path.ts";
import type { FilePath, HasSettings } from "../lib/src/common/types.ts";
import { createBinaryBlob, isDocContentSame } from "../lib/src/common/utils.ts";
import type { InternalFileInfo } from "../common/types.ts";
import { markChangesAreSame } from "../common/utils.ts";
import { type App, TFile, type DataWriteOptions, TFolder, TAbstractFile } from "../../../deps.ts";
import { serialized } from "../../../lib/src/concurrency/lock.ts";
import { Logger } from "../../../lib/src/common/logger.ts";
import { isPlainText } from "../../../lib/src/string_and_binary/path.ts";
import type { FilePath, HasSettings, UXFileInfoStub } from "../../../lib/src/common/types.ts";
import { createBinaryBlob, isDocContentSame } from "../../../lib/src/common/utils.ts";
import type { InternalFileInfo } from "../../../common/types.ts";
import { markChangesAreSame } from "../../../common/utils.ts";
import { type UXFileInfo } from "../../../lib/src/common/types.ts";
function getFileLockKey(file: TFile | TFolder | string) {
function getFileLockKey(file: TFile | TFolder | string | UXFileInfo) {
return `fl:${typeof (file) == "string" ? file : file.path}`;
}
function toArrayBuffer(arr: Uint8Array | ArrayBuffer | DataView): ArrayBufferLike {
@@ -20,15 +21,19 @@ function toArrayBuffer(arr: Uint8Array | ArrayBuffer | DataView): ArrayBufferLik
return arr;
}
// function isFile(file: TFile | TFolder | string | UXFileInfo): boolean {
// file instanceof TFile;
// }
async function processReadFile<T>(file: TFile | TFolder | string, proc: () => Promise<T>) {
async function processReadFile<T>(file: TFile | TFolder | string | UXFileInfo, proc: () => Promise<T>) {
const ret = await serialized(getFileLockKey(file), () => proc());
return ret;
}
async function processWriteFile<T>(file: TFile | TFolder | string, proc: () => Promise<T>) {
async function processWriteFile<T>(file: TFile | TFolder | string | UXFileInfo, proc: () => Promise<T>) {
const ret = await serialized(getFileLockKey(file), () => proc());
return ret;
}
export class SerializedFileAccess {
app: App
plugin: HasSettings<{ handleFilenameCaseSensitive: boolean }>
@@ -98,8 +103,8 @@ export class SerializedFileAccess {
return await processWriteFile(file, async () => {
const oldData = await this.app.vault.read(file);
if (data === oldData) {
if (options && options.mtime) markChangesAreSame(file, file.stat.mtime, options.mtime);
return false
if (options && options.mtime) markChangesAreSame(file.path, file.stat.mtime, options.mtime);
return true;
}
await this.app.vault.modify(file, data, options)
return true;
@@ -109,8 +114,8 @@ export class SerializedFileAccess {
return await processWriteFile(file, async () => {
const oldData = await this.app.vault.readBinary(file);
if (await isDocContentSame(createBinaryBlob(oldData), createBinaryBlob(data))) {
if (options && options.mtime) markChangesAreSame(file, file.stat.mtime, options.mtime);
return false;
if (options && options.mtime) markChangesAreSame(file.path, file.stat.mtime, options.mtime);
return true;
}
await this.app.vault.modifyBinary(file, toArrayBuffer(data), options)
return true;
@@ -193,8 +198,8 @@ export class SerializedFileAccess {
this.touchedFiles.unshift(key);
this.touchedFiles = this.touchedFiles.slice(0, 100);
}
recentlyTouched(file: TFile | InternalFileInfo) {
const key = file instanceof TFile ? `${file.path}-${file.stat.mtime}-${file.stat.size}` : `${file.path}-${file.mtime}-${file.size}`;
recentlyTouched(file: TFile | InternalFileInfo | UXFileInfoStub) {
const key = "stat" in file ? `${file.path}-${file.stat.mtime}-${file.stat.size}` : `${file.path}-${file.mtime}-${file.size}`;
if (this.touchedFiles.indexOf(key) == -1) return false;
return true;
}

220
src/storages/StorageEventManager.ts → src/modules/coreObsidian/storageLib/StorageEventManager.ts
View File

@@ -1,19 +1,21 @@
import type { SerializedFileAccess } from "./SerializedFileAccess.ts";
import { Plugin, TAbstractFile, TFile, TFolder } from "../deps.ts";
import { Logger } from "../lib/src/common/logger.ts";
import { shouldBeIgnored } from "../lib/src/string_and_binary/path.ts";
import { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, type FilePath, type ObsidianLiveSyncSettings } from "../lib/src/common/types.ts";
import { delay, fireAndForget } from "../lib/src/common/utils.ts";
import { type FileEventItem, type FileEventType, type FileInfo, type InternalFileInfo } from "../common/types.ts";
import { skipIfDuplicated } from "../lib/src/concurrency/lock.ts";
import { finishAllWaitingForTimeout, finishWaitingForTimeout, isWaitingForTimeout, waitForTimeout } from "../lib/src/concurrency/task.ts";
import { reactiveSource, type ReactiveSource } from "../lib/src/dataobject/reactive.ts";
import { Semaphore } from "../lib/src/concurrency/semaphore.ts";
import { TAbstractFile, TFile, TFolder } from "../../../deps.ts";
import { Logger } from "../../../lib/src/common/logger.ts";
import { shouldBeIgnored } from "../../../lib/src/string_and_binary/path.ts";
import { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, type FilePath, type FilePathWithPrefix, type UXFileInfoStub, type UXInternalFileInfoStub } from "../../../lib/src/common/types.ts";
import { delay, fireAndForget } from "../../../lib/src/common/utils.ts";
import { type FileEventItem, type FileEventType } from "../../../common/types.ts";
import { serialized, skipIfDuplicated } from "../../../lib/src/concurrency/lock.ts";
import { finishAllWaitingForTimeout, finishWaitingForTimeout, isWaitingForTimeout, waitForTimeout } from "../../../lib/src/concurrency/task.ts";
import { Semaphore } from "../../../lib/src/concurrency/semaphore.ts";
import type { LiveSyncCore } from "../../../main.ts";
import { InternalFileToUXFileInfoStub, TFileToUXFileInfoStub } from "./utilObsidian.ts";
import ObsidianLiveSyncPlugin from "../../../main.ts";
// import { InternalFileToUXFileInfo } from "../platforms/obsidian.ts";
export type FileEvent = {
type: FileEventType;
file: TAbstractFile | InternalFileInfo;
file: UXFileInfoStub | UXInternalFileInfoStub;
oldPath?: string;
cachedData?: string;
skipBatchWait?: boolean;
@@ -23,37 +25,17 @@ export type FileEvent = {
export abstract class StorageEventManager {
abstract beginWatch(): void;
abstract flushQueue(): void;
abstract appendQueue(items: FileEvent[], ctx?: any): void;
abstract appendQueue(items: FileEvent[], ctx?: any): Promise<void>;
abstract cancelQueue(key: string): void;
abstract isWaiting(filename: FilePath): boolean;
abstract totalQueued: ReactiveSource<number>;
abstract batched: ReactiveSource<number>;
abstract processing: ReactiveSource<number>;
}
type LiveSyncForStorageEventManager = Plugin &
{
settings: ObsidianLiveSyncSettings
ignoreFiles: string[],
vaultAccess: SerializedFileAccess
shouldBatchSave: boolean
batchSaveMinimumDelay: number;
batchSaveMaximumDelay: number;
} & {
isTargetFile: (file: string | TAbstractFile) => Promise<boolean>,
// fileEventQueue: QueueProcessor<FileEventItem, any>,
handleFileEvent: (queue: FileEventItem) => Promise<any>,
isFileSizeExceeded: (size: number) => boolean;
};
export class StorageEventManagerObsidian extends StorageEventManager {
totalQueued = reactiveSource(0);
batched = reactiveSource(0);
processing = reactiveSource(0);
plugin: LiveSyncForStorageEventManager;
plugin: ObsidianLiveSyncPlugin;
core: LiveSyncCore;
get shouldBatchSave() {
return this.plugin.shouldBatchSave;
@@ -64,9 +46,10 @@ export class StorageEventManagerObsidian extends StorageEventManager {
get batchSaveMaximumDelay(): number {
return this.plugin.batchSaveMaximumDelay
}
constructor(plugin: LiveSyncForStorageEventManager) {
constructor(plugin: ObsidianLiveSyncPlugin, core: LiveSyncCore) {
super();
this.plugin = plugin;
this.core = core;
}
beginWatch() {
const plugin = this.plugin;
@@ -75,39 +58,79 @@ export class StorageEventManagerObsidian extends StorageEventManager {
this.watchVaultDelete = this.watchVaultDelete.bind(this);
this.watchVaultRename = this.watchVaultRename.bind(this);
this.watchVaultRawEvents = this.watchVaultRawEvents.bind(this);
this.watchEditorChange = this.watchEditorChange.bind(this);
plugin.registerEvent(plugin.app.vault.on("modify", this.watchVaultChange));
plugin.registerEvent(plugin.app.vault.on("delete", this.watchVaultDelete));
plugin.registerEvent(plugin.app.vault.on("rename", this.watchVaultRename));
plugin.registerEvent(plugin.app.vault.on("create", this.watchVaultCreate));
//@ts-ignore : Internal API
plugin.registerEvent(plugin.app.vault.on("raw", this.watchVaultRawEvents));
plugin.registerEvent(plugin.app.workspace.on("editor-change", this.watchEditorChange));
// plugin.fileEventQueue.startPipeline();
}
watchEditorChange(editor: any, info: any) {
if (!("path" in info)) {
return;
}
if (!this.shouldBatchSave) {
return;
}
const file = info?.file as TFile;
if (!file) return;
if (!this.isWaiting(file.path as FilePath)) {
return;
}
const data = info?.data as string;
const fi: FileEvent = {
type: "CHANGED", file: TFileToUXFileInfoStub(file), cachedData: data,
}
void this.appendQueue([
fi])
}
watchVaultCreate(file: TAbstractFile, ctx?: any) {
this.appendQueue([{ type: "CREATE", file }], ctx);
if (file instanceof TFolder) return;
const fileInfo = TFileToUXFileInfoStub(file);
void this.appendQueue([{ type: "CREATE", file: fileInfo }], ctx);
}
watchVaultChange(file: TAbstractFile, ctx?: any) {
this.appendQueue([{ type: "CHANGED", file }], ctx);
if (file instanceof TFolder) return;
const fileInfo = TFileToUXFileInfoStub(file);
void this.appendQueue([{ type: "CHANGED", file: fileInfo }], ctx);
}
watchVaultDelete(file: TAbstractFile, ctx?: any) {
this.appendQueue([{ type: "DELETE", file }], ctx);
if (file instanceof TFolder) return;
const fileInfo = TFileToUXFileInfoStub(file, true);
void this.appendQueue([{ type: "DELETE", file: fileInfo }], ctx);
}
watchVaultRename(file: TAbstractFile, oldFile: string, ctx?: any) {
if (file instanceof TFile) {
this.appendQueue([
{ type: "DELETE", file: { path: oldFile as FilePath, mtime: file.stat.mtime, ctime: file.stat.ctime, size: file.stat.size, deleted: true }, skipBatchWait: true },
{ type: "CREATE", file, skipBatchWait: true },
], ctx);
const fileInfo = TFileToUXFileInfoStub(file);
void this.appendQueue([
{
type: "DELETE", file: {
path: oldFile as FilePath, name: file.name, stat: {
mtime: file.stat.mtime,
ctime: file.stat.ctime,
size: file.stat.size,
type: "file"
}, deleted: true
}, skipBatchWait: true
}, { type: "CREATE", file: fileInfo, skipBatchWait: true },], ctx);
}
}
// Watch raw events (Internal API)
watchVaultRawEvents(path: FilePath) {
if (this.plugin.settings.useIgnoreFiles && this.plugin.ignoreFiles.some(e => path.endsWith(e.trim()))) {
// Only for internal files.
if (!this.plugin.settings) return;
// if (this.plugin.settings.useIgnoreFiles && this.plugin.ignoreFiles.some(e => path.endsWith(e.trim()))) {
if (this.plugin.settings.useIgnoreFiles) {
// If it is one of ignore files, refresh the cached one.
this.plugin.isTargetFile(path).then(() => this._watchVaultRawEvents(path));
// (Calling$$isTargetFile will refresh the cache)
void this.plugin.$$isTargetFile(path).then(() => this._watchVaultRawEvents(path));
} else {
this._watchVaultRawEvents(path);
}
@@ -121,32 +144,45 @@ export class StorageEventManagerObsidian extends StorageEventManager {
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
if (ignorePatterns.some(e => path.match(e))) return;
this.appendQueue(
[{
type: "INTERNAL",
file: { path, mtime: 0, ctime: 0, size: 0 }
if (path.endsWith("/")) {
// Folder
return;
}
void this.appendQueue([
{
type: "INTERNAL", file: InternalFileToUXFileInfoStub(path),
}], null);
}
// Cache file and waiting to can be proceed.
async appendQueue(params: FileEvent[], ctx?: any) {
if (!this.plugin.settings.isConfigured) return;
if (this.plugin.settings.suspendFileWatching) return;
this.plugin.totalFileEventCount++;
// Flag up to be reload
const processFiles = new Set<FilePath>();
for (const param of params) {
if (shouldBeIgnored(param.file.path)) {
continue;
}
const atomicKey = [0, 0, 0, 0, 0, 0].map(e => `${Math.floor(Math.random() * 100000)}`).join("-");
const atomicKey = [
0,
0,
0,
0,
0,
0].map(e => `${Math.floor(Math.random() * 100000)}`).join("-");
const type = param.type;
const file = param.file;
const oldPath = param.oldPath;
const size = file instanceof TFile ? file.stat.size : (file as InternalFileInfo)?.size ?? 0;
if (this.plugin.isFileSizeExceeded(size) && (type == "CREATE" || type == "CHANGED")) {
Logger(`The storage file has been changed but exceeds the maximum size. Skipping: ${param.file.path}`, LOG_LEVEL_NOTICE);
continue;
if (type !== "INTERNAL") {
const size = (file as UXFileInfoStub).stat.size;
if (this.plugin.$$isFileSizeExceeded(size) && (type == "CREATE" || type == "CHANGED")) {
Logger(`The storage file has been changed but exceeds the maximum size. Skipping: ${param.file.path}`, LOG_LEVEL_NOTICE);
continue;
}
}
if (file instanceof TFolder) continue;
if (!await this.plugin.isTargetFile(file.path)) continue;
if (!await this.plugin.$$isTargetFile(file.path)) continue;
// Stop cache using to prevent the corruption;
// let cache: null | string | ArrayBuffer;
@@ -154,31 +190,19 @@ export class StorageEventManagerObsidian extends StorageEventManager {
if (file instanceof TFile && (type == "CREATE" || type == "CHANGED")) {
// Wait for a bit while to let the writer has marked `touched` at the file.
await delay(10);
if (this.plugin.vaultAccess.recentlyTouched(file)) {
if (this.plugin.storageAccess.recentlyTouched(file)) {
continue;
}
}
const fileInfo = file instanceof TFile ? {
ctime: file.stat.ctime,
mtime: file.stat.mtime,
file: file,
path: file.path,
size: file.stat.size
} as FileInfo : file as InternalFileInfo;
let cache: string | undefined = undefined;
if (param.cachedData) {
cache = param.cachedData
}
this.enqueue({
type,
args: {
file: fileInfo,
oldPath,
cache,
ctx,
},
skipBatchWait: param.skipBatchWait,
key: atomicKey
type, args: {
file: file, oldPath, cache, ctx,
}, skipBatchWait: param.skipBatchWait, key: atomicKey
})
processFiles.add(file.path as FilePath);
if (oldPath) {
@@ -199,12 +223,12 @@ export class StorageEventManagerObsidian extends StorageEventManager {
}
this.bufferedQueuedItems.push(newItem);
// When deleting or renaming, the queue must be flushed once before processing subsequent processes to prevent unexpected race condition.
if (newItem.type == "DELETE" || newItem.type == "RENAME") {
if (newItem.type == "DELETE") {
return this.flushQueue();
}
}
concurrentProcessing = Semaphore(5);
waitedSince = new Map<FilePath, number>();
waitedSince = new Map<FilePath | FilePathWithPrefix, number>();
async startStandingBy(filename: FilePath) {
// If waited, cancel previous waiting.
await skipIfDuplicated(`storage-event-manager-${filename}`, async () => {
@@ -260,7 +284,7 @@ export class StorageEventManagerObsidian extends StorageEventManager {
Logger(`Processing ${filename}:Requested to perform immediately ${filename}: ${operationType}`, LOG_LEVEL_DEBUG)
}
Logger(`Processing ${filename}: Request main to process: ${operationType}`, LOG_LEVEL_DEBUG)
this.requestProcessQueue(target);
await this.requestProcessQueue(target);
} while (!noMoreFiles)
} finally {
release()
@@ -280,7 +304,7 @@ export class StorageEventManagerObsidian extends StorageEventManager {
this.bufferedQueuedItems.remove(fei);
this.updateStatus()
this.waitedSince.delete(fei.args.file.path);
await this.plugin.handleFileEvent(fei);
await this.handleFileEvent(fei);
} finally {
this.processingCount--;
this.updateStatus()
@@ -300,8 +324,44 @@ export class StorageEventManagerObsidian extends StorageEventManager {
}
updateStatus() {
const allItems = this.bufferedQueuedItems.filter(e => !e.cancelled)
this.batched.value = allItems.filter(e => e.batched && !e.skipBatchWait).length;
this.processing.value = this.processingCount;
this.totalQueued.value = allItems.length - this.batched.value;
const batchedCount = allItems.filter(e => e.batched && !e.skipBatchWait).length;
this.core.batched.value = batchedCount
this.core.processing.value = this.processingCount;
this.core.totalQueued.value = allItems.length - batchedCount;
}
async handleFileEvent(queue: FileEventItem): Promise<any> {
const file = queue.args.file;
const lockKey = `handleFile:${file.path}`;
return await serialized(lockKey, async () => {
// TODO CHECK
const key = `file-last-proc-${queue.type}-${file.path}`;
const last = Number(await this.core.kvDB.get(key) || 0);
if (queue.type == "INTERNAL" || file.isInternal) {
await this.plugin.$anyProcessOptionalFileEvent(file.path as unknown as FilePath);
} else {
// let mtime = file.stat.mtime;
if (queue.type == "DELETE") {
await this.plugin.$anyHandlerProcessesFileEvent(queue);
} else {
if (file.stat.mtime == last) {
Logger(`File has been already scanned on ${queue.type}, skip: ${file.path}`, LOG_LEVEL_VERBOSE);
// Should Cancel the relative operations? (e.g. rename)
// this.cancelRelativeEvent(queue);
return;
}
if (!await this.plugin.$anyHandlerProcessesFileEvent(queue)) {
Logger(`STORAGE -> DB: Handler failed, cancel the relative operations: ${file.path}`, LOG_LEVEL_INFO);
// cancel running queues and remove one of atomic operation (e.g. rename)
this.cancelRelativeEvent(queue);
return;
}
}
}
});
}
cancelRelativeEvent(item: FileEventItem): void {
this.cancelQueue(item.key);
}
}

109
src/modules/coreObsidian/storageLib/utilObsidian.ts Normal file
View File

@@ -0,0 +1,109 @@
// Obsidian to LiveSync Utils
import { TFile, type TAbstractFile, type TFolder } from "../../../deps.ts";
import { ICHeader } from "../../../common/types.ts";
import type { SerializedFileAccess } from "./SerializedFileAccess.ts";
import { addPrefix, isPlainText } from "../../../lib/src/string_and_binary/path.ts";
import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
import { createBlob } from "../../../lib/src/common/utils.ts";
import type { FilePath, FilePathWithPrefix, UXFileInfo, UXFileInfoStub, UXFolderInfo, UXInternalFileInfoStub } from "../../../lib/src/common/types.ts";
import type { LiveSyncCore } from "../../../main.ts";
export async function TFileToUXFileInfo(core: LiveSyncCore, file: TFile, prefix?: string, deleted?: boolean): Promise<UXFileInfo> {
const isPlain = isPlainText(file.name);
const possiblyLarge = !isPlain;
let content: Blob;
if (deleted) {
content = new Blob();
} else {
if (possiblyLarge) Logger(`Reading : ${file.path}`, LOG_LEVEL_VERBOSE);
content = createBlob(await core.storageAccess.readFileAuto(file.path));
if (possiblyLarge) Logger(`Processing: ${file.path}`, LOG_LEVEL_VERBOSE);
}
// const datatype = determineTypeFromBlob(content);
const bareFullPath = file.path as FilePathWithPrefix;
const fullPath = prefix ? addPrefix(bareFullPath, prefix) : bareFullPath;
return {
name: file.name,
path: fullPath,
stat: {
size: content.size,
ctime: file.stat.ctime,
mtime: file.stat.mtime,
type: "file",
},
body: content,
}
}
export async function InternalFileToUXFileInfo(fullPath: string, vaultAccess: SerializedFileAccess, prefix: string = ICHeader): Promise<UXFileInfo> {
const name = fullPath.split("/").pop() as string;
const stat = await vaultAccess.adapterStat(fullPath);
if (stat == null) throw new Error(`File not found: ${fullPath}`);
if (stat.type == "folder") throw new Error(`File not found: ${fullPath}`);
const file = await vaultAccess.adapterReadAuto(fullPath);
const isPlain = isPlainText(name);
const possiblyLarge = !isPlain;
if (possiblyLarge) Logger(`Reading : ${fullPath}`, LOG_LEVEL_VERBOSE);
const content = createBlob(file);
if (possiblyLarge) Logger(`Processing: ${fullPath}`, LOG_LEVEL_VERBOSE);
// const datatype = determineTypeFromBlob(content);
const bareFullPath = fullPath as FilePathWithPrefix;
const saveFullPath = prefix ? addPrefix(bareFullPath, prefix) : bareFullPath;
return {
name: name,
path: saveFullPath,
stat: {
size: content.size,
ctime: stat.ctime,
mtime: stat.mtime,
type: "file",
},
body: content,
}
}
export function TFileToUXFileInfoStub(file: TFile | TAbstractFile, deleted?: boolean): UXFileInfoStub {
if (!(file instanceof TFile)) {
throw new Error("Invalid file type");
}
const ret: UXFileInfoStub = {
name: file.name,
path: file.path as FilePathWithPrefix,
isFolder: false,
stat: {
size: file.stat.size,
mtime: file.stat.mtime,
ctime: file.stat.ctime,
type: "file",
},
deleted: deleted
}
return ret;
}
export function InternalFileToUXFileInfoStub(filename: FilePathWithPrefix, deleted?: boolean): UXInternalFileInfoStub {
const name = filename.split("/").pop() as string;
const ret: UXInternalFileInfoStub = {
name: name,
path: filename,
isFolder: false,
stat: undefined,
isInternal: true,
deleted
}
return ret;
}
export function TFolderToUXFileInfoStub(file: TFolder): UXFolderInfo {
const ret: UXFolderInfo = {
name: file.name,
path: file.path as FilePathWithPrefix,
parent: file.parent?.path as FilePath | undefined,
isFolder: true,
children: file.children.map(e => TFileToUXFileInfoStub(e)),
}
return ret;
}

328
src/modules/essential/ModuleInitializerFile.ts Normal file
View File

@@ -0,0 +1,328 @@
import { unique } from "octagonal-wheels/collection";
import { QueueProcessor } from "octagonal-wheels/concurrency/processor";
import { throttle } from "octagonal-wheels/function";
import { eventHub } from "../../common/events.ts";
import { BASE_IS_NEW, compareFileFreshness, EVEN, getPath, isValidPath, TARGET_IS_NEW } from "../../common/utils.ts";
import { type FilePathWithPrefixLC, type FilePathWithPrefix, type MetaEntry, isMetaEntry, type EntryDoc, LOG_LEVEL_VERBOSE, LOG_LEVEL_NOTICE, LOG_LEVEL_INFO, LOG_LEVEL_DEBUG, type UXFileInfoStub } from "../../lib/src/common/types.ts";
import { isAnyNote } from "../../lib/src/common/utils.ts";
import { stripAllPrefixes } from "../../lib/src/string_and_binary/path.ts";
import { AbstractModule } from "../AbstractModule.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleInitializerFile extends AbstractModule implements ICoreModule {
async $$performFullScan(showingNotice?: boolean): Promise<void> {
this._log("Opening the key-value database", LOG_LEVEL_VERBOSE);
const isInitialized = await (this.core.kvDB.get<boolean>("initialized")) || false;
// synchronize all files between database and storage.
if (!this.settings.isConfigured) {
if (showingNotice) {
this._log("LiveSync is not configured yet. Synchronising between the storage and the local database is now prevented.", LOG_LEVEL_NOTICE, "syncAll");
}
return;
}
if (showingNotice) {
this._log("Initializing", LOG_LEVEL_NOTICE, "syncAll");
}
this._log("Initialize and checking database files");
this._log("Checking deleted files");
await this.collectDeletedFiles();
this._log("Collecting local files on the storage", LOG_LEVEL_VERBOSE);
const filesStorageSrc = this.core.storageAccess.getFiles();
const _filesStorage = [] as typeof filesStorageSrc;
for (const f of filesStorageSrc) {
if (await this.core.$$isTargetFile(f.path, f != filesStorageSrc[0])) {
_filesStorage.push(f);
}
}
const convertCase = <FilePathWithPrefix>(path: FilePathWithPrefix): FilePathWithPrefixLC => {
if (this.settings.handleFilenameCaseSensitive) {
return path as FilePathWithPrefixLC;
}
return (path as string).toLowerCase() as FilePathWithPrefixLC;
}
// If handleFilenameCaseSensitive is enabled, `FilePathWithPrefixLC` is the same as `FilePathWithPrefix`.
const storageFileNameMap = Object.fromEntries(_filesStorage.map((e) => [
e.path, e] as [FilePathWithPrefix, UXFileInfoStub]));
const storageFileNames = Object.keys(storageFileNameMap) as FilePathWithPrefix[];
const storageFileNameCapsPair = storageFileNames.map((e) => [
e, convertCase(e)] as [FilePathWithPrefix, FilePathWithPrefixLC]);
// const storageFileNameCS2CI = Object.fromEntries(storageFileNameCapsPair) as Record<FilePathWithPrefix, FilePathWithPrefixLC>;
const storageFileNameCI2CS = Object.fromEntries(storageFileNameCapsPair.map(e => [
e[1], e[0]])) as Record<FilePathWithPrefixLC, FilePathWithPrefix>;
this._log("Collecting local files on the DB", LOG_LEVEL_VERBOSE);
const _DBEntries = [] as MetaEntry[];
// const _DBEntriesTask = [] as (() => Promise<MetaEntry | false>)[];
let count = 0;
for await (const doc of this.localDatabase.findAllNormalDocs()) {
count++;
if (count % 25 == 0) this._log(`Collecting local files on the DB: ${count}`, showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO, "syncAll");
const path = getPath(doc);
if (isValidPath(path) && await this.core.$$isTargetFile(path, true)) {
if (!isMetaEntry(doc)) {
this._log(`Invalid entry: ${path}`, LOG_LEVEL_INFO);
continue;
}
_DBEntries.push(doc);
}
}
const databaseFileNameMap = Object.fromEntries(_DBEntries.map((e) => [
getPath(e), e] as [FilePathWithPrefix, MetaEntry]));
const databaseFileNames = Object.keys(databaseFileNameMap) as FilePathWithPrefix[];
const databaseFileNameCapsPair = databaseFileNames.map((e) => [
e, convertCase(e)] as [FilePathWithPrefix, FilePathWithPrefixLC]);
// const databaseFileNameCS2CI = Object.fromEntries(databaseFileNameCapsPair) as Record<FilePathWithPrefix, FilePathWithPrefixLC>;
const databaseFileNameCI2CS = Object.fromEntries(databaseFileNameCapsPair.map(e => [
e[1], e[0]])) as Record<FilePathWithPrefix, FilePathWithPrefixLC>;
const allFiles = unique([
...Object.keys(databaseFileNameCI2CS),
...Object.keys(storageFileNameCI2CS)]) as FilePathWithPrefixLC[];
this._log(`Total files in the database: ${databaseFileNames.length}`, LOG_LEVEL_VERBOSE, "syncAll");
this._log(`Total files in the storage: ${storageFileNames.length}`, LOG_LEVEL_VERBOSE, "syncAll");
this._log(`Total files: ${allFiles.length}`, LOG_LEVEL_VERBOSE, "syncAll");
const filesExistOnlyInStorage = allFiles.filter((e) => !databaseFileNameCI2CS[e]);
const filesExistOnlyInDatabase = allFiles.filter((e) => !storageFileNameCI2CS[e]);
const filesExistBoth = allFiles.filter((e) => databaseFileNameCI2CS[e] && storageFileNameCI2CS[e]);
this._log(`Files exist only in storage: ${filesExistOnlyInStorage.length}`, LOG_LEVEL_VERBOSE, "syncAll");
this._log(`Files exist only in database: ${filesExistOnlyInDatabase.length}`, LOG_LEVEL_VERBOSE, "syncAll");
this._log(`Files exist both in storage and database: ${filesExistBoth.length}`, LOG_LEVEL_VERBOSE, "syncAll");
this._log("Synchronising...");
const processStatus = {} as Record<string, string>;
const logLevel = showingNotice ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
const updateLog = throttle((key: string, msg: string) => {
processStatus[key] = msg;
const log = Object.values(processStatus).join("\n");
this._log(log, logLevel, "syncAll");
}, 25);
const initProcess = [];
const runAll = async <T>(procedureName: string, objects: T[], callback: (arg: T) => Promise<void>) => {
if (objects.length == 0) {
this._log(`${procedureName}: Nothing to do`);
return;
}
this._log(procedureName);
if (!this.localDatabase.isReady) throw Error("Database is not ready!");
let success = 0;
let failed = 0;
const step = 10;
const processor = new QueueProcessor(async (e) => {
try {
await callback(e[0]);
success++;
// return
} catch (ex) {
this._log(`Error while ${procedureName}`, LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
failed++;
}
if ((success + failed) % step == 0) {
const msg = `${procedureName}: DONE:${success}, FAILED:${failed}, LAST:${processor._queue.length}`;
updateLog(procedureName, msg);
}
return;
}, {
batchSize: 1,
concurrentLimit: 10,
delay: 0,
suspended: true,
maintainDelay: false,
interval: 0
}, objects)
await processor.waitForAllDoneAndTerminate();
const msg = `${procedureName} All done: DONE:${success}, FAILED:${failed}`;
updateLog(procedureName, msg)
}
initProcess.push(runAll("UPDATE DATABASE", filesExistOnlyInStorage, async (e) => {
// console.warn("UPDATE DATABASE", e);
const file = storageFileNameMap[storageFileNameCI2CS[e]];
if (!this.core.$$isFileSizeExceeded(file.stat.size)) {
const path = file.path;
await this.core.fileHandler.storeFileToDB(file);
// fireAndForget(() => this.checkAndApplySettingFromMarkdown(path, true));
eventHub.emitEvent("event-file-changed", { file: path, automated: true });
} else {
this._log(`UPDATE DATABASE: ${e} has been skipped due to file size exceeding the limit`, logLevel);
}
}));
initProcess.push(runAll("UPDATE STORAGE", filesExistOnlyInDatabase, async (e) => {
const w = databaseFileNameMap[databaseFileNameCI2CS[e]];
const path = getPath(w) ?? e;
if (w && !(w.deleted || w._deleted)) {
if (!this.core.$$isFileSizeExceeded(w.size)) {
// await this.pullFile(path, undefined, false, undefined, false);
// Memo: No need to force
await this.core.fileHandler.dbToStorage(path, null, true);
// fireAndForget(() => this.checkAndApplySettingFromMarkdown(e, true));
eventHub.emitEvent("event-file-changed", {
file: e, automated: true
});
this._log(`Check or pull from db:${path} OK`);
} else {
this._log(`UPDATE STORAGE: ${path} has been skipped due to file size exceeding the limit`, logLevel);
}
} else if (w) {
this._log(`Deletion history skipped: ${path}`, LOG_LEVEL_VERBOSE);
} else {
this._log(`entry not found: ${path}`);
}
}));
const fileMap = filesExistBoth.map(path => {
const file = storageFileNameMap[storageFileNameCI2CS[path]];
const doc = databaseFileNameMap[databaseFileNameCI2CS[path]];
return { file, doc }
})
initProcess.push(runAll("SYNC DATABASE AND STORAGE", fileMap, async (e) => {
const { file, doc } = e;
if (!this.core.$$isFileSizeExceeded(file.stat.size) && !this.core.$$isFileSizeExceeded(doc.size)) {
await this.syncFileBetweenDBandStorage(file, doc);
// fireAndForget(() => this.checkAndApplySettingFromMarkdown(getPath(doc), true));
eventHub.emitEvent("event-file-changed", {
file: getPath(doc), automated: true
});
} else {
this._log(`SYNC DATABASE AND STORAGE: ${getPath(doc)} has been skipped due to file size exceeding the limit`, logLevel);
}
}))
await Promise.all(initProcess);
// this.setStatusBarText(`NOW TRACKING!`);
this._log("Initialized, NOW TRACKING!");
if (!isInitialized) {
await (this.core.kvDB.set("initialized", true))
}
if (showingNotice) {
this._log("Initialize done!", LOG_LEVEL_NOTICE, "syncAll");
}
}
async syncFileBetweenDBandStorage(file: UXFileInfoStub, doc: MetaEntry) {
if (!doc) {
throw new Error(`Missing doc:${(file as any).path}`)
}
if ("path" in file) {
const w = this.core.storageAccess.getFileStub((file as any).path);
if (w) {
file = w;
} else {
throw new Error(`Missing file:${(file as any).path}`)
}
}
const compareResult = compareFileFreshness(file, doc);
switch (compareResult) {
case BASE_IS_NEW:
if (!this.core.$$isFileSizeExceeded(file.stat.size)) {
this._log("STORAGE -> DB :" + file.path);
await this.core.fileHandler.storeFileToDB(file);
eventHub.emitEvent("event-file-changed", {
file: file.path, automated: true
});
} else {
this._log(`STORAGE -> DB : ${file.path} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
}
break;
case TARGET_IS_NEW:
if (!this.core.$$isFileSizeExceeded(doc.size)) {
this._log("STORAGE <- DB :" + file.path);
if (!await this.core.fileHandler.dbToStorage(doc, stripAllPrefixes(file.path), true)) {
this._log(`STORAGE <- DB : Cloud not read ${file.path}, possibly deleted`, LOG_LEVEL_NOTICE);
}
return caches;
} else {
this._log(`STORAGE <- DB : ${file.path} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
}
break;
case EVEN:
this._log("STORAGE == DB :" + file.path + "", LOG_LEVEL_DEBUG);
break;
default:
this._log("STORAGE ?? DB :" + file.path + " Something got weird");
}
}
// This method uses an old version of database accessor, which is not recommended.
// TODO: Fix
async collectDeletedFiles() {
const limitDays = this.settings.automaticallyDeleteMetadataOfDeletedFiles;
if (limitDays <= 0) return;
this._log(`Checking expired file history`);
const limit = Date.now() - (86400 * 1000 * limitDays);
const notes: {
path: string,
mtime: number,
ttl: number,
doc: PouchDB.Core.ExistingDocument<EntryDoc & PouchDB.Core.AllDocsMeta>
}[] = [];
for await (const doc of this.localDatabase.findAllDocs({ conflicts: true })) {
if (isAnyNote(doc)) {
if (doc.deleted && (doc.mtime - limit) < 0) {
notes.push({
path: getPath(doc),
mtime: doc.mtime,
ttl: (doc.mtime - limit) / 1000 / 86400,
doc: doc
});
}
}
}
if (notes.length == 0) {
this._log("There are no old documents");
this._log(`Checking expired file history done`);
return;
}
for (const v of notes) {
this._log(`Deletion history expired: ${v.path}`);
const delDoc = v.doc;
delDoc._deleted = true;
await this.localDatabase.putRaw(delDoc);
}
this._log(`Checking expired file history done`);
}
async $$initializeDatabase(showingNotice: boolean = false, reopenDatabase = true): Promise<boolean> {
this.core.isReady = false;
if ((!reopenDatabase) || await this.core.$$openDatabase()) {
if (this.localDatabase.isReady) {
await this.core.$$performFullScan(showingNotice);
}
if (!await this.core.$everyOnDatabaseInitialized(showingNotice)) {
this._log(`Initializing database has been failed on some module`, LOG_LEVEL_NOTICE);
return false;
}
this.core.isReady = true;
// run queued event once.
await this.core.$everyCommitPendingFileEvent();
return true;
} else {
this.core.isReady = false;
return false;
}
}
}

90
src/modules/essential/ModuleKeyValueDB.ts Normal file
View File

@@ -0,0 +1,90 @@
import { delay, yieldMicrotask } from "octagonal-wheels/promises";
import { OpenKeyValueDatabase } from "../../common/KeyValueDB.ts";
import type { LiveSyncLocalDB } from "../../lib/src/pouchdb/LiveSyncLocalDB.ts";
import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { AbstractModule } from "../AbstractModule.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
export class ModuleKeyValueDB extends AbstractModule implements ICoreModule {
tryCloseKvDB() {
try {
this.core.kvDB?.close();
return true;
} catch (e) {
this._log("Failed to close KeyValueDB", LOG_LEVEL_VERBOSE);
this._log(e);
return false;
}
}
async openKeyValueDB(): Promise<boolean> {
await delay(10);
try {
this.tryCloseKvDB();
await delay(10);
await yieldMicrotask();
this.core.kvDB = await OpenKeyValueDatabase(this.core.$$getVaultName() + "-livesync-kv");
await yieldMicrotask();
await delay(100);
} catch (e) {
this.core.kvDB = undefined!;
this._log("Failed to open KeyValueDB", LOG_LEVEL_NOTICE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
return true;
}
$allOnDBUnload(db: LiveSyncLocalDB): void {
if (this.core.kvDB) this.core.kvDB.close();
}
$allOnDBClose(db: LiveSyncLocalDB): void {
if (this.core.kvDB) this.core.kvDB.close();
}
async $everyOnloadAfterLoadSettings(): Promise<boolean> {
if (!await this.openKeyValueDB()) {
return false;
}
this.core.simpleStore = this.core.$$getSimpleStore<any>("os");
return Promise.resolve(true);
}
$$getSimpleStore<T>(kind: string) {
const prefix = `${kind}-`;
return {
get: async (key: string): Promise<T> => {
return await this.core.kvDB.get(`${prefix}${key}`);
},
set: async (key: string, value: any): Promise<void> => {
await this.core.kvDB.set(`${prefix}${key}`, value);
},
delete: async (key: string): Promise<void> => {
await this.core.kvDB.del(`${prefix}${key}`);
},
keys: async (from: string | undefined, to: string | undefined, count?: number | undefined): Promise<string[]> => {
const ret = this.core.kvDB.keys(IDBKeyRange.bound(`${prefix}${from || ""}`, `${prefix}${to || ""}`), count);
return (await ret).map(e => e.toString()).filter(e => e.startsWith(prefix)).map(e => e.substring(prefix.length));
}
}
}
$everyOnInitializeDatabase(db: LiveSyncLocalDB): Promise<boolean> {
return this.openKeyValueDB();
}
async $everyOnResetDatabase(db: LiveSyncLocalDB): Promise<boolean> {
try {
const kvDBKey = "queued-files"
await this.core.kvDB.del(kvDBKey);
// localStorage.removeItem(lsKey);
await this.core.kvDB.destroy();
await yieldMicrotask();
this.core.kvDB = await OpenKeyValueDatabase(this.core.$$getVaultName() + "-livesync-kv");
await delay(100);
} catch (e) {
this.core.kvDB = undefined!;
this._log("Failed to reset KeyValueDB", LOG_LEVEL_NOTICE);
this._log(e, LOG_LEVEL_VERBOSE);
return false;
}
return true;
}
}

240
src/modules/essential/ModuleMigration.ts Normal file
View File

@@ -0,0 +1,240 @@
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from 'octagonal-wheels/common/logger.js';
import { SETTING_VERSION_SUPPORT_CASE_INSENSITIVE } from '../../lib/src/common/types.js';
import { EVENT_REQUEST_OPEN_SETTING_WIZARD, EVENT_REQUEST_OPEN_SETTINGS, EVENT_REQUEST_OPEN_SETUP_URI, eventHub } from '../../common/events.ts';
import { AbstractModule } from "../AbstractModule.ts";
import type { ICoreModule } from "../ModuleTypes.ts";
const URI_DOC = "https://github.com/vrtmrz/obsidian-livesync/blob/main/README.md#how-to-use";
export class ModuleMigration extends AbstractModule implements ICoreModule {
async migrateDisableBulkSend() {
if (this.settings.sendChunksBulk) {
this._log("Send chunks in bulk has been enabled, however, this feature had been corrupted. Sorry for your inconvenience. Automatically disabled.", LOG_LEVEL_NOTICE);
this.settings.sendChunksBulk = false;
this.settings.sendChunksBulkMaxSize = 1;
await this.saveSettings();
}
}
async migrationCheck() {
const old = this.settings.settingVersion;
const current = SETTING_VERSION_SUPPORT_CASE_INSENSITIVE;
// Check each migrations(old -> current)
if (!await this.migrateToCaseInsensitive(old, current)) {
this._log(`Migration failed or cancelled from ${old} to ${current}`, LOG_LEVEL_NOTICE);
return;
}
}
async migrateToCaseInsensitive(old: number, current: number) {
if (this.settings.handleFilenameCaseSensitive !== undefined && this.settings.doNotUseFixedRevisionForChunks !== undefined) {
if (current < SETTING_VERSION_SUPPORT_CASE_INSENSITIVE) {
this.settings.settingVersion = SETTING_VERSION_SUPPORT_CASE_INSENSITIVE;
await this.saveSettings();
}
return true;
}
if (old >= SETTING_VERSION_SUPPORT_CASE_INSENSITIVE && this.settings.handleFilenameCaseSensitive !== undefined && this.settings.doNotUseFixedRevisionForChunks !== undefined) {
return true;
}
let remoteHandleFilenameCaseSensitive: undefined | boolean = undefined;
let remoteDoNotUseFixedRevisionForChunks: undefined | boolean = undefined;
let remoteChecked = false;
try {
const remoteInfo = await this.core.replicator.getRemotePreferredTweakValues(this.settings);
if (remoteInfo) {
remoteHandleFilenameCaseSensitive = "handleFilenameCaseSensitive" in remoteInfo ? remoteInfo.handleFilenameCaseSensitive : false;
remoteDoNotUseFixedRevisionForChunks = "doNotUseFixedRevisionForChunks" in remoteInfo ? remoteInfo.doNotUseFixedRevisionForChunks : false;
if (remoteHandleFilenameCaseSensitive !== undefined || remoteDoNotUseFixedRevisionForChunks !== undefined) {
remoteChecked = true;
}
} else {
this._log("Failed to fetch remote tweak values", LOG_LEVEL_INFO);
}
} catch (ex) {
this._log("Could not get remote tweak values", LOG_LEVEL_INFO);
this._log(ex, LOG_LEVEL_VERBOSE);
}
if (remoteChecked) {
// The case that the remote could be checked.
if (remoteHandleFilenameCaseSensitive && remoteDoNotUseFixedRevisionForChunks) {
// Migrated, but configured as same as old behaviour.
this.settings.handleFilenameCaseSensitive = true;
this.settings.doNotUseFixedRevisionForChunks = true;
this.settings.settingVersion = SETTING_VERSION_SUPPORT_CASE_INSENSITIVE;
this._log(`Migrated to db:${current} with the same behaviour as before`, LOG_LEVEL_INFO);
await this.saveSettings();
return true;
}
const message = `As you may already know, the self-hosted LiveSync has changed its default behaviour and database structure.
And thankfully, with your time and efforts, the remote database appears to have already been migrated. Congratulations!
However, we need a bit more. The configuration of this device is not compatible with the remote database. We will need to fetch the remote database again. Should we fetch from the remote again now?
___Note: We cannot synchronise until the configuration has been changed and the database has been fetched again.___
___Note2: The chunks are completely immutable, we can fetch only the metadata and difference.___
`;
const OPTION_FETCH = "Yes, fetch again";
const DISMISS = "No, please ask again";
const options = [OPTION_FETCH, DISMISS];
const ret = await this.core.confirm.confirmWithMessage("Case Sensitivity", message, options, "No, please ask again", 40);
if (ret == OPTION_FETCH) {
this.settings.handleFilenameCaseSensitive = remoteHandleFilenameCaseSensitive || false;
this.settings.doNotUseFixedRevisionForChunks = remoteDoNotUseFixedRevisionForChunks || false;
this.settings.settingVersion = SETTING_VERSION_SUPPORT_CASE_INSENSITIVE;
await this.saveSettings();
try {
await this.core.rebuilder.scheduleFetch();
return;
} catch (ex) {
this._log("Failed to create redflag2", LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
return false;
} else {
return false;
}
}
const ENABLE_BOTH = "Enable both";
const ENABLE_FILENAME_CASE_INSENSITIVE = "Enable only #1";
const ENABLE_FIXED_REVISION_FOR_CHUNKS = "Enable only #2";
const ADJUST_TO_REMOTE = "Adjust to remote";
const DISMISS = "Decide it later";
const KEEP = "Keep previous behaviour";
const message = `Since v0.23.21, the self-hosted LiveSync has changed the default behaviour and database structure. The following changes have been made:
1. **Case sensitivity of filenames**
The handling of filenames is now case-insensitive. This is a beneficial change for most platforms, other than Linux and iOS, which do not manage filename case sensitivity effectively.
(On These, a warning will be displayed for files with the same name but different cases).
2. **Revision handling of the chunks**
Chunks are immutable, which allows their revisions to be fixed. This change will enhance the performance of file saving.
___However, to enable either of these changes, both remote and local databases need to be rebuilt. This process takes a few minutes, and we recommend doing it when you have ample time.___
- If you wish to maintain the previous behaviour, you can skip this process by using \`${KEEP}\`.
- If you do not have enough time, please choose \`${DISMISS}\`. You will be prompted again later.
- If you have rebuilt the database on another device, please select \`${DISMISS}\` and try synchronizing again. Since a difference has been detected, you will be prompted again.
`;
const options = [
ENABLE_BOTH,
ENABLE_FILENAME_CASE_INSENSITIVE,
ENABLE_FIXED_REVISION_FOR_CHUNKS];
if (remoteChecked) {
options.push(ADJUST_TO_REMOTE);
}
options.push(KEEP, DISMISS);
const ret = await this.core.confirm.confirmWithMessage("Case Sensitivity", message, options, DISMISS, 40);
console.dir(ret);
switch (ret) {
case ENABLE_BOTH:
this.settings.handleFilenameCaseSensitive = false;
this.settings.doNotUseFixedRevisionForChunks = false;
break;
case ENABLE_FILENAME_CASE_INSENSITIVE:
this.settings.handleFilenameCaseSensitive = false;
this.settings.doNotUseFixedRevisionForChunks = true;
break;
case ENABLE_FIXED_REVISION_FOR_CHUNKS:
this.settings.doNotUseFixedRevisionForChunks = false;
this.settings.handleFilenameCaseSensitive = true;
break;
case KEEP:
this.settings.handleFilenameCaseSensitive = true;
this.settings.doNotUseFixedRevisionForChunks = true;
this.settings.settingVersion = SETTING_VERSION_SUPPORT_CASE_INSENSITIVE;
await this.saveSettings();
return true;
case DISMISS:
default:
return false;
}
this.settings.settingVersion = SETTING_VERSION_SUPPORT_CASE_INSENSITIVE;
await this.saveSettings();
await this.core.rebuilder.scheduleRebuild();
await this.core.$$performRestart();
}
async initialMessage() {
const message = `Your device has **not been set up yet**. Let me guide you through the setup process.
Please keep in mind that every dialogue content can be copied to the clipboard. If you need to refer to it later, you can paste it into a note in Obsidian. You can also translate it into your language using a translation tool.
First, do you have **Setup URI**?
Note: If you do not know what it is, please refer to the [documentation](${URI_DOC}).
`;
const USE_SETUP = "Yes, I have";
const NEXT = "No, I do not have";
const ret = await this.core.confirm.askSelectStringDialogue(message, [
USE_SETUP, NEXT], {
title: "Welcome to Self-hosted LiveSync",
defaultAction: USE_SETUP
});
if (ret === USE_SETUP) {
eventHub.emitEvent(EVENT_REQUEST_OPEN_SETUP_URI);
return false;
}
else if (ret == NEXT) {
return true;
}
return false;
}
async askAgainForSetupURI() {
const message = `We strongly recommend that you generate a set-up URI and use it.
If you do not have knowledge about it, please refer to the [documentation](${URI_DOC}) (Sorry again, but it is important).
How do you want to set it up manually?`;
const USE_MINIMAL = "Take me into the setup wizard";
const USE_SETUP = "Set it up all manually";
const NEXT = "Remind me at the next launch";
const ret = await this.core.confirm.askSelectStringDialogue(message, [
USE_MINIMAL, USE_SETUP, NEXT], {
title: "Recommendation to use Setup URI",
defaultAction: USE_MINIMAL
});
if (ret === USE_MINIMAL) {
eventHub.emitEvent(EVENT_REQUEST_OPEN_SETTING_WIZARD);
return false;
}
if (ret === USE_SETUP) {
eventHub.emitEvent(EVENT_REQUEST_OPEN_SETTINGS);
return false;
}
else if (ret == NEXT) {
return false;
}
return false;
}
async $everyOnFirstInitialize(): Promise<boolean> {
if (!this.localDatabase.isReady) {
this._log(`Something went wrong! The local database is not ready`, LOG_LEVEL_NOTICE);
return false;
}
if (this.settings.isConfigured) {
await this.migrationCheck();
await this.migrateDisableBulkSend();
}
if (!this.settings.isConfigured) {
// Case sensitivity
if (!await this.initialMessage() || !await this.askAgainForSetupURI()) {
this._log("The setup has been cancelled, Self-hosted LiveSync waiting for your setup!", LOG_LEVEL_NOTICE);
return false;
}
}
return true;
}
}

2
src/common/ObsHttpHandler.ts → src/modules/essentialObsidian/APILib/ObsHttpHandler.ts
View File

@@ -11,7 +11,7 @@ import { HttpRequest, HttpResponse, type HttpHandlerOptions } from "@smithy/prot
//@ts-ignore
import { requestTimeout } from "@smithy/fetch-http-handler/dist-es/request-timeout";
import { buildQueryString } from "@smithy/querystring-builder";
import { requestUrl, type RequestUrlParam } from "../deps.ts";
import { requestUrl, type RequestUrlParam } from "../../../deps.ts";
////////////////////////////////////////////////////////////////////////////////
// special handler using Obsidian requestUrl
////////////////////////////////////////////////////////////////////////////////

207
src/modules/essentialObsidian/ModuleObsidianAPI.ts Normal file
View File

@@ -0,0 +1,207 @@
import { AbstractObsidianModule, type IObsidianModule } from '../AbstractObsidianModule.ts';
import { LOG_LEVEL_DEBUG, LOG_LEVEL_VERBOSE } from 'octagonal-wheels/common/logger';
import { Notice, requestUrl, type RequestUrlParam, type RequestUrlResponse } from '../../deps.ts';
import { type EntryDoc, type FilePathWithPrefix } from '../../lib/src/common/types.ts';
import { getPathFromTFile } from '../../common/utils.ts';
import { disableEncryption, enableEncryption, isCloudantURI, isValidRemoteCouchDBURI, replicationFilter } from '../../lib/src/pouchdb/utils_couchdb.ts';
import { setNoticeClass } from '../../lib/src/mock_and_interop/wrapper.ts';
import { ObsHttpHandler } from './APILib/ObsHttpHandler.ts';
import { PouchDB } from '../../lib/src/pouchdb/pouchdb-browser.ts';
import { reactive, reactiveSource } from 'octagonal-wheels/dataobject/reactive';
setNoticeClass(Notice);
async function fetchByAPI(request: RequestUrlParam): Promise<RequestUrlResponse> {
const ret = await requestUrl(request);
if (ret.status - (ret.status % 100) !== 200) {
const er: Error & { status?: number } = new Error(`Request Error:${ret.status}`);
if (ret.json) {
er.message = ret.json.reason;
er.name = `${ret.json.error ?? ""}:${ret.json.message ?? ""}`;
}
er.status = ret.status;
throw er;
}
return ret;
}
export class ModuleObsidianAPI extends AbstractObsidianModule implements IObsidianModule {
_customHandler!: ObsHttpHandler;
authHeaderSource = reactiveSource<string>("");
authHeader = reactive(() =>
this.authHeaderSource.value == "" ? "" : "Basic " + window.btoa(this.authHeaderSource.value));
$$customFetchHandler(): ObsHttpHandler {
if (!this._customHandler) this._customHandler = new ObsHttpHandler(undefined, undefined);
return this._customHandler;
}
async $$connectRemoteCouchDB(uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | false, useDynamicIterationCount: boolean, performSetup: boolean, skipInfo: boolean, compression: boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> {
if (!isValidRemoteCouchDBURI(uri)) return "Remote URI is not valid";
if (uri.toLowerCase() != uri) return "Remote URI and database name could not contain capital letters.";
if (uri.indexOf(" ") !== -1) return "Remote URI and database name could not contain spaces.";
const userNameAndPassword = (auth.username && auth.password) ? `${auth.username}:${auth.password}` : "";
if (this.authHeaderSource.value != userNameAndPassword) {
this.authHeaderSource.value = userNameAndPassword;
}
const authHeader = this.authHeader.value;
// const _this = this;
const conf: PouchDB.HttpAdapter.HttpAdapterConfiguration = {
adapter: "http",
auth,
skip_setup: !performSetup,
fetch: async (url: string | Request, opts?: RequestInit) => {
let size = "";
const localURL = url.toString().substring(uri.length);
const method = opts?.method ?? "GET";
if (opts?.body) {
const opts_length = opts.body.toString().length;
if (opts_length > 1000 * 1000 * 10) {
// over 10MB
if (isCloudantURI(uri)) {
this.plugin.last_successful_post = false;
this._log("This request should fail on IBM Cloudant.", LOG_LEVEL_VERBOSE);
throw new Error("This request should fail on IBM Cloudant.");
}
}
size = ` (${opts_length})`;
}
if (!disableRequestURI && typeof url == "string" && typeof (opts?.body ?? "") == "string") {
const body = opts?.body as string;
const transformedHeaders = { ...(opts?.headers as Record<string, string>) };
if (authHeader != "") transformedHeaders["authorization"] = authHeader;
delete transformedHeaders["host"];
delete transformedHeaders["Host"];
delete transformedHeaders["content-length"];
delete transformedHeaders["Content-Length"];
const requestParam: RequestUrlParam = {
url,
method: opts?.method,
body: body,
headers: transformedHeaders,
contentType: "application/json",
// contentType: opts.headers,
};
try {
this.plugin.requestCount.value = this.plugin.requestCount.value + 1;
const r = await fetchByAPI(requestParam);
if (method == "POST" || method == "PUT") {
this.plugin.last_successful_post = r.status - (r.status % 100) == 200;
} else {
this.plugin.last_successful_post = true;
}
this._log(`HTTP:${method}${size} to:${localURL} -> ${r.status}`, LOG_LEVEL_DEBUG);
return new Response(r.arrayBuffer, {
headers: r.headers,
status: r.status,
statusText: `${r.status}`,
});
} catch (ex) {
this._log(`HTTP:${method}${size} to:${localURL} -> failed`, LOG_LEVEL_VERBOSE);
// limit only in bulk_docs.
if (url.toString().indexOf("_bulk_docs") !== -1) {
this.plugin.last_successful_post = false;
}
this._log(ex);
throw ex;
} finally {
this.plugin.responseCount.value = this.plugin.responseCount.value + 1;
}
}
try {
if (this.settings.enableDebugTools) {
// Issue #407
(opts!.headers as Headers).append("ngrok-skip-browser-warning", "123");
}
this.plugin.requestCount.value = this.plugin.requestCount.value + 1;
const response: Response = await fetch(url, opts);
if (method == "POST" || method == "PUT") {
this.plugin.last_successful_post = response.ok;
} else {
this.plugin.last_successful_post = true;
}
this._log(`HTTP:${method}${size} to:${localURL} -> ${response.status}`, LOG_LEVEL_DEBUG);
if (Math.floor(response.status / 100) !== 2) {
if (method != "GET" && localURL.indexOf("/_local/") === -1 && !localURL.endsWith("/")) {
const r = response.clone();
this._log(`The request may have failed. The reason sent by the server: ${r.status}: ${r.statusText}`);
try {
this._log(await (await r.blob()).text(), LOG_LEVEL_VERBOSE);
} catch (_) {
this._log("Cloud not parse response", LOG_LEVEL_VERBOSE);
this._log(_, LOG_LEVEL_VERBOSE);
}
} else {
this._log(`Just checkpoint or some server information has been missing. The 404 error shown above is not an error.`, LOG_LEVEL_VERBOSE)
}
}
return response;
} catch (ex) {
this._log(`HTTP:${method}${size} to:${localURL} -> failed`, LOG_LEVEL_VERBOSE);
// limit only in bulk_docs.
if (url.toString().indexOf("_bulk_docs") !== -1) {
this.plugin.last_successful_post = false;
}
this._log(ex);
throw ex;
} finally {
this.plugin.responseCount.value = this.plugin.responseCount.value + 1;
}
// return await fetch(url, opts);
},
};
const db: PouchDB.Database<EntryDoc> = new PouchDB<EntryDoc>(uri, conf);
replicationFilter(db, compression);
disableEncryption();
if (passphrase !== "false" && typeof passphrase === "string") {
enableEncryption(db, passphrase, useDynamicIterationCount, false);
}
if (skipInfo) {
return { db: db, info: { db_name: "", doc_count: 0, update_seq: "" } };
}
try {
const info = await db.info();
return { db: db, info: info };
} catch (ex: any) {
let msg = `${ex?.name}:${ex?.message}`;
if (ex?.name == "TypeError" && ex?.message == "Failed to fetch") {
msg += "\n**Note** This error caused by many reasons. The only sure thing is you didn't touch the server.\nTo check details, open inspector.";
}
this._log(ex, LOG_LEVEL_VERBOSE);
return msg;
}
}
$$isMobile(): boolean {
//@ts-ignore : internal API
return this.app.isMobile;
}
$$vaultName(): string {
return this.app.vault.getName();
}
$$getVaultName(): string {
return this.core.$$vaultName() + (this.settings?.additionalSuffixOfDatabaseName ? ("-" + this.settings.additionalSuffixOfDatabaseName) : "");
}
$$getActiveFilePath(): FilePathWithPrefix | undefined {
const file = this.app.workspace.getActiveFile();
if (file) {
return getPathFromTFile(file);
}
return undefined;
}
$anyGetAppId(): Promise<string | undefined> {
return Promise.resolve(`${("appId" in this.app ? this.app.appId : "")}`);
}
}

217
src/modules/essentialObsidian/ModuleObsidianEvents.ts Normal file
View File

@@ -0,0 +1,217 @@
import { AbstractObsidianModule, type IObsidianModule } from '../AbstractObsidianModule.ts';
import { EVENT_FILE_RENAMED, EVENT_LEAF_ACTIVE_CHANGED, eventHub } from '../../common/events.js';
import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from 'octagonal-wheels/common/logger';
import { scheduleTask } from 'octagonal-wheels/concurrency/task';
import { type TFile } from '../../deps.ts';
import { fireAndForget } from 'octagonal-wheels/promises';
import { type FilePathWithPrefix } from '../../lib/src/common/types.ts';
import { reactive, reactiveSource } from 'octagonal-wheels/dataobject/reactive';
import { collectingChunks, pluginScanningCount, hiddenFilesEventCount, hiddenFilesProcessingCount } from '../../lib/src/mock_and_interop/stores.ts';
export class ModuleObsidianEvents extends AbstractObsidianModule implements IObsidianModule {
$everyOnloadStart(): Promise<boolean> {
// this.registerEvent(this.app.workspace.on("editor-change", ));
this.plugin.registerEvent(this.app.vault.on("rename", (file, oldPath) => {
eventHub.emitEvent(EVENT_FILE_RENAMED, { newPath: file.path, old: oldPath });
}));
this.plugin.registerEvent(this.app.workspace.on("active-leaf-change", () => eventHub.emitEvent(EVENT_LEAF_ACTIVE_CHANGED)));
return Promise.resolve(true);
}
$$performRestart(): void {
this._performAppReload();
}
_performAppReload() {
//@ts-ignore
this.app.commands.executeCommandById("app:reload");
}
initialCallback: any;
swapSaveCommand() {
this._log("Modifying callback of the save command", LOG_LEVEL_VERBOSE);
const saveCommandDefinition = (this.app as any).commands?.commands?.[
"editor:save-file"
];
const save = saveCommandDefinition?.callback;
if (typeof save === "function") {
this.initialCallback = save;
saveCommandDefinition.callback = () => {
scheduleTask("syncOnEditorSave", 250, () => {
if (this.plugin._unloaded) {
this._log("Unload and remove the handler.", LOG_LEVEL_VERBOSE);
saveCommandDefinition.callback = this.initialCallback;
this.initialCallback = undefined;
} else {
if (this.settings.syncOnEditorSave) {
this._log("Sync on Editor Save.", LOG_LEVEL_VERBOSE);
fireAndForget(() => this.core.$$replicate());
}
}
});
save();
};
}
// eslint-disable-next-line @typescript-eslint/no-this-alias
const _this = this;
//@ts-ignore
window.CodeMirrorAdapter.commands.save = () => {
//@ts-ignore
_this.app.commands.executeCommandById('editor:save-file')
// _this.app.performCommand('editor:save-file');
};
}
registerWatchEvents() {
this.setHasFocus = this.setHasFocus.bind(this);
this.watchWindowVisibility = this.watchWindowVisibility.bind(this);
this.watchWorkspaceOpen = this.watchWorkspaceOpen.bind(this);
this.watchOnline = this.watchOnline.bind(this);
this.plugin.registerEvent(this.app.workspace.on("file-open", this.watchWorkspaceOpen));
this.plugin.registerDomEvent(document, "visibilitychange", this.watchWindowVisibility);
this.plugin.registerDomEvent(window, "focus", () => this.setHasFocus(true));
this.plugin.registerDomEvent(window, "blur", () => this.setHasFocus(false));
this.plugin.registerDomEvent(window, "online", this.watchOnline);
this.plugin.registerDomEvent(window, "offline", this.watchOnline);
}
hasFocus = true;
isLastHidden = false;
setHasFocus(hasFocus: boolean) {
this.hasFocus = hasFocus;
this.watchWindowVisibility();
}
watchWindowVisibility() {
scheduleTask("watch-window-visibility", 100, () => fireAndForget(() => this.watchWindowVisibilityAsync()));
}
watchOnline() {
scheduleTask("watch-online", 500, () => fireAndForget(() => this.watchOnlineAsync()));
}
async watchOnlineAsync() {
// If some files were failed to retrieve, scan files again.
// TODO:FIXME AT V0.17.31, this logic has been disabled.
if (navigator.onLine && this.localDatabase.needScanning) {
this.localDatabase.needScanning = false;
await this.plugin.$$performFullScan();
}
}
async watchWindowVisibilityAsync() {
if (this.settings.suspendFileWatching) return;
if (!this.settings.isConfigured) return;
if (!this.plugin.isReady) return;
if (this.isLastHidden && !this.hasFocus) {
// NO OP while non-focused after made hidden;
return;
}
const isHidden = document.hidden;
if (this.isLastHidden === isHidden) {
return;
}
this.isLastHidden = isHidden;
await this.core.$everyCommitPendingFileEvent();
if (isHidden) {
await this.core.$everyBeforeSuspendProcess();
} else {
// suspend all temporary.
if (this.plugin.suspended) return;
if (!this.hasFocus) return;
await this.core.$everyOnResumeProcess();
await this.core.$everyAfterResumeProcess();
}
}
watchWorkspaceOpen(file: TFile | null) {
if (this.settings.suspendFileWatching) return;
if (!this.settings.isConfigured) return;
if (!this.plugin.isReady) return;
if (!file) return;
scheduleTask("watch-workspace-open", 500, () => fireAndForget(() => this.watchWorkspaceOpenAsync(file)));
}
async watchWorkspaceOpenAsync(file: TFile) {
if (this.settings.suspendFileWatching) return;
if (!this.settings.isConfigured) return;
if (!this.plugin.isReady) return;
await this.core.$everyCommitPendingFileEvent();
if (file == null) {
return;
}
if (this.settings.syncOnFileOpen && !this.plugin.suspended) {
await this.core.$$replicate();
}
await this.core.$$queueConflictCheckIfOpen(file.path as FilePathWithPrefix);
}
$everyOnLayoutReady(): Promise<boolean> {
this.swapSaveCommand();
this.registerWatchEvents();
return Promise.resolve(true);
}
$$askReload(message?: string) {
if (this.core.isReloadingScheduled) {
this._log(`Reloading is already scheduled`, LOG_LEVEL_VERBOSE);
return;
}
scheduleTask("configReload", 250, async () => {
const RESTART_NOW = "Yes, restart immediately";
const RESTART_AFTER_STABLE = "Yes, schedule a restart after stabilisation";
const RETRY_LATER = "No, Leave it to me";
const ret = await this.core.confirm.askSelectStringDialogue(message || "Do you want to restart and reload Obsidian now?", [
RESTART_AFTER_STABLE,
RESTART_NOW,
RETRY_LATER], { defaultAction: RETRY_LATER });
if (ret == RESTART_NOW) {
this._performAppReload();
} else if (ret == RESTART_AFTER_STABLE) {
this.core.$$scheduleAppReload();
}
})
}
$$scheduleAppReload() {
if (!this.core._totalProcessingCount) {
const __tick = reactiveSource(0);
this.core._totalProcessingCount = reactive(() => {
const dbCount = this.core.databaseQueueCount.value;
const replicationCount = this.core.replicationResultCount.value;
const storageApplyingCount = this.core.storageApplyingCount.value;
const chunkCount = collectingChunks.value;
const pluginScanCount = pluginScanningCount.value;
const hiddenFilesCount = hiddenFilesEventCount.value + hiddenFilesProcessingCount.value;
const conflictProcessCount = this.core.conflictProcessQueueCount.value;
const e = this.core.pendingFileEventCount.value;
const proc = this.core.processingFileEventCount.value;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const __ = __tick.value;
return dbCount + replicationCount + storageApplyingCount + chunkCount + pluginScanCount + hiddenFilesCount + conflictProcessCount + e + proc;
})
this.plugin.registerInterval(setInterval(() => {
__tick.value++;
}, 1000) as unknown as number);
let stableCheck = 3;
this.core._totalProcessingCount.onChanged(e => {
if (e.value == 0) {
if (stableCheck-- <= 0) {
this._performAppReload();
}
this._log(`Obsidian will be restarted soon! (Within ${stableCheck} seconds)`, LOG_LEVEL_NOTICE, "restart-notice");
} else {
stableCheck = 3;
}
})
}
}
}

136
src/modules/essentialObsidian/ModuleObsidianMenu.ts Normal file
View File

@@ -0,0 +1,136 @@
import { fireAndForget } from "octagonal-wheels/promises";
import { addIcon, type Editor, type MarkdownFileInfo, type MarkdownView } from "../../deps.ts";
import { LOG_LEVEL_NOTICE, type FilePathWithPrefix } from "../../lib/src/common/types.ts";
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
export class ModuleObsidianMenu extends AbstractObsidianModule implements IObsidianModule {
$everyOnloadStart(): Promise<boolean> {
// UI
addIcon(
"replicate",
`<g transform="matrix(1.15 0 0 1.15 -8.31 -9.52)" fill="currentColor" fill-rule="evenodd">
<path d="m85 22.2c-0.799-4.74-4.99-8.37-9.88-8.37-0.499 0-1.1 0.101-1.6 0.101-2.4-3.03-6.09-4.94-10.3-4.94-6.09 0-11.2 4.14-12.8 9.79-5.59 1.11-9.78 6.05-9.78 12 0 6.76 5.39 12.2 12 12.2h29.9c5.79 0 10.1-4.74 10.1-10.6 0-4.84-3.29-8.88-7.68-10.2zm-2.99 14.7h-29.5c-2.3-0.202-4.29-1.51-5.29-3.53-0.899-2.12-0.699-4.54 0.698-6.46 1.2-1.61 2.99-2.52 4.89-2.52 0.299 0 0.698 0 0.998 0.101l1.8 0.303v-2.02c0-3.63 2.4-6.76 5.89-7.57 0.599-0.101 1.2-0.202 1.8-0.202 2.89 0 5.49 1.62 6.79 4.24l0.598 1.21 1.3-0.504c0.599-0.202 1.3-0.303 2-0.303 1.3 0 2.5 0.404 3.59 1.11 1.6 1.21 2.6 3.13 2.6 5.15v1.61h2c2.6 0 4.69 2.12 4.69 4.74-0.099 2.52-2.2 4.64-4.79 4.64z"/>
<path d="m53.2 49.2h-41.6c-1.8 0-3.2 1.4-3.2 3.2v28.6c0 1.8 1.4 3.2 3.2 3.2h15.8v4h-7v6h24v-6h-7v-4h15.8c1.8 0 3.2-1.4 3.2-3.2v-28.6c0-1.8-1.4-3.2-3.2-3.2zm-2.8 29h-36v-23h36z"/>
<path d="m73 49.2c1.02 1.29 1.53 2.97 1.53 4.56 0 2.97-1.74 5.65-4.39 7.04v-4.06l-7.46 7.33 7.46 7.14v-4.06c7.66-1.98 12.2-9.61 10-17-0.102-0.297-0.205-0.595-0.307-0.892z"/>
<path d="m24.1 43c-0.817-0.991-1.53-2.97-1.53-4.56 0-2.97 1.74-5.65 4.39-7.04v4.06l7.46-7.33-7.46-7.14v4.06c-7.66 1.98-12.2 9.61-10 17 0.102 0.297 0.205 0.595 0.307 0.892z"/>
</g>`
);
this.addRibbonIcon("replicate", "Replicate", async () => {
await this.plugin.$$replicate(true);
}).addClass("livesync-ribbon-replicate");
this.addCommand({
id: "livesync-replicate",
name: "Replicate now",
callback: async () => {
await this.plugin.$$replicate();
},
});
this.addCommand({
id: "livesync-dump",
name: "Dump information of this doc ",
callback: () => {
const file = this.plugin.$$getActiveFilePath();
if (!file) return;
fireAndForget(() => this.localDatabase.getDBEntry(file, {}, true, false));
},
});
this.addCommand({
id: "livesync-checkdoc-conflicted",
name: "Resolve if conflicted.",
editorCallback: (editor: Editor, view: MarkdownView | MarkdownFileInfo) => {
const file = view.file;
if (!file) return;
void this.plugin.$$queueConflictCheckIfOpen(file.path as FilePathWithPrefix);
},
});
this.addCommand({
id: "livesync-toggle",
name: "Toggle LiveSync",
callback: async () => {
if (this.settings.liveSync) {
this.settings.liveSync = false;
this._log("LiveSync Disabled.", LOG_LEVEL_NOTICE);
} else {
this.settings.liveSync = true;
this._log("LiveSync Enabled.", LOG_LEVEL_NOTICE);
}
await this.plugin.realizeSettingSyncMode();
await this.plugin.saveSettings();
},
});
this.addCommand({
id: "livesync-suspendall",
name: "Toggle All Sync.",
callback: async () => {
if (this.plugin.suspended) {
this.plugin.suspended = false;
this._log("Self-hosted LiveSync resumed", LOG_LEVEL_NOTICE);
} else {
this.plugin.suspended = true;
this._log("Self-hosted LiveSync suspended", LOG_LEVEL_NOTICE);
}
await this.plugin.realizeSettingSyncMode();
await this.plugin.saveSettings();
},
});
this.addCommand({
id: "livesync-scan-files",
name: "Scan storage and database again",
callback: async () => {
await this.plugin.$$performFullScan(true)
}
})
this.addCommand({
id: "livesync-runbatch",
name: "Run pended batch processes",
callback: async () => {
await this.core.$everyCommitPendingFileEvent();
},
})
// TODO, Replicator is possibly one of features. It should be moved to features.
this.addCommand({
id: "livesync-abortsync",
name: "Abort synchronization immediately",
callback: () => {
this.plugin.replicator.terminateSync();
},
})
return Promise.resolve(true);
}
$everyOnload(): Promise<boolean> {
this.app.workspace.onLayoutReady(this.plugin.onLiveSyncReady.bind(this.plugin));
// eslint-disable-next-line no-unused-labels
return Promise.resolve(true);
}
async $$showView(viewType: string) {
const leaves = this.app.workspace.getLeavesOfType(viewType);
if (leaves.length == 0) {
await this.app.workspace.getLeaf(true).setViewState({
type: viewType,
active: true,
});
} else {
await leaves[0].setViewState({
type: viewType,
active: true,
})
}
if (leaves.length > 0) {
this.app.workspace.revealLeaf(
leaves[0]
);
}
}
}

114
src/modules/extras/ModuleDev.ts Normal file
View File

@@ -0,0 +1,114 @@
import { fireAndForget } from "octagonal-wheels/promises";
import { __onMissingTranslation } from "../../lib/src/common/i18n";
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { eventHub } from "../../common/events";
import { enableTestFunction } from "./devUtil/testUtils.ts";
import { TestPaneView, VIEW_TYPE_TEST } from "./devUtil/TestPaneView.ts";
import { writable } from "svelte/store";
export class ModuleDev extends AbstractObsidianModule implements IObsidianModule {
$everyOnloadStart(): Promise<boolean> {
__onMissingTranslation(() => { });
return Promise.resolve(true);
}
$everyOnloadAfterLoadSettings(): Promise<boolean> {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
// eslint-disable-next-line no-unused-labels
__onMissingTranslation((key) => {
const now = new Date();
const filename = `missing-translation-`
const time = now.toISOString().split("T")[0];
const outFile = `${filename}${time}.jsonl`;
const piece = JSON.stringify(
{
[key]: {}
}
)
const writePiece = piece.substring(1, piece.length - 1) + ",";
fireAndForget(async () => {
try {
await this.core.storageAccess.ensureDir(this.app.vault.configDir + "/ls-debug/");
await this.core.storageAccess.appendHiddenFile(this.app.vault.configDir + "/ls-debug/" + outFile, writePiece + "\n")
} catch (ex) {
this._log(`Could not write ${outFile}`, LOG_LEVEL_VERBOSE);
this._log(`Missing translation: ${writePiece}`, LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
});
})
type STUB = {
toc: Set<string>,
stub: { [key: string]: { [key: string]: Map<string, Record<string, string>> } }
};
eventHub.onEvent("document-stub-created", (e: CustomEvent<STUB>) => {
fireAndForget(async () => {
const stub = e.detail.stub;
const toc = e.detail.toc;
const stubDocX =
Object.entries(stub).map(([key, value]) => {
return [`## ${key}`, Object.entries(value).
map(([key2, value2]) => {
return [`### ${key2}`,
([...(value2.entries())].map(([key3, value3]) => {
// return `#### ${key3}` + "\n" + JSON.stringify(value3);
const isObsolete = value3["is_obsolete"] ? " (obsolete)" : "";
const desc = value3["desc"] ?? "";
const key = value3["key"] ? "Setting key: " + value3["key"] + "\n" : "";
return `#### ${key3}${isObsolete}\n${key}${desc}\n`
}))].flat()
}).flat()].flat()
}).flat();
const stubDocMD = `
| Icon | Description |
| :---: | ----------------------------------------------------------------- |
` +
[...toc.values()].map(e => `${e}`).join("\n") + "\n\n" +
stubDocX.join("\n");
await this.core.storageAccess.writeHiddenFileAuto(this.app.vault.configDir + "/ls-debug/stub-doc.md", stubDocMD);
})
});
enableTestFunction(this.plugin);
this.registerView(
VIEW_TYPE_TEST,
(leaf) => new TestPaneView(leaf, this.plugin, this)
);
this.addCommand({
id: "view-test",
name: "Open Test dialogue",
callback: () => {
void this.core.$$showView(VIEW_TYPE_TEST);
}
});
return Promise.resolve(true);
}
async $everyOnLayoutReady(): Promise<boolean> {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
if (await this.core.storageAccess.isExistsIncludeHidden("_SHOWDIALOGAUTO.md")) {
void this.core.$$showView(VIEW_TYPE_TEST);
}
return true;
}
testResults = writable<[boolean, string, string][]>([]);
// testResults: string[] = [];
$$addTestResult(name: string, key: string, result: boolean, summary?: string, message?: string): void {
const logLine = `${name}: ${key} ${summary ?? ""}`;
this.testResults.update((results) => {
results.push([result, logLine, message ?? ""]);
return results;
});
}
$everyModuleTest(): Promise<boolean> {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
// this.core.$$addTestResult("DevModule", "Test", true);
// return Promise.resolve(true);
// this.addTestResult("Test of test1", true, "Just OK", "This is a test of test");
// this.addTestResult("Test of test2", true, "Just OK?");
// this.addTestResult("Test of test3", true);
return this.testDone();
}
}

441
src/modules/extras/ModuleIntegratedTest.ts Normal file
View File

@@ -0,0 +1,441 @@
import { delay } from "octagonal-wheels/promises";
import { LOG_LEVEL_NOTICE, REMOTE_MINIO, type FilePathWithPrefix } from "src/lib/src/common/types";
import { shareRunningResult } from "octagonal-wheels/concurrency/lock";
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule";
export class ModuleIntegratedTest extends AbstractObsidianModule implements IObsidianModule {
async waitFor(proc: () => Promise<boolean>, timeout = 10000): Promise<boolean> {
await delay(100);
const start = Date.now();
while (!await proc()) {
if (timeout > 0) {
if (Date.now() - start > timeout) {
this._log(`Timeout`);
return false;
}
}
await delay(500);
}
return true;
}
waitWithReplicating(proc: () => Promise<boolean>, timeout = 10000): Promise<boolean> {
return this.waitFor(async () => {
await this.tryReplicate();
return await proc();
}, timeout);
}
async storageContentIsEqual(file: string, content: string): Promise<boolean> {
try {
const fileContent = await this.readStorageContent(file as FilePathWithPrefix);
if (fileContent === content) {
return true;
} else {
// this._log(`Content is not same \n Expected:${content}\n Actual:${fileContent}`, LOG_LEVEL_VERBOSE);
return false;
}
} catch (e) {
this._log(`Error: ${e}`);
return false;
}
}
async assert(proc: () => Promise<boolean>): Promise<boolean> {
if (!await proc()) {
this._log(`Assertion failed`);
return false;
}
return true;
}
async _orDie(key: string, proc: () => Promise<boolean>): Promise<true> | never {
if (!await this._test(key, proc)) {
throw new Error(`${key}`);
}
return true;
}
tryReplicate() {
if (!this.settings.liveSync) {
return shareRunningResult("replicate-test", async () => { await this.core.$$replicate() });
}
}
async readStorageContent(file: FilePathWithPrefix): Promise<string | undefined> {
if (!await this.core.storageAccess.isExistsIncludeHidden(file)) {
return undefined;
}
return await this.core.storageAccess.readHiddenFileText(file);
}
async _proceed(no: number, title: string): Promise<boolean> {
const stepFile = "_STEP.md" as FilePathWithPrefix;
const stepAckFile = "_STEP_ACK.md" as FilePathWithPrefix;
const stepContent = `Step ${no}`;
await this.core.$anyResolveConflictByNewest(stepFile);
await this.core.storageAccess.writeFileAuto(stepFile, stepContent);
await this._orDie(`Wait for acknowledge ${no}`, async () => {
if (!await this.waitWithReplicating(
async () => {
return await this.storageContentIsEqual(stepAckFile, stepContent)
}, 20000)
) return false;
return true;
})
return true;
}
async _join(no: number, title: string): Promise<boolean> {
const stepFile = "_STEP.md" as FilePathWithPrefix;
const stepAckFile = "_STEP_ACK.md" as FilePathWithPrefix;
// const otherStepFile = `_STEP_${isLeader ? "R" : "L"}.md` as FilePathWithPrefix;
const stepContent = `Step ${no}`;
await this._orDie(`Wait for step ${no} (${title})`, async () => {
if (!await this.waitWithReplicating(
async () => {
return await this.storageContentIsEqual(stepFile, stepContent)
}, 20000)
) return false;
return true;
}
)
await this.core.$anyResolveConflictByNewest(stepAckFile);
await this.core.storageAccess.writeFileAuto(stepAckFile, stepContent);
await this.tryReplicate();
return true;
}
async performStep({
step,
title,
isGameChanger,
proc,
check
}: {
step: number,
title: string,
isGameChanger: boolean,
proc: () => Promise<any>,
check: () => Promise<boolean>,
}): Promise<boolean> {
if (isGameChanger) {
await this._proceed(step, title);
try {
await proc();
} catch (e) {
this._log(`Error: ${e}`);
return false;
}
return await this._orDie(`Step ${step} - ${title}`,
async () => await this.waitWithReplicating(check)
);
} else {
return await this._join(step, title);
}
}
// // see scenario.md
// async testLeader(testMain: (testFileName: FilePathWithPrefix) => Promise<boolean>): Promise<boolean> {
// }
// async testReceiver(testMain: (testFileName: FilePathWithPrefix) => Promise<boolean>): Promise<boolean> {
// }
async nonLiveTestRunner(isLeader: boolean, testMain: (testFileName: FilePathWithPrefix, isLeader: boolean) => Promise<boolean>): Promise<boolean> {
const storage = this.core.storageAccess;
// const database = this.core.databaseFileAccess;
// const _orDie = this._orDie.bind(this);
const testCommandFile = "IT.md" as FilePathWithPrefix;
const textCommandResponseFile = "ITx.md" as FilePathWithPrefix;
let testFileName: FilePathWithPrefix;
this.addTestResult("-------Starting ... ", true, `Test as ${isLeader ? "Leader" : "Receiver"} command file ${testCommandFile}`);
if (isLeader) {
await this._proceed(0, "start");
}
await this.tryReplicate();
await this.performStep({
step: 0,
title: "Make sure that command File Not Exists",
isGameChanger: isLeader,
proc: async () => await storage.removeHidden(testCommandFile),
check: async () => !(await storage.isExistsIncludeHidden(testCommandFile)),
})
await this.performStep({
step: 1,
title: "Make sure that command File Not Exists On Receiver",
isGameChanger: !isLeader,
proc: async () => await storage.removeHidden(textCommandResponseFile),
check: async () => !(await storage.isExistsIncludeHidden(textCommandResponseFile)),
})
await this.performStep({
step: 2,
title: "Decide the test file name",
isGameChanger: isLeader,
proc: async () => {
testFileName = (Date.now() + "-" + Math.ceil(Math.random() * 1000) + ".md") as FilePathWithPrefix;
const testCommandFile = "IT.md" as FilePathWithPrefix;
await storage.writeFileAuto(testCommandFile, testFileName);
},
check: () => Promise.resolve(true),
})
await this.performStep({
step: 3,
title: "Wait for the command file to be arrived",
isGameChanger: !isLeader,
proc: async () => { },
check: async () => await storage.isExistsIncludeHidden(testCommandFile),
})
await this.performStep({
step: 4,
title: "Send the response file",
isGameChanger: !isLeader,
proc: async () => {
await storage.writeHiddenFileAuto(textCommandResponseFile, "!");
},
check: () => Promise.resolve(true),
})
await this.performStep({
step: 5,
title: "Wait for the response file to be arrived",
isGameChanger: isLeader,
proc: async () => { },
check: async () => await storage.isExistsIncludeHidden(textCommandResponseFile),
})
await this.performStep({
step: 6,
title: "Proceed to begin the test",
isGameChanger: isLeader,
proc: async () => {
},
check: () => Promise.resolve(true),
});
await this.performStep({
step: 6,
title: "Begin the test",
isGameChanger: !false,
proc: async () => {
},
check: () => {
return Promise.resolve(true);
},
})
// await this.step(0, isLeader, true);
try {
this.addTestResult("** Main------", true, ``);
if (isLeader) {
return await testMain(testFileName!, true);
} else {
const testFileName = await this.readStorageContent(testCommandFile);
this.addTestResult("testFileName", true, `Request client to use :${testFileName!}`);
return await testMain(testFileName! as FilePathWithPrefix, false);
}
} finally {
this.addTestResult("Teardown", true, `Deleting ${testFileName!}`);
await storage.removeHidden(testFileName!);
}
return true;
// Make sure the
}
async testBasic(filename: FilePathWithPrefix, isLeader: boolean): Promise<boolean> {
const storage = this.core.storageAccess;
const database = this.core.databaseFileAccess;
await this.addTestResult(`---**Starting Basic Test**---`, true, `Test as ${isLeader ? "Leader" : "Receiver"} command file ${filename}`);
// if (isLeader) {
// await this._proceed(0);
// }
// await this.tryReplicate();
await this.performStep({
step: 0,
title: "Make sure that file is not exist",
isGameChanger: !isLeader,
proc: async () => { },
check: async () => !(await storage.isExists(filename)),
})
await this.performStep({
step: 1,
title: "Write a file",
isGameChanger: isLeader,
proc: async () => await storage.writeFileAuto(filename, "Hello World"),
check: async () => await storage.isExists(filename),
})
await this.performStep({
step: 2,
title: "Make sure the file is arrived",
isGameChanger: !isLeader,
proc: async () => { },
check: async () => await storage.isExists(filename),
})
await this.performStep({
step: 3,
title: "Update to Hello World 2",
isGameChanger: isLeader,
proc: async () => await storage.writeFileAuto(filename, "Hello World 2"),
check: async () => await this.storageContentIsEqual(filename, "Hello World 2"),
})
await this.performStep({
step: 4,
title: "Make sure the modified file is arrived",
isGameChanger: !isLeader,
proc: async () => { },
check: async () => await this.storageContentIsEqual(filename, "Hello World 2"),
})
await this.performStep({
step: 5,
title: "Update to Hello World 3",
isGameChanger: !isLeader,
proc: async () => await storage.writeFileAuto(filename, "Hello World 3"),
check: async () => await this.storageContentIsEqual(filename, "Hello World 3"),
})
await this.performStep({
step: 6,
title: "Make sure the modified file is arrived",
isGameChanger: isLeader,
proc: async () => { },
check: async () => await this.storageContentIsEqual(filename, "Hello World 3"),
})
const multiLineContent = `Line1:A
Line2:B
Line3:C
Line4:D`
await this.performStep({
step: 7,
title: "Update to Multiline",
isGameChanger: isLeader,
proc: async () => await storage.writeFileAuto(filename, multiLineContent),
check: async () => await this.storageContentIsEqual(filename, multiLineContent),
})
await this.performStep({
step: 8,
title: "Make sure the modified file is arrived",
isGameChanger: !isLeader,
proc: async () => { },
check: async () => await this.storageContentIsEqual(filename, multiLineContent),
})
// While LiveSync, possibly cannot cause the conflict.
if (!this.settings.liveSync) {
// Step 9 Make Conflict But Resolvable
const multiLineContentL = `Line1:A
Line2:B
Line3:C!
Line4:D`
const multiLineContentC = `Line1:A
Line2:bbbbb
Line3:C
Line4:D`
await this.performStep({
step: 9,
title: "Progress to be conflicted",
isGameChanger: isLeader,
proc: async () => { },
check: () => Promise.resolve(true),
})
await storage.writeFileAuto(filename, isLeader ? multiLineContentL : multiLineContentC);
await this.performStep({
step: 10,
title: "Update As Conflicted",
isGameChanger: !isLeader,
proc: async () => { },
check: () => Promise.resolve(true),
})
await this.performStep({
step: 10,
title: "Make sure Automatically resolved",
isGameChanger: isLeader,
proc: async () => { },
check: async () => (await database.getConflictedRevs(filename)).length === 0,
})
await this.performStep({
step: 11,
title: "Make sure Automatically resolved",
isGameChanger: !isLeader,
proc: async () => { },
check: async () => (await database.getConflictedRevs(filename)).length === 0,
})
const sensiblyMergedContent = `Line1:A
Line2:bbbbb
Line3:C!
Line4:D`
await this.performStep({
step: 12,
title: "Make sure Sensibly Merged on Leader",
isGameChanger: isLeader,
proc: async () => { },
check: async () => await this.storageContentIsEqual(filename, sensiblyMergedContent),
})
await this.performStep({
step: 13,
title: "Make sure Sensibly Merged on Receiver",
isGameChanger: !isLeader,
proc: async () => { },
check: async () => await this.storageContentIsEqual(filename, sensiblyMergedContent),
})
}
await this.performStep({
step: 14,
title: "Delete File",
isGameChanger: isLeader,
proc: async () => { await storage.removeHidden(filename) },
check: async () => !await storage.isExists(filename),
})
await this.performStep({
step: 15,
title: "Make sure File is deleted",
isGameChanger: !isLeader,
proc: async () => { },
check: async () => !await storage.isExists(filename),
})
this._log(`The Basic Test has been completed`, LOG_LEVEL_NOTICE);
return true;
}
async testBasicEvent(isLeader: boolean) {
this.settings.liveSync = false;
await this.saveSettings();
await this._test("basic", async () => await this.nonLiveTestRunner(isLeader, (t, l) => this.testBasic(t, l)));
}
async testBasicLive(isLeader: boolean) {
this.settings.liveSync = true;
await this.saveSettings();
await this._test("basic", async () => await this.nonLiveTestRunner(isLeader, (t, l) => this.testBasic(t, l)));
}
async $everyModuleTestMultiDevice(): Promise<boolean> {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
const isLeader = this.core.$$vaultName().indexOf("recv") === -1;
this.addTestResult("-------", true, `Test as ${isLeader ? "Leader" : "Receiver"}`);
try {
this._log(`Starting Test`);
await this.testBasicEvent(isLeader);
if (this.settings.remoteType == REMOTE_MINIO) await this.testBasicLive(isLeader);
} catch (e) {
this._log(e)
this._log(`Error: ${e}`);
return Promise.resolve(false);
}
return Promise.resolve(true);
}
}

528
src/modules/extras/ModuleReplicateTest.ts Normal file
View File

@@ -0,0 +1,528 @@
import { delay } from "octagonal-wheels/promises";
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
import { eventHub } from "../../common/events";
import { webcrypto } from "crypto";
import { uint8ArrayToHexString } from "octagonal-wheels/binary/hex";
import { parseYaml, requestUrl, stringifyYaml } from "obsidian";
import type { FilePath } from "../../lib/src/common/types.ts";
import { scheduleTask } from "octagonal-wheels/concurrency/task";
export class ModuleReplicateTest extends AbstractObsidianModule implements IObsidianModule {
testRootPath = "_test/";
testInfoPath = "_testinfo/";
get isLeader() {
return this.core.$$getVaultName().indexOf("dev") >= 0 && this.core.$$vaultName().indexOf("recv") < 0;
}
get nameByKind() {
if (!this.isLeader) {
return "RECV";
} else if (this.isLeader) {
return "LEADER";
}
}
get pairName() {
if (this.isLeader) {
return "RECV";
} else if (!this.isLeader) {
return "LEADER";
}
}
watchIsSynchronised = false;
statusBarSyncStatus?: HTMLElement;
async readFileContent(file: string) {
try {
return await this.core.storageAccess.readHiddenFileText(file);
} catch {
return "";
}
}
async dumpList() {
if (this.settings.syncInternalFiles) {
this._log("Write file list (Include Hidden)");
await this._dumpFileListIncludeHidden("files.md");
} else {
this._log("Write file list");
await this._dumpFileList("files.md");
}
}
async $everyBeforeReplicate(showMessage: boolean): Promise<boolean> {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
await this.dumpList();
return true;
}
$everyOnloadAfterLoadSettings(): Promise<boolean> {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
this.addCommand({
id: "dump-file-structure-normal",
name: `Dump Structure (Normal)`,
callback: () => {
void this._dumpFileList("files.md").finally(() => {
void this.refreshSyncStatus();
});
}
})
this.addCommand({
id: "dump-file-structure-ih",
name: "Dump Structure (Include Hidden)",
callback: () => {
const d = "files.md";
void this._dumpFileListIncludeHidden(d);
}
})
this.addCommand({
id: "dump-file-structure-auto",
name: "Dump Structure",
callback: () => {
void this.dumpList();
}
})
this.addCommand({
id: "dump-file-test",
name: `Perform Test (Dev) ${this.isLeader ? "(Leader)" : "(Recv)"}`,
callback: () => {
void this.performTestManually();
}
})
this.addCommand({
id: "watch-sync-result",
name: `Watch sync result is matched between devices`,
callback: () => {
this.watchIsSynchronised = !this.watchIsSynchronised;
void this.refreshSyncStatus();
}
})
this.app.vault.on("modify", async (file) => {
if (file.path.startsWith(this.testInfoPath)) {
await this.refreshSyncStatus();
} else {
scheduleTask("dumpStatus", 125, async () => {
await this.dumpList();
return true;
});
}
})
this.statusBarSyncStatus = this.plugin.addStatusBarItem();
return Promise.resolve(true);
}
async getSyncStatusAsText() {
const fileMine = this.testInfoPath + this.nameByKind + "/" + "files.md";
const filePair = this.testInfoPath + this.pairName + "/" + "files.md";
const mine = parseYaml(await this.readFileContent(fileMine));
const pair = parseYaml(await this.readFileContent(filePair));
const result = [] as string[];
if (mine.length != pair.length) {
result.push(`File count is different: ${mine.length} vs ${pair.length}`);
}
const filesAll = new Set([...mine.map((e: any) => e.path), ...pair.map((e: any) => e.path)]);
for (const file of filesAll) {
const mineFile = mine.find((e: any) => e.path == file);
const pairFile = pair.find((e: any) => e.path == file);
if (!mineFile || !pairFile) {
result.push(`File not found: ${file}`);
} else {
if (mineFile.size != pairFile.size) {
result.push(`Size is different: ${file} ${mineFile.size} vs ${pairFile.size}`);
}
if (mineFile.hash != pairFile.hash) {
result.push(`Hash is different: ${file} ${mineFile.hash} vs ${pairFile.hash}`);
}
}
}
eventHub.emitEvent("debug-sync-status", result);
return result.join("\n");
}
async refreshSyncStatus() {
if (this.watchIsSynchronised) {
// Normal Files
const syncStatus = await this.getSyncStatusAsText();
if (syncStatus) {
this.statusBarSyncStatus!.setText(`Sync Status: Having Error`);
this._log(`Sync Status: Having Error\n${syncStatus}`, LOG_LEVEL_INFO);
} else {
this.statusBarSyncStatus!.setText(`Sync Status: Synchronised`);
}
} else {
this.statusBarSyncStatus!.setText("");
}
}
async _dumpFileList(outFile?: string) {
const files = this.core.storageAccess.getFiles();
const out = [] as any[];
for (const file of files) {
if (!await this.core.$$isTargetFile(file.path)) {
continue;
}
if (file.path.startsWith(this.testInfoPath)) continue;
const stat = await this.core.storageAccess.stat(file.path);
if (stat) {
const hashSrc = await this.core.storageAccess.readHiddenFileBinary(file.path);
const hash = await webcrypto.subtle.digest("SHA-1", hashSrc);
const hashStr = uint8ArrayToHexString(new Uint8Array(hash));
const item = {
path: file.path,
name: file.name,
size: stat.size,
mtime: stat.mtime,
hash: hashStr
}
// const fileLine = `-${file.path}:${stat.size}:${stat.mtime}:${hashStr}`;
out.push(item);
}
}
out.sort((a, b) => a.path.localeCompare(b.path));
if (outFile) {
outFile = this.testInfoPath + this.nameByKind + "/" + outFile;
await this.core.storageAccess.ensureDir(outFile);
await this.core.storageAccess.writeHiddenFileAuto(outFile, stringifyYaml(out));
} else {
// console.dir(out);
}
this._log(`Dumped ${out.length} files`, LOG_LEVEL_INFO);
}
async _dumpFileListIncludeHidden(outFile?: string) {
const ignorePatterns = this.settings.syncInternalFilesIgnorePatterns
.replace(/\n| /g, "")
.split(",").filter(e => e).map(e => new RegExp(e, "i"));
const out = [] as any[];
const files = await this.core.storageAccess.getFilesIncludeHidden("", undefined, ignorePatterns);
console.dir(files);
for (const file of files) {
// if (!await this.core.$$isTargetFile(file)) {
// continue;
// }
if (file.startsWith(this.testInfoPath)) continue;
const stat = await this.core.storageAccess.statHidden(file);
if (stat) {
const hashSrc = await this.core.storageAccess.readHiddenFileBinary(file);
const hash = await webcrypto.subtle.digest("SHA-1", hashSrc);
const hashStr = uint8ArrayToHexString(new Uint8Array(hash));
const item = {
path: file,
name: file.split("/").pop(),
size: stat.size,
mtime: stat.mtime,
hash: hashStr
}
// const fileLine = `-${file.path}:${stat.size}:${stat.mtime}:${hashStr}`;
out.push(item);
}
}
out.sort((a, b) => a.path.localeCompare(b.path));
if (outFile) {
outFile = this.testInfoPath + this.nameByKind + "/" + outFile;
await this.core.storageAccess.ensureDir(outFile);
await this.core.storageAccess.writeHiddenFileAuto(outFile, stringifyYaml(out));
} else {
// console.dir(out);
}
this._log(`Dumped ${out.length} files`, LOG_LEVEL_NOTICE);
}
async collectTestFiles() {
const remoteTopDir = "https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/refs/heads/main/";
const files = [
"README.md",
"docs/adding_translations.md",
"docs/design_docs_of_journalsync.md",
"docs/design_docs_of_keep_newborn_chunks.md",
"docs/design_docs_of_prefixed_hidden_file_sync.md",
"docs/design_docs_of_sharing_tweak_value.md",
"docs/quick_setup_cn.md",
"docs/quick_setup_ja.md",
"docs/quick_setup.md",
"docs/settings_ja.md",
"docs/settings.md",
"docs/setup_cloudant_ja.md",
"docs/setup_cloudant.md",
"docs/setup_flyio.md",
"docs/setup_own_server_cn.md",
"docs/setup_own_server_ja.md",
"docs/setup_own_server.md",
"docs/tech_info_ja.md",
"docs/tech_info.md",
"docs/terms.md",
"docs/troubleshooting.md",
'images/1.png',
'images/2.png',
'images/corrupted_data.png',
'images/hatch.png',
'images/lock_pattern1.png',
'images/lock_pattern2.png',
'images/quick_setup_1.png',
'images/quick_setup_2.png',
'images/quick_setup_3.png',
'images/quick_setup_3b.png',
'images/quick_setup_4.png',
'images/quick_setup_5.png',
'images/quick_setup_6.png',
'images/quick_setup_7.png',
'images/quick_setup_8.png',
'images/quick_setup_9_1.png',
'images/quick_setup_9_2.png',
'images/quick_setup_10.png',
'images/remote_db_setting.png',
'images/write_logs_into_the_file.png',
]
for (const file of files) {
const remote = remoteTopDir + file;
const local = this.testRootPath + file;
try {
const f = (await requestUrl(remote)).arrayBuffer;
await this.core.storageAccess.ensureDir(local);
await this.core.storageAccess.writeHiddenFileAuto(local, f);
} catch (ex) {
this._log(`Could not fetch ${remote}`, LOG_LEVEL_VERBOSE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
}
await this.dumpList();
}
async waitFor(proc: () => Promise<boolean>, timeout = 10000): Promise<boolean> {
await delay(100);
const start = Date.now();
while (!await proc()) {
if (timeout > 0) {
if (Date.now() - start > timeout) {
this._log(`Timeout`);
return false;
}
}
await delay(500);
}
return true;
}
async testConflictedManually1() {
await this.core.$$replicate();
const commonFile = `Resolve!
*****, the amazing chocolatier!!`;
if (this.isLeader) {
await this.core.storageAccess.writeHiddenFileAuto(this.testRootPath + "wonka.md", commonFile);
}
await this.core.$$replicate();
await this.core.$$replicate();
if (await this.core.confirm.askYesNoDialog("Ready to begin the test conflict Manually 1?", { timeout: 30, defaultOption: "Yes" }) == "no") {
return;
}
const fileA = `Resolve to KEEP THIS
Willy Wonka, Willy Wonka, the amazing chocolatier!!`
const fileB = `Resolve to DISCARD THIS
Charlie Bucket, Charlie Bucket, the amazing chocolatier!!`
if (this.isLeader) {
await this.core.storageAccess.writeHiddenFileAuto(this.testRootPath + "wonka.md", fileA);
} else {
await this.core.storageAccess.writeHiddenFileAuto(this.testRootPath + "wonka.md", fileB);
}
if (await this.core.confirm.askYesNoDialog("Ready to check the result of Manually 1?", { timeout: 30, defaultOption: "Yes" }) == "no") {
return;
}
await this.core.$$replicate();
await this.core.$$replicate();
if (!await this.waitFor(async () => {
await this.core.$$replicate();
return await this.__assertStorageContent(this.testRootPath + "wonka.md" as FilePath, fileA, false, true) == true;
}, 30000)) {
return await this.__assertStorageContent(this.testRootPath + "wonka.md" as FilePath, fileA, false, true);
}
return true;
// We have to check the result
}
async testConflictedManually2() {
await this.core.$$replicate();
const commonFile = `Resolve To concatenate
ABCDEFG`;
if (this.isLeader) {
await this.core.storageAccess.writeHiddenFileAuto(this.testRootPath + "concat.md", commonFile);
}
await this.core.$$replicate();
await this.core.$$replicate();
if (await this.core.confirm.askYesNoDialog("Ready to begin the test conflict Manually 2?", { timeout: 30, defaultOption: "Yes" }) == "no") {
return;
}
const fileA = `Resolve to Concatenate
ABCDEFGHIJKLMNOPQRSTYZ`
const fileB = `Resolve to Concatenate
AJKLMNOPQRSTUVWXYZ`
const concatenated = `Resolve to Concatenate
ABCDEFGHIJKLMNOPQRSTUVWXYZ`
if (this.isLeader) {
await this.core.storageAccess.writeHiddenFileAuto(this.testRootPath + "concat.md", fileA);
} else {
await this.core.storageAccess.writeHiddenFileAuto(this.testRootPath + "concat.md", fileB);
}
if (await this.core.confirm.askYesNoDialog("Ready to test conflict Manually 2?", { timeout: 30, defaultOption: "Yes" }) == "no") {
return;
}
await this.core.$$replicate();
await this.core.$$replicate();
if (!await this.waitFor(async () => {
await this.core.$$replicate();
return await this.__assertStorageContent(this.testRootPath + "concat.md" as FilePath, concatenated, false, true) == true;
}, 30000)) {
return await this.__assertStorageContent(this.testRootPath + "concat.md" as FilePath, concatenated, false, true);
}
return true;
}
async testConflictAutomatic() {
if (this.isLeader) {
const baseDoc = `Tasks!
- [ ] Task 1
- [ ] Task 2
- [ ] Task 3
- [ ] Task 4
`
await this.core.storageAccess.writeHiddenFileAuto(this.testRootPath + "task.md", baseDoc);
}
await delay(100)
await this.core.$$replicate();
await this.core.$$replicate();
if (await this.core.confirm.askYesNoDialog("Ready to test conflict?", { timeout: 30, defaultOption: "Yes" }) == "no") {
return;
}
const mod1Doc = `Tasks!
- [ ] Task 1
- [v] Task 2
- [ ] Task 3
- [ ] Task 4
`
const mod2Doc = `Tasks!
- [ ] Task 1
- [ ] Task 2
- [v] Task 3
- [ ] Task 4
`
if (this.isLeader) {
await this.core.storageAccess.writeHiddenFileAuto(this.testRootPath + "task.md", mod1Doc);
} else {
await this.core.storageAccess.writeHiddenFileAuto(this.testRootPath + "task.md", mod2Doc);
}
await this.core.$$replicate();
await this.core.$$replicate();
await delay(1000);
if (await this.core.confirm.askYesNoDialog("Ready to check result?", { timeout: 30, defaultOption: "Yes" }) == "no") {
return;
}
await this.core.$$replicate();
await this.core.$$replicate();
const mergedDoc = `Tasks!
- [ ] Task 1
- [v] Task 2
- [v] Task 3
- [ ] Task 4
`
return this.__assertStorageContent(this.testRootPath + "task.md" as FilePath, mergedDoc, false, true);
}
async checkConflictResolution() {
this._log("Before testing conflicted files, resolve all once", LOG_LEVEL_NOTICE);
await this.core.rebuilder.resolveAllConflictedFilesByNewerOnes();
await this.core.rebuilder.resolveAllConflictedFilesByNewerOnes();
await this.core.$$replicate();
await delay(1000);
if (!await this.testConflictAutomatic()) {
this._log("Conflict resolution (Auto) failed", LOG_LEVEL_NOTICE);
return false;
}
if (!await this.testConflictedManually1()) {
this._log("Conflict resolution (Manual1) failed", LOG_LEVEL_NOTICE);
return false;
}
if (!await this.testConflictedManually2()) {
this._log("Conflict resolution (Manual2) failed", LOG_LEVEL_NOTICE);
return false;
}
return true;
}
async __assertStorageContent(fileName: FilePath, content: string, inverted = false, showResult = false): Promise<boolean | string> {
try {
const fileContent = await this.core.storageAccess.readHiddenFileText(fileName);
let result = fileContent === content;
if (inverted) {
result = !result;
}
if (result) {
return true;
} else {
if (showResult) {
this._log(`Content is not same \n Expected:${content}\n Actual:${fileContent}`, LOG_LEVEL_VERBOSE);
}
return `Content is not same \n Expected:${content}\n Actual:${fileContent}`;
}
} catch (e) {
this._log(`Cannot assert storage content: ${e}`);
return false;
}
}
async performTestManually() {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
await this.checkConflictResolution();
// await this.collectTestFiles();
}
// testResults = writable<[boolean, string, string][]>([]);
// testResults: string[] = [];
// $$addTestResult(name: string, key: string, result: boolean, summary?: string, message?: string): void {
// const logLine = `${name}: ${key} ${summary ?? ""}`;
// this.testResults.update((results) => {
// results.push([result, logLine, message ?? ""]);
// return results;
// });
// }
async $everyModuleTestMultiDevice(): Promise<boolean> {
if (!this.settings.enableDebugTools) return Promise.resolve(true);
// this.core.$$addTestResult("DevModule", "Test", true);
// return Promise.resolve(true);
await this._test("Conflict resolution", async () => await this.checkConflictResolution());
return this.testDone();
}
}

119
src/modules/extras/devUtil/TestPane.svelte Normal file
View File

@@ -0,0 +1,119 @@
<script lang="ts">
import { onDestroy, onMount } from "svelte";
import type ObsidianLiveSyncPlugin from "../../../main.ts";
import { perf_trench } from "./tests.ts";
import { MarkdownRenderer, Notice } from "../../../deps.ts";
import type { ModuleDev } from "../ModuleDev.ts";
import { fireAndForget } from "octagonal-wheels/promises";
import { EVENT_LAYOUT_READY, eventHub } from "../../../common/events.ts";
import { writable } from "svelte/store";
export let plugin: ObsidianLiveSyncPlugin;
export let moduleDev: ModuleDev;
let performanceTestResult = "";
let functionCheckResult = "";
let testRunning = false;
let prefTestResultEl: HTMLDivElement;
let isReady = false;
$: {
if (performanceTestResult != "" && isReady) {
MarkdownRenderer.render(plugin.app, performanceTestResult, prefTestResultEl, "/", plugin);
}
}
async function performTest() {
try {
testRunning = true;
performanceTestResult = await perf_trench(plugin);
} finally {
testRunning = false;
}
}
function clearResult() {
moduleDev.testResults.update((v) => {
v = [];
return v;
});
}
function clearPerfTestResult() {
prefTestResultEl.empty();
}
onMount(async () => {
isReady = true;
// performTest();
eventHub.once(EVENT_LAYOUT_READY, async () => {
if (await plugin.storageAccess.isExistsIncludeHidden("_AUTO_TEST.md")) {
new Notice("Auto test file found, running tests...");
fireAndForget(async () => {
await allTest();
});
} else {
// new Notice("No auto test file found, skipping tests...");
}
});
});
let moduleTesting = false;
function moduleMultiDeviceTest() {
if (moduleTesting) return;
moduleTesting = true;
plugin.$everyModuleTestMultiDevice().finally(() => {
moduleTesting = false;
});
}
function moduleSingleDeviceTest() {
if (moduleTesting) return;
moduleTesting = true;
plugin.$everyModuleTest().finally(() => {
moduleTesting = false;
});
}
async function allTest() {
if (moduleTesting) return;
moduleTesting = true;
try {
await plugin.$everyModuleTest();
await plugin.$everyModuleTestMultiDevice();
} finally {
moduleTesting = false;
}
}
const results = moduleDev.testResults;
$: resultLines = $results;
let syncStatus = [] as string[];
eventHub.on<string[]>("debug-sync-status", (status) => {
syncStatus = [...status];
});
</script>
<h2>TESTING BENCH: Self-hosted LiveSync</h2>
<h3>Module Checks</h3>
<button on:click={() => moduleMultiDeviceTest()} disabled={moduleTesting}>MultiDevice Test</button>
<button on:click={() => moduleSingleDeviceTest()} disabled={moduleTesting}>SingleDevice Test</button>
<button on:click={() => allTest()} disabled={moduleTesting}>All Test</button>
<button on:click={() => clearResult()}>Clear</button>
{#each resultLines as [result, line, message]}
<details open={!result}>
<summary>[{result ? "PASS" : "FAILED"}] {line}</summary>
<pre>{message}</pre>
</details>
{/each}
<h3>Synchronisation Result Status</h3>
<pre>{syncStatus.join("\n")}</pre>
<h3>Performance test</h3>
<button on:click={() => performTest()} disabled={testRunning}>Test!</button>
<button on:click={() => clearPerfTestResult()}>Clear</button>
<div bind:this={prefTestResultEl}></div>
<style>
* {
box-sizing: border-box;
}
</style>

12
src/tests/TestPaneView.ts → src/modules/extras/devUtil/TestPaneView.ts
View File

@@ -3,13 +3,15 @@ import {
WorkspaceLeaf
} from "obsidian";
import TestPaneComponent from "./TestPane.svelte"
import type ObsidianLiveSyncPlugin from "../main"
import type ObsidianLiveSyncPlugin from "../../../main.ts"
import type { ModuleDev } from "../ModuleDev.ts";
export const VIEW_TYPE_TEST = "ols-pane-test";
//Log view
export class TestPaneView extends ItemView {
component?: TestPaneComponent;
plugin: ObsidianLiveSyncPlugin;
moduleDev: ModuleDev;
icon = "view-log";
title: string = "Self-hosted LiveSync Test and Results"
navigation = true;
@@ -18,9 +20,10 @@ export class TestPaneView extends ItemView {
return "view-log";
}
constructor(leaf: WorkspaceLeaf, plugin: ObsidianLiveSyncPlugin) {
constructor(leaf: WorkspaceLeaf, plugin: ObsidianLiveSyncPlugin, moduleDev: ModuleDev) {
super(leaf);
this.plugin = plugin;
this.moduleDev = moduleDev;
}
@@ -37,13 +40,16 @@ export class TestPaneView extends ItemView {
this.component = new TestPaneComponent({
target: this.contentEl,
props: {
plugin: this.plugin
plugin: this.plugin,
moduleDev: this.moduleDev
},
});
await Promise.resolve();
}
// eslint-disable-next-line require-await
async onClose() {
this.component?.$destroy();
await Promise.resolve();
}
}

11
src/tests/testUtils.ts → src/modules/extras/devUtil/testUtils.ts
View File

@@ -1,6 +1,6 @@
import { fireAndForget } from "src/lib/src/common/utils";
import { serialized } from "src/lib/src/concurrency/lock";
import type ObsidianLiveSyncPlugin from "src/main";
import { fireAndForget } from "../../../lib/src/common/utils.ts";
import { serialized } from "../../../lib/src/concurrency/lock.ts";
import type ObsidianLiveSyncPlugin from "../../../main.ts";
let plugin: ObsidianLiveSyncPlugin;
export function enableTestFunction(plugin_: ObsidianLiveSyncPlugin) {
@@ -37,8 +37,9 @@ export function addDebugFileLog(message: any, stackLog = false) {
// const out = "--" + timestamp + "--\n" + messageContent + " " + (stack || "");
// const out
try {
await plugin.vaultAccess.adapterAppend(plugin.app.vault.configDir + "/ls-debug/" + outFile, JSON.stringify(out) + "\n")
} catch (ex) {
await plugin.storageAccess.appendHiddenFile(plugin.app.vault.configDir + "/ls-debug/" + outFile, JSON.stringify(out) + "\n")
} catch {
//NO OP
}
}));

13
src/tests/tests.ts → src/modules/extras/devUtil/tests.ts
View File

@@ -1,5 +1,5 @@
import { Trench } from "../lib/src/memory/memutil.ts";
import type ObsidianLiveSyncPlugin from "../main.ts";
import { Trench } from "../../../lib/src/memory/memutil.ts";
import type ObsidianLiveSyncPlugin from "../../../main.ts";
type MeasureResult = [times: number, spent: number];
type NamedMeasureResult = [name: string, result: MeasureResult];
const measures = new Map<string, MeasureResult>();
@@ -30,9 +30,10 @@ async function measure(name: string, proc: () => (void | Promise<void>), times:
return [name, measures.get(name) as MeasureResult];
}
// eslint-disable-next-line require-await
// eslint-disable-next-line require-await, @typescript-eslint/require-await
async function formatPerfResults(items: NamedMeasureResult[]) {
return `| Name | Runs | Each | Total |\n| --- | --- | --- | --- | \n` + items.map(e => `| ${e[0]} | ${e[1][0]} | ${e[1][0] != 0 ? formatNumber(e[1][1] / e[1][0]) : "-"} | ${formatNumber(e[1][0])} |`).join("\n");
}
export async function perf_trench(plugin: ObsidianLiveSyncPlugin) {
clearResult("trench");
@@ -43,7 +44,7 @@ export async function perf_trench(plugin: ObsidianLiveSyncPlugin) {
await p();
}));
{
const testBinary = await plugin.vaultAccess.adapterReadBinary("testdata/10kb.png");
const testBinary = await plugin.storageAccess.readHiddenFileBinary("testdata/10kb.png");
const uint8Array = new Uint8Array(testBinary);
result.push(await measure("trench-binary-10kb", async () => {
const p = trench.evacuate(uint8Array);
@@ -51,7 +52,7 @@ export async function perf_trench(plugin: ObsidianLiveSyncPlugin) {
}));
}
{
const testBinary = await plugin.vaultAccess.adapterReadBinary("testdata/100kb.jpeg");
const testBinary = await plugin.storageAccess.readHiddenFileBinary("testdata/100kb.jpeg");
const uint8Array = new Uint8Array(testBinary);
result.push(await measure("trench-binary-100kb", async () => {
const p = trench.evacuate(uint8Array);
@@ -59,7 +60,7 @@ export async function perf_trench(plugin: ObsidianLiveSyncPlugin) {
}));
}
{
const testBinary = await plugin.vaultAccess.adapterReadBinary("testdata/1mb.png");
const testBinary = await plugin.storageAccess.readHiddenFileBinary("testdata/1mb.png");
const uint8Array = new Uint8Array(testBinary);
result.push(await measure("trench-binary-1mb", async () => {
const p = trench.evacuate(uint8Array);

72
src/ui/DocumentHistoryModal.ts → src/modules/features/DocumentHistory/DocumentHistoryModal.ts
View File

@@ -1,12 +1,13 @@
import { TFile, Modal, App, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "../deps.ts";
import { getPathFromTFile, isValidPath } from "../common/utils.ts";
import { decodeBinary, escapeStringToHTML, readString } from "../lib/src/string_and_binary/convert.ts";
import ObsidianLiveSyncPlugin from "../main.ts";
import { type DocumentID, type FilePathWithPrefix, type LoadedEntry, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "../lib/src/common/types.ts";
import { Logger } from "../lib/src/common/logger.ts";
import { isErrorOfMissingDoc } from "../lib/src/pouchdb/utils_couchdb.ts";
import { getDocData, readContent } from "../lib/src/common/utils.ts";
import { isPlainText, stripPrefix } from "../lib/src/string_and_binary/path.ts";
import { TFile, Modal, App, DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT, diff_match_patch } from "../../../deps.ts";
import { getPathFromTFile, isValidPath } from "../../../common/utils.ts";
import { decodeBinary, escapeStringToHTML, readString } from "../../../lib/src/string_and_binary/convert.ts";
import ObsidianLiveSyncPlugin from "../../../main.ts";
import { type DocumentID, type FilePathWithPrefix, type LoadedEntry, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "../../../lib/src/common/types.ts";
import { Logger } from "../../../lib/src/common/logger.ts";
import { isErrorOfMissingDoc } from "../../../lib/src/pouchdb/utils_couchdb.ts";
import { fireAndForget, getDocData, readContent } from "../../../lib/src/common/utils.ts";
import { isPlainText, stripPrefix } from "../../../lib/src/string_and_binary/path.ts";
import { scheduleOnceIfDuplicated } from "octagonal-wheels/concurrency/lock";
function isImage(path: string) {
const ext = path.split(".").splice(-1)[0].toLowerCase();
@@ -31,6 +32,7 @@ function readDocument(w: LoadedEntry) {
try {
return readString(new Uint8Array(decodeBinary(w.data)));
} catch (ex) {
Logger(ex, LOG_LEVEL_VERBOSE);
// NO OP.
}
return getDocData(w.data);
@@ -59,7 +61,7 @@ export class DocumentHistoryModal extends Modal {
this.id = id;
this.initialRev = revision;
if (!file && id) {
this.file = this.plugin.id2path(id);
this.file = this.plugin.$$id2path(id);
}
if (localStorage.getItem("ols-history-highlightdiff") == "1") {
this.showDiff = true;
@@ -68,7 +70,7 @@ export class DocumentHistoryModal extends Modal {
async loadFile(initialRev?: string) {
if (!this.id) {
this.id = await this.plugin.path2id(this.file);
this.id = await this.plugin.$$path2id(this.file);
}
const db = this.plugin.localDatabase;
try {
@@ -207,10 +209,10 @@ export class DocumentHistoryModal extends Modal {
divView.createEl("input", { type: "range" }, (e) => {
this.range = e;
e.addEventListener("change", (e) => {
this.loadRevs();
void scheduleOnceIfDuplicated("loadRevs", () => this.loadRevs());
});
e.addEventListener("input", (e) => {
this.loadRevs();
void scheduleOnceIfDuplicated("loadRevs", () => this.loadRevs());
});
});
contentEl
@@ -224,7 +226,7 @@ export class DocumentHistoryModal extends Modal {
checkbox.addEventListener("input", (evt: any) => {
this.showDiff = checkbox.checked;
localStorage.setItem("ols-history-highlightdiff", this.showDiff == true ? "1" : "");
this.loadRevs();
void scheduleOnceIfDuplicated("loadRevs", () => this.loadRevs());
});
})
);
@@ -234,7 +236,7 @@ export class DocumentHistoryModal extends Modal {
.addClass("op-info");
this.info = contentEl.createDiv("");
this.info.addClass("op-info");
this.loadFile(this.initialRev);
fireAndForget(async () => await this.loadFile(this.initialRev));
const div = contentEl.createDiv({ text: "Loading old revisions..." });
this.contentView = div;
div.addClass("op-scrollable");
@@ -242,9 +244,11 @@ export class DocumentHistoryModal extends Modal {
const buttons = contentEl.createDiv("");
buttons.createEl("button", { text: "Copy to clipboard" }, (e) => {
e.addClass("mod-cta");
e.addEventListener("click", async () => {
await navigator.clipboard.writeText(this.currentText);
Logger(`Old content copied to clipboard`, LOG_LEVEL_NOTICE);
e.addEventListener("click", () => {
fireAndForget(async () => {
await navigator.clipboard.writeText(this.currentText);
Logger(`Old content copied to clipboard`, LOG_LEVEL_NOTICE);
});
});
});
const focusFile = async (path: string) => {
@@ -258,21 +262,23 @@ export class DocumentHistoryModal extends Modal {
}
buttons.createEl("button", { text: "Back to this revision" }, (e) => {
e.addClass("mod-cta");
e.addEventListener("click", async () => {
// const pathToWrite = this.plugin.id2path(this.id, true);
const pathToWrite = stripPrefix(this.file);
if (!isValidPath(pathToWrite)) {
Logger("Path is not valid to write content.", LOG_LEVEL_INFO);
return;
}
if (!this.currentDoc) {
Logger("No active file loaded.", LOG_LEVEL_INFO);
return;
}
const d = readContent(this.currentDoc);
await this.plugin.vaultAccess.adapterWrite(pathToWrite, d);
await focusFile(pathToWrite);
this.close();
e.addEventListener("click", () => {
fireAndForget(async () => {
// const pathToWrite = this.plugin.id2path(this.id, true);
const pathToWrite = stripPrefix(this.file);
if (!isValidPath(pathToWrite)) {
Logger("Path is not valid to write content.", LOG_LEVEL_INFO);
return;
}
if (!this.currentDoc) {
Logger("No active file loaded.", LOG_LEVEL_INFO);
return;
}
const d = readContent(this.currentDoc);
await this.plugin.storageAccess.writeHiddenFileAuto(pathToWrite, d);
await focusFile(pathToWrite);
this.close();
});
});
});
}

29
src/ui/GlobalHistory.svelte → src/modules/features/GlobalHistory/GlobalHistory.svelte
View File

@@ -1,12 +1,13 @@
<script lang="ts">
import ObsidianLiveSyncPlugin from "../main";
import ObsidianLiveSyncPlugin from "../../../main.ts";
import { onDestroy, onMount } from "svelte";
import type { AnyEntry, FilePathWithPrefix } from "../lib/src/common/types";
import { getDocData, isAnyNote, isDocContentSame, readAsBlob } from "../lib/src/common/utils";
import { diff_match_patch } from "../deps";
import { DocumentHistoryModal } from "./DocumentHistoryModal";
import { isPlainText, stripAllPrefixes } from "../lib/src/string_and_binary/path";
import { TFile } from "../deps";
import type { AnyEntry, FilePathWithPrefix } from "../../../lib/src/common/types.ts";
import { getDocData, isAnyNote, isDocContentSame, readAsBlob } from "../../../lib/src/common/utils.ts";
import { diff_match_patch } from "../../../deps.ts";
import { DocumentHistoryModal } from "../DocumentHistory/DocumentHistoryModal.ts";
import { isPlainText, stripAllPrefixes } from "../../../lib/src/string_and_binary/path.ts";
import { TFile } from "../../../deps.ts";
import { getPath } from "../../../common/utils.ts";
export let plugin: ObsidianLiveSyncPlugin;
let showDiffInfo = false;
@@ -54,7 +55,7 @@
continue;
}
if (!isAnyNote(docA)) continue;
const path = plugin.getPath(docA as AnyEntry);
const path = getPath(docA as AnyEntry);
const isPlain = isPlainText(docA.path);
const revs = await db.getRaw(docA._id, { revs_info: true });
let p: string | undefined = undefined;
@@ -89,12 +90,12 @@
const diff = dmp.diff_main(p, data);
dmp.diff_cleanupSemantic(diff);
p = data;
const pxinit = {
const pxInit = {
[DIFF_DELETE]: 0,
[DIFF_EQUAL]: 0,
[DIFF_INSERT]: 0,
} as { [key: number]: number };
const px = diff.reduce((p, c) => ({ ...p, [c[0]]: (p[c[0]] ?? 0) + c[1].length }), pxinit);
const px = diff.reduce((p, c) => ({ ...p, [c[0]]: (p[c[0]] ?? 0) + c[1].length }), pxInit);
diffDetail = `-${px[DIFF_DELETE]}, +${px[DIFF_INSERT]}`;
}
}
@@ -104,9 +105,9 @@
}
if (rev == docA._rev) {
if (checkStorageDiff) {
const abs = plugin.vaultAccess.getAbstractFileByPath(stripAllPrefixes(plugin.getPath(docA)));
if (abs instanceof TFile) {
const data = await plugin.vaultAccess.adapterReadAuto(abs);
const isExist = await plugin.storageAccess.isExists(stripAllPrefixes(getPath(docA)));
if (isExist) {
const data = await plugin.storageAccess.readFileAuto(stripAllPrefixes(getPath(docA)));
const d = readAsBlob(doc);
const result = await isDocContentSame(data, d);
if (result) {
@@ -117,7 +118,7 @@
}
}
}
const docPath = plugin.getPath(doc as AnyEntry);
const docPath = getPath(doc as AnyEntry);
const [filename, ...pathItems] = docPath.split("/").reverse();
let chunksStatus = "";

8
src/ui/GlobalHistoryView.ts → src/modules/features/GlobalHistory/GlobalHistoryView.ts
View File

@@ -1,9 +1,9 @@
import {
ItemView,
WorkspaceLeaf
} from "../deps.ts";
} from "../../../deps.ts";
import GlobalHistoryComponent from "./GlobalHistory.svelte";
import type ObsidianLiveSyncPlugin from "../main.ts";
import type ObsidianLiveSyncPlugin from "../../../main.ts";
export const VIEW_TYPE_GLOBAL_HISTORY = "global-history";
export class GlobalHistoryView extends ItemView {
@@ -32,7 +32,6 @@ export class GlobalHistoryView extends ItemView {
return "Vault history";
}
// eslint-disable-next-line require-await
async onOpen() {
this.component = new GlobalHistoryComponent({
target: this.contentEl,
@@ -40,10 +39,11 @@ export class GlobalHistoryView extends ItemView {
plugin: this.plugin,
},
});
await Promise.resolve();
}
// eslint-disable-next-line require-await
async onClose() {
this.component?.$destroy();
await Promise.resolve();
}
}

22
src/ui/ConflictResolveModal.ts → src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts
View File

@@ -1,8 +1,8 @@
import { App, Modal } from "../deps.ts";
import { App, Modal } from "../../../deps.ts";
import { DIFF_DELETE, DIFF_EQUAL, DIFF_INSERT } from "diff-match-patch";
import { CANCELLED, LEAVE_TO_SUBSEQUENT, RESULT_TIMED_OUT, type diff_result } from "../lib/src/common/types.ts";
import { escapeStringToHTML } from "../lib/src/string_and_binary/convert.ts";
import { delay, sendValue, waitForValue } from "../lib/src/common/utils.ts";
import { CANCELLED, LEAVE_TO_SUBSEQUENT, RESULT_TIMED_OUT, type diff_result } from "../../../lib/src/common/types.ts";
import { escapeStringToHTML } from "../../../lib/src/string_and_binary/convert.ts";
import { delay, fireAndForget, sendValue, waitForValue } from "../../../lib/src/common/utils.ts";
export type MergeDialogResult = typeof LEAVE_TO_SUBSEQUENT | typeof CANCELLED | string;
export class ConflictResolveModal extends Modal {
@@ -40,12 +40,14 @@ export class ConflictResolveModal extends Modal {
// Send cancel signal for the previous merge dialogue
// if not there, simply be ignored.
sendValue("cancel-resolve-conflict:" + this.filename, true);
setTimeout(async () => {
const forceClose = await waitForValue("cancel-resolve-conflict:" + this.filename);
// debugger;
if (forceClose) {
this.sendResponse(CANCELLED);
}
setTimeout(() => {
fireAndForget(async () => {
const forceClose = await waitForValue("cancel-resolve-conflict:" + this.filename);
// debugger;
if (forceClose) {
this.sendResponse(CANCELLED);
}
});
}, 10)
// sendValue("close-resolve-conflict:" + this.filename, false);
this.titleEl.setText(this.title);

6
src/ui/LogPane.svelte → src/modules/features/Log/LogPane.svelte
View File

@@ -1,8 +1,8 @@
<script lang="ts">
import { onDestroy, onMount } from "svelte";
import { logMessages } from "../lib/src/mock_and_interop/stores";
import type { ReactiveInstance } from "../lib/src/dataobject/reactive";
import { Logger } from "../lib/src/common/logger";
import { logMessages } from "../../../lib/src/mock_and_interop/stores";
import type { ReactiveInstance } from "../../../lib/src/dataobject/reactive";
import { Logger } from "../../../lib/src/common/logger";
let unsubscribe: () => void;
let messages = [] as string[];

6
src/ui/LogPaneView.ts → src/modules/features/Log/LogPaneView.ts
View File

@@ -3,7 +3,7 @@ import {
WorkspaceLeaf
} from "obsidian";
import LogPaneComponent from "./LogPane.svelte";
import type ObsidianLiveSyncPlugin from "../main.ts";
import type ObsidianLiveSyncPlugin from "../../../main.ts";
export const VIEW_TYPE_LOG = "log-log";
//Log view
export class LogPaneView extends ItemView {
@@ -32,17 +32,17 @@ export class LogPaneView extends ItemView {
return "Self-hosted LiveSync Log";
}
// eslint-disable-next-line require-await
async onOpen() {
this.component = new LogPaneComponent({
target: this.contentEl,
props: {
},
});
await Promise.resolve();
}
// eslint-disable-next-line require-await
async onClose() {
this.component?.$destroy();
await Promise.resolve();
}
}

29
src/modules/features/ModuleGlobalHistory.ts Normal file
View File

@@ -0,0 +1,29 @@
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
import { VIEW_TYPE_GLOBAL_HISTORY, GlobalHistoryView } from "./GlobalHistory/GlobalHistoryView.ts";
export class ModuleObsidianGlobalHistory extends AbstractObsidianModule implements IObsidianModule {
$everyOnloadStart(): Promise<boolean> {
this.addCommand({
id: "livesync-global-history",
name: "Show vault history",
callback: () => {
this.showGlobalHistory()
}
})
this.registerView(
VIEW_TYPE_GLOBAL_HISTORY,
(leaf) => new GlobalHistoryView(leaf, this.plugin)
);
return Promise.resolve(true);
}
showGlobalHistory() {
void this.core.$$showView(VIEW_TYPE_GLOBAL_HISTORY);
}
}

134
src/modules/features/ModuleInteractiveConflictResolver.ts Normal file
View File

@@ -0,0 +1,134 @@
import { CANCELLED, LEAVE_TO_SUBSEQUENT, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE, MISSING_OR_ERROR, type DocumentID, type FilePathWithPrefix, type diff_result } from "../../lib/src/common/types.ts";
import { ConflictResolveModal } from "./InteractiveConflictResolving/ConflictResolveModal.ts";
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
import { displayRev, getPath, getPathWithoutPrefix } from "../../common/utils.ts";
import { fireAndForget } from "octagonal-wheels/promises";
export class ModuleInteractiveConflictResolver extends AbstractObsidianModule implements IObsidianModule {
$everyOnloadStart(): Promise<boolean> {
this.addCommand({
id: "livesync-conflictcheck",
name: "Pick a file to resolve conflict",
callback: async () => {
await this.pickFileForResolve();
},
})
this.addCommand({
id: "livesync-all-conflictcheck",
name: "Resolve all conflicted files",
callback: async () => {
await this.allConflictCheck();
},
})
return Promise.resolve(true);
}
async $anyResolveConflictByUI(filename: FilePathWithPrefix, conflictCheckResult: diff_result): Promise<boolean> {
this._log("Merge:open conflict dialog", LOG_LEVEL_VERBOSE);
const dialog = new ConflictResolveModal(this.app, filename, conflictCheckResult);
dialog.open();
const selected = await dialog.waitForResult();
if (selected === CANCELLED) {
// Cancelled by UI, or another conflict.
this._log(`Merge: Cancelled ${filename}`, LOG_LEVEL_INFO);
return false;
}
const testDoc = await this.localDatabase.getDBEntry(filename, { conflicts: true }, false, true, true);
if (testDoc === false) {
this._log(`Merge: Could not read ${filename} from the local database`, LOG_LEVEL_VERBOSE);
return false;
}
if (!testDoc._conflicts) {
this._log(`Merge: Nothing to do ${filename}`, LOG_LEVEL_VERBOSE);
return false;
}
const toDelete = selected;
// const toKeep = conflictCheckResult.left.rev != toDelete ? conflictCheckResult.left.rev : conflictCheckResult.right.rev;
if (toDelete === LEAVE_TO_SUBSEQUENT) {
// Concatenate both conflicted revisions.
// Create a new file by concatenating both conflicted revisions.
const p = conflictCheckResult.diff.map((e) => e[1]).join("");
const delRev = testDoc._conflicts[0];
if (!await this.core.databaseFileAccess.storeContent(filename, p)) {
this._log(`Concatenated content cannot be stored:${filename}`, LOG_LEVEL_NOTICE);
return false;
}
// 2. As usual, delete the conflicted revision and if there are no conflicts, write the resolved content to the storage.
if (await this.core.$$resolveConflictByDeletingRev(filename, delRev, "UI Concatenated") == MISSING_OR_ERROR) {
this._log(`Concatenated saved, but cannot delete conflicted revisions: ${filename}, (${displayRev(delRev)})`, LOG_LEVEL_NOTICE);
return false;
}
} else if (typeof toDelete === "string") {
// Select one of the conflicted revision to delete.
if (await this.core.$$resolveConflictByDeletingRev(filename, toDelete, "UI Selected") == MISSING_OR_ERROR) {
this._log(`Merge: Something went wrong: ${filename}, (${toDelete})`, LOG_LEVEL_NOTICE);
return false;
}
} else {
this._log(`Merge: Something went wrong: ${filename}, (${toDelete})`, LOG_LEVEL_NOTICE);
return false;
}
// In here, some merge has been processed.
// So we have to run replication if configured.
// TODO: Make this is as a event request
if (this.settings.syncAfterMerge && !this.plugin.suspended) {
await this.core.$$waitForReplicationOnce();
}
// And, check it again.
await this.core.$$queueConflictCheck(filename);
return false;
}
async allConflictCheck() {
while (await this.pickFileForResolve());
}
async pickFileForResolve() {
const notes: { id: DocumentID, path: FilePathWithPrefix, dispPath: string, mtime: number }[] = [];
for await (const doc of this.localDatabase.findAllDocs({ conflicts: true })) {
if (!("_conflicts" in doc)) continue;
notes.push({ id: doc._id, path: getPath(doc), dispPath: getPathWithoutPrefix(doc), mtime: doc.mtime });
}
notes.sort((a, b) => b.mtime - a.mtime);
const notesList = notes.map(e => e.dispPath);
if (notesList.length == 0) {
this._log("There are no conflicted documents", LOG_LEVEL_NOTICE);
return false;
}
const target = await this.plugin.confirm.askSelectString("File to resolve conflict", notesList);
if (target) {
const targetItem = notes.find(e => e.dispPath == target)!;
await this.core.$$queueConflictCheck(targetItem.path);
await this.core.$$waitForAllConflictProcessed();
return true;
}
return false;
}
async $allScanStat(): Promise<boolean> {
const notes: { path: string, mtime: number }[] = [];
this._log(`Checking conflicted files`, LOG_LEVEL_VERBOSE);
for await (const doc of this.localDatabase.findAllDocs({ conflicts: true })) {
if (!("_conflicts" in doc)) continue;
notes.push({ path: getPath(doc), mtime: doc.mtime });
}
if (notes.length > 0) {
this.plugin.confirm.askInPopup(`conflicting-detected-on-safety`, `Some files have been left conflicted! Press {HERE} to resolve them, or you can do it later by "Pick a file to resolve conflict`, (anchor) => {
anchor.text = "HERE";
anchor.addEventListener("click", () => {
fireAndForget(() => this.allConflictCheck())
});
}
);
this._log(`Some files have been left conflicted! Please resolve them by "Pick a file to resolve conflict". The list is written in the log.`, LOG_LEVEL_VERBOSE);
for (const note of notes) {
this._log(`Conflicted: ${note.path}`);
}
} else {
this._log(`There are no conflicted files`, LOG_LEVEL_VERBOSE);
}
return true;
}
}

247
src/features/CmdStatusInsideEditor.ts → src/modules/features/ModuleLog.ts
View File

@@ -1,13 +1,37 @@
import { computed, reactive, reactiveSource, type ReactiveValue } from "octagonal-wheels/dataobject/reactive";
import type { DatabaseConnectingStatus, EntryDoc } from "../lib/src/common/types";
import { LiveSyncCommands } from "./LiveSyncCommands";
import { scheduleTask } from "octagonal-wheels/concurrency/task";
import { isDirty, throttle } from "../lib/src/common/utils";
import { collectingChunks, pluginScanningCount, hiddenFilesEventCount, hiddenFilesProcessingCount } from "../lib/src/mock_and_interop/stores";
import { eventHub } from "../lib/src/hub/hub";
import { EVENT_FILE_RENAMED, EVENT_LAYOUT_READY, EVENT_LEAF_ACTIVE_CHANGED } from "../common/events";
import { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_VERBOSE, PREFIXMD_LOGFILE, type DatabaseConnectingStatus, type LOG_LEVEL } from "../../lib/src/common/types.ts";
import { cancelTask, scheduleTask } from "octagonal-wheels/concurrency/task";
import { fireAndForget, isDirty, throttle } from "../../lib/src/common/utils.ts";
import { collectingChunks, pluginScanningCount, hiddenFilesEventCount, hiddenFilesProcessingCount, type LogEntry, logStore, logMessages } from "../../lib/src/mock_and_interop/stores.ts";
import { eventHub } from "../../lib/src/hub/hub.ts";
import { EVENT_FILE_RENAMED, EVENT_LAYOUT_READY, EVENT_LEAF_ACTIVE_CHANGED } from "../../common/events.ts";
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
import { addIcon, normalizePath, Notice } from "../../deps.ts";
import { LOG_LEVEL_NOTICE, setGlobalLogFunction } from "octagonal-wheels/common/logger";
import { QueueProcessor } from "octagonal-wheels/concurrency/processor";
import { LogPaneView, VIEW_TYPE_LOG } from "./Log/LogPaneView.ts";
import { serialized } from "octagonal-wheels/concurrency/lock";
export class LogAddOn extends LiveSyncCommands {
// This module cannot be a core module because it depends on the Obsidian UI.
// DI the log again.
setGlobalLogFunction((message: any, level?: number, key?: string) => {
const entry = { message, level, key } as LogEntry;
logStore.enqueue(entry);
});
let recentLogs = [] as string[];
// Recent log splicer
const recentLogProcessor = new QueueProcessor((logs: string[]) => {
recentLogs = [...recentLogs, ...logs].splice(-200);
logMessages.value = recentLogs;
}, { batchSize: 25, delay: 10, suspended: false, concurrentLimit: 1 }).resumePipeLine();
// logStore.intercept(e => e.slice(Math.min(e.length - 200, 0)));
export class ModuleLog extends AbstractObsidianModule implements IObsidianModule {
registerView = this.plugin.registerView.bind(this.plugin);
statusBar?: HTMLElement;
@@ -18,6 +42,8 @@ export class LogAddOn extends LiveSyncCommands {
messageArea?: HTMLDivElement;
statusBarLabels!: ReactiveValue<{ message: string, status: string }>;
statusLog = reactiveSource("");
notifies: { [key: string]: { notice: Notice; count: number } } = {};
observeForLogs() {
const padSpaces = `\u{2007}`.repeat(10);
@@ -102,9 +128,9 @@ export class LogAddOn extends LiveSyncCommands {
}
return { w, sent, pushLast, arrived, pullLast };
})
const labelProc = padLeftSpComputed(this.plugin.vaultManager.processing, `⏳`);
const labelPend = padLeftSpComputed(this.plugin.vaultManager.totalQueued, `🛫`);
const labelInBatchDelay = padLeftSpComputed(this.plugin.vaultManager.batched, `📬`);
const labelProc = padLeftSpComputed(this.plugin.processing, `⏳`);
const labelPend = padLeftSpComputed(this.plugin.totalQueued, `🛫`);
const labelInBatchDelay = padLeftSpComputed(this.plugin.batched, `📬`);
const waitingLabel = computed(() => {
return `${labelProc()}${labelPend()}${labelInBatchDelay()}`;
})
@@ -120,7 +146,7 @@ export class LogAddOn extends LiveSyncCommands {
const statusBarLabels = reactive(() => {
const scheduleMessage = this.plugin.isReloadingScheduled ? `WARNING! RESTARTING OBSIDIAN IS SCHEDULED!\n` : "";
const { message } = statusLineLabel();
const status = scheduleMessage + this.plugin.statusLog.value;
const status = scheduleMessage + this.statusLog.value;
return {
message, status
@@ -136,6 +162,11 @@ export class LogAddOn extends LiveSyncCommands {
statusBarLabels.onChanged(label => applyToDisplay(label.value))
}
$everyOnload(): Promise<boolean> {
eventHub.onEvent(EVENT_LEAF_ACTIVE_CHANGED, () => this.onActiveLeafChange());
eventHub.onceEvent(EVENT_LAYOUT_READY, () => this.onActiveLeafChange());
return Promise.resolve(true);
}
adjustStatusDivPosition() {
const mdv = this.app.workspace.getMostRecentLeaf();
if (mdv && this.statusDiv) {
@@ -146,41 +177,29 @@ export class LogAddOn extends LiveSyncCommands {
}
}
onunload() {
if (this.statusDiv) {
this.statusDiv.remove();
async getActiveFileStatus() {
const thisFile = this.app.workspace.getActiveFile();
if (!thisFile) return "";
// Case Sensitivity
if (this.core.shouldCheckCaseInsensitive) {
const f = this.core.storageAccess.getFiles().map(e => e.path).filter(e => e.toLowerCase() == thisFile.path.toLowerCase());
if (f.length > 1) return "Not synchronised: There are multiple files with the same name";
}
document.querySelectorAll(`.livesync-status`)?.forEach(e => e.remove());
if (!await this.core.$$isTargetFile(thisFile.path)) return "Not synchronised: not a target file";
if (this.core.$$isFileSizeExceeded(thisFile.stat.size)) return "Not synchronised: File size exceeded";
return "";
}
async setFileStatus() {
this.messageArea!.innerText = await this.plugin.getActiveFileStatus();
this.messageArea!.innerText = await this.getActiveFileStatus();
}
onActiveLeafChange() {
this.adjustStatusDivPosition();
this.setFileStatus();
}
onload(): void | Promise<void> {
eventHub.onEvent(EVENT_FILE_RENAMED, (evt: CustomEvent<{ oldPath: string, newPath: string }>) => {
this.setFileStatus();
fireAndForget(async () => {
this.adjustStatusDivPosition();
await this.setFileStatus();
});
eventHub.onEvent(EVENT_LEAF_ACTIVE_CHANGED, () => this.onActiveLeafChange());
const w = document.querySelectorAll(`.livesync-status`);
w.forEach(e => e.remove());
this.observeForLogs();
this.adjustStatusDivPosition();
this.statusDiv = this.app.workspace.containerEl.createDiv({ cls: "livesync-status" });
this.statusLine = this.statusDiv.createDiv({ cls: "livesync-status-statusline" });
this.messageArea = this.statusDiv.createDiv({ cls: "livesync-status-messagearea" });
this.logMessage = this.statusDiv.createDiv({ cls: "livesync-status-logmessage" });
this.logHistory = this.statusDiv.createDiv({ cls: "livesync-status-loghistory" });
eventHub.onEvent(EVENT_LAYOUT_READY, () => this.adjustStatusDivPosition());
if (this.settings.showStatusOnStatusbar) {
this.statusBar = this.plugin.addStatusBarItem();
this.statusBar.addClass("syncstatusbar");
}
}
nextFrameQueue: ReturnType<typeof requestAnimationFrame> | undefined = undefined;
logLines: { ttl: number, message: string }[] = [];
@@ -193,13 +212,14 @@ export class LogAddOn extends LiveSyncCommands {
const { message, status } = this.statusBarLabels.value;
// const recent = logMessages.value;
const newMsg = message;
const newLog = this.settings.showOnlyIconsOnEditor ? "" : status;
let newLog = this.settings?.showOnlyIconsOnEditor ? "" : status;
const moduleTagEnd = newLog.indexOf(`]\u{200A}`);
if (moduleTagEnd != -1) {
newLog = newLog.substring(moduleTagEnd + 2);
}
this.statusBar?.setText(newMsg.split("\n")[0]);
if (this.settings.showStatusOnEditor && this.statusDiv) {
// const root = activeDocument.documentElement;
// root.style.setProperty("--sls-log-text", "'" + (newMsg + "\\A " + newLog) + "'");
// this.statusDiv.innerText = newMsg + "\\A " + newLog;
if (this.settings?.showStatusOnEditor && this.statusDiv) {
if (this.settings.showLongerLogInsideEditor) {
const now = new Date().getTime();
this.logLines = this.logLines.filter(e => e.ttl > now);
@@ -217,18 +237,141 @@ export class LogAddOn extends LiveSyncCommands {
}
});
scheduleTask("log-hide", 3000, () => { this.plugin.statusLog.value = "" });
scheduleTask("log-hide", 3000, () => { this.statusLog.value = "" });
}
onInitializeDatabase(showNotice: boolean) { }
beforeReplicate(showNotice: boolean) { }
onResume() { }
parseReplicationResultItem(docs: PouchDB.Core.ExistingDocument<EntryDoc>): boolean | Promise<boolean> {
return false;
$allStartOnUnload(): Promise<boolean> {
if (this.statusDiv) {
this.statusDiv.remove();
}
document.querySelectorAll(`.livesync-status`)?.forEach(e => e.remove());
return Promise.resolve(true);
}
async realizeSettingSyncMode() { }
$everyOnloadStart(): Promise<boolean> {
addIcon(
"view-log",
`<g transform="matrix(1.28 0 0 1.28 -131 -411)" fill="currentColor" fill-rule="evenodd">
<path d="m103 330h76v12h-76z"/>
<path d="m106 346v44h70v-44zm45 16h-20v-8h20z"/>
</g>`
);
this.addRibbonIcon("view-log", "Show log", () => {
void this.core.$$showView(VIEW_TYPE_LOG);
}).addClass("livesync-ribbon-showlog");
this.addCommand({
id: "view-log",
name: "Show log",
callback: () => {
void this.core.$$showView(VIEW_TYPE_LOG);
}
});
this.registerView(
VIEW_TYPE_LOG,
(leaf) => new LogPaneView(leaf, this.plugin)
);
return Promise.resolve(true);
}
$everyOnloadAfterLoadSettings(): Promise<boolean> {
logStore.pipeTo(new QueueProcessor(logs => logs.forEach(e => this.core.$$addLog(e.message, e.level, e.key)), { suspended: false, batchSize: 20, concurrentLimit: 1, delay: 0 })).startPipeline();
eventHub.onEvent(EVENT_FILE_RENAMED, (evt: CustomEvent<{ oldPath: string, newPath: string }>) => {
void this.setFileStatus();
});
const w = document.querySelectorAll(`.livesync-status`);
w.forEach(e => e.remove());
this.observeForLogs();
this.statusDiv = this.app.workspace.containerEl.createDiv({ cls: "livesync-status" });
this.statusLine = this.statusDiv.createDiv({ cls: "livesync-status-statusline" });
this.messageArea = this.statusDiv.createDiv({ cls: "livesync-status-messagearea" });
this.logMessage = this.statusDiv.createDiv({ cls: "livesync-status-logmessage" });
this.logHistory = this.statusDiv.createDiv({ cls: "livesync-status-loghistory" });
eventHub.onEvent(EVENT_LAYOUT_READY, () => this.adjustStatusDivPosition());
if (this.settings?.showStatusOnStatusbar) {
this.statusBar = this.plugin.addStatusBarItem();
this.statusBar.addClass("syncstatusbar");
}
this.adjustStatusDivPosition();
return Promise.resolve(true);
}
writeLogToTheFile(now: Date, vaultName: string, newMessage: string) {
fireAndForget(() => serialized("writeLog", async () => {
const time = now.toISOString().split("T")[0];
const logDate = `${PREFIXMD_LOGFILE}${time}.md`;
const file = await this.core.storageAccess.isExists(normalizePath(logDate));
if (!file) {
await this.core.storageAccess.appendHiddenFile(normalizePath(logDate), "```\n");
}
await this.core.storageAccess.appendHiddenFile(normalizePath(logDate), vaultName + ":" + newMessage + "\n");
}));
}
$$addLog(message: any, level: LOG_LEVEL = LOG_LEVEL_INFO, key = ""): void {
if (level == LOG_LEVEL_DEBUG) {
return;
}
if (level < LOG_LEVEL_INFO && this.settings && this.settings.lessInformationInLog) {
return;
}
if (this.settings && !this.settings.showVerboseLog && level == LOG_LEVEL_VERBOSE) {
return;
}
const vaultName = this.plugin.$$getVaultName();
const now = new Date();
const timestamp = now.toLocaleString();
const messageContent = typeof message == "string" ? message : message instanceof Error ? `${message.name}:${message.message}` : JSON.stringify(message, null, 2);
if (message instanceof Error) {
// debugger;
console.dir(message.stack);
}
const newMessage = timestamp + "->" + messageContent;
console.log(vaultName + ":" + newMessage);
if (!this.settings?.showOnlyIconsOnEditor) {
this.statusLog.value = messageContent;
}
if (this.settings?.writeLogToTheFile) {
this.writeLogToTheFile(now, vaultName, newMessage);
}
recentLogProcessor.enqueue(newMessage);
this.logLines.push({ ttl: now.getTime() + 3000, message: newMessage });
if (level >= LOG_LEVEL_NOTICE) {
if (!key) key = messageContent;
if (key in this.notifies) {
// @ts-ignore
const isShown = this.notifies[key].notice.noticeEl?.isShown()
if (!isShown) {
this.notifies[key].notice = new Notice(messageContent, 0);
}
cancelTask(`notify-${key}`);
if (key == messageContent) {
this.notifies[key].count++;
this.notifies[key].notice.setMessage(`(${this.notifies[key].count}):${messageContent}`);
} else {
this.notifies[key].notice.setMessage(`${messageContent}`);
}
} else {
const notify = new Notice(messageContent, 0);
this.notifies[key] = {
count: 0,
notice: notify,
};
}
const timeout = 5000;
scheduleTask(`notify-${key}`, timeout, () => {
const notify = this.notifies[key].notice;
delete this.notifies[key];
try {
notify.hide();
} catch {
// NO OP
}
})
}
}
}

56
src/modules/features/ModuleObsidianDocumentHistory.ts Normal file
View File

@@ -0,0 +1,56 @@
import { type TFile } from "obsidian";
import { eventHub, EVENT_REQUEST_SHOW_HISTORY } from "../../common/events.ts";
import type { FilePathWithPrefix, LoadedEntry, DocumentID } from "../../lib/src/common/types.ts";
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
import { DocumentHistoryModal } from "./DocumentHistory/DocumentHistoryModal.ts";
import { getPath } from "../../common/utils.ts";
import { fireAndForget } from "octagonal-wheels/promises";
export class ModuleObsidianDocumentHistory extends AbstractObsidianModule implements IObsidianModule {
$everyOnloadStart(): Promise<boolean> {
this.addCommand({
id: "livesync-history",
name: "Show history",
callback: () => {
const file = this.core.$$getActiveFilePath();
if (file) this.showHistory(file, undefined);
}
});
this.addCommand({
id: "livesync-filehistory",
name: "Pick a file to show history",
callback: () => {
fireAndForget(async () => await this.fileHistory());
},
});
eventHub.on(EVENT_REQUEST_SHOW_HISTORY, ({ file, fileOnDB }: { file: TFile, fileOnDB: LoadedEntry }) => {
this.showHistory(file, fileOnDB._id);
})
return Promise.resolve(true);
}
showHistory(file: TFile | FilePathWithPrefix, id?: DocumentID) {
new DocumentHistoryModal(this.app, this.plugin, file, id).open();
}
async fileHistory() {
const notes: { id: DocumentID, path: FilePathWithPrefix, dispPath: string, mtime: number }[] = [];
for await (const doc of this.localDatabase.findAllDocs()) {
notes.push({ id: doc._id, path: getPath(doc), dispPath: getPath(doc), mtime: doc.mtime });
}
notes.sort((a, b) => b.mtime - a.mtime);
const notesList = notes.map(e => e.dispPath);
const target = await this.plugin.confirm.askSelectString("File to view History", notesList);
if (target) {
const targetId = notes.find(e => e.dispPath == target)!;
this.showHistory(targetId.path, targetId.id);
}
}
}

207
src/modules/features/ModuleObsidianSetting.ts Normal file
View File

@@ -0,0 +1,207 @@
import { type IObsidianModule, AbstractObsidianModule } from "../AbstractObsidianModule.ts";
// import { PouchDB } from "../../lib/src/pouchdb/pouchdb-browser";
import { EVENT_REQUEST_RELOAD_SETTING_TAB, EVENT_SETTING_SAVED, eventHub } from "../../common/events";
import { type BucketSyncSetting, type ConfigPassphraseStore, type CouchDBConnection, DEFAULT_SETTINGS, type ObsidianLiveSyncSettings, SALT_OF_PASSPHRASE } from "../../lib/src/common/types";
import { LOG_LEVEL_NOTICE, LOG_LEVEL_URGENT } from "octagonal-wheels/common/logger";
import { encrypt, tryDecrypt } from "octagonal-wheels/encryption";
import { setLang } from "../../lib/src/common/i18n";
import { isCloudantURI } from "../../lib/src/pouchdb/utils_couchdb";
export class ModuleObsidianSettings extends AbstractObsidianModule implements IObsidianModule {
getPassphrase(settings: ObsidianLiveSyncSettings) {
const methods: Record<ConfigPassphraseStore, (() => Promise<string | false>)> = {
"": () => Promise.resolve("*"),
"LOCALSTORAGE": () => Promise.resolve(localStorage.getItem("ls-setting-passphrase") ?? false),
"ASK_AT_LAUNCH": () => this.plugin.confirm.askString("Passphrase", "passphrase", "")
}
const method = settings.configPassphraseStore;
const methodFunc = method in methods ? methods[method] : methods[""];
return methodFunc();
}
$$saveDeviceAndVaultName(): void {
const lsKey = "obsidian-live-sync-vaultanddevicename-" + this.plugin.$$getVaultName();
localStorage.setItem(lsKey, this.plugin.deviceAndVaultName || "");
}
usedPassphrase = "";
$$clearUsedPassphrase(): void {
this.usedPassphrase = "";
}
async decryptConfigurationItem(encrypted: string, passphrase: string) {
const dec = await tryDecrypt(encrypted, passphrase + SALT_OF_PASSPHRASE, false);
if (dec) {
this.usedPassphrase = passphrase;
return dec;
}
return false;
}
async encryptConfigurationItem(src: string, settings: ObsidianLiveSyncSettings) {
if (this.usedPassphrase != "") {
return await encrypt(src, this.usedPassphrase + SALT_OF_PASSPHRASE, false);
}
const passphrase = await this.getPassphrase(settings);
if (passphrase === false) {
this._log("Could not determine passphrase to save data.json! You probably make the configuration sure again!", LOG_LEVEL_URGENT);
return "";
}
const dec = await encrypt(src, passphrase + SALT_OF_PASSPHRASE, false);
if (dec) {
this.usedPassphrase = passphrase;
return dec;
}
return "";
}
get appId() {
return `${("appId" in this.app ? this.app.appId : "")}`;
}
async $$saveSettingData() {
this.plugin.$$saveDeviceAndVaultName();
const settings = { ...this.settings };
settings.deviceAndVaultName = "";
if (this.usedPassphrase == "" && !await this.getPassphrase(settings)) {
this._log("Could not determine passphrase for saving data.json! Our data.json have insecure items!", LOG_LEVEL_NOTICE);
} else {
if (settings.couchDB_PASSWORD != "" || settings.couchDB_URI != "" || settings.couchDB_USER != "" || settings.couchDB_DBNAME) {
const connectionSetting: CouchDBConnection & BucketSyncSetting = {
couchDB_DBNAME: settings.couchDB_DBNAME,
couchDB_PASSWORD: settings.couchDB_PASSWORD,
couchDB_URI: settings.couchDB_URI,
couchDB_USER: settings.couchDB_USER,
accessKey: settings.accessKey,
bucket: settings.bucket,
endpoint: settings.endpoint,
region: settings.region,
secretKey: settings.secretKey,
useCustomRequestHandler: settings.useCustomRequestHandler
};
settings.encryptedCouchDBConnection = await this.encryptConfigurationItem(JSON.stringify(connectionSetting), settings);
settings.couchDB_PASSWORD = "";
settings.couchDB_DBNAME = "";
settings.couchDB_URI = "";
settings.couchDB_USER = "";
settings.accessKey = "";
settings.bucket = "";
settings.region = "";
settings.secretKey = "";
settings.endpoint = "";
}
if (settings.encrypt && settings.passphrase != "") {
settings.encryptedPassphrase = await this.encryptConfigurationItem(settings.passphrase, settings);
settings.passphrase = "";
}
}
await this.core.saveData(settings);
eventHub.emitEvent(EVENT_SETTING_SAVED, settings);
}
tryDecodeJson(encoded: string | false): object | false {
try {
if (!encoded) return false;
return JSON.parse(encoded);
} catch {
return false;
}
}
async $$loadSettings(): Promise<void> {
const settings = Object.assign({}, DEFAULT_SETTINGS, await this.core.loadData()) as ObsidianLiveSyncSettings;
if (typeof settings.isConfigured == "undefined") {
// If migrated, mark true
if (JSON.stringify(settings) !== JSON.stringify(DEFAULT_SETTINGS)) {
settings.isConfigured = true;
} else {
settings.additionalSuffixOfDatabaseName = this.appId;
settings.isConfigured = false;
}
}
const passphrase = await this.getPassphrase(settings);
if (passphrase === false) {
this._log("Could not determine passphrase for reading data.json! DO NOT synchronize with the remote before making sure your configuration is!", LOG_LEVEL_URGENT);
} else {
if (settings.encryptedCouchDBConnection) {
const keys = [
"couchDB_URI",
"couchDB_USER",
"couchDB_PASSWORD",
"couchDB_DBNAME",
"accessKey",
"bucket",
"endpoint",
"region",
"secretKey"] as (keyof CouchDBConnection | keyof BucketSyncSetting)[];
const decrypted = this.tryDecodeJson(await this.decryptConfigurationItem(settings.encryptedCouchDBConnection, passphrase)) as (CouchDBConnection & BucketSyncSetting);
if (decrypted) {
for (const key of keys) {
if (key in decrypted) {
//@ts-ignore
settings[key] = decrypted[key]
}
}
} else {
this._log("Could not decrypt passphrase for reading data.json! DO NOT synchronize with the remote before making sure your configuration is!", LOG_LEVEL_URGENT);
for (const key of keys) {
//@ts-ignore
settings[key] = "";
}
}
}
if (settings.encrypt && settings.encryptedPassphrase) {
const encrypted = settings.encryptedPassphrase;
const decrypted = await this.decryptConfigurationItem(encrypted, passphrase);
if (decrypted) {
settings.passphrase = decrypted;
} else {
this._log("Could not decrypt passphrase for reading data.json! DO NOT synchronize with the remote before making sure your configuration is!", LOG_LEVEL_URGENT);
settings.passphrase = "";
}
}
}
this.settings = settings;
setLang(this.settings.displayLanguage);
if ("workingEncrypt" in this.settings) delete this.settings.workingEncrypt;
if ("workingPassphrase" in this.settings) delete this.settings.workingPassphrase;
// Delete this feature to avoid problems on mobile.
this.settings.disableRequestURI = true;
// GC is disabled.
this.settings.gcDelay = 0;
// So, use history is always enabled.
this.settings.useHistory = true;
const lsKey = "obsidian-live-sync-vaultanddevicename-" + this.plugin.$$getVaultName();
if (this.settings.deviceAndVaultName != "") {
if (!localStorage.getItem(lsKey)) {
this.core.deviceAndVaultName = this.settings.deviceAndVaultName;
localStorage.setItem(lsKey, this.core.deviceAndVaultName);
this.settings.deviceAndVaultName = "";
}
}
if (isCloudantURI(this.settings.couchDB_URI) && this.settings.customChunkSize != 0) {
this._log("Configuration verification founds problems with your configuration. This has been fixed automatically. But you may already have data that cannot be synchronised. If this is the case, please rebuild everything.", LOG_LEVEL_NOTICE)
this.settings.customChunkSize = 0;
}
this.core.deviceAndVaultName = localStorage.getItem(lsKey) || "";
if (this.core.deviceAndVaultName == "") {
if (this.settings.usePluginSync) {
this._log("Device name is not set. Plug-in sync has been disabled.", LOG_LEVEL_NOTICE);
this.settings.usePluginSync = false;
}
}
// this.core.ignoreFiles = this.settings.ignoreFiles.split(",").map(e => e.trim());
eventHub.emitEvent(EVENT_REQUEST_RELOAD_SETTING_TAB);
}
}

212
src/modules/features/ModuleObsidianSettingAsMarkdown.ts Normal file
View File

@@ -0,0 +1,212 @@
import { type IObsidianModule, AbstractObsidianModule } from "../AbstractObsidianModule.ts";
// import { PouchDB } from "../../lib/src/pouchdb/pouchdb-browser";
import { isObjectDifferent } from "octagonal-wheels/object";
import { EVENT_SETTING_SAVED, eventHub } from "../../common/events";
import { fireAndForget } from "octagonal-wheels/promises";
import { DEFAULT_SETTINGS, type FilePathWithPrefix, type ObsidianLiveSyncSettings } from "../../lib/src/common/types";
import { parseYaml, stringifyYaml } from "../../deps";
import { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
const SETTING_HEADER = "````yaml:livesync-setting\n";
const SETTING_FOOTER = "\n````";
export class ModuleObsidianSettingsAsMarkdown extends AbstractObsidianModule implements IObsidianModule {
$everyOnloadStart(): Promise<boolean> {
this.addCommand({
id: "livesync-export-config",
name: "Write setting markdown manually",
checkCallback: (checking) => {
if (checking) {
return this.settings.settingSyncFile != "";
}
fireAndForget(async () => {
await this.plugin.$$saveSettingData();
});
}
})
this.addCommand({
id: "livesync-import-config",
name: "Parse setting file",
editorCheckCallback: (checking, editor, ctx) => {
if (checking) {
const doc = editor.getValue();
const ret = this.extractSettingFromWholeText(doc);
return ret.body != "";
}
if (ctx.file) {
const file = ctx.file
fireAndForget(async () => await this.checkAndApplySettingFromMarkdown(file.path, false));
}
},
})
eventHub.on("event-file-changed", (info: {
file: FilePathWithPrefix, automated: boolean
}) => {
fireAndForget(() => this.checkAndApplySettingFromMarkdown(info.file, info.automated));
});
eventHub.onEvent(EVENT_SETTING_SAVED, (evt: CustomEvent<ObsidianLiveSyncSettings>) => {
const settings = evt.detail;
if (settings.settingSyncFile != "") {
fireAndForget(() => this.saveSettingToMarkdown(settings.settingSyncFile));
}
})
return Promise.resolve(true);
}
extractSettingFromWholeText(data: string): {
preamble: string, body: string, postscript: string
} {
if (data.indexOf(SETTING_HEADER) === -1) {
return {
preamble: data, body: "", postscript: ""
}
}
const startMarkerPos = data.indexOf(SETTING_HEADER);
const dataStartPos = startMarkerPos == -1 ? data.length : startMarkerPos;
const endMarkerPos = startMarkerPos == -1 ? data.length : data.indexOf(SETTING_FOOTER, dataStartPos);
const dataEndPos = endMarkerPos == -1 ? data.length : endMarkerPos;
const body = data.substring(dataStartPos + SETTING_HEADER.length, dataEndPos);
const ret = {
preamble: data.substring(0, dataStartPos),
body,
postscript: data.substring(dataEndPos + SETTING_FOOTER.length + 1)
}
return ret;
}
async parseSettingFromMarkdown(filename: string, data?: string) {
const file = await this.core.storageAccess.isExists(filename);
if (!file) return {
preamble: "", body: "", postscript: "",
};
if (data) {
return this.extractSettingFromWholeText(data);
}
const parseData = data ?? await this.core.storageAccess.readFileText(filename);
return this.extractSettingFromWholeText(parseData);
}
async checkAndApplySettingFromMarkdown(filename: string, automated?: boolean) {
if (automated && !this.settings.notifyAllSettingSyncFile) {
if (!this.settings.settingSyncFile || this.settings.settingSyncFile != filename) {
this._log(`Setting file (${filename}) is not matched to the current configuration. skipped.`, LOG_LEVEL_DEBUG);
return;
}
}
const { body } = await this.parseSettingFromMarkdown(filename);
let newSetting = {} as Partial<ObsidianLiveSyncSettings>;
try {
newSetting = parseYaml(body);
} catch (ex) {
this._log("Could not parse YAML", LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
return;
}
if ("settingSyncFile" in newSetting && newSetting.settingSyncFile != filename) {
this._log("This setting file seems to backed up one. Please fix the filename or settingSyncFile value.", automated ? LOG_LEVEL_INFO : LOG_LEVEL_NOTICE);
return;
}
let settingToApply = { ...DEFAULT_SETTINGS } as ObsidianLiveSyncSettings;
settingToApply = { ...settingToApply, ...newSetting }
if (!(settingToApply?.writeCredentialsForSettingSync)) {
//New setting does not contains credentials.
settingToApply.couchDB_USER = this.settings.couchDB_USER;
settingToApply.couchDB_PASSWORD = this.settings.couchDB_PASSWORD;
settingToApply.passphrase = this.settings.passphrase;
}
const oldSetting = this.generateSettingForMarkdown(this.settings, settingToApply.writeCredentialsForSettingSync);
if (!isObjectDifferent(oldSetting, this.generateSettingForMarkdown(settingToApply))) {
this._log("Setting markdown has been detected, but not changed.", automated ? LOG_LEVEL_INFO : LOG_LEVEL_NOTICE);
return
}
const addMsg = this.settings.settingSyncFile != filename ? " (This is not-active file)" : "";
this.plugin.confirm.askInPopup("apply-setting-from-md", `Setting markdown ${filename}${addMsg} has been detected. Apply this from {HERE}.`, (anchor) => {
anchor.text = "HERE";
anchor.addEventListener("click", () => {
fireAndForget(async () => {
const APPLY_ONLY = "Apply settings";
const APPLY_AND_RESTART = "Apply settings and restart obsidian";
const APPLY_AND_REBUILD = "Apply settings and restart obsidian with red_flag_rebuild.md";
const APPLY_AND_FETCH = "Apply settings and restart obsidian with red_flag_fetch.md";
const CANCEL = "Cancel";
const result = await this.plugin.confirm.askSelectStringDialogue("Ready for apply the setting.", [
APPLY_AND_RESTART,
APPLY_ONLY,
APPLY_AND_FETCH,
APPLY_AND_REBUILD,
CANCEL], { defaultAction: APPLY_AND_RESTART });
if (result == APPLY_ONLY || result == APPLY_AND_RESTART || result == APPLY_AND_REBUILD || result == APPLY_AND_FETCH) {
this.plugin.settings = settingToApply;
await this.plugin.$$saveSettingData();
if (result == APPLY_ONLY) {
this._log("Loaded settings have been applied!", LOG_LEVEL_NOTICE);
return;
}
if (result == APPLY_AND_REBUILD) {
await this.plugin.rebuilder.scheduleRebuild();
}
if (result == APPLY_AND_FETCH) {
await this.plugin.rebuilder.scheduleFetch();
}
this.plugin.$$performRestart();
}
})
})
})
}
generateSettingForMarkdown(settings?: ObsidianLiveSyncSettings, keepCredential?: boolean): Partial<ObsidianLiveSyncSettings> {
const saveData = { ...(settings ? settings : this.settings) } as Partial<ObsidianLiveSyncSettings>;
delete saveData.encryptedCouchDBConnection;
delete saveData.encryptedPassphrase;
delete saveData.additionalSuffixOfDatabaseName;
if (!saveData.writeCredentialsForSettingSync && !keepCredential) {
delete saveData.couchDB_USER;
delete saveData.couchDB_PASSWORD;
delete saveData.passphrase;
}
return saveData;
}
async saveSettingToMarkdown(filename: string) {
const saveData = this.generateSettingForMarkdown();
const file = await this.core.storageAccess.isExists(filename);
if (!file) {
await this.core.storageAccess.ensureDir(filename);
const initialContent = `This file contains Self-hosted LiveSync settings as YAML.
Except for the \`livesync-setting\` code block, we can add a note for free.
If the name of this file matches the value of the "settingSyncFile" setting inside the \`livesync-setting\` block, LiveSync will tell us whenever the settings change. We can decide to accept or decline the remote setting. (In other words, we can back up this file by renaming it to another name).
We can perform a command in this file.
- \`Parse setting file\` : load the setting from the file.
**Note** Please handle it with all of your care if you have configured to write credentials in.
`
await this.core.storageAccess.writeFileAuto(filename, initialContent + SETTING_HEADER + "\n" + SETTING_FOOTER);
}
// if (!(file instanceof TFile)) {
// this._log(`Markdown Setting: ${filename} already exists as a folder`, LOG_LEVEL_NOTICE);
// return;
// }
const data = await this.core.storageAccess.readFileText(filename);
const { preamble, body, postscript } = this.extractSettingFromWholeText(data);
const newBody = stringifyYaml(saveData);
if (newBody == body) {
this._log("Markdown setting: Nothing had been changed", LOG_LEVEL_VERBOSE);
} else {
await this.core.storageAccess.writeFileAuto(filename, preamble + SETTING_HEADER + newBody + SETTING_FOOTER + postscript);
this._log(`Markdown setting: ${filename} has been updated!`, LOG_LEVEL_VERBOSE);
}
}
}

35
src/modules/features/ModuleObsidianSettingTab.ts Normal file
View File

@@ -0,0 +1,35 @@
import { ObsidianLiveSyncSettingTab } from "./SettingDialogue/ObsidianLiveSyncSettingTab.ts";
import { type IObsidianModule, AbstractObsidianModule } from "../AbstractObsidianModule.ts";
// import { PouchDB } from "../../lib/src/pouchdb/pouchdb-browser";
import { EVENT_REQUEST_OPEN_SETTING_WIZARD, EVENT_REQUEST_OPEN_SETTINGS, eventHub } from "../../common/events.ts";
export class ModuleObsidianSettingDialogue extends AbstractObsidianModule implements IObsidianModule {
settingTab!: ObsidianLiveSyncSettingTab;
$everyOnloadStart(): Promise<boolean> {
this.settingTab = new ObsidianLiveSyncSettingTab(this.app, this.plugin);
this.plugin.addSettingTab(this.settingTab);
eventHub.onEvent(EVENT_REQUEST_OPEN_SETTINGS, () => this.openSetting());
eventHub.onEvent(EVENT_REQUEST_OPEN_SETTING_WIZARD, () => {
this.openSetting();
void this.settingTab.enableMinimalSetup();
});
return Promise.resolve(true);
}
openSetting() {
// Undocumented API
//@ts-ignore
this.app.setting.open();
//@ts-ignore
this.app.setting.openTabById("obsidian-livesync");
}
get appId() {
return `${("appId" in this.app ? this.app.appId : "")}`;
}
}

205
src/modules/features/ModuleSetupObsidian.ts Normal file
View File

@@ -0,0 +1,205 @@
import { type ObsidianLiveSyncSettings, DEFAULT_SETTINGS, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "../../lib/src/common/types.ts";
import { configURIBase } from "../../common/types.ts";
// import { PouchDB } from "../../lib/src/pouchdb/pouchdb-browser.js";
import { decrypt, encrypt } from "../../lib/src/encryption/e2ee_v2.ts";
import { fireAndForget } from "../../lib/src/common/utils.ts";
import { EVENT_REQUEST_COPY_SETUP_URI, EVENT_REQUEST_OPEN_SETUP_URI, eventHub } from "../../common/events.ts";
import { AbstractObsidianModule, type IObsidianModule } from "../AbstractObsidianModule.ts";
export class ModuleSetupObsidian extends AbstractObsidianModule implements IObsidianModule {
$everyOnload(): Promise<boolean> {
this.registerObsidianProtocolHandler("setuplivesync", async (conf: any) => await this.setupWizard(conf.settings));
this.addCommand({
id: "livesync-copysetupuri",
name: "Copy settings as a new setup URI",
callback: () => fireAndForget(this.command_copySetupURI()),
});
this.addCommand({
id: "livesync-copysetupuri-short",
name: "Copy settings as a new setup URI (With customization sync)",
callback: () => fireAndForget(this.command_copySetupURIWithSync()),
});
this.addCommand({
id: "livesync-copysetupurifull",
name: "Copy settings as a new setup URI (Full)",
callback: () => fireAndForget(this.command_copySetupURIFull()),
});
this.addCommand({
id: "livesync-opensetupuri",
name: "Use the copied setup URI (Formerly Open setup URI)",
callback: () => fireAndForget(this.command_openSetupURI()),
});
eventHub.onEvent(EVENT_REQUEST_OPEN_SETUP_URI, () => fireAndForget(() => this.command_openSetupURI()));
eventHub.onEvent(EVENT_REQUEST_COPY_SETUP_URI, () => fireAndForget(() => this.command_copySetupURI()));
return Promise.resolve(true);
}
async command_copySetupURI(stripExtra = true) {
const encryptingPassphrase = await this.core.confirm.askString("Encrypt your settings", "The passphrase to encrypt the setup URI", "", true);
if (encryptingPassphrase === false)
return;
const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" } as Partial<ObsidianLiveSyncSettings>;
if (stripExtra) {
delete setting.pluginSyncExtendedSetting;
}
const keys = Object.keys(setting) as (keyof ObsidianLiveSyncSettings)[];
for (const k of keys) {
if (JSON.stringify(k in setting ? setting[k] : "") == JSON.stringify(k in DEFAULT_SETTINGS ? DEFAULT_SETTINGS[k] : "*")) {
delete setting[k];
}
}
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri);
this._log("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);
}
async command_copySetupURIFull() {
const encryptingPassphrase = await this.core.confirm.askString("Encrypt your settings", "The passphrase to encrypt the setup URI", "", true);
if (encryptingPassphrase === false)
return;
const setting = { ...this.settings, configPassphraseStore: "", encryptedCouchDBConnection: "", encryptedPassphrase: "" };
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(setting), encryptingPassphrase, false));
const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri);
this._log("Setup URI copied to clipboard", LOG_LEVEL_NOTICE);
}
async command_copySetupURIWithSync() {
await this.command_copySetupURI(false);
}
async command_openSetupURI() {
const setupURI = await this.core.confirm.askString("Easy setup", "Set up URI", `${configURIBase}aaaaa`);
if (setupURI === false)
return;
if (!setupURI.startsWith(`${configURIBase}`)) {
this._log("Set up URI looks wrong.", LOG_LEVEL_NOTICE);
return;
}
const config = decodeURIComponent(setupURI.substring(configURIBase.length));
console.dir(config);
await this.setupWizard(config);
}
async setupWizard(confString: string) {
try {
const oldConf = JSON.parse(JSON.stringify(this.settings));
const encryptingPassphrase = await this.core.confirm.askString("Passphrase", "The passphrase to decrypt your setup URI", "", true);
if (encryptingPassphrase === false)
return;
const newConf = await JSON.parse(await decrypt(confString, encryptingPassphrase, false));
if (newConf) {
const result = await this.core.confirm.askYesNoDialog("Importing Configuration from the Setup-URI. Are you sure to proceed?", {});
if (result == "yes") {
const newSettingW = Object.assign({}, DEFAULT_SETTINGS, newConf) as ObsidianLiveSyncSettings;
this.core.replicator.closeReplication();
this.settings.suspendFileWatching = true;
console.dir(newSettingW);
// Back into the default method once.
newSettingW.configPassphraseStore = "";
newSettingW.encryptedPassphrase = "";
newSettingW.encryptedCouchDBConnection = "";
newSettingW.additionalSuffixOfDatabaseName = `${("appId" in this.app ? this.app.appId : "")}`
const setupJustImport = "Just import setting";
const setupAsNew = "Set it up as secondary or subsequent device";
const setupAsMerge = "Secondary device but try keeping local changes";
const setupAgain = "Reconfigure and reconstitute the data";
const setupManually = "Leave everything to me";
newSettingW.syncInternalFiles = false;
newSettingW.usePluginSync = false;
newSettingW.isConfigured = true;
// Migrate completely obsoleted configuration.
if (!newSettingW.useIndexedDBAdapter) {
newSettingW.useIndexedDBAdapter = true;
}
const setupType = await this.core.confirm.askSelectStringDialogue("How would you like to set it up?", [setupAsNew, setupAgain, setupAsMerge, setupJustImport, setupManually], { defaultAction: setupAsNew });
if (setupType == setupJustImport) {
this.core.settings = newSettingW;
this.core.$$clearUsedPassphrase();
await this.core.saveSettings();
} else if (setupType == setupAsNew) {
this.core.settings = newSettingW;
this.core.$$clearUsedPassphrase();
await this.core.rebuilder.$fetchLocal();
} else if (setupType == setupAsMerge) {
this.core.settings = newSettingW;
this.core.$$clearUsedPassphrase();
await this.core.rebuilder.$fetchLocal(true);
} else if (setupType == setupAgain) {
const confirm = "I know this operation will rebuild all my databases with files on this device, and files that are on the remote database and I didn't synchronize to any other devices will be lost and want to proceed indeed.";
if (await this.core.confirm.askSelectStringDialogue("Do you really want to do this?", ["Cancel", confirm], { defaultAction: "Cancel" }) != confirm) {
return;
}
this.core.settings = newSettingW;
this.core.$$clearUsedPassphrase();
await this.core.rebuilder.$rebuildEverything();
} else if (setupType == setupManually) {
const keepLocalDB = await this.core.confirm.askYesNoDialog("Keep local DB?", { defaultOption: "No" });
const keepRemoteDB = await this.core.confirm.askYesNoDialog("Keep remote DB?", { defaultOption: "No" });
if (keepLocalDB == "yes" && keepRemoteDB == "yes") {
// nothing to do. so peaceful.
this.core.settings = newSettingW;
this.core.$$clearUsedPassphrase();
await this.core.$allSuspendAllSync();
await this.core.$allSuspendExtraSync();
await this.core.saveSettings();
const replicate = await this.core.confirm.askYesNoDialog("Unlock and replicate?", { defaultOption: "Yes" });
if (replicate == "yes") {
await this.core.$$replicate(true);
await this.core.$$markRemoteUnlocked();
}
this._log("Configuration loaded.", LOG_LEVEL_NOTICE);
return;
}
if (keepLocalDB == "no" && keepRemoteDB == "no") {
const reset = await this.core.confirm.askYesNoDialog("Drop everything?", { defaultOption: "No" });
if (reset != "yes") {
this._log("Cancelled", LOG_LEVEL_NOTICE);
this.core.settings = oldConf;
return;
}
}
let initDB;
this.core.settings = newSettingW;
this.core.$$clearUsedPassphrase();
await this.core.saveSettings();
if (keepLocalDB == "no") {
await this.core.$$resetLocalDatabase();
await this.core.localDatabase.initializeDatabase();
const rebuild = await this.core.confirm.askYesNoDialog("Rebuild the database?", { defaultOption: "Yes" });
if (rebuild == "yes") {
initDB = this.core.$$initializeDatabase(true);
} else {
await this.core.$$markRemoteResolved();
}
}
if (keepRemoteDB == "no") {
await this.core.$$tryResetRemoteDatabase();
await this.core.$$markRemoteLocked();
}
if (keepLocalDB == "no" || keepRemoteDB == "no") {
const replicate = await this.core.confirm.askYesNoDialog("Replicate once?", { defaultOption: "Yes" });
if (replicate == "yes") {
if (initDB != null) {
await initDB;
}
await this.core.$$replicate(true);
}
}
}
}
this._log("Configuration loaded.", LOG_LEVEL_NOTICE);
} else {
this._log("Cancelled.", LOG_LEVEL_NOTICE);
}
} catch (ex) {
this._log("Couldn't parse or decrypt configuration uri.", LOG_LEVEL_NOTICE);
this._log(ex, LOG_LEVEL_VERBOSE);
}
}
}

14
src/ui/components/LiveSyncSetting.ts → src/modules/features/SettingDialogue/LiveSyncSetting.ts
View File

@@ -1,8 +1,8 @@
import { Setting, TextComponent, type ToggleComponent, type DropdownComponent, ButtonComponent, type TextAreaComponent, type ValueComponent } from "obsidian";
import { unique } from "octagonal-wheels/collection";
import { LEVEL_ADVANCED, LEVEL_POWER_USER, statusDisplay, type ConfigurationItem } from "../../lib/src/common/types";
import { type ObsidianLiveSyncSettingTab, type AutoWireOption, wrapMemo, type OnUpdateResult, createStub, findAttrFromParent } from "../ObsidianLiveSyncSettingTab";
import { type AllSettingItemKey, getConfig, type AllSettings, type AllStringItemKey, type AllNumericItemKey, type AllBooleanItemKey } from "../settingConstants";
import { LEVEL_ADVANCED, LEVEL_POWER_USER, statusDisplay, type ConfigurationItem } from "../../../lib/src/common/types.ts";
import { type ObsidianLiveSyncSettingTab, type AutoWireOption, wrapMemo, type OnUpdateResult, createStub, findAttrFromParent } from "./ObsidianLiveSyncSettingTab.ts";
import { type AllSettingItemKey, getConfig, type AllSettings, type AllStringItemKey, type AllNumericItemKey, type AllBooleanItemKey } from "./settingConstants.ts";
export class LiveSyncSetting extends Setting {
@@ -113,7 +113,7 @@ export class LiveSyncSetting extends Setting {
const conf = this.autoWireSetting(key, opt);
this.addText(text => {
this.autoWiredComponent = text;
const setValue = wrapMemo((value: string) => text.setValue(value));
const setValue = wrapMemo((value: string) => { text.setValue(value) });
this.invalidateValue = () => setValue(`${LiveSyncSetting.env.editingSettings[key]}`);
this.invalidateValue();
text.onChange(async (value) => {
@@ -131,7 +131,7 @@ export class LiveSyncSetting extends Setting {
const conf = this.autoWireSetting(key, opt);
this.addTextArea(text => {
this.autoWiredComponent = text;
const setValue = wrapMemo((value: string) => text.setValue(value));
const setValue = wrapMemo((value: string) => { text.setValue(value) });
this.invalidateValue = () => setValue(`${LiveSyncSetting.env.editingSettings[key]}`);
this.invalidateValue();
text.onChange(async (value) => {
@@ -156,7 +156,7 @@ export class LiveSyncSetting extends Setting {
text.inputEl.setAttribute("max", `${opt.clampMax}`);
}
let lastError = false;
const setValue = wrapMemo((value: string) => text.setValue(value));
const setValue = wrapMemo((value: string) => { text.setValue(value) });
this.invalidateValue = () => {
if (!lastError) setValue(`${LiveSyncSetting.env.editingSettings[key]}`);
};
@@ -195,7 +195,7 @@ export class LiveSyncSetting extends Setting {
const conf = this.autoWireSetting(key, opt);
this.addToggle(toggle => {
this.autoWiredComponent = toggle;
const setValue = wrapMemo((value: boolean) => toggle.setValue(opt?.invert ? !value : value));
const setValue = wrapMemo((value: boolean) => { toggle.setValue(opt?.invert ? !value : value) });
this.invalidateValue = () => setValue(LiveSyncSetting.env.editingSettings[key] ?? false);
this.invalidateValue();

27
src/ui/components/MultipleRegExpControl.svelte → src/modules/features/SettingDialogue/MultipleRegExpControl.svelte
View File

@@ -12,14 +12,14 @@
function checkRegExp(pattern: string) {
if (pattern.trim() == "") return "";
try {
const _ = new RegExp(pattern);
new RegExp(pattern);
return CHECK_OK;
} catch (ex) {
return CHECK_NG;
}
}
$: status = patterns.map((e) => checkRegExp(e));
$: modified = patterns.map((e, i) => (e != originals?.[i] ?? "" ? MARK_MODIFIED : ""));
$: statusName = patterns.map((e) => checkRegExp(e));
$: modified = patterns.map((e, i) => (e != (originals?.[i] ?? "") ? MARK_MODIFIED : ""));
function remove(idx: number) {
patterns[idx] = "";
@@ -32,14 +32,19 @@
<ul>
{#each patterns as pattern, idx}
<!-- svelte-ignore a11y-label-has-associated-control -->
<li><label>{modified[idx]}{status[idx]}</label><input type="text" bind:value={pattern} class={modified[idx]} /><button class="iconbutton" on:click={() => remove(idx)}>🗑</button></li>
<li>
<label>{modified[idx]}{statusName[idx]}</label><input type="text" bind:value={pattern} class={modified[idx]} />
<button class="iconbutton" on:click={() => remove(idx)}>🗑</button>
</li>
{/each}
<li>
<label><button on:click={() => add()}>Add</button></label>
<label>
<button on:click={() => add()}>Add</button>
</label>
</li>
<li class="buttons">
<button on:click={() => apply(patterns)} disabled={status.some((e) => e == CHECK_NG) || modified.every((e) => e == "")}>Apply</button>
<button on:click={() => revert()} disabled={status.some((e) => e == CHECK_NG) || modified.every((e) => e == "")}>Revert</button>
<button on:click={() => apply(patterns)} disabled={statusName.some((e) => e === CHECK_NG) || modified.every((e) => e === "")}>Apply </button>
<button on:click={() => revert()} disabled={statusName.some((e) => e === CHECK_NG) || modified.every((e) => e === "")}>Revert </button>
</li>
</ul>
@@ -51,6 +56,7 @@
flex-direction: row;
justify-content: flex-end;
}
ul {
flex-grow: 1;
display: inline-flex;
@@ -58,10 +64,11 @@
list-style-type: none;
margin-block-start: 0;
margin-block-end: 0;
margin-inline-start: 0px;
margin-inline-end: 0px;
margin-inline-start: 0;
margin-inline-end: 0;
padding-inline-start: 0;
}
li {
padding: var(--size-2-1) var(--size-4-1);
display: inline-flex;
@@ -70,9 +77,11 @@
justify-content: flex-end;
gap: var(--size-4-2);
}
li input {
min-width: 10em;
}
button.iconbutton {
max-width: 4em;
}

2116
src/ui/ObsidianLiveSyncSettingTab.ts → src/modules/features/SettingDialogue/ObsidianLiveSyncSettingTab.ts
View File

File diff suppressed because it is too large Load Diff

8
src/ui/settingConstants.ts → src/modules/features/SettingDialogue/settingConstants.ts
View File

@@ -1,5 +1,5 @@
import { $t } from "src/lib/src/common/i18n";
import { DEFAULT_SETTINGS, configurationNames, type ConfigurationItem, type FilterBooleanKeys, type FilterNumberKeys, type FilterStringKeys, type ObsidianLiveSyncSettings } from "src/lib/src/common/types";
import { $t } from "../../../lib/src/common/i18n.ts";
import { DEFAULT_SETTINGS, configurationNames, type ConfigurationItem, type FilterBooleanKeys, type FilterNumberKeys, type FilterStringKeys, type ObsidianLiveSyncSettings } from "../../../lib/src/common/types.ts";
export type OnDialogSettings = {
configPassphrase: string,
@@ -354,6 +354,10 @@ export const SettingInformation: Partial<Record<keyof AllSettings, Configuration
useEdgeCaseMode: {
name: "Enable edge case treatment features",
},
"enableDebugTools": {
name: "Enable Developers' Debug Tools.",
desc: "You need a restart to apply this setting."
}
}
function translateInfo(infoSrc: ConfigurationItem | undefined | false) {
if (!infoSrc) return false;

14
src/modules/interfaces/Confirm.ts Normal file
View File

@@ -0,0 +1,14 @@
export interface Confirm {
askYesNo(message: string): Promise<"yes" | "no">;
askString(title: string, key: string, placeholder: string, isPassword?: boolean): Promise<string | false>;
askYesNoDialog(message: string, opt: { title?: string, defaultOption?: "Yes" | "No", timeout?: number }): Promise<"yes" | "no">;
askSelectString(message: string, items: string[]): Promise<string>
askSelectStringDialogue(message: string, buttons: string[], opt: { title?: string, defaultAction: (typeof buttons)[number], timeout?: number }): Promise<(typeof buttons)[number] | false>;
askInPopup(key: string, dialogText: string, anchorCallback: (anchor: HTMLAnchorElement) => void): void;
confirmWithMessage(title: string, contentMd: string, buttons: string[], defaultAction: (typeof buttons)[number], timeout?: number): Promise<(typeof buttons)[number] | false>;
}

18
src/modules/interfaces/DatabaseFileAccess.ts Normal file
View File

@@ -0,0 +1,18 @@
import type { FilePathWithPrefix, LoadedEntry, MetaEntry, UXFileInfo, UXFileInfoStub } from "../../lib/src/common/types";
export interface DatabaseFileAccess {
delete: (file: UXFileInfoStub | FilePathWithPrefix, rev?: string) => Promise<boolean>;
store: (file: UXFileInfo, force?: boolean, skipCheck?: boolean) => Promise<boolean>;
storeContent(path: FilePathWithPrefix, content: string): Promise<boolean>;
createChunks: (file: UXFileInfo, force?: boolean, skipCheck?: boolean) => Promise<boolean>;
fetch: (file: UXFileInfoStub | FilePathWithPrefix,
rev?: string, waitForReady?: boolean, skipCheck?: boolean) => Promise<UXFileInfo | false>;
fetchEntryFromMeta: (meta: MetaEntry,
waitForReady?: boolean, skipCheck?: boolean) => Promise<LoadedEntry | false>;
fetchEntryMeta: (file: UXFileInfoStub | FilePathWithPrefix,
rev?: string, skipCheck?: boolean) => Promise<MetaEntry | false>;
fetchEntry: (file: UXFileInfoStub | FilePathWithPrefix,
rev?: string, waitForReady?: boolean, skipCheck?: boolean) => Promise<LoadedEntry | false>;
getConflictedRevs: (file: UXFileInfoStub | FilePathWithPrefix) => Promise<string[]>;
// storeFromStorage: (file: UXFileInfoStub | FilePathWithPrefix, force?: boolean) => Promise<boolean>;
}

11
src/modules/interfaces/DatabaseRebuilder.ts Normal file
View File

@@ -0,0 +1,11 @@
export interface Rebuilder {
$performRebuildDB(method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice" | "localOnlyWithChunks"): Promise<void>;
$rebuildRemote(): Promise<void>;
$rebuildEverything(): Promise<void>;
$fetchLocal(makeLocalChunkBeforeSync?: boolean): Promise<void>;
scheduleRebuild(): Promise<void>;
scheduleFetch(): Promise<void>;
resolveAllConflictedFilesByNewerOnes(): Promise<void>;
}

48
src/modules/interfaces/StorageAccess.ts Normal file
View File

@@ -0,0 +1,48 @@
import type { FilePath, FilePathWithPrefix, UXDataWriteOptions, UXFileInfo, UXFileInfoStub, UXFolderInfo, UXStat } from "../../lib/src/common/types"
export interface StorageAccess {
deleteVaultItem(file: FilePathWithPrefix | UXFileInfoStub | UXFolderInfo): Promise<void>
writeFileAuto(path: string, data: string | ArrayBuffer, opt?: UXDataWriteOptions): Promise<boolean>
readFileAuto(path: string): Promise<string | ArrayBuffer>
readFileText(path: string): Promise<string>
isExists(path: string): Promise<boolean>
writeHiddenFileAuto(path: string, data: string | ArrayBuffer, opt?: UXDataWriteOptions): Promise<boolean>
appendHiddenFile(path: string, data: string, opt?: UXDataWriteOptions): Promise<boolean>
stat(path: string): Promise<UXStat | null>
statHidden(path: string): Promise<UXStat | null>
removeHidden(path: string): Promise<boolean>
readHiddenFileAuto(path: string): Promise<string | ArrayBuffer>
readHiddenFileBinary(path: string): Promise<ArrayBuffer>
readHiddenFileText(path: string): Promise<string>
isExistsIncludeHidden(path: string): Promise<boolean>
// This could be work also for the hidden files.
ensureDir(path: string): Promise<boolean>
triggerFileEvent(event: string, path: string): void
triggerHiddenFile(path: string): Promise<void>
getFileStub(path: string): UXFileInfoStub | null
readStubContent(stub: UXFileInfoStub): Promise<UXFileInfo | false>;
getStub(path: string): UXFileInfoStub | UXFolderInfo | null
getFiles(): UXFileInfoStub[]
getFileNames(): FilePathWithPrefix[]
touched(file: UXFileInfoStub | FilePathWithPrefix): void
recentlyTouched(file: UXFileInfoStub | FilePathWithPrefix): boolean
clearTouched(): void
// -- Low-Level
delete(file: FilePathWithPrefix | UXFileInfoStub | string, force: boolean): Promise<void>
trash(file: FilePathWithPrefix | UXFileInfoStub | string, system: boolean): Promise<void>
getFilesIncludeHidden(
basePath: string,
includeFilter?: RegExp[],
excludeFilter?: RegExp[],
skipFolder?: string[]
): Promise<FilePath[]>;
}

45
src/modules/modules.md Normal file
View File

@@ -0,0 +1,45 @@
# Dynamic Load Modules
## Introduction
Self-hosted LiveSync has gradually but steadily become very feature-rich and they have created a very heavy `Main` class. This is very difficult to understand and maintain especially new contributors or futures contributors.
And some of the features are not used by all users, we should limit the inter-dependencies between modules. And also inter-effects between modules.
Hence, to make the code more readable and maintainable, I decided to split the code into multiple modules.
I also got a little greedy here, but I have an another objective now, which is to reduce the difficulty when porting to other platforms.
Therefore, almost all feature of the plug-in can be implemented as a module. And the `Main` class will be responsible for loading these modules.
## Modules
### Sorts
Modules can be sorted into two categories in some sorts:
- `CoreModule` and `ObsidianModule`
- `Core`, `Essential`, and `Feature` ...
### How it works
After instancing `Core` and Modules, you should call `injectModules`. Then, the specific function will be injected into the stub of it of `Core` class by following rules:
| Function prefix | Description |
| --------------- | ----------------------------------------------------------------- |
| `$$` | Completely overridden functions. |
| `$all` | Process all modules and return all results. |
| `$every` | Process all modules until the first failure. |
| `$any` | Process all modules until the first success. |
| `$` | Other interceptive points. You should manually assign the module. |
Note1: `Core` class should implement the same function as the module. If not, the module will be ignored.
And, basically, the Module has a `Core` class as `core` property. You should call any of inject functions by `this.core.$xxxxxx`. This rule is also applied to the function which implemented itself. Because some other modules possibly injects the function again, for the specific purpose.
### CoreModule
This Module is independent from Obsidian, and can be used in any platform. However, it can be call (or use) functions which has implemented in `ObsidianModule`.
To porting, you should implement shim functions for `ObsidianModule`.
### ObsidianModule
(TBW)

50
src/tests/TestPane.svelte
View File

@@ -1,50 +0,0 @@
<script lang="ts">
import { onDestroy, onMount } from "svelte";
import type ObsidianLiveSyncPlugin from "../main";
import { perf_trench } from "./tests";
import { MarkdownRenderer } from "../deps";
export let plugin: ObsidianLiveSyncPlugin;
let performanceTestResult = "";
let functionCheckResult = "";
let testRunning = false;
let prefTestResultEl: HTMLDivElement;
let isReady = false;
$: {
if (performanceTestResult != "" && isReady) {
MarkdownRenderer.render(plugin.app, performanceTestResult, prefTestResultEl, "/", plugin);
}
}
async function performTest() {
try {
testRunning = true;
performanceTestResult = await perf_trench(plugin);
} finally {
testRunning = false;
}
}
function clearPerfTestResult() {
prefTestResultEl.empty();
}
onMount(() => {
isReady = true;
// performTest();
});
</script>
<h2>TESTBENCH: Self-hosted LiveSync</h2>
<h3>Function check</h3>
<pre>{functionCheckResult}</pre>
<h3>Performance test</h3>
<button on:click={() => performTest()} disabled={testRunning}>Test!</button>
<button on:click={() => clearPerfTestResult()}>Clear</button>
<div bind:this={prefTestResultEl}></div>
<style>
* {
box-sizing: border-box;
}
</style>

4
terser.config.mjs
View File

@@ -12,8 +12,8 @@ const terserOption = {
}
: {},
format: {
indent_level: 2,
beautify: true,
// indent_level: 2,
// beautify: true,
comments: "some",
ecma: 2018,
preamble: banner,

28
tsconfig.json
View File

@@ -8,30 +8,16 @@
"allowJs": true,
"noImplicitAny": true,
"moduleResolution": "node",
"types": [
"svelte",
"node"
],
"types": ["svelte", "node"],
// "importsNotUsedAsValues": "error",
"importHelpers": false,
"alwaysStrict": true,
"allowImportingTsExtensions": true,
"noEmit": true,
"lib": [
"es2018",
"DOM",
"ES5",
"ES6",
"ES7",
"es2019.array",
"ES2020.BigInt",
"ESNext.Intl"
]
"lib": ["es2018", "DOM", "ES5", "ES6", "ES7", "es2019.array", "ES2021.WeakRef", "ES2020.BigInt", "ESNext.Intl"],
"strictBindCallApply": true,
"strictFunctionTypes": true
},
"include": [
"**/*.ts"
],
"exclude": [
"pouchdb-browser-webpack"
]
}
"include": ["**/*.ts"],
"exclude": ["pouchdb-browser-webpack", "utils"]
}

206
updates.md
View File

@@ -1,97 +1,123 @@
### 0.23.0
Incredibly new features!
## 0.24.0 RC Release Note
Now, we can use object storage (MinIO, S3, R2 or anything you like) for synchronising! Moreover, despite that, we can use all the features as if we were using CouchDB.
Note: As this is a pretty experimental feature, hence we have some limitations.
- This is built on the append-only architecture. It will not shrink used storage if we do not perform a rebuild.
- A bit fragile. However, our version x.yy.0 is always so.
- When the first synchronisation, the entire history to date is transferred. For this reason, it is preferable to do this under the WiFi network.
- Do not worry, from the second synchronisation, we always transfer only differences.
**Note:** This will be rewritten with the stable release. I confess, before you take the time, this is quite long.
I hope this feature empowers users to maintain independence and self-host their data, offering an alternative for those who prefer to manage their own storage solutions and avoid being stuck on the right side of a sudden change in business model.
Over the past three years since the inception of the plugin, various features have been implemented to address diverse user needs. This is so honourable and I am grateful for your years of support.
However, However, this process has resulted in a codebase that has become increasingly disorganised, with features becoming entangled.
Of course, I use Self-hosted MinIO for testing and recommend this. It is for the same reason as using CouchDB. -- open, controllable, auditable and indeed already audited by numerous eyes.
Consequently, this has led to a situation where bugs can go unnoticed or resolving one issue may inadvertently introduce another.
Let me write one more acknowledgement.
In 0.24.0, I reorganised the previously disjointed main codebase into clearly defined modules. Although I anticipated that the overall volume of code would not increase, I discovered that it has, in fact, expanded. While the complexity may still be considerable, the refactoring has enhanced clarity regarding the current structure of the code. (The next focus may involve a review of dependencies).
I have a lot of respect for that plugin, even though it is sometimes treated as if it is a competitor, remotely-save. I think it is a great architecture that embodies a different approach to my approach of recreating history. This time, with all due respect, I have used some of its code as a reference.
Hooray for open source, and generous licences, and the sharing of knowledge by experts.
Throughout this process, a significant number of bugs have been resolved. And it may be worth mentioning that these bugs may had given rise to other bugs. I kindly request that you verify whether your issues have been addressed. At least conflict resolution and related issues have improved significantly.
#### Version history
- 0.23.23:
- Refined:
- Setting dialogue very slightly refined.
- The hodgepodge inside the `Hatch` pane has been sorted into more explicit categorised panes.
- Now we have new panes for:
- `Selector`
- `Advanced`
- `Power users`
- `Patches (Edge case)`
- Applying the settings will now be more informative.
- The header bar will be shown for applying the settings which needs a database rebuild.
- Applying methods are now more clearly navigated.
- Definitely, drastic change. I hope this will be more user-friendly. However, if you notice any issues, please let me know. I hope that nothing missed.
- New features:
- Word-segmented chunk building on users language
- Chunks can now be built with word-segmented data, enhancing efficiency for markdown files which contains the multiple sentences in a single line.
- This feature is enabled by default through `Use Segmented-splitter`.
- (Default: Disabled, Please be relived, I have learnt).
- Fixed:
- Sending chunks on `Send chunk in bulk` are now buffered to avoid the out-of-memory error.
- `Send chunk in bulk` is back to default disabled. (Sorry, not applied to the migrated users; I did not think we should deepen the wound any further "automatically").
- Merging conflicts of JSON files are now works fine even if it contains `null`.
- Development:
- Implemented the logic for automatically generating the stub of document for the setting dialogue.
- 0.23.22:
- Fixed:
- Case-insensitive file handling
- Full-lower-case files are no longer created during database checking.
- Bulk chunk transfer
- The default value will automatically adjust to an acceptable size when using IBM Cloudant.
- 0.23.21:
- New Features:
- Case-insensitive file handling
- Files can now be handled case-insensitively.
- This behaviour can be modified in the settings under `Handle files as Case-Sensitive` (Default: Prompt, Enabled for previous behaviour).
- Improved chunk revision fixing
- Revisions for chunks can now be fixed for faster chunk creation.
- This can be adjusted in the settings under `Compute revisions for chunks` (Default: Prompt, Enabled for previous behaviour).
- Bulk chunk transfer
- Chunks can now be transferred in bulk during uploads.
- This feature is enabled by default through `Send chunks in bulk`.
- Creation of missing chunks without
- Missing chunks can be created without storing notes, enhancing efficiency for first synchronisation or after prolonged periods without synchronisation.
- Improvements:
- File status scanning on the startup
- Quite significant performance improvements.
- No more missing scans of some files.
- Status in editor enhancements
- Significant performance improvements in the status display within the editor.
- Notifications for files that will not be synchronised will now be properly communicated.
- Encryption and Decryption
- These processes are now performed in background threads to ensure fast and stable transfers.
- Verify and repair all files
- Got faster through parallel checking.
- Migration on update
- Migration messages and wizards have become more helpful.
- Behavioural changes:
- Chunk size adjustments
- Large chunks will no longer be created for older, stable files, addressing storage consumption issues.
- Flag file automation
- Confirmation will be shown and we can cancel it.
- Fixed:
- Database File Scanning
- All files in the database will now be enumerated correctly.
- Miscellaneous
- Dependency updated.
- Now, tree shaking is left to terser, from esbuild.
- 0.23.20:
- Fixed:
- Customisation Sync now checks the difference while storing or applying the configuration.
- No longer storing the same configuration multiple times.
- Time difference in the dialogue has been fixed.
- Remote Storage Limit Notification dialogue has been fixed, now the chosen value is saved.
- Improved:
- The Enlarging button on the enlarging threshold dialogue now displays the new value.
It is also the first step towards a fully-fledged-fancy LiveSync, not just a plug-in from Obsidian. Of course, it will still be a plug-in as a first class and foremost, but this development marks a significant step towards the self-hosting concept.
Older notes is in [updates_old.md](https://github.com/vrtmrz/obsidian-livesync/blob/main/updates_old.md).
This dev release is very close to the beta version that I had previously indicated would not be released. As a result, I have faced challenges in maintaining the main branch while working on this dev release. Regrettably, I have not been able to make any commits to the main branch in the last three weeks. Thus, the dev branch will remain reserved for major changes only.
The Release Candidate will be available for a few days and will only be officially released once users, including myself, have confirmed that there are no issues.
Finally, I would like to once again express my respect and gratitude to all of you once again. Thank you for your interest in the development version. Your contributions and dedication are greatly appreciated through testing.
Thank you, and I hope your troubles will be resolved!
---
## 0.24.0.dev-rc8
### Fixed
- Now the deletion of hidden files are correctly synchronised.
## 0.24.0.dev-rc7
### Fixed
- Verifying files between the local database and storage is now working correctly.
### New Features
- We can verify and resolve also the hidden files now.
## 0.24.0.dev-rc6
### Fixed
- We can resolve the conflict of the JSON file correctly now.
- This would be the final Release Candidate.
## 0.24.0.dev-rc5
### Improved
- A note relating to device names has been added to Customisation Sync on the setting dialogue.
- Logs of Hidden File Sync and Customisation Sync have been prefixed with the respective feature names.
### Fixed
- Hidden file sync is now working correctly.
- Customisation Sync is now working correctly together with hidden file sync
- No longer database suffix is stored in the setting sharing markdown.
## 0.24.0.dev-rc4
### Improved
- The welcome message is now more simple to encourage the use of the Setup-URI.
- And the secondary message is also simpler to guide users to Minimal Setup.
- But Setup-URI will be recommended again, due to its importance.
- These dialogues contain a link to the documentation which can be clicked.
- The minimal setup is more minimal now. And, the setup is more user-friendly.
- Now the Configuration of the remote database is checked more robust, but we can ignore the warning and proceed with the setup.
- Before we are asked about each feature, we are asked if we want to use optional features in the first place.
- This is to prevent the user from being overwhelmed by the features.
- And made it clear that it is not recommended for new users.
- Many messages have been improved for better understanding.
- Ridiculous messages have been (carefully) refined.
- Dialogues are more informative and friendly.
- A lot of messages have been mostly rewritten, leveraging Markdown.
- Especially auto-closing dialogues are now explicitly labelled: `To stop the countdown, tap anywhere on the dialogue`.
- Now if the is plugin configured to ignore some events, we will get a chance to fix it, in addition to the warning.
- And why that has happened is also explained in the dialogue.
### Fixed
- While restarting the plug-in, the shown dialogues will be automatically closed to avoid unexpected behaviour.
- Replicated documents that the local device has configured to ignore are now correctly ignored.
- The chunks of the document on the local device during the first transfer will be created correctly.
- And why we should create them is now explained in the dialogue.
- If optional features have been enabled in the wizard, `Enable advanced features` will be toggled correctly.
### Changed
- Some default settings have been changed for easier new user experience.
- Preventing the meaningless migration of the settings.
### Tidied
- Commented-out codes have been gradually removed.
## 0.24.0.dev-rc3
### Fixed
- No longer Missing Translation Warning is shown in the console.
- Fixed the issue where some functions were not working properly (`_` started functions).
## 0.24.0.dev-rc2
### Fixed
- Some status icons is now shown correctly.
## 0.24.0-rc1
### Fixed
- A fair numbers of bugs have been fixed.
### Tiding
- The codebase has been reorganised into clearly defined modules.
Older notes is in [updates_old.md](https://github.com/vrtmrz/obsidian-livesync/blob/main/updates_old.md).

75
updates_old.md
View File

@@ -18,6 +18,81 @@ I have a lot of respect for that plugin, even though it is sometimes treated as
Hooray for open source, and generous licences, and the sharing of knowledge by experts.
#### Version history
- 0.23.23:
- Refined:
- Setting dialogue very slightly refined.
- The hodgepodge inside the `Hatch` pane has been sorted into more explicit categorised panes.
- Now we have new panes for:
- `Selector`
- `Advanced`
- `Power users`
- `Patches (Edge case)`
- Applying the settings will now be more informative.
- The header bar will be shown for applying the settings which needs a database rebuild.
- Applying methods are now more clearly navigated.
- Definitely, drastic change. I hope this will be more user-friendly. However, if you notice any issues, please let me know. I hope that nothing missed.
- New features:
- Word-segmented chunk building on users language
- Chunks can now be built with word-segmented data, enhancing efficiency for markdown files which contains the multiple sentences in a single line.
- This feature is enabled by default through `Use Segmented-splitter`.
- (Default: Disabled, Please be relived, I have learnt).
- Fixed:
- Sending chunks on `Send chunk in bulk` are now buffered to avoid the out-of-memory error.
- `Send chunk in bulk` is back to default disabled. (Sorry, not applied to the migrated users; I did not think we should deepen the wound any further "automatically").
- Merging conflicts of JSON files are now works fine even if it contains `null`.
- Development:
- Implemented the logic for automatically generating the stub of document for the setting dialogue.
- 0.23.22:
- Fixed:
- Case-insensitive file handling
- Full-lower-case files are no longer created during database checking.
- Bulk chunk transfer
- The default value will automatically adjust to an acceptable size when using IBM Cloudant.
- 0.23.21:
- New Features:
- Case-insensitive file handling
- Files can now be handled case-insensitively.
- This behaviour can be modified in the settings under `Handle files as Case-Sensitive` (Default: Prompt, Enabled for previous behaviour).
- Improved chunk revision fixing
- Revisions for chunks can now be fixed for faster chunk creation.
- This can be adjusted in the settings under `Compute revisions for chunks` (Default: Prompt, Enabled for previous behaviour).
- Bulk chunk transfer
- Chunks can now be transferred in bulk during uploads.
- This feature is enabled by default through `Send chunks in bulk`.
- Creation of missing chunks without
- Missing chunks can be created without storing notes, enhancing efficiency for first synchronisation or after prolonged periods without synchronisation.
- Improvements:
- File status scanning on the startup
- Quite significant performance improvements.
- No more missing scans of some files.
- Status in editor enhancements
- Significant performance improvements in the status display within the editor.
- Notifications for files that will not be synchronised will now be properly communicated.
- Encryption and Decryption
- These processes are now performed in background threads to ensure fast and stable transfers.
- Verify and repair all files
- Got faster through parallel checking.
- Migration on update
- Migration messages and wizards have become more helpful.
- Behavioural changes:
- Chunk size adjustments
- Large chunks will no longer be created for older, stable files, addressing storage consumption issues.
- Flag file automation
- Confirmation will be shown and we can cancel it.
- Fixed:
- Database File Scanning
- All files in the database will now be enumerated correctly.
- Miscellaneous
- Dependency updated.
- Now, tree shaking is left to terser, from esbuild.
- 0.23.20:
- Fixed:
- Customisation Sync now checks the difference while storing or applying the configuration.
- No longer storing the same configuration multiple times.
- Time difference in the dialogue has been fixed.
- Remote Storage Limit Notification dialogue has been fixed, now the chosen value is saved.
- Improved:
- The Enlarging button on the enlarging threshold dialogue now displays the new value.
- 0.23.19:
- Not released.
- 0.23.18: