public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-02-23 12:38:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: public:0.2.1
Branches Tags
public:main public:beta public:snyk-upgrade-7fd31e6f474568014502e7205847f257 public:snyk-fix-90f3b71450f7543a1c61724d8afa0e71 public:refactor_services_2 public:refactor_services public:dev public:snyk-upgrade-fe9625385aa4be849c7157033fd2abb0 public:snyk-upgrade-99a31e70c822894595544b0c9813b11a public:svelteui public:disenchant public:snyk-fix-61f76ef0bce2a276df9552c2fc80a14f public:snyk-upgrade-2baa7938ba318464c3bad5c3fd5cbda5 public:snyk-upgrade-7af62a575eab20e8fb4e9565bf1adcfc public:snyk-upgrade-b32f281b31a81483f126b8fb6c54a957 public:snyk-upgrade-bd61b1103442d3faecc9fc726c34e9a4 public:snyk-upgrade-5b2d6584430c061b5187611a5d7fb64a public:snyk-fix-0e9fbbcb374fcb8d5e8cb027d8414fb7 public:snyk-upgrade-5242daf91e706d1ae500c9ea6545636e public:snyk-upgrade-70434a6822fbad83424135dfad9df909 public:snyk-upgrade-8fa01ccad0352125f1866d41e57d35e1 public:snyk-upgrade-39eb97b4655e70091f718a88fccd57c1 public:snyk-upgrade-2b99d54d19e207887e36c3071b796925 public:forprivacy public:snyk-upgrade-dad9088d0a333677e0157e3deef7fc38 public:snyk-upgrade-b9a76d74b3e5723759af5b3319aabb80 public:snyk-upgrade-7da6459d8ff5d293d80de9d80061d912 public:snyk-upgrade-6a3786fc4cbe394542e9d1b6058c46e7 public:snyk-upgrade-8fc9e62771dc41b0f23dc0efefb5c949
public:0.25.43-patched-9 public:0.25.43-patched-8 public:0.25.43-patched-7 public:0.25.43-patched-6 public:0.25.43-patched-5 public:0.25.43-patched-4 public:0.25.43-patched-3 public:0.25.43-patched-2 public:0.25.43-patched-1 public:0.25.43 public:0.25.42 public:0.25.41-patched-3 public:0.25.41-patched-2 public:0.25.41-patched-1 public:0.25.41 public:0.25.40 public:0.25.39 public:0.25.38 public:0.25.37 public:0.25.36 public:0.25.35 public:0.25.34 public:0.25.33 public:0.25.32 public:0.25.31 public:0.25.30 public:0.25.30-dev public:0.25.29 public:0.25.28 public:0.25.27 public:0.25.26 public:0.25.25 public:0.25.24 public:0.25.24.beta3 public:0.25.24.beta2 public:0.25.24.beta1 public:0.25.23 public:0.25.23.beta1 public:0.25.22 public:0.25.21 public:0.25.21.beta2 public:0.25.21.beta1 public:0.25.20 public:0.25.19 public:0.25.18 public:0.25.17 public:0.25.16 public:0.25.15 public:0.25.14 public:0.25.13 public:0.25.12 public:0.25.11 public:0.25.10 public:0.25.9 public:0.25.8 public:0.25.7 public:0.25.6 public:0.25.5 public:0.25.4 public:0.25.3 public:0.25.2 public:0.25.1 public:0.25.0 public:0.25.0-beta1 public:0.24.31 public:0.24.30 public:0.24.29 public:0.24.28 public:0.24.27 public:0.24.26 public:0.24.25 public:0.24.24 public:0.24.23 public:0.24.22 public:0.24.21 public:0.24.20 public:0.24.19 public:0.24.18 public:0.24.17 public:0.24.16 public:0.24.15 public:0.24.14 public:0.24.13 public:0.24.12 public:0.24.11 public:0.24.10 public:0.24.9 public:0.24.8 public:0.24.7 public:0.24.6 public:0.24.5 public:0.24.4 public:0.24.3 public:0.24.2 public:0.24.1 public:0.24.0 public:0.24.0.dev-rc8 public:0.24.0.dev-rc7 public:0.24.0.dev-rc6 public:0.24.0.dev-rc5 public:0.24.0.dev-rc4 public:0.24.0.dev-rc3 public:0.24.0.dev-rc2 public:0.23.23 public:0.23.22 public:0.23.21 public:0.23.20 public:0.23.19 public:0.23.18 public:0.23.17 public:0.23.16 public:0.23.15 public:0.23.14 public:0.23.13 public:0.23.12 public:0.23.11 public:0.23.10 public:0.23.9 public:0.23.8 public:0.23.7 public:0.23.6 public:0.23.5 public:0.23.4 public:0.23.3 public:0.23.2 public:0.23.1 public:0.23.0 public:0.22.19 public:0.22.18 public:0.22.17 public:0.22.16 public:0.22.15 public:0.22.14 public:0.22.13 public:0.22.12 public:0.22.11 public:0.22.10 public:0.22.9 public:0.22.8 public:0.22.7 public:0.22.6 public:0.22.5 public:0.22.4 public:0.22.3 public:0.22.2 public:0.22.1 public:0.22.0 public:0.21.5 public:0.21.4 public:0.21.3 public:0.21.2 public:0.21.1 public:0.21.0 public:0.20.7 public:0.20.6 public:0.20.5 public:0.20.4 public:0.20.3 public:0.20.2 public:0.20.1 public:0.20.0 public:0.19.23 public:0.19.22 public:0.19.21 public:0.19.20 public:0.19.19 public:0.19.18 public:0.19.17 public:0.19.16 public:0.19.15 public:0.19.14 public:0.19.13 public:0.19.12 public:0.19.11 public:0.19.10 public:0.19.9 public:0.19.8 public:0.19.7 public:0.19.6 public:0.19.5 public:0.19.4 public:0.19.3 public:0.19.2 public:0.19.1 public:0.19.0 public:0.18.6 public:0.18.5 public:0.18.4 public:0.18.3 public:0.18.2 public:0.18.1 public:0.18.0 public:0.17.34 public:0.17.33 public:0.17.32 public:0.17.31 public:0.17.30 public:0.17.29 public:0.17.28 public:0.17.27 public:0.17.26 public:0.17.25 public:0.17.24 public:0.17.23 public:0.17.22 public:0.17.21 public:0.17.20 public:0.17.19 public:0.17.18 public:0.17.17 public:0.17.16 public:0.17.15 public:0.17.14 public:0.17.13 public:0.17.12 public:0.17.11 public:0.17.10 public:0.17.9 public:0.17.8 public:0.17.7 public:0.17.6 public:0.17.5 public:0.17.4 public:0.17.3 public:0.17.2 public:0.17.1 public:0.17.0 public:0.16.8 public:0.16.7 public:0.16.6 public:0.16.5 public:0.16.4 public:0.16.3 public:0.16.2 public:0.16.1 public:0.16.0 public:0.15.10 public:0.15.9 public:0.15.8 public:0.15.7 public:0.15.6 public:0.15.5 public:0.15.4 public:0.15.3 public:0.15.2 public:0.15.1 public:0.15.0 public:0.14.8 public:0.14.7 public:0.14.6 public:0.14.5 public:0.14.4 public:0.14.3 public:0.14.2 public:0.14.1 public:0.14.0 public:0.13.4 public:0.13.3 public:0.13.2 public:0.13.1 public:0.13.0 public:0.12.5 public:0.12.4 public:0.12.3 public:0.12.2 public:0.12.1 public:0.12.0 public:0.11.10 public:0.11.9 public:0.11.8 public:0.11.7 public:0.11.6 public:0.11.5 public:0.11.4 public:0.11.3 public:0.11.2 public:0.11.1 public:0.11.0 public:0.10.1 public:0.10.0 public:0.9.0 public:0.8.8 public:0.8.7 public:0.8.6 public:0.8.5 public:0.8.3 public:0.8.2 public:0.8.1 public:0.8.0 public:0.7.2 public:0.7.1 public:0.7.0 public:0.6.1 public:0.6.0 public:0.5.0 public:0.4.1 public:0.4.0 public:0.3.1 public:0.3.0 public:0.2.1 public:0.2.0 public:0.1.26 public:0.1.25 public:0.1.24 public:0.1.23 public:0.1.22 public:0.1.21 public:0.1.20 public:0.1.19 public:0.1.18 public:0.1.17 public:0.1.16 public:0.1.15 public:0.1.14 public:0.1.13 public:0.1.12 public:0.1.11 public:0.1.10 public:0.1.9 public:0.1.8 public:0.1.7 public:0.1.6 public:0.1.5 public:0.1.4 public:0.1.3 public:0.1.2 public:0.1.1 public:0.1.0 public:0.0.9 public:0.0.8 public:0.0.7 public:0.0.6
..
compare: public:0.4.0
Branches Tags
public:beta public:main public:snyk-upgrade-7fd31e6f474568014502e7205847f257 public:snyk-fix-90f3b71450f7543a1c61724d8afa0e71 public:refactor_services_2 public:refactor_services public:dev public:snyk-upgrade-fe9625385aa4be849c7157033fd2abb0 public:snyk-upgrade-99a31e70c822894595544b0c9813b11a public:svelteui public:disenchant public:snyk-fix-61f76ef0bce2a276df9552c2fc80a14f public:snyk-upgrade-2baa7938ba318464c3bad5c3fd5cbda5 public:snyk-upgrade-7af62a575eab20e8fb4e9565bf1adcfc public:snyk-upgrade-b32f281b31a81483f126b8fb6c54a957 public:snyk-upgrade-bd61b1103442d3faecc9fc726c34e9a4 public:snyk-upgrade-5b2d6584430c061b5187611a5d7fb64a public:snyk-fix-0e9fbbcb374fcb8d5e8cb027d8414fb7 public:snyk-upgrade-5242daf91e706d1ae500c9ea6545636e public:snyk-upgrade-70434a6822fbad83424135dfad9df909 public:snyk-upgrade-8fa01ccad0352125f1866d41e57d35e1 public:snyk-upgrade-39eb97b4655e70091f718a88fccd57c1 public:snyk-upgrade-2b99d54d19e207887e36c3071b796925 public:forprivacy public:snyk-upgrade-dad9088d0a333677e0157e3deef7fc38 public:snyk-upgrade-b9a76d74b3e5723759af5b3319aabb80 public:snyk-upgrade-7da6459d8ff5d293d80de9d80061d912 public:snyk-upgrade-6a3786fc4cbe394542e9d1b6058c46e7 public:snyk-upgrade-8fc9e62771dc41b0f23dc0efefb5c949
public:0.25.43-patched-9 public:0.25.43-patched-8 public:0.25.43-patched-7 public:0.25.43-patched-6 public:0.25.43-patched-5 public:0.25.43-patched-4 public:0.25.43-patched-3 public:0.25.43-patched-2 public:0.25.43-patched-1 public:0.25.43 public:0.25.42 public:0.25.41-patched-3 public:0.25.41-patched-2 public:0.25.41-patched-1 public:0.25.41 public:0.25.40 public:0.25.39 public:0.25.38 public:0.25.37 public:0.25.36 public:0.25.35 public:0.25.34 public:0.25.33 public:0.25.32 public:0.25.31 public:0.25.30 public:0.25.30-dev public:0.25.29 public:0.25.28 public:0.25.27 public:0.25.26 public:0.25.25 public:0.25.24 public:0.25.24.beta3 public:0.25.24.beta2 public:0.25.24.beta1 public:0.25.23 public:0.25.23.beta1 public:0.25.22 public:0.25.21 public:0.25.21.beta2 public:0.25.21.beta1 public:0.25.20 public:0.25.19 public:0.25.18 public:0.25.17 public:0.25.16 public:0.25.15 public:0.25.14 public:0.25.13 public:0.25.12 public:0.25.11 public:0.25.10 public:0.25.9 public:0.25.8 public:0.25.7 public:0.25.6 public:0.25.5 public:0.25.4 public:0.25.3 public:0.25.2 public:0.25.1 public:0.25.0 public:0.25.0-beta1 public:0.24.31 public:0.24.30 public:0.24.29 public:0.24.28 public:0.24.27 public:0.24.26 public:0.24.25 public:0.24.24 public:0.24.23 public:0.24.22 public:0.24.21 public:0.24.20 public:0.24.19 public:0.24.18 public:0.24.17 public:0.24.16 public:0.24.15 public:0.24.14 public:0.24.13 public:0.24.12 public:0.24.11 public:0.24.10 public:0.24.9 public:0.24.8 public:0.24.7 public:0.24.6 public:0.24.5 public:0.24.4 public:0.24.3 public:0.24.2 public:0.24.1 public:0.24.0 public:0.24.0.dev-rc8 public:0.24.0.dev-rc7 public:0.24.0.dev-rc6 public:0.24.0.dev-rc5 public:0.24.0.dev-rc4 public:0.24.0.dev-rc3 public:0.24.0.dev-rc2 public:0.23.23 public:0.23.22 public:0.23.21 public:0.23.20 public:0.23.19 public:0.23.18 public:0.23.17 public:0.23.16 public:0.23.15 public:0.23.14 public:0.23.13 public:0.23.12 public:0.23.11 public:0.23.10 public:0.23.9 public:0.23.8 public:0.23.7 public:0.23.6 public:0.23.5 public:0.23.4 public:0.23.3 public:0.23.2 public:0.23.1 public:0.23.0 public:0.22.19 public:0.22.18 public:0.22.17 public:0.22.16 public:0.22.15 public:0.22.14 public:0.22.13 public:0.22.12 public:0.22.11 public:0.22.10 public:0.22.9 public:0.22.8 public:0.22.7 public:0.22.6 public:0.22.5 public:0.22.4 public:0.22.3 public:0.22.2 public:0.22.1 public:0.22.0 public:0.21.5 public:0.21.4 public:0.21.3 public:0.21.2 public:0.21.1 public:0.21.0 public:0.20.7 public:0.20.6 public:0.20.5 public:0.20.4 public:0.20.3 public:0.20.2 public:0.20.1 public:0.20.0 public:0.19.23 public:0.19.22 public:0.19.21 public:0.19.20 public:0.19.19 public:0.19.18 public:0.19.17 public:0.19.16 public:0.19.15 public:0.19.14 public:0.19.13 public:0.19.12 public:0.19.11 public:0.19.10 public:0.19.9 public:0.19.8 public:0.19.7 public:0.19.6 public:0.19.5 public:0.19.4 public:0.19.3 public:0.19.2 public:0.19.1 public:0.19.0 public:0.18.6 public:0.18.5 public:0.18.4 public:0.18.3 public:0.18.2 public:0.18.1 public:0.18.0 public:0.17.34 public:0.17.33 public:0.17.32 public:0.17.31 public:0.17.30 public:0.17.29 public:0.17.28 public:0.17.27 public:0.17.26 public:0.17.25 public:0.17.24 public:0.17.23 public:0.17.22 public:0.17.21 public:0.17.20 public:0.17.19 public:0.17.18 public:0.17.17 public:0.17.16 public:0.17.15 public:0.17.14 public:0.17.13 public:0.17.12 public:0.17.11 public:0.17.10 public:0.17.9 public:0.17.8 public:0.17.7 public:0.17.6 public:0.17.5 public:0.17.4 public:0.17.3 public:0.17.2 public:0.17.1 public:0.17.0 public:0.16.8 public:0.16.7 public:0.16.6 public:0.16.5 public:0.16.4 public:0.16.3 public:0.16.2 public:0.16.1 public:0.16.0 public:0.15.10 public:0.15.9 public:0.15.8 public:0.15.7 public:0.15.6 public:0.15.5 public:0.15.4 public:0.15.3 public:0.15.2 public:0.15.1 public:0.15.0 public:0.14.8 public:0.14.7 public:0.14.6 public:0.14.5 public:0.14.4 public:0.14.3 public:0.14.2 public:0.14.1 public:0.14.0 public:0.13.4 public:0.13.3 public:0.13.2 public:0.13.1 public:0.13.0 public:0.12.5 public:0.12.4 public:0.12.3 public:0.12.2 public:0.12.1 public:0.12.0 public:0.11.10 public:0.11.9 public:0.11.8 public:0.11.7 public:0.11.6 public:0.11.5 public:0.11.4 public:0.11.3 public:0.11.2 public:0.11.1 public:0.11.0 public:0.10.1 public:0.10.0 public:0.9.0 public:0.8.8 public:0.8.7 public:0.8.6 public:0.8.5 public:0.8.3 public:0.8.2 public:0.8.1 public:0.8.0 public:0.7.2 public:0.7.1 public:0.7.0 public:0.6.1 public:0.6.0 public:0.5.0 public:0.4.1 public:0.4.0 public:0.3.1 public:0.3.0 public:0.2.1 public:0.2.0 public:0.1.26 public:0.1.25 public:0.1.24 public:0.1.23 public:0.1.22 public:0.1.21 public:0.1.20 public:0.1.19 public:0.1.18 public:0.1.17 public:0.1.16 public:0.1.15 public:0.1.14 public:0.1.13 public:0.1.12 public:0.1.11 public:0.1.10 public:0.1.9 public:0.1.8 public:0.1.7 public:0.1.6 public:0.1.5 public:0.1.4 public:0.1.3 public:0.1.2 public:0.1.1 public:0.1.0 public:0.0.9 public:0.0.8 public:0.0.7 public:0.0.6

3 Commits
0.2.1 ... 0.4.0

Author SHA1 Message Date
vorotamoroz
c3464a4e9c New feature: 
- Bootup sequence prevention implemented.

Touched the docs up:
 2021-12-28 11:30:19 +09:00
vorotamoroz
55545da45f Fixed: 
- Fixed problems about saving or deleting files to the local database.
- Disable version up warning.
- Fixed error on folder renaming.
- Merge dialog is now shown one by one.
- Fixed icons of queued files.
- Handled sync issue of Folder to File
- Fixed the messages in the setting dialog.
- Fixed deadlock.
 2021-12-24 17:05:57 +09:00
vorotamoroz
96165b4f9b Fixed and implemented: 
- New configuration to solve synchronization failure on large vault.
- Preset misconfigurations
- Sometimes hanged on replication.
- Wrote documents
 2021-12-23 13:22:46 +09:00
13 changed files with 703 additions and 331 deletions
Expand all files Collapse all files

1
README.md
View File

@@ -81,6 +81,7 @@ Synchronization status is shown in statusbar.
- When synchronized, files are compared by their modified times and overwritten by the newer ones once. Then plugin checks the conflicts and if a merge is needed, the dialog will open.
- Rarely, the file in the database would be broken. The plugin will not write storage when it looks broken, so some old files must be on your device. If you edit the file, it will be cured. But if the file does not exist on any device, can not rescue it. So you can delete these items from the setting dialog.
- If your database looks corrupted, try "Drop History". Usually, It is the easiest way.
- To stop the bootup sequence for fixing problems on databases, you can put `redflag.md` on top of your vault.
- Q: Database is growing, how can I shrink it up?
A: each of the docs is saved with their old 100 revisions to detect and resolve confliction. Picture yourself that one device has been off the line for a while, and joined again. The device has to check his note and remote saved note. If exists in revision histories of remote notes even though the device's note is a little different from the latest one, it could be merged safely. Even if that is not in revision histories, we only have to check differences after the revision that both devices commonly have. This is like The git's conflict resolving method. So, We have to make the database again like an enlarged git repo if you want to solve the root of the problem.
- And more technical Information are in the [Technical Information](docs/tech_info.md)

1
README_ja.md
View File

@@ -83,6 +83,7 @@ Self-hosted LiveSync用にWebClipperも作りました。Chrome Web Storeから
- ファイルは同期された後、タイムスタンプを比較して新しければいったん新しい方で上書きされます。その後、衝突が発生したかによって、マージが行われます。
- まれにファイルが破損することがあります。破損したファイルに関してはディスクへの反映を試みないため、実際には使用しているデバイスには少し古いファイルが残っていることが多いです。そのファイルを再度更新してもらうと、データベースが更新されて問題なくなるケースがあります。ファイルがどの端末にも存在しない場合は、設定画面から、削除できます。
- データベースが変。そういうときは、いったんデータベースをDrop Historyのapply and sendで再初期化してみてください。だいたい直ります。
- データベースの復旧中に再起動した場合など、うまくローカルデータベースを修正できない際には、Vaultのトップに`redflag.md`というファイルを置いてください。起動時のシーケンスがスキップされます。
- データベースが大きくなってきてるんだけど、小さくできる？→各ノートは、それぞれの古い100リビジョンとともに保存されています。例えば、しばらくオフラインだったあるデバイスが、久しぶりに同期したと想定してみてください。そのとき、そのデバイスは最新とは少し異なるリビジョンを持ってるはずです。その場合でも、リモートのリビジョン履歴にリモートのものが存在した場合、安全にマージできます。もしリビジョン履歴に存在しなかった場合、確認しなければいけない差分も、対象を存在して持っている共通のリビジョン以降のみに絞れます。ちょうどGitのような方法で、衝突を解決している形になるのです。そのため、肥大化したリポジトリの解消と同様に、本質的にデータベースを小さくしたい場合は、データベースの作り直しが必要です。
- その他の技術的なお話は、[技術的な内容](docs/tech_info_ja.md)に書いてあります。

86
docs/settings.md
View File

@@ -23,7 +23,7 @@ The Database name to synchronize.
## Local Database Configurations
"Local Database" is created inside your obsidian.
### Batch database update (beta)
### Batch database update
Delay database update until raise replication, open another file, window visibility changed, or file events except for file modification.
This option can not be used with LiveSync at the same time.
@@ -78,6 +78,27 @@ When running these operations, every synchronization settings is disabled.
**And even your passphrase is wrong, It doesn't be checked before the plugin really decrypts. So If you set the wrong passphrase and run "Apply and Receive", you will get an amount of decryption error. But, this is the specification.**
### minimum chunk size and LongLine threshold
The configuration of chunk splitting.
Self-hosted LiveSync splits the note into chunks for efficient synchronization. This chunk should be longer than "Minimum chunk size".
Specifically, the length of the chunk is determined by the following orders.
1. Find the nearest newline character, and if it is farther than LongLineThreshold, this piece becomes an independent chunk.
2. If not, find nearest to these items.
1. Newline character
2. Empty line (Windows style)
3. Empty line (non-Windows style)
3. Compare the farther in these 3 positions and next "\[newline\]#" position, pick a shorter piece to as chunk.
This rule was made empirically from my dataset. If this rule acts as badly on your data. Please give me the information.
You can dump saved note structure to `Dump informations of this doc`. Replace every character to x except newline and "#" when sending information to me.
Default values are 20 letters and 250 letters.
## General Settings
### Do not show low-priority log
@@ -122,32 +143,58 @@ Self-hosted LiveSync will delete the folder when the folder becomes empty. If th
### Use newer file if conflicted (beta)
Always use the newer file to resolve and overwrite when conflict has occurred.
### minimum chunk size and LongLine threshold
The configuration of chunk splitting.
### Advanced settings
Self-hosted LiveSync using PouchDB and synchronizes with the remote by [this protocol](https://docs.couchdb.org/en/stable/replication/protocol.html).
So, it splits every entry into chunks to be acceptable by the database with limited payload size and document size.
Self-hosted LiveSync splits the note into chunks for efficient synchronization. This chunk should be longer than "Minimum chunk size".
However, it was not enough.
According to [2.4.2.5.2. Upload Batch of Changed Documents](https://docs.couchdb.org/en/stable/replication/protocol.html#upload-batch-of-changed-documents) in [Replicate Changes](https://docs.couchdb.org/en/stable/replication/protocol.html#replicate-changes), it might become a bigger request.
Specifically, the length of the chunk is determined by the following orders.
Unfortunately, there is no way to deal with this automatically by size for every request.
Therefore, I made it possible to configure this.
1. Find the nearest newline character, and if it is farther than LongLineThreshold, this piece becomes an independent chunk.
Note: If you set these values lower number, the number of requests will increase.
Therefore, if you are far from the server, the total throughput will be low, and the traffic will increase.
2. If not, find nearest to these items.
1. Newline character
2. Empty line (Windows style)
3. Empty line (non-Windows style)
3. Compare the farther in these 3 positions and next "\[newline\]#" position, pick a shorter piece to as chunk.
### Batch size
Number of change feed items to process at a time. Defaults to 250.
This rule was made empirically from my dataset. If this rule acts as badly on your data. Please give me the information.
You can dump saved note structure to `Dump informations of this doc`. Replace every character to x except newline and "#" when sending information to me.
Default values are 20 letters and 250 letters.
### Batch limit
Number of batches to process at a time. Defaults to 40. This along with batch size controls how many docs are kept in memory at a time.
## Miscellaneous
### Show status inside editor
Show information inside the editor pane.
It would be useful for mobile.
### Check integrity on saving
Check all chunks are correctly saved on saving.
### Presets
You can set synchronization method at once as these pattern:
- LiveSync
- LiveSync : enabled
- Batch database update : disabled
- Periodic Sync : disabled
- Sync on Save : disabled
- Sync on File Open : disabled
- Sync on Start : disabled
- Periodic w/ batch
- LiveSync : disabled
- Batch database update : enabled
- Periodic Sync : enabled
- Sync on Save : disabled
- Sync on File Open : enabled
- Sync on Start : enabled
- Disable all sync
- LiveSync : disabled
- Batch database update : disabled
- Periodic Sync : disabled
- Sync on Save : disabled
- Sync on File Open : disabled
- Sync on Start : disabled
## Hatch
From here, everything is under the hood. Please handle it with care.
@@ -165,8 +212,11 @@ The remote database indicates that has been unlocked Pattern 1.
When you mark all devices as resolved, you can unlock the database.
But, there's no problem even if you leave it as it is.
### Reread all files
Reread all files in the vault, and update them into the database if there's diff or could not read from the database.
### Verify and repair all files
read all files in the vault, and update them into the database if there's diff or could not read from the database.
### Sanity check
Make sure that all the files on the local database have all chunks.
### Drop history
Drop all histories on the local database and the remote database, and initialize When synchronization time has been prolonged to the new device or new vault, or database size became to be much larger. Try this.

59
docs/settings_ja.md
View File

@@ -25,7 +25,7 @@ CouchDBのURIを入力します。Cloudantの場合は「External Endpoint(prefe
## Local Database Configurations
端末内に作成されるデータベースの設定です。
### Batch database update (beta)
### Batch database update
データベースの更新を以下の事象が発生するまで遅延させます。
- レプリケーションが発生する
- 他のファイルを開く
@@ -78,6 +78,24 @@ End to End 暗号化を行うに当たって、異なるパスフレーズで暗
どちらのオペレーションも、実行するとすべての同期設定が無効化されます。
**また、パスフレーズのチェックは、実際に復号するまで行いません。そのため、パスフレーズを間違えて設定し、Apply and receiveで同期を行うと、大量のエラーが発生します。これは仕様です。**
### minimum chunk size と LongLine threshold
チャンクの分割についての設定です。
Self-hosted LiveSyncは一つのチャンクのサイズを最低minimum chunk size文字確保した上で、できるだけ効率的に同期できるよう、ノートを分割してチャンクを作成します。
これは、同期を行う際に、一定の文字数で分割した場合、先頭の方を編集すると、その後の分割位置がすべてずれ、結果としてほぼまるごとのファイルのファイル送受信を行うことになっていた問題を避けるために実装されました。
具体的には、先頭から順に直近の下記の箇所を検索し、一番長く切れたものを一つのチャンクとします。
1. 次の改行を探し、それがLongLine Thresholdより先であれば、一つのチャンクとして確定します。
2. そうではない場合は、下記を順に探します。
1. 改行
2. windowsでの空行がある所
3. 非Windowsでの空行がある所
3. この三つのうち一番遠い場所と、 「改行後、#から始まる所」を比べ、短い方をチャンクとします。
このルールは経験則的に作りました。実データが偏っているため。もし思わぬ挙動をしている場合は、是非コマンドから`Dump informations of this doc`を選択し、情報をください。
改行文字と#を除き、すべて●に置換しても、アルゴリズムは有効に働きます。
デフォルトは20文字と、250文字です。
## General Settings
一般的な設定です。
@@ -124,30 +142,34 @@ Self-hosted LiveSyncは通常、フォルダ内のファイルがすべて削除
### Use newer file if conflicted (beta)
競合が発生したとき、常に新しいファイルを使用して競合を自動的に解決します。
### minimum chunk size と LongLine threshold
チャンクの分割についての設定です。
Self-hosted LiveSyncは一つのチャンクのサイズを最低minimum chunk size文字確保した上で、できるだけ効率的に同期できるよう、ノートを分割してチャンクを作成します。
これは、同期を行う際に、一定の文字数で分割した場合、先頭の方を編集すると、その後の分割位置がすべてずれ、結果としてほぼまるごとのファイルのファイル送受信を行うことになっていた問題を避けるために実装されました。
具体的には、先頭から順に直近の下記の箇所を検索し、一番長く切れたものを一つのチャンクとします。
### Advanced settings
Self-hosted LiveSyncはPouchDBを使用し、リモートと[このプロトコル](https://docs.couchdb.org/en/stable/replication/protocol.html)で同期しています。
そのため、全てのノートなどはデータベースが許容するペイロードサイズやドキュメントサイズに併せてチャンクに分割されています。
1. 次の改行を探し、それがLongLine Thresholdより先であれば、一つのチャンクとして確定します。
しかしながら、それだけでは不十分なケースがあり、[Replicate Changes](https://docs.couchdb.org/en/stable/replication/protocol.html#replicate-changes)の[2.4.2.5.2. Upload Batch of Changed Documents](https://docs.couchdb.org/en/stable/replication/protocol.html#upload-batch-of-changed-documents)を参照すると、このリクエストは巨大になる可能性がありました。
2. そうではない場合は、下記を順に探します。
1. 改行
2. windowsでの空行がある所
3. 非Windowsでの空行がある所
3. この三つのうち一番遠い場所と、 「改行後、#から始まる所」を比べ、短い方をチャンクとします。
残念ながら、このサイズを呼び出しごとに自動的に調整する方法はありません。
そのため、設定を変更できるように機能追加いたしました。
このルールは経験則的に作りました。実データが偏っているため。もし思わぬ挙動をしている場合は、是非コマンドから`Dump informations of this doc`を選択し、情報をください。
改行文字と#を除き、すべて●に置換しても、アルゴリズムは有効に働きます。
デフォルトは20文字と、250文字です。
備考：もし小さな値を設定した場合、リクエスト数は増えます。
もしサーバから遠い場合、トータルのスループットは遅くなり、転送量は増えます。
### Batch size
一度に処理するChange feedの数です。デフォルトは250です。
### Batch limit
一度に処理するBatchの数です。デフォルトは40です。
## Miscellaneous
その他の設定です
### Show status inside editor
同期の情報をエディター内に表示します。
同期の情報をエディター内に表示します。
モバイルで便利です。
### Check integrity on saving
保存時にデータが全て保存できたかチェックを行います。
## Hatch
ここから先は、困ったときに開ける蓋の中身です。注意して使用してください。
@@ -166,9 +188,12 @@ Self-hosted LiveSyncは一つのチャンクのサイズを最低minimum chunk s
ご使用のすべてのデバイスでロックを解除した場合は、データベースのロックを解除することができます。
ただし、このまま放置しても問題はありません。
### Reread all files
### Verify and repair all files
Vault内のファイルを全て読み込み直し、もし差分があったり、データベースから正常に読み込めなかったものに関して、データベースに反映します。
### Sanity check
ローカルデータベースに保存されている全てのファイルが正しくチャンクを持っていることを確認します。
### Drop history
データベースに記録されている履歴を削除し、データベースを初期化します。
新しい端末や新しいVaultへの同期にやたらと時間がかかったり、データベースサイズが肥大化したりしてきた際に使用してください。

2
manifest.json
View File

@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Self-hosted LiveSync",
"version": "0.2.1",
"version": "0.4.0",
"minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",

120
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "obsidian-livesync",
"version": "0.2.1",
"version": "0.4.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "obsidian-livesync",
"version": "0.2.1",
"version": "0.4.0",
"license": "MIT",
"dependencies": {
"diff-match-patch": "^1.0.5",
@@ -23,7 +23,7 @@
"eslint": "^7.32.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.25.2",
"obsidian": "^0.12.0",
"obsidian": "^0.13.11",
"rollup": "^2.32.1",
"tslib": "^2.2.0",
"typescript": "^4.2.4"
@@ -132,6 +132,43 @@
"node": ">=4"
}
},
"node_modules/@codemirror/rangeset": {
"version": "0.19.5",
"resolved": "https://registry.npmjs.org/@codemirror/rangeset/-/rangeset-0.19.5.tgz",
"integrity": "sha512-L3b+RIwIRKOJ3pJLOtpkxCUjGnxZKFyPb0CjYWKnVLuzEIaEExWWK7sp6rsejxOy8RjYzfCHlFhYB4UdQN7brw==",
"dev": true,
"dependencies": {
"@codemirror/state": "^0.19.0"
}
},
"node_modules/@codemirror/state": {
"version": "0.19.6",
"resolved": "https://registry.npmjs.org/@codemirror/state/-/state-0.19.6.tgz",
"integrity": "sha512-sqIQZE9VqwQj7D4c2oz9mfLhlT1ElAzGB5lO1lE33BPyrdNy1cJyCIOecT4cn4VeJOFrnjOeu+IftZ3zqdFETw==",
"dev": true,
"dependencies": {
"@codemirror/text": "^0.19.0"
}
},
"node_modules/@codemirror/text": {
"version": "0.19.5",
"resolved": "https://registry.npmjs.org/@codemirror/text/-/text-0.19.5.tgz",
"integrity": "sha512-Syu5Xc7tZzeUAM/y4fETkT0zgGr48rDG+w4U38bPwSIUr+L9S/7w2wDE1WGNzjaZPz12F6gb1gxWiSTg9ocLow==",
"dev": true
},
"node_modules/@codemirror/view": {
"version": "0.19.37",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-0.19.37.tgz",
"integrity": "sha512-SLuLx9p0O1ZHXLehvl5MwSvUrQRcsNGemzTgJ0zRajmc3BBsNigI1PXxdo7tvBhO5DcAzRRBXoke9DZFUR6Qqg==",
"dev": true,
"dependencies": {
"@codemirror/rangeset": "^0.19.5",
"@codemirror/state": "^0.19.3",
"@codemirror/text": "^0.19.0",
"style-mod": "^4.0.0",
"w3c-keyname": "^2.2.4"
}
},
"node_modules/@eslint/eslintrc": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
@@ -2185,11 +2222,13 @@
}
},
"node_modules/obsidian": {
"version": "0.12.17",
"resolved": "https://registry.npmjs.org/obsidian/-/obsidian-0.12.17.tgz",
"integrity": "sha512-YvCAlRym9D8zNPXt6Ez8QubSTVGoChx6lb58zqI13Dcrz3l1lgUO+pcOGDiD5Qa67nzDZLXo3aV2rqkCCpTvGQ==",
"version": "0.13.11",
"resolved": "https://registry.npmjs.org/obsidian/-/obsidian-0.13.11.tgz",
"integrity": "sha512-KxOvAh4CG5vzcukmHvyuK9hUIr6ZFlM9FQfGZEwrrEV8VG2/W2Tk5cWrg0VM7EkGE3QBmjX6owjIDIO8QDXVUQ==",
"dev": true,
"dependencies": {
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.31",
"@types/codemirror": "0.0.108",
"moment": "2.29.1"
}
@@ -2645,6 +2684,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/style-mod": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.0.0.tgz",
"integrity": "sha512-OPhtyEjyyN9x3nhPsu76f52yUGXiZcgvsrFVtvTkyGRQJ0XK+GPc6ov1z+lRpbeabka+MYEQxOYRnt5nF30aMw==",
"dev": true
},
"node_modules/supports-color": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
@@ -2819,6 +2864,12 @@
"integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==",
"dev": true
},
"node_modules/w3c-keyname": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.4.tgz",
"integrity": "sha512-tOhfEwEzFLJzf6d1ZPkYfGj+FWhIpBux9ppoP3rlclw3Z0BZv3N7b7030Z1kYth+6rDuAsXUFr+d0VE6Ed1ikw==",
"dev": true
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@@ -2962,6 +3013,43 @@
}
}
},
"@codemirror/rangeset": {
"version": "0.19.5",
"resolved": "https://registry.npmjs.org/@codemirror/rangeset/-/rangeset-0.19.5.tgz",
"integrity": "sha512-L3b+RIwIRKOJ3pJLOtpkxCUjGnxZKFyPb0CjYWKnVLuzEIaEExWWK7sp6rsejxOy8RjYzfCHlFhYB4UdQN7brw==",
"dev": true,
"requires": {
"@codemirror/state": "^0.19.0"
}
},
"@codemirror/state": {
"version": "0.19.6",
"resolved": "https://registry.npmjs.org/@codemirror/state/-/state-0.19.6.tgz",
"integrity": "sha512-sqIQZE9VqwQj7D4c2oz9mfLhlT1ElAzGB5lO1lE33BPyrdNy1cJyCIOecT4cn4VeJOFrnjOeu+IftZ3zqdFETw==",
"dev": true,
"requires": {
"@codemirror/text": "^0.19.0"
}
},
"@codemirror/text": {
"version": "0.19.5",
"resolved": "https://registry.npmjs.org/@codemirror/text/-/text-0.19.5.tgz",
"integrity": "sha512-Syu5Xc7tZzeUAM/y4fETkT0zgGr48rDG+w4U38bPwSIUr+L9S/7w2wDE1WGNzjaZPz12F6gb1gxWiSTg9ocLow==",
"dev": true
},
"@codemirror/view": {
"version": "0.19.37",
"resolved": "https://registry.npmjs.org/@codemirror/view/-/view-0.19.37.tgz",
"integrity": "sha512-SLuLx9p0O1ZHXLehvl5MwSvUrQRcsNGemzTgJ0zRajmc3BBsNigI1PXxdo7tvBhO5DcAzRRBXoke9DZFUR6Qqg==",
"dev": true,
"requires": {
"@codemirror/rangeset": "^0.19.5",
"@codemirror/state": "^0.19.3",
"@codemirror/text": "^0.19.0",
"style-mod": "^4.0.0",
"w3c-keyname": "^2.2.4"
}
},
"@eslint/eslintrc": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
@@ -4511,11 +4599,13 @@
}
},
"obsidian": {
"version": "0.12.17",
"resolved": "https://registry.npmjs.org/obsidian/-/obsidian-0.12.17.tgz",
"integrity": "sha512-YvCAlRym9D8zNPXt6Ez8QubSTVGoChx6lb58zqI13Dcrz3l1lgUO+pcOGDiD5Qa67nzDZLXo3aV2rqkCCpTvGQ==",
"version": "0.13.11",
"resolved": "https://registry.npmjs.org/obsidian/-/obsidian-0.13.11.tgz",
"integrity": "sha512-KxOvAh4CG5vzcukmHvyuK9hUIr6ZFlM9FQfGZEwrrEV8VG2/W2Tk5cWrg0VM7EkGE3QBmjX6owjIDIO8QDXVUQ==",
"dev": true,
"requires": {
"@codemirror/state": "^0.19.6",
"@codemirror/view": "^0.19.31",
"@types/codemirror": "0.0.108",
"moment": "2.29.1"
}
@@ -4822,6 +4912,12 @@
"integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==",
"dev": true
},
"style-mod": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.0.0.tgz",
"integrity": "sha512-OPhtyEjyyN9x3nhPsu76f52yUGXiZcgvsrFVtvTkyGRQJ0XK+GPc6ov1z+lRpbeabka+MYEQxOYRnt5nF30aMw==",
"dev": true
},
"supports-color": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
@@ -4962,6 +5058,12 @@
"integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==",
"dev": true
},
"w3c-keyname": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.4.tgz",
"integrity": "sha512-tOhfEwEzFLJzf6d1ZPkYfGj+FWhIpBux9ppoP3rlclw3Z0BZv3N7b7030Z1kYth+6rDuAsXUFr+d0VE6Ed1ikw==",
"dev": true
},
"which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

4
package.json
View File

@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "0.2.1",
"version": "0.4.0",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"scripts": {
@@ -22,7 +22,7 @@
"eslint": "^7.32.0",
"eslint-config-airbnb-base": "^14.2.1",
"eslint-plugin-import": "^2.25.2",
"obsidian": "^0.12.0",
"obsidian": "^0.13.11",
"rollup": "^2.32.1",
"tslib": "^2.2.0",
"typescript": "^4.2.4"

9
src/ConflictResolveModal.ts
View File

@@ -7,6 +7,7 @@ export class ConflictResolveModal extends Modal {
// result: Array<[number, string]>;
result: diff_result;
callback: (remove_rev: string) => Promise<void>;
constructor(app: App, diff: diff_result, callback: (remove_rev: string) => Promise<void>) {
super(app);
this.result = diff;
@@ -45,18 +46,21 @@ export class ConflictResolveModal extends Modal {
contentEl.createEl("button", { text: "Keep A" }, (e) => {
e.addEventListener("click", async () => {
await this.callback(this.result.right.rev);
this.callback = null;
this.close();
});
});
contentEl.createEl("button", { text: "Keep B" }, (e) => {
e.addEventListener("click", async () => {
await this.callback(this.result.left.rev);
this.callback = null;
this.close();
});
});
contentEl.createEl("button", { text: "Concat both" }, (e) => {
e.addEventListener("click", async () => {
await this.callback(null);
await this.callback("");
this.callback = null;
this.close();
});
});
@@ -70,5 +74,8 @@ export class ConflictResolveModal extends Modal {
onClose() {
const { contentEl } = this;
contentEl.empty();
if (this.callback != null) {
this.callback(null);
}
}
}

351
src/LocalPouchDB.ts
View File

@@ -403,41 +403,44 @@ export class LocalPouchDB {
async deleteDBEntry(path: string, opt?: PouchDB.Core.GetOptions): Promise<boolean> {
await this.waitForGCComplete();
const id = path2id(path);
try {
let obj: EntryDocResponse = null;
if (opt) {
obj = await this.localDatabase.get(id, opt);
} else {
obj = await this.localDatabase.get(id);
}
return await runWithLock("file:" + id, false, async () => {
if (opt) {
obj = await this.localDatabase.get(id, opt);
} else {
obj = await this.localDatabase.get(id);
}
if (obj.type && obj.type == "leaf") {
//do nothing for leaf;
return false;
}
//Check it out and fix docs to regular case
if (!obj.type || (obj.type && obj.type == "notes")) {
obj._deleted = true;
const r = await this.localDatabase.put(obj);
this.updateRecentModifiedDocs(r.id, r.rev, true);
if (typeof this.corruptedEntries[obj._id] != "undefined") {
delete this.corruptedEntries[obj._id];
if (obj.type && obj.type == "leaf") {
//do nothing for leaf;
return false;
}
return true;
// simple note
}
if (obj.type == "newnote" || obj.type == "plain") {
obj._deleted = true;
const r = await this.localDatabase.put(obj);
Logger(`entry removed:${obj._id}-${r.rev}`);
this.updateRecentModifiedDocs(r.id, r.rev, true);
if (typeof this.corruptedEntries[obj._id] != "undefined") {
delete this.corruptedEntries[obj._id];
//Check it out and fix docs to regular case
if (!obj.type || (obj.type && obj.type == "notes")) {
obj._deleted = true;
const r = await this.localDatabase.put(obj);
this.updateRecentModifiedDocs(r.id, r.rev, true);
if (typeof this.corruptedEntries[obj._id] != "undefined") {
delete this.corruptedEntries[obj._id];
}
return true;
// simple note
}
return true;
} else {
return false;
}
if (obj.type == "newnote" || obj.type == "plain") {
obj._deleted = true;
const r = await this.localDatabase.put(obj);
Logger(`entry removed:${obj._id}-${r.rev}`);
this.updateRecentModifiedDocs(r.id, r.rev, true);
if (typeof this.corruptedEntries[obj._id] != "undefined") {
delete this.corruptedEntries[obj._id];
}
return true;
} else {
return false;
}
});
} catch (ex) {
if (ex.status && ex.status == 404) {
return false;
@@ -478,10 +481,13 @@ export class LocalPouchDB {
let notfound = 0;
for (const v of delDocs) {
try {
const item = await this.localDatabase.get(v);
item._deleted = true;
await this.localDatabase.put(item);
this.updateRecentModifiedDocs(item._id, item._rev, true);
await runWithLock("file:" + v, false, async () => {
const item = await this.localDatabase.get(v);
item._deleted = true;
await this.localDatabase.put(item);
this.updateRecentModifiedDocs(item._id, item._rev, true);
});
deleteCount++;
} catch (ex) {
if (ex.status && ex.status == 404) {
@@ -540,7 +546,6 @@ export class LocalPouchDB {
cPieceSize = 0;
// lookup for next splittion .
// we're standing on "\n"
// debugger
do {
const n1 = leftData.indexOf("\n", cPieceSize + 1);
const n2 = leftData.indexOf("\n\n", cPieceSize + 1);
@@ -691,33 +696,35 @@ export class LocalPouchDB {
type: plainSplit ? "plain" : "newnote",
};
// Here for upsert logic,
try {
const old = await this.localDatabase.get(newDoc._id);
if (!old.type || old.type == "notes" || old.type == "newnote" || old.type == "plain") {
// simple use rev for new doc
newDoc._rev = old._rev;
await runWithLock("file:" + newDoc._id, false, async () => {
try {
const old = await this.localDatabase.get(newDoc._id);
if (!old.type || old.type == "notes" || old.type == "newnote" || old.type == "plain") {
// simple use rev for new doc
newDoc._rev = old._rev;
}
} catch (ex) {
if (ex.status && ex.status == 404) {
// NO OP/
} else {
throw ex;
}
}
} catch (ex) {
if (ex.status && ex.status == 404) {
// NO OP/
const r = await this.localDatabase.put(newDoc, { force: true });
this.updateRecentModifiedDocs(r.id, r.rev, newDoc._deleted);
if (typeof this.corruptedEntries[note._id] != "undefined") {
delete this.corruptedEntries[note._id];
}
if (this.settings.checkIntegrityOnSave) {
if (!this.sanCheck(await this.localDatabase.get(r.id))) {
Logger("note save failed!", LOG_LEVEL.NOTICE);
} else {
Logger(`note has been surely saved:${newDoc._id}:${r.rev}`);
}
} else {
throw ex;
Logger(`note saved:${newDoc._id}:${r.rev}`);
}
}
const r = await this.localDatabase.put(newDoc, { force: true });
this.updateRecentModifiedDocs(r.id, r.rev, newDoc._deleted);
if (typeof this.corruptedEntries[note._id] != "undefined") {
delete this.corruptedEntries[note._id];
}
if (this.settings.checkIntegrityOnSave) {
if (!this.sanCheck(await this.localDatabase.get(r.id))) {
Logger("note save failed!", LOG_LEVEL.NOTICE);
} else {
Logger(`note has been surely saved:${newDoc._id}:${r.rev}`);
}
} else {
Logger(`note saved:${newDoc._id}:${r.rev}`);
}
});
} else {
Logger(`note coud not saved:${note._id}`);
}
@@ -756,8 +763,8 @@ export class LocalPouchDB {
}
const syncOptionBase: PouchDB.Replication.SyncOptions = {
batch_size: 250,
batches_limit: 40,
batches_limit: setting.batches_limit,
batch_size: setting.batch_size,
};
const db = dbret.db;
@@ -837,6 +844,15 @@ export class LocalPouchDB {
locked: false,
accepted_nodes: [this.nodeid],
};
// const remoteInfo = dbret.info;
// const localInfo = await this.localDatabase.info();
// const remoteDocsCount = remoteInfo.doc_count;
// const localDocsCount = localInfo.doc_count;
// const remoteUpdSeq = typeof remoteInfo.update_seq == "string" ? Number(remoteInfo.update_seq.split("-")[0]) : remoteInfo.update_seq;
// const localUpdSeq = typeof localInfo.update_seq == "string" ? Number(localInfo.update_seq.split("-")[0]) : localInfo.update_seq;
// Logger(`Database diffences: remote:${remoteDocsCount} docs / last update ${remoteUpdSeq}`);
// Logger(`Database diffences: local :${localDocsCount} docs / last update ${localUpdSeq}`);
const remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defMilestonePoint);
this.remoteLocked = remoteMilestone.locked;
@@ -851,12 +867,12 @@ export class LocalPouchDB {
}
const syncOptionBase: PouchDB.Replication.SyncOptions = {
batch_size: 250,
batches_limit: 40,
batches_limit: setting.batches_limit,
batch_size: setting.batch_size,
};
const syncOption: PouchDB.Replication.SyncOptions = keepAlive ? { live: true, retry: true, heartbeat: 30000, ...syncOptionBase } : { ...syncOptionBase };
return { db: dbret.db, syncOptionBase, syncOption };
return { db: dbret.db, info: dbret.info, syncOptionBase, syncOption };
}
async openReplication(setting: ObsidianLiveSyncSettings, keepAlive: boolean, showResult: boolean, callback: (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => Promise<void>): Promise<boolean> {
@@ -876,87 +892,89 @@ export class LocalPouchDB {
//replicate once
this.syncStatus = "STARTED";
return new Promise<boolean>(async (res, rej) => {
let resolved = false;
const _openReplicationSync = () => {
this.syncHandler = this.cancelHandler(this.syncHandler);
this.syncHandler = this.localDatabase.sync<EntryDoc>(db, syncOption);
this.syncHandler
.on("active", () => {
this.syncStatus = "CONNECTED";
this.updateInfo();
Logger("Replication activated");
})
.on("change", async (e) => {
try {
if (e.direction == "pull") {
// console.log(`pulled data:${e.change.docs.map((e) => e._id).join(",")}`);
await callback(e.change.docs);
Logger(`replicated ${e.change.docs_read} doc(s)`);
this.docArrived += e.change.docs.length;
} else {
// console.log(`put data:${e.change.docs.map((e) => e._id).join(",")}`);
this.docSent += e.change.docs.length;
}
if (notice != null) {
notice.setMessage(`↑${e.change.docs_written} ↓${e.change.docs_read}`);
}
this.updateInfo();
} catch (ex) {
Logger("Replication callback error");
Logger(ex);
}
})
.on("complete", (e) => {
this.syncStatus = "COMPLETED";
this.updateInfo();
Logger("Replication completed", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO);
if (notice != null) notice.hide();
if (!keepAlive) {
this.syncHandler = this.cancelHandler(this.syncHandler);
// if keep alive runnning, resolve here,
res(true);
}
})
.on("denied", (e) => {
this.syncStatus = "ERRORED";
this.updateInfo();
this.syncHandler = this.cancelHandler(this.syncHandler);
if (notice != null) notice.hide();
Logger("Replication denied", LOG_LEVEL.NOTICE);
// Logger(e);
rej(e);
})
.on("error", (e) => {
this.syncStatus = "ERRORED";
this.syncHandler = this.cancelHandler(this.syncHandler);
this.updateInfo();
if (notice != null) notice.hide();
Logger("Replication error", LOG_LEVEL.NOTICE);
// Logger(e);
rej(e);
})
.on("paused", (e) => {
this.syncStatus = "PAUSED";
this.updateInfo();
if (notice != null) notice.hide();
Logger("replication paused", LOG_LEVEL.VERBOSE);
if (keepAlive && !resolved) {
// if keep alive runnning, resolve here,
resolved = true;
res(true);
}
// Logger(e);
});
};
if (!keepAlive) {
return await _openReplicationSync();
}
let resolved = false;
const docArrivedOnStart = this.docArrived;
const docSentOnStart = this.docSent;
const _openReplicationSync = () => {
Logger("Sync Main Started");
this.syncHandler = this.cancelHandler(this.syncHandler);
Logger("Pull before replicate.");
Logger(await this.localDatabase.info(), LOG_LEVEL.VERBOSE);
Logger(await db.info(), LOG_LEVEL.VERBOSE);
const replicate = this.localDatabase.replicate.from(db, syncOptionBase);
this.syncHandler = this.localDatabase.sync<EntryDoc>(db, syncOption);
this.syncHandler
.on("active", () => {
this.syncStatus = "CONNECTED";
this.updateInfo();
Logger("Replication activated");
})
.on("change", async (e) => {
try {
if (e.direction == "pull") {
// console.log(`pulled data:${e.change.docs.map((e) => e._id).join(",")}`);
await callback(e.change.docs);
Logger(`replicated ${e.change.docs_read} doc(s)`);
this.docArrived += e.change.docs.length;
} else {
// console.log(`put data:${e.change.docs.map((e) => e._id).join(",")}`);
this.docSent += e.change.docs.length;
}
if (notice != null) {
notice.setMessage(`↑${this.docSent - docSentOnStart} ↓${this.docArrived - docArrivedOnStart}`);
}
this.updateInfo();
} catch (ex) {
Logger("Replication callback error");
Logger(ex);
}
})
.on("complete", (e) => {
this.syncStatus = "COMPLETED";
this.updateInfo();
Logger("Replication completed", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO);
if (notice != null) notice.hide();
if (!keepAlive) {
this.syncHandler = this.cancelHandler(this.syncHandler);
// if keep alive runnning, resolve here,
}
})
.on("denied", (e) => {
this.syncStatus = "ERRORED";
this.updateInfo();
this.syncHandler = this.cancelHandler(this.syncHandler);
if (notice != null) notice.hide();
Logger("Replication denied", LOG_LEVEL.NOTICE);
Logger(e);
})
.on("error", (e) => {
this.syncStatus = "ERRORED";
this.syncHandler = this.cancelHandler(this.syncHandler);
this.updateInfo();
if (notice != null) notice.hide();
Logger("Replication error", LOG_LEVEL.NOTICE);
Logger(e);
})
.on("paused", (e) => {
this.syncStatus = "PAUSED";
this.updateInfo();
if (notice != null) notice.hide();
Logger("replication paused", LOG_LEVEL.VERBOSE);
if (keepAlive && !resolved) {
// if keep alive runnning, resolve here,
resolved = true;
}
// Logger(e);
});
return this.syncHandler;
};
if (!keepAlive) {
await _openReplicationSync();
return true;
}
this.syncHandler = this.cancelHandler(this.syncHandler);
Logger("Pull before replicate.");
Logger(await this.localDatabase.info(), LOG_LEVEL.VERBOSE);
Logger(await db.info(), LOG_LEVEL.VERBOSE);
let replicate: PouchDB.Replication.Replication<EntryDoc>;
try {
replicate = this.localDatabase.replicate.from(db, syncOptionBase);
replicate
.on("active", () => {
this.syncStatus = "CONNECTED";
@@ -979,36 +997,23 @@ export class LocalPouchDB {
Logger("Replication callback error");
Logger(ex);
}
})
.on("complete", async (info) => {
this.syncStatus = "COMPLETED";
this.updateInfo();
this.cancelHandler(replicate);
this.syncHandler = this.cancelHandler(this.syncHandler);
Logger("Replication pull completed.");
await _openReplicationSync();
})
.on("denied", (e) => {
this.syncStatus = "ERRORED";
this.updateInfo();
if (notice != null) notice.hide();
Logger("Pulling Replication denied", LOG_LEVEL.NOTICE);
this.cancelHandler(replicate);
this.syncHandler = this.cancelHandler(this.syncHandler);
rej(e);
})
.on("error", (e) => {
this.syncStatus = "ERRORED";
this.updateInfo();
Logger("Pulling Replication error", LOG_LEVEL.INFO);
this.cancelHandler(replicate);
this.syncHandler = this.cancelHandler(this.syncHandler);
if (notice != null) notice.hide();
// debugger;
Logger(e);
rej(e);
});
});
this.syncStatus = "COMPLETED";
this.updateInfo();
this.cancelHandler(replicate);
this.syncHandler = this.cancelHandler(this.syncHandler);
Logger("Replication pull completed.");
_openReplicationSync();
return true;
} catch (ex) {
this.syncStatus = "ERRORED";
this.updateInfo();
Logger("Pulling Replication error", LOG_LEVEL.NOTICE);
this.cancelHandler(replicate);
this.syncHandler = this.cancelHandler(this.syncHandler);
if (notice != null) notice.hide();
throw ex;
}
}
closeReplication() {

81
src/ObsidianLiveSyncSettingTab.ts
View File

@@ -1,4 +1,4 @@
import { App, Notice, PluginSettingTab, Setting } from "obsidian";
import { App, Notice, PluginSettingTab, Setting, sanitizeHTMLToDom } from "obsidian";
import { EntryDoc, LOG_LEVEL } from "./types";
import { escapeStringToHTML, versionNumberString2Number, path2id, id2path, runWithLock } from "./utils";
import { Logger } from "./logger";
@@ -72,7 +72,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
const containerRemoteDatabaseEl = containerEl.createDiv();
containerRemoteDatabaseEl.createEl("h3", { text: "Remote Database configuration" });
const syncWarn = containerRemoteDatabaseEl.createEl("div", { text: "The remote configuration is locked while any synchronization is enabled." });
const syncWarn = containerRemoteDatabaseEl.createEl("div", { text: `These settings are kept locked while automatic synchronization options are enabled. Disable these options in the "Sync Settings" tab to unlock.` });
syncWarn.addClass("op-warn");
syncWarn.addClass("sls-hidden");
@@ -82,6 +82,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
if (this.plugin.settings.syncOnFileOpen) return true;
if (this.plugin.settings.syncOnSave) return true;
if (this.plugin.settings.syncOnStart) return true;
if (this.plugin.localDatabase.syncStatus == "CONNECTED") return true;
if (this.plugin.localDatabase.syncStatus == "PAUSED") return true;
return false;
};
const applyDisplayEnabled = () => {
@@ -304,6 +306,44 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
})
);
containerLocalDatabaseEl.createEl("div", {
text: sanitizeHTMLToDom(`Advanced settings<br>
Configuration of how LiveSync makes chunks from the file.`),
});
new Setting(containerLocalDatabaseEl)
.setName("Minimum chunk size")
.setDesc("(letters), minimum chunk size.")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.minimumChunkSize + "")
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v < 10 || v > 1000) {
v = 10;
}
this.plugin.settings.minimumChunkSize = v;
await this.plugin.saveSettings();
});
text.inputEl.setAttribute("type", "number");
});
new Setting(containerLocalDatabaseEl)
.setName("LongLine Threshold")
.setDesc("(letters), If the line is longer than this, make the line to chunk")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.longLineThreshold + "")
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v < 10 || v > 1000) {
v = 10;
}
this.plugin.settings.longLineThreshold = v;
await this.plugin.saveSettings();
});
text.inputEl.setAttribute("type", "number");
});
addScreenElement("10", containerLocalDatabaseEl);
const containerGeneralSettingsEl = containerEl.createDiv();
containerGeneralSettingsEl.createEl("h3", { text: "General Settings" });
@@ -458,35 +498,40 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
await this.plugin.saveSettings();
})
);
containerSyncSettingEl.createEl("div", {
text: sanitizeHTMLToDom(`Advanced settings<br>
If you reached the payload size limit when using IBM Cloudant, please set batch size and batch limit to a lower value.`),
});
new Setting(containerSyncSettingEl)
.setName("Minimum chunk size")
.setDesc("(letters), minimum chunk size.")
.setName("Batch size")
.setDesc("Number of change feed items to process at a time. Defaults to 250.")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.minimumChunkSize + "")
.setValue(this.plugin.settings.batch_size + "")
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v < 10 || v > 1000) {
if (isNaN(v) || v < 10) {
v = 10;
}
this.plugin.settings.minimumChunkSize = v;
this.plugin.settings.batch_size = v;
await this.plugin.saveSettings();
});
text.inputEl.setAttribute("type", "number");
});
new Setting(containerSyncSettingEl)
.setName("LongLine Threshold")
.setDesc("(letters), If the line is longer than this, make the line to chunk")
.setName("Batch limit")
.setDesc("Number of batches to process at a time. Defaults to 40. This along with batch size controls how many docs are kept in memory at a time.")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.longLineThreshold + "")
.setValue(this.plugin.settings.batches_limit + "")
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v < 10 || v > 1000) {
if (isNaN(v) || v < 10) {
v = 10;
}
this.plugin.settings.longLineThreshold = v;
this.plugin.settings.batches_limit = v;
await this.plugin.saveSettings();
});
text.inputEl.setAttribute("type", "number");
@@ -544,10 +589,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
Logger("Synchronization setting configured as LiveSync.", LOG_LEVEL.NOTICE);
} else if (currentPrest == "PERIODIC") {
this.plugin.settings.batchSave = true;
this.plugin.settings.periodicReplication = false;
this.plugin.settings.periodicReplication = true;
this.plugin.settings.syncOnSave = false;
this.plugin.settings.syncOnStart = false;
this.plugin.settings.syncOnFileOpen = false;
this.plugin.settings.syncOnStart = true;
this.plugin.settings.syncOnFileOpen = true;
Logger("Synchronization setting configured as Periodic sync with batch database update.", LOG_LEVEL.NOTICE);
} else {
Logger("All synchronization disabled.", LOG_LEVEL.NOTICE);
@@ -590,6 +635,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
c.addClass("op-warn");
}
}
const hatchWarn = containerHatchEl.createEl("div", { text: `To stop the bootup sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` });
hatchWarn.addClass("op-warn");
const dropHistory = async (sendToServer: boolean) => {
this.plugin.settings.liveSync = false;
this.plugin.settings.periodicReplication = false;
@@ -718,6 +765,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
})
);
containerHatchEl.createEl("div", {
text: sanitizeHTMLToDom(`Advanced buttons<br>
These buttons could break your database easily.`),
});
new Setting(containerHatchEl)
.setName("Reset remote database")
.setDesc("Reset remote database, this affects only database. If you replicate again, remote database will restored by local database.")

290
src/main.ts
View File

@@ -1,8 +1,24 @@
import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, PluginManifest } from "obsidian";
import { diff_match_patch } from "diff-match-patch";
import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, PluginDataEntry, LOG_LEVEL, VER, PERIODIC_PLUGIN_SWEEP, DEFAULT_SETTINGS, PluginList, DevicePluginList } from "./types";
import { base64ToString, arrayBufferToBase64, base64ToArrayBuffer, isValidPath, versionNumberString2Number, id2path, path2id, runWithLock } from "./utils";
import {
EntryDoc,
LoadedEntry,
ObsidianLiveSyncSettings,
diff_check_result,
diff_result_leaf,
EntryBody,
PluginDataEntry,
LOG_LEVEL,
VER,
PERIODIC_PLUGIN_SWEEP,
DEFAULT_SETTINGS,
PluginList,
DevicePluginList,
diff_result,
FLAGMD_REDFLAG,
} from "./types";
import { base64ToString, arrayBufferToBase64, base64ToArrayBuffer, isValidPath, versionNumberString2Number, id2path, path2id, runWithLock, shouldBeIgnored } from "./utils";
import { Logger, setLogger } from "./logger";
import { LocalPouchDB } from "./LocalPouchDB";
import { LogDisplayModal } from "./LogDisplayModal";
@@ -22,13 +38,20 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.registerInterval(timer);
return timer;
}
isRedFlagRaised(): boolean {
const redflag = this.app.vault.getAbstractFileByPath(normalizePath(FLAGMD_REDFLAG));
if (redflag != null) {
return true;
}
return false;
}
async onload() {
setLogger(this.addLog.bind(this)); // Logger moved to global.
Logger("loading plugin");
const lsname = "obsidian-live-sync-ver" + this.app.vault.getName();
const last_version = localStorage.getItem(lsname);
await this.loadSettings();
if (!last_version || Number(last_version) < VER) {
if (last_version && Number(last_version) < VER) {
this.settings.liveSync = false;
this.settings.syncOnSave = false;
this.settings.syncOnStart = false;
@@ -90,7 +113,27 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.app.workspace.onLayoutReady(async () => {
try {
await this.initializeDatabase();
if (this.isRedFlagRaised()) {
this.settings.batchSave = false;
this.settings.liveSync = false;
this.settings.periodicReplication = false;
this.settings.syncOnSave = false;
this.settings.syncOnStart = false;
this.settings.syncOnFileOpen = false;
this.settings.autoSweepPlugins = false;
this.settings.usePluginSync = false;
this.settings.suspendFileWatching = true;
await this.saveSettings();
await this.openDatabase();
const warningMessage = "The red flag is raised! The whole initialize steps are skipped, and any file changes are not captured.";
Logger(warningMessage, LOG_LEVEL.NOTICE);
this.setStatusBarText(warningMessage);
} else {
if (this.settings.suspendFileWatching) {
Logger("'Suspend file watching' turned on. Are you sure this is what you intended? Every modification on the vault will be ignored.", LOG_LEVEL.NOTICE);
}
await this.initializeDatabase();
}
await this.realizeSettingSyncMode();
this.registerWatchEvents();
if (this.settings.syncOnStart) {
@@ -116,6 +159,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.localDatabase.getDBEntry(view.file.path, {}, true, false);
},
});
this.addCommand({
id: "livesync-checkdoc-conflicted",
name: "Resolve if conflicted.",
editorCallback: async (editor: Editor, view: MarkdownView) => {
await this.showIfConflicted(view.file);
},
});
this.addCommand({
id: "livesync-gc",
name: "garbage collect now",
@@ -284,18 +334,18 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.watchVaultChangeAsync(file, ...args);
}
async applyBatchChange() {
if (!this.settings.batchSave || this.batchFileChange.length == 0) {
return [];
}
return await runWithLock("batchSave", false, async () => {
const batchItems = JSON.parse(JSON.stringify(this.batchFileChange)) as string[];
this.batchFileChange = [];
const files = this.app.vault.getFiles();
const promises = batchItems.map(async (e) => {
try {
if (await this.app.vault.adapter.exists(normalizePath(e))) {
const f = files.find((f) => f.path == e);
if (f) {
await this.updateIntoDB(f);
Logger(`Batch save:${e}`);
}
const f = this.app.vault.getAbstractFileByPath(normalizePath(e));
if (f && f instanceof TFile) {
await this.updateIntoDB(f);
Logger(`Batch save:${e}`);
}
} catch (ex) {
Logger(`Batch save error:${e}`, LOG_LEVEL.NOTICE);
@@ -358,25 +408,37 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async watchVaultRenameAsync(file: TAbstractFile, oldFile: any) {
Logger(`${oldFile} renamed to ${file.path}`, LOG_LEVEL.VERBOSE);
await this.applyBatchChange();
try {
await this.applyBatchChange();
} catch (ex) {
Logger(ex);
}
if (file instanceof TFolder) {
const newFiles = this.GetAllFilesRecursively(file);
// for guard edge cases. this won't happen and each file's event will be raise.
for (const i of newFiles) {
const newFilePath = normalizePath(this.getFilePath(i));
const newFile = this.app.vault.getAbstractFileByPath(newFilePath);
if (newFile instanceof TFile) {
Logger(`save ${newFile.path} into db`);
await this.updateIntoDB(newFile);
try {
const newFilePath = normalizePath(this.getFilePath(i));
const newFile = this.app.vault.getAbstractFileByPath(newFilePath);
if (newFile instanceof TFile) {
Logger(`save ${newFile.path} into db`);
await this.updateIntoDB(newFile);
}
} catch (ex) {
Logger(ex);
}
}
Logger(`delete below ${oldFile} from db`);
await this.deleteFromDBbyPath(oldFile);
} else if (file instanceof TFile) {
Logger(`file save ${file.path} into db`);
await this.updateIntoDB(file);
Logger(`deleted ${oldFile} into db`);
await this.deleteFromDBbyPath(oldFile);
try {
Logger(`file save ${file.path} into db`);
await this.updateIntoDB(file);
Logger(`deleted ${oldFile} into db`);
await this.deleteFromDBbyPath(oldFile);
} catch (ex) {
Logger(ex);
}
}
this.gcHook();
}
@@ -398,9 +460,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.logMessage = [].concat(this.logMessage).concat([newmessage]).slice(-100);
console.log(valutName + ":" + newmessage);
// if (this.statusBar2 != null) {
// this.statusBar2.setText(newmessage.substring(0, 60));
// }
if (level >= LOG_LEVEL.NOTICE) {
if (messagecontent in this.notifies) {
@@ -454,6 +513,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
async doc2storage_create(docEntry: EntryBody, force?: boolean) {
const pathSrc = id2path(docEntry._id);
if (shouldBeIgnored(pathSrc)) {
return;
}
const doc = await this.localDatabase.getDBEntry(pathSrc, { rev: docEntry._rev });
if (doc === false) return;
const path = id2path(doc._id);
@@ -468,7 +530,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
try {
const newfile = await this.app.vault.createBinary(normalizePath(path), bin, { ctime: doc.ctime, mtime: doc.mtime });
Logger("live : write to local (newfile:b) " + path);
await this.app.vault.trigger("create", newfile);
this.app.vault.trigger("create", newfile);
} catch (ex) {
Logger("could not write to local (newfile:bin) " + path, LOG_LEVEL.NOTICE);
Logger(ex, LOG_LEVEL.VERBOSE);
@@ -483,7 +545,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
try {
const newfile = await this.app.vault.create(normalizePath(path), doc.data, { ctime: doc.ctime, mtime: doc.mtime });
Logger("live : write to local (newfile:p) " + path);
await this.app.vault.trigger("create", newfile);
this.app.vault.trigger("create", newfile);
} catch (ex) {
Logger("could not write to local (newfile:plain) " + path, LOG_LEVEL.NOTICE);
Logger(ex, LOG_LEVEL.VERBOSE);
@@ -511,6 +573,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async doc2storate_modify(docEntry: EntryBody, file: TFile, force?: boolean) {
const pathSrc = id2path(docEntry._id);
if (shouldBeIgnored(pathSrc)) {
return;
}
if (docEntry._deleted) {
//basically pass.
//but if there're no docs left, delete file.
@@ -544,7 +609,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
try {
await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
Logger(msg);
await this.app.vault.trigger("modify", file);
this.app.vault.trigger("modify", file);
} catch (ex) {
Logger("could not write to local (modify:bin) " + path, LOG_LEVEL.NOTICE);
}
@@ -558,7 +623,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
try {
await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
Logger(msg);
await this.app.vault.trigger("modify", file);
this.app.vault.trigger("modify", file);
} catch (ex) {
Logger("could not write to local (modify:plain) " + path, LOG_LEVEL.NOTICE);
}
@@ -574,20 +639,20 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
}
async handleDBChanged(change: EntryBody) {
const allfiles = this.app.vault.getFiles();
const targetFiles = allfiles.filter((e) => e.path == id2path(change._id));
if (targetFiles.length == 0) {
const targetFile = this.app.vault.getAbstractFileByPath(id2path(change._id));
if (targetFile == null) {
if (change._deleted) {
return;
}
const doc = change;
await this.doc2storage_create(doc);
}
if (targetFiles.length == 1) {
} else if (targetFile instanceof TFile) {
const doc = change;
const file = targetFiles[0];
const file = targetFile;
await this.doc2storate_modify(doc, file);
await this.showIfConflicted(file);
this.queueConflictedCheck(file);
} else {
Logger(`${id2path(change._id)} is already exist as the folder`);
}
}
@@ -720,7 +785,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
let waiting = "";
if (this.settings.batchSave) {
waiting = " " + this.batchFileChange.map((e) => "🛫").join("");
waiting = waiting.replace(/🛫{10}/g, "🚀");
waiting = waiting.replace(/(🛫){10}/g, "🚀");
}
const message = `Sync:${w} ↑${sent} ↓${arrived}${waiting}`;
this.setStatusBarText(message);
@@ -909,7 +974,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
// --> conflict resolving
async getConflictedDoc(path: string, rev: string): Promise<false | diff_result_leaf> {
try {
const doc = await this.localDatabase.getDBEntry(path, { rev: rev });
const doc = await this.localDatabase.getDBEntry(path, { rev: rev }, false, false);
if (doc === false) return false;
let data = doc.data;
if (doc.datatype == "newnote") {
@@ -936,7 +1001,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
* @returns true -> resolved, false -> nothing to do, or check result.
*/
async getConflictedStatus(path: string): Promise<diff_check_result> {
const test = await this.localDatabase.getDBEntry(path, { conflicts: true });
const test = await this.localDatabase.getDBEntry(path, { conflicts: true }, false, false);
if (test === false) return false;
if (test == null) return false;
if (!test._conflicts) return false;
@@ -990,69 +1055,110 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
diff: diff,
};
}
showMergeDialog(file: TFile, conflictCheckResult: diff_result): Promise<boolean> {
return new Promise((res, rej) => {
Logger("open conflict dialog", LOG_LEVEL.VERBOSE);
new ConflictResolveModal(this.app, conflictCheckResult, async (selected) => {
const testDoc = await this.localDatabase.getDBEntry(file.path, { conflicts: true });
if (testDoc === false) {
Logger("Missing file..", LOG_LEVEL.VERBOSE);
return res(true);
}
if (!testDoc._conflicts) {
Logger("Nothing have to do with this conflict", LOG_LEVEL.VERBOSE);
return res(true);
}
const toDelete = selected;
const toKeep = conflictCheckResult.left.rev != toDelete ? conflictCheckResult.left.rev : conflictCheckResult.right.rev;
if (toDelete == "") {
//concat both,
// write data,and delete both old rev.
const p = conflictCheckResult.diff.map((e) => e[1]).join("");
await this.app.vault.modify(file, p);
await this.updateIntoDB(file);
await this.localDatabase.deleteDBEntry(file.path, { rev: conflictCheckResult.left.rev });
await this.localDatabase.deleteDBEntry(file.path, { rev: conflictCheckResult.right.rev });
await this.pullFile(file.path);
Logger("concat both file");
setTimeout(() => {
//resolved, check again.
this.showIfConflicted(file);
}, 500);
} else if (toDelete == null) {
Logger("Leave it still conflicted");
} else {
Logger(`resolved conflict:${file.path}`);
await this.localDatabase.deleteDBEntry(file.path, { rev: toDelete });
await this.pullFile(file.path, null, true, toKeep);
setTimeout(() => {
//resolved, check again.
this.showIfConflicted(file);
}, 500);
}
return res(true);
}).open();
});
}
conflictedCheckFiles: string[] = [];
// queueing the conflicted file check
conflictedCheckTimer: number;
queueConflictedCheck(file: TFile) {
this.conflictedCheckFiles = this.conflictedCheckFiles.filter((e) => e != file.path);
this.conflictedCheckFiles.push(file.path);
if (this.conflictedCheckTimer != null) {
window.clearTimeout(this.conflictedCheckTimer);
}
this.conflictedCheckTimer = window.setTimeout(async () => {
this.conflictedCheckTimer = null;
const checkFiles = JSON.parse(JSON.stringify(this.conflictedCheckFiles)) as string[];
for (const filename of checkFiles) {
try {
const file = this.app.vault.getAbstractFileByPath(filename);
if (file != null && file instanceof TFile) {
await this.showIfConflicted(file);
}
} catch (ex) {
Logger(ex);
}
}
}, 1000);
}
async showIfConflicted(file: TFile) {
await runWithLock("conflicted", false, async () => {
const conflictCheckResult = await this.getConflictedStatus(file.path);
if (conflictCheckResult === false) return; //nothign to do.
if (conflictCheckResult === false) {
//nothign to do.
return;
}
if (conflictCheckResult === true) {
//auto resolved, but need check again;
Logger("conflict:Automatically merged, but we have to check it again");
setTimeout(() => {
this.showIfConflicted(file);
}, 500);
return;
}
//there conflicts, and have to resolve ;
const leaf = this.app.workspace.activeLeaf;
if (leaf) {
new ConflictResolveModal(this.app, conflictCheckResult, async (selected) => {
const testDoc = await this.localDatabase.getDBEntry(file.path, { conflicts: true });
if (testDoc === false) return;
if (!testDoc._conflicts) {
Logger("something went wrong on merging.", LOG_LEVEL.NOTICE);
return;
}
const toDelete = selected;
if (toDelete == null) {
//concat both,
// write data,and delete both old rev.
const p = conflictCheckResult.diff.map((e) => e[1]).join("");
await this.app.vault.modify(file, p);
await this.localDatabase.deleteDBEntry(file.path, { rev: conflictCheckResult.left.rev });
await this.localDatabase.deleteDBEntry(file.path, { rev: conflictCheckResult.right.rev });
return;
}
if (toDelete == "") {
return;
}
Logger(`resolved conflict:${file.path}`);
await this.localDatabase.deleteDBEntry(file.path, { rev: toDelete });
await this.pullFile(file.path, null, true);
setTimeout(() => {
//resolved, check again.
this.showIfConflicted(file);
}, 500);
}).open();
}
await this.showMergeDialog(file, conflictCheckResult);
});
}
async pullFile(filename: string, fileList?: TFile[], force?: boolean, rev?: string, waitForReady = true) {
if (!fileList) {
fileList = this.app.vault.getFiles();
}
const targetFiles = fileList.filter((e) => e.path == id2path(filename));
if (targetFiles.length == 0) {
const targetFile = this.app.vault.getAbstractFileByPath(id2path(filename));
if (targetFile == null) {
//have to create;
const doc = await this.localDatabase.getDBEntry(filename, rev ? { rev: rev } : null, false, waitForReady);
if (doc === false) return;
await this.doc2storage_create(doc, force);
} else if (targetFiles.length == 1) {
} else if (targetFile instanceof TFile) {
//normal case
const file = targetFiles[0];
const file = targetFile;
const doc = await this.localDatabase.getDBEntry(filename, rev ? { rev: rev } : null, false, waitForReady);
if (doc === false) return;
await this.doc2storate_modify(doc, file, force);
} else {
Logger(`target files:${filename} is two or more files in your vault`);
Logger(`target files:${filename} is exists as the folder`);
//something went wrong..
}
//when to opened file;
@@ -1084,6 +1190,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}
async updateIntoDB(file: TFile) {
if (shouldBeIgnored(file.path)) {
return;
}
await this.localDatabase.waitForGCComplete();
let content = "";
let datatype: "plain" | "newnote" = "newnote";
@@ -1105,17 +1214,21 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
children: [],
datatype: datatype,
};
//From here
const old = await this.localDatabase.getDBEntry(fullpath, null, false, false);
if (old !== false) {
const oldData = { data: old.data, deleted: old._deleted };
const newData = { data: d.data, deleted: d._deleted };
if (JSON.stringify(oldData) == JSON.stringify(newData)) {
Logger("not changed:" + fullpath + (d._deleted ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
return;
//upsert should locked
const isNotChanged = await runWithLock("file:" + fullpath, false, async () => {
const old = await this.localDatabase.getDBEntry(fullpath, null, false, false);
if (old !== false) {
const oldData = { data: old.data, deleted: old._deleted };
const newData = { data: d.data, deleted: d._deleted };
if (JSON.stringify(oldData) == JSON.stringify(newData)) {
Logger("not changed:" + fullpath + (d._deleted ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
return true;
}
// d._rev = old._rev;
}
// d._rev = old._rev;
}
return false;
});
if (isNotChanged) return;
await this.localDatabase.putDBEntry(d);
Logger("put database:" + fullpath + "(" + datatype + ") ");
@@ -1167,7 +1280,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
return { plugins, allPlugins, thisDevicePlugins };
}
async sweepPlugin(showMessage = false) {
console.log(`pluginSync:${this.settings.usePluginSync}`);
if (!this.settings.usePluginSync) return;
await runWithLock("sweepplugin", false, async () => {
const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO;

6
src/types.ts
View File

@@ -56,6 +56,8 @@ export interface ObsidianLiveSyncSettings {
autoSweepPluginsPeriodic: boolean;
notifyPluginOrSettingUpdated: boolean;
checkIntegrityOnSave: boolean;
batch_size: number;
batches_limit: number;
}
export const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
@@ -94,6 +96,8 @@ export const DEFAULT_SETTINGS: ObsidianLiveSyncSettings = {
autoSweepPluginsPeriodic: false,
notifyPluginOrSettingUpdated: false,
checkIntegrityOnSave: false,
batch_size: 250,
batches_limit: 40,
};
export const PERIODIC_PLUGIN_SWEEP = 60;
@@ -222,3 +226,5 @@ export interface PluginList {
export interface DevicePluginList {
[key: string]: PluginDataEntry;
}
export const FLAGMD_REDFLAG = "redflag.md";

24
src/utils.ts
View File

@@ -1,6 +1,6 @@
import { normalizePath } from "obsidian";
import { Logger } from "./logger";
import { LOG_LEVEL } from "./types";
import { FLAGMD_REDFLAG, LOG_LEVEL } from "./types";
export function arrayBufferToBase64(buffer: ArrayBuffer): Promise<string> {
return new Promise((res) => {
@@ -79,6 +79,13 @@ export function isValidPath(filename: string): boolean {
return sx == filename;
}
export function shouldBeIgnored(filename: string): boolean {
if (filename == FLAGMD_REDFLAG) {
return true;
}
return false;
}
export function versionNumberString2Number(version: string): number {
return version // "1.23.45"
.split(".") // 1 23 45
@@ -114,16 +121,16 @@ function objectToKey(key: any): string {
const keys = Object.keys(key).sort((a, b) => a.localeCompare(b));
return keys.map((e) => e + objectToKey(key[e])).join(":");
}
// Just run some async/await as like transacion SERIALIZABLE
// Just run async/await as like transacion ISOLATION SERIALIZABLE
export function runWithLock<T>(key: unknown, ignoreWhenRunning: boolean, proc: () => Promise<T>): Promise<T> {
Logger(`Lock:${key}:enter`, LOG_LEVEL.VERBOSE);
// Logger(`Lock:${key}:enter`, LOG_LEVEL.VERBOSE);
const lockKey = typeof key === "string" ? key : objectToKey(key);
const handleNextProcs = () => {
if (typeof pendingProcs[lockKey] === "undefined") {
//simply unlock
runningProcs.remove(lockKey);
Logger(`Lock:${lockKey}:released`, LOG_LEVEL.VERBOSE);
// Logger(`Lock:${lockKey}:released`, LOG_LEVEL.VERBOSE);
} else {
Logger(`Lock:${lockKey}:left ${pendingProcs[lockKey].length}`, LOG_LEVEL.VERBOSE);
let nextProc = null;
@@ -143,6 +150,10 @@ export function runWithLock<T>(key: unknown, ignoreWhenRunning: boolean, proc: (
handleNextProcs();
});
});
} else {
if (pendingProcs && lockKey in pendingProcs && pendingProcs[lockKey].length == 0) {
delete pendingProcs[lockKey];
}
}
}
};
@@ -164,7 +175,7 @@ export function runWithLock<T>(key: unknown, ignoreWhenRunning: boolean, proc: (
new Promise<void>((res, rej) => {
proc()
.then((v) => {
Logger(`Lock:${key}:processed`, LOG_LEVEL.VERBOSE);
// Logger(`Lock:${key}:processed`, LOG_LEVEL.VERBOSE);
handleNextProcs();
responderRes(v);
res();
@@ -178,10 +189,11 @@ export function runWithLock<T>(key: unknown, ignoreWhenRunning: boolean, proc: (
});
pendingProcs[lockKey].push(subproc);
// Logger(`Lock:${lockKey}:queud:left${pendingProcs[lockKey].length}`, LOG_LEVEL.VERBOSE);
return responder;
} else {
runningProcs.push(lockKey);
Logger(`Lock:${lockKey}:aqquired`, LOG_LEVEL.VERBOSE);
// Logger(`Lock:${lockKey}:aqquired`, LOG_LEVEL.VERBOSE);
return new Promise((res, rej) => {
proc()
.then((v) => {