public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-05-12 10:35:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: public:0.11.2
Branches Tags
public:main public:test_real_obsidian public:feat-userhashsalt public:p2p-rpc public:auto_conf_2 public:auto_conf public:beta public:alpha public:refactor_services_2 public:refactor_services public:dev public:svelteui public:forprivacy
public:0.25.60 public:0.25.59 public:0.25.58 public:0.25.57 public:0.25.56+patched5 public:0.25.56+patched4 public:0.25.56+patched3 public:0.25.56+patched2 public:0.25.56+patched1 public:0.25.56 public:0.25.55 public:0.25.54 public:0.25.53 public:0.25.52-patched-3 public:0.25.52-patched-2 public:0.25.52-patched-1 public:0.25.52 public:0.25.51 public:0.25.50 public:0.25.49 public:0.25.48 public:0.25.47 public:0.25.46 public:0.25.45 public:0.25.44 public:0.25.43-patched-9 public:0.25.43-patched-8 public:0.25.43-patched-7 public:0.25.43-patched-6 public:0.25.43-patched-5 public:0.25.43-patched-4 public:0.25.43-patched-3 public:0.25.43-patched-2 public:0.25.43-patched-1 public:0.25.43 public:0.25.42 public:0.25.41-patched-3 public:0.25.41-patched-2 public:0.25.41-patched-1 public:0.25.41 public:0.25.40 public:0.25.39 public:0.25.38 public:0.25.37 public:0.25.36 public:0.25.35 public:0.25.34 public:0.25.33 public:0.25.32 public:0.25.31 public:0.25.30 public:0.25.30-dev public:0.25.29 public:0.25.28 public:0.25.27 public:0.25.26 public:0.25.25 public:0.25.24 public:0.25.24.beta3 public:0.25.24.beta2 public:0.25.24.beta1 public:0.25.23 public:0.25.23.beta1 public:0.25.22 public:0.25.21 public:0.25.21.beta2 public:0.25.21.beta1 public:0.25.20 public:0.25.19 public:0.25.18 public:0.25.17 public:0.25.16 public:0.25.15 public:0.25.14 public:0.25.13 public:0.25.12 public:0.25.11 public:0.25.10 public:0.25.9 public:0.25.8 public:0.25.7 public:0.25.6 public:0.25.5 public:0.25.4 public:0.25.3 public:0.25.2 public:0.25.1 public:0.25.0 public:0.25.0-beta1 public:0.24.31 public:0.24.30 public:0.24.29 public:0.24.28 public:0.24.27 public:0.24.26 public:0.24.25 public:0.24.24 public:0.24.23 public:0.24.22 public:0.24.21 public:0.24.20 public:0.24.19 public:0.24.18 public:0.24.17 public:0.24.16 public:0.24.15 public:0.24.14 public:0.24.13 public:0.24.12 public:0.24.11 public:0.24.10 public:0.24.9 public:0.24.8 public:0.24.7 public:0.24.6 public:0.24.5 public:0.24.4 public:0.24.3 public:0.24.2 public:0.24.1 public:0.24.0 public:0.24.0.dev-rc8 public:0.24.0.dev-rc7 public:0.24.0.dev-rc6 public:0.24.0.dev-rc5 public:0.24.0.dev-rc4 public:0.24.0.dev-rc3 public:0.24.0.dev-rc2 public:0.23.23 public:0.23.22 public:0.23.21 public:0.23.20 public:0.23.19 public:0.23.18 public:0.23.17 public:0.23.16 public:0.23.15 public:0.23.14 public:0.23.13 public:0.23.12 public:0.23.11 public:0.23.10 public:0.23.9 public:0.23.8 public:0.23.7 public:0.23.6 public:0.23.5 public:0.23.4 public:0.23.3 public:0.23.2 public:0.23.1 public:0.23.0 public:0.22.19 public:0.22.18 public:0.22.17 public:0.22.16 public:0.22.15 public:0.22.14 public:0.22.13 public:0.22.12 public:0.22.11 public:0.22.10 public:0.22.9 public:0.22.8 public:0.22.7 public:0.22.6 public:0.22.5 public:0.22.4 public:0.22.3 public:0.22.2 public:0.22.1 public:0.22.0 public:0.21.5 public:0.21.4 public:0.21.3 public:0.21.2 public:0.21.1 public:0.21.0 public:0.20.7 public:0.20.6 public:0.20.5 public:0.20.4 public:0.20.3 public:0.20.2 public:0.20.1 public:0.20.0 public:0.19.23 public:0.19.22 public:0.19.21 public:0.19.20 public:0.19.19 public:0.19.18 public:0.19.17 public:0.19.16 public:0.19.15 public:0.19.14 public:0.19.13 public:0.19.12 public:0.19.11 public:0.19.10 public:0.19.9 public:0.19.8 public:0.19.7 public:0.19.6 public:0.19.5 public:0.19.4 public:0.19.3 public:0.19.2 public:0.19.1 public:0.19.0 public:0.18.6 public:0.18.5 public:0.18.4 public:0.18.3 public:0.18.2 public:0.18.1 public:0.18.0 public:0.17.34 public:0.17.33 public:0.17.32 public:0.17.31 public:0.17.30 public:0.17.29 public:0.17.28 public:0.17.27 public:0.17.26 public:0.17.25 public:0.17.24 public:0.17.23 public:0.17.22 public:0.17.21 public:0.17.20 public:0.17.19 public:0.17.18 public:0.17.17 public:0.17.16 public:0.17.15 public:0.17.14 public:0.17.13 public:0.17.12 public:0.17.11 public:0.17.10 public:0.17.9 public:0.17.8 public:0.17.7 public:0.17.6 public:0.17.5 public:0.17.4 public:0.17.3 public:0.17.2 public:0.17.1 public:0.17.0 public:0.16.8 public:0.16.7 public:0.16.6 public:0.16.5 public:0.16.4 public:0.16.3 public:0.16.2 public:0.16.1 public:0.16.0 public:0.15.10 public:0.15.9 public:0.15.8 public:0.15.7 public:0.15.6 public:0.15.5 public:0.15.4 public:0.15.3 public:0.15.2 public:0.15.1 public:0.15.0 public:0.14.8 public:0.14.7 public:0.14.6 public:0.14.5 public:0.14.4 public:0.14.3 public:0.14.2 public:0.14.1 public:0.14.0 public:0.13.4 public:0.13.3 public:0.13.2 public:0.13.1 public:0.13.0 public:0.12.5 public:0.12.4 public:0.12.3 public:0.12.2 public:0.12.1 public:0.12.0 public:0.11.10 public:0.11.9 public:0.11.8 public:0.11.7 public:0.11.6 public:0.11.5 public:0.11.4 public:0.11.3 public:0.11.2 public:0.11.1 public:0.11.0 public:0.10.1 public:0.10.0 public:0.9.0 public:0.8.8 public:0.8.7 public:0.8.6 public:0.8.5 public:0.8.3 public:0.8.2 public:0.8.1 public:0.8.0 public:0.7.2 public:0.7.1 public:0.7.0 public:0.6.1 public:0.6.0 public:0.5.0 public:0.4.1 public:0.4.0 public:0.3.1 public:0.3.0 public:0.2.1 public:0.2.0 public:0.1.26 public:0.1.25 public:0.1.24 public:0.1.23 public:0.1.22 public:0.1.21 public:0.1.20 public:0.1.19 public:0.1.18 public:0.1.17 public:0.1.16 public:0.1.15 public:0.1.14 public:0.1.13 public:0.1.12 public:0.1.11 public:0.1.10 public:0.1.9 public:0.1.8 public:0.1.7 public:0.1.6 public:0.1.5 public:0.1.4 public:0.1.3 public:0.1.2 public:0.1.1 public:0.1.0 public:0.0.9 public:0.0.8 public:0.0.7 public:0.0.6
...
compare: public:0.11.9
Branches Tags
public:test_real_obsidian public:main public:feat-userhashsalt public:p2p-rpc public:auto_conf_2 public:auto_conf public:beta public:alpha public:refactor_services_2 public:refactor_services public:dev public:svelteui public:forprivacy
public:0.25.60 public:0.25.59 public:0.25.58 public:0.25.57 public:0.25.56+patched5 public:0.25.56+patched4 public:0.25.56+patched3 public:0.25.56+patched2 public:0.25.56+patched1 public:0.25.56 public:0.25.55 public:0.25.54 public:0.25.53 public:0.25.52-patched-3 public:0.25.52-patched-2 public:0.25.52-patched-1 public:0.25.52 public:0.25.51 public:0.25.50 public:0.25.49 public:0.25.48 public:0.25.47 public:0.25.46 public:0.25.45 public:0.25.44 public:0.25.43-patched-9 public:0.25.43-patched-8 public:0.25.43-patched-7 public:0.25.43-patched-6 public:0.25.43-patched-5 public:0.25.43-patched-4 public:0.25.43-patched-3 public:0.25.43-patched-2 public:0.25.43-patched-1 public:0.25.43 public:0.25.42 public:0.25.41-patched-3 public:0.25.41-patched-2 public:0.25.41-patched-1 public:0.25.41 public:0.25.40 public:0.25.39 public:0.25.38 public:0.25.37 public:0.25.36 public:0.25.35 public:0.25.34 public:0.25.33 public:0.25.32 public:0.25.31 public:0.25.30 public:0.25.30-dev public:0.25.29 public:0.25.28 public:0.25.27 public:0.25.26 public:0.25.25 public:0.25.24 public:0.25.24.beta3 public:0.25.24.beta2 public:0.25.24.beta1 public:0.25.23 public:0.25.23.beta1 public:0.25.22 public:0.25.21 public:0.25.21.beta2 public:0.25.21.beta1 public:0.25.20 public:0.25.19 public:0.25.18 public:0.25.17 public:0.25.16 public:0.25.15 public:0.25.14 public:0.25.13 public:0.25.12 public:0.25.11 public:0.25.10 public:0.25.9 public:0.25.8 public:0.25.7 public:0.25.6 public:0.25.5 public:0.25.4 public:0.25.3 public:0.25.2 public:0.25.1 public:0.25.0 public:0.25.0-beta1 public:0.24.31 public:0.24.30 public:0.24.29 public:0.24.28 public:0.24.27 public:0.24.26 public:0.24.25 public:0.24.24 public:0.24.23 public:0.24.22 public:0.24.21 public:0.24.20 public:0.24.19 public:0.24.18 public:0.24.17 public:0.24.16 public:0.24.15 public:0.24.14 public:0.24.13 public:0.24.12 public:0.24.11 public:0.24.10 public:0.24.9 public:0.24.8 public:0.24.7 public:0.24.6 public:0.24.5 public:0.24.4 public:0.24.3 public:0.24.2 public:0.24.1 public:0.24.0 public:0.24.0.dev-rc8 public:0.24.0.dev-rc7 public:0.24.0.dev-rc6 public:0.24.0.dev-rc5 public:0.24.0.dev-rc4 public:0.24.0.dev-rc3 public:0.24.0.dev-rc2 public:0.23.23 public:0.23.22 public:0.23.21 public:0.23.20 public:0.23.19 public:0.23.18 public:0.23.17 public:0.23.16 public:0.23.15 public:0.23.14 public:0.23.13 public:0.23.12 public:0.23.11 public:0.23.10 public:0.23.9 public:0.23.8 public:0.23.7 public:0.23.6 public:0.23.5 public:0.23.4 public:0.23.3 public:0.23.2 public:0.23.1 public:0.23.0 public:0.22.19 public:0.22.18 public:0.22.17 public:0.22.16 public:0.22.15 public:0.22.14 public:0.22.13 public:0.22.12 public:0.22.11 public:0.22.10 public:0.22.9 public:0.22.8 public:0.22.7 public:0.22.6 public:0.22.5 public:0.22.4 public:0.22.3 public:0.22.2 public:0.22.1 public:0.22.0 public:0.21.5 public:0.21.4 public:0.21.3 public:0.21.2 public:0.21.1 public:0.21.0 public:0.20.7 public:0.20.6 public:0.20.5 public:0.20.4 public:0.20.3 public:0.20.2 public:0.20.1 public:0.20.0 public:0.19.23 public:0.19.22 public:0.19.21 public:0.19.20 public:0.19.19 public:0.19.18 public:0.19.17 public:0.19.16 public:0.19.15 public:0.19.14 public:0.19.13 public:0.19.12 public:0.19.11 public:0.19.10 public:0.19.9 public:0.19.8 public:0.19.7 public:0.19.6 public:0.19.5 public:0.19.4 public:0.19.3 public:0.19.2 public:0.19.1 public:0.19.0 public:0.18.6 public:0.18.5 public:0.18.4 public:0.18.3 public:0.18.2 public:0.18.1 public:0.18.0 public:0.17.34 public:0.17.33 public:0.17.32 public:0.17.31 public:0.17.30 public:0.17.29 public:0.17.28 public:0.17.27 public:0.17.26 public:0.17.25 public:0.17.24 public:0.17.23 public:0.17.22 public:0.17.21 public:0.17.20 public:0.17.19 public:0.17.18 public:0.17.17 public:0.17.16 public:0.17.15 public:0.17.14 public:0.17.13 public:0.17.12 public:0.17.11 public:0.17.10 public:0.17.9 public:0.17.8 public:0.17.7 public:0.17.6 public:0.17.5 public:0.17.4 public:0.17.3 public:0.17.2 public:0.17.1 public:0.17.0 public:0.16.8 public:0.16.7 public:0.16.6 public:0.16.5 public:0.16.4 public:0.16.3 public:0.16.2 public:0.16.1 public:0.16.0 public:0.15.10 public:0.15.9 public:0.15.8 public:0.15.7 public:0.15.6 public:0.15.5 public:0.15.4 public:0.15.3 public:0.15.2 public:0.15.1 public:0.15.0 public:0.14.8 public:0.14.7 public:0.14.6 public:0.14.5 public:0.14.4 public:0.14.3 public:0.14.2 public:0.14.1 public:0.14.0 public:0.13.4 public:0.13.3 public:0.13.2 public:0.13.1 public:0.13.0 public:0.12.5 public:0.12.4 public:0.12.3 public:0.12.2 public:0.12.1 public:0.12.0 public:0.11.10 public:0.11.9 public:0.11.8 public:0.11.7 public:0.11.6 public:0.11.5 public:0.11.4 public:0.11.3 public:0.11.2 public:0.11.1 public:0.11.0 public:0.10.1 public:0.10.0 public:0.9.0 public:0.8.8 public:0.8.7 public:0.8.6 public:0.8.5 public:0.8.3 public:0.8.2 public:0.8.1 public:0.8.0 public:0.7.2 public:0.7.1 public:0.7.0 public:0.6.1 public:0.6.0 public:0.5.0 public:0.4.1 public:0.4.0 public:0.3.1 public:0.3.0 public:0.2.1 public:0.2.0 public:0.1.26 public:0.1.25 public:0.1.24 public:0.1.23 public:0.1.22 public:0.1.21 public:0.1.20 public:0.1.19 public:0.1.18 public:0.1.17 public:0.1.16 public:0.1.15 public:0.1.14 public:0.1.13 public:0.1.12 public:0.1.11 public:0.1.10 public:0.1.9 public:0.1.8 public:0.1.7 public:0.1.6 public:0.1.5 public:0.1.4 public:0.1.3 public:0.1.2 public:0.1.1 public:0.1.0 public:0.0.9 public:0.0.8 public:0.0.7 public:0.0.6

13 Commits
0.11.2 ... 0.11.9

Author SHA1 Message Date
vorotamoroz
7d6b83a1cb Fixed 
- Saving notes with wrong type.
 2022-07-07 17:21:23 +09:00
vorotamoroz
41034d7d92 Create release.yml 2022-06-30 18:18:31 +09:00
vorotamoroz
2455ff6ee1 bump 2022-06-30 18:17:09 +09:00
vorotamoroz
89de551fd7 Fixed: 
- Unexpected massive palallel running of file checking in boot sequence is solved.
- Batch file change is not  missing changes now.
- Ignore changes caused by the plug-ins themselves.
- Garbage collection is completely disabled.
- Fixed sometimes fails initial replication after dropping local DB.
Improved:
- a bit more understandable messages
- Save the file into the big chunk on initial scan.
- Use history is always enabled.
- Boot sequence got faster.
 2022-06-30 17:46:42 +09:00
vorotamoroz
124a49b80f Fixed and implemented 
- Readme tidied.
- More faster e2ee.
- URI and databasename check improved.
 2022-06-23 18:26:43 +09:00
vorotamoroz
3e76292aa7 Implemented: 
- Encrypting setup URI by passphrse.
(Note: You have to make the setup URI again)

Fixed:
- Setup procedure fixed.
- Status text fixed.
- Documentation fixed.
 2022-06-19 15:36:36 +09:00
vorotamoroz
4634ab73b1 - Automatic garbage collection disabled 
- Fixed database unloading problem
 2022-06-19 14:09:11 +09:00
vorotamoroz
359c10f1d7 Correction of wording 2022-06-15 17:59:10 +09:00
vorotamoroz
59ebac3efc Correction of wording 2022-06-15 17:54:35 +09:00
vorotamoroz
b4edca3a99 Fixed format. 2022-06-15 17:51:07 +09:00
vorotamoroz
4b76b10a6f Add some note. 2022-06-15 17:45:46 +09:00
vorotamoroz
d4b53280e3 Fixed status message and lag on boot time scan. 2022-06-15 17:45:37 +09:00
vorotamoroz
dbd9b17b20 Fixed: 
- Fixed ignoring changes on replicating.
- Disabled `Skip old files on sync` temporary (so fragile between multiple devices)
 2022-06-14 19:49:21 +09:00
13 changed files with 948 additions and 708 deletions
Expand all files Collapse all files

94
.github/workflows/release.yml vendored Normal file
View File

@@ -0,0 +1,94 @@
name: Release Obsidian Plugin
on:
push:
# Sequence of patterns matched against refs/tags
tags:
- '*' # Push events to matching any tag format, i.e. 1.0, 20.15.10
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
submodules: recursive
- name: Use Node.js
uses: actions/setup-node@v1
with:
node-version: '14.x' # You might need to adjust this value to your own version
# Get the version number and put it in a variable
- name: Get Version
id: version
run: |
echo "::set-output name=tag::$(git describe --abbrev=0)"
# Build the plugin
- name: Build
id: build
run: |
npm ci
npm run build --if-present
# Package the required files into a zip
- name: Package
run: |
mkdir ${{ github.event.repository.name }}
cp main.js manifest.json styles.css README.md ${{ github.event.repository.name }}
zip -r ${{ github.event.repository.name }}.zip ${{ github.event.repository.name }}
# Create the release on github
- name: Create Release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERSION: ${{ github.ref }}
with:
tag_name: ${{ github.ref }}
release_name: ${{ github.ref }}
draft: true
prerelease: false
# Upload the packaged release file
- name: Upload zip file
id: upload-zip
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./${{ github.event.repository.name }}.zip
asset_name: ${{ github.event.repository.name }}-${{ steps.version.outputs.tag }}.zip
asset_content_type: application/zip
# Upload the main.js
- name: Upload main.js
id: upload-main
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./main.js
asset_name: main.js
asset_content_type: text/javascript
# Upload the manifest.json
- name: Upload manifest.json
id: upload-manifest
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./manifest.json
asset_name: manifest.json
asset_content_type: application/json
# Upload the style.css
- name: Upload styles.css
id: upload-css
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./styles.css
asset_name: styles.css
asset_content_type: text/css
# TODO: release notes???

123
README.md
View File

@@ -1,32 +1,25 @@
# Self-hosted LiveSync # Self-hosted LiveSync
Sorry for late! [Japanese docs](./README_ja.md) is also coming up. [Japanese docs](./README_ja.md).
**Renamed from: obsidian-livesync** Self-hosted LiveSync is a community implemented synchronization plugin.
It uses Self-hosted or you procured CouchDB as the server. Available on every obsidian installed devices.
Using a self-hosted database, live-sync to multi-devices bidirectionally. Note: It has no compatibilities with official "Sync".
Runs in Mac, Android, Windows, and iOS. Perhaps available on Linux too.
Community implementation, not compatible with official "Sync".
![obsidian_live_sync_demo](https://user-images.githubusercontent.com/45774780/137355323-f57a8b09-abf2-4501-836c-8cb7d2ff24a3.gif) ![obsidian_live_sync_demo](https://user-images.githubusercontent.com/45774780/137355323-f57a8b09-abf2-4501-836c-8cb7d2ff24a3.gif)
**It's getting almost stable now, But Please make sure to back your vault up!** If you install or upgrade LiveSync, please back your vault up.
Limitations: ~~Folder deletion handling is not completed.~~ **It would work now.** ## Features
## This plugin enables... - Visual conflict resolver included.
- Synchronize with other devices bidirectionally near-real-time
- Runs in Windows, Mac, iPad, iPhone, Android, Chromebook - You can use CouchDB or its compatibles like IBM Cloudant.
- Synchronize to Self-hosted Database - End-to-End encryption.
- Replicate to/from other devices bidirectionally near-real-time - Plugin synchronization(Beta)
- Resolving synchronizing conflicts in the Obsidian.
- You can use CouchDB or its compatibles like IBM Cloudant. CouchDB is OSS, and IBM Cloudant has the terms and certificates about security. Your notes are yours.
- Off-line sync is also available.
- End-to-End encryption is available (beta).
- Receive WebClip from [obsidian-livesync-webclip](https://chrome.google.com/webstore/detail/obsidian-livesync-webclip/jfpaflmpckblieefkegjncjoceapakdf) (End-to-End encryption will not be applicable.) - Receive WebClip from [obsidian-livesync-webclip](https://chrome.google.com/webstore/detail/obsidian-livesync-webclip/jfpaflmpckblieefkegjncjoceapakdf) (End-to-End encryption will not be applicable.)
It must be useful for the Researcher, Engineer, Developer who has to keep NDA or something like agreement. It must be useful for the Researcher, Engineer, Developer who has to keep NDA or something like agreement. Especially, in some companies, people have to store all data to their fully controlled host, even End-to-End encryption applied.
Especially, in some companies, people have to store all data to their fully controlled host, even End-to-End encryption applied.
## IMPORTANT NOTICE ## IMPORTANT NOTICE
@@ -36,35 +29,68 @@ Especially, in some companies, people have to store all data to their fully cont
- When the device's storage has been run out, Database corruption may happen. - When the device's storage has been run out, Database corruption may happen.
- When editing hidden files or any other invisible files from obsidian, the file wouldn't be kept in the database. (**Or be deleted.**) - When editing hidden files or any other invisible files from obsidian, the file wouldn't be kept in the database. (**Or be deleted.**)
## Supplements
- When the file has been deleted, the deletion of the file is replicated to other devices.
- When the folder became empty by replication, The folder will be deleted in the default setting. But you can change this behaivour. Check the [Settings](docs/settings.md).
- LiveSync drains many batteries in mobile devices.
- Mobile Obsidian can not connect to the non-secure(HTTP) or local CA-signed servers, even though the certificate is stored in the device store.
- There are no 'exclude_folders' like configurations.
## How to use ## How to use
1. Install from Obsidian, or download from this repo's releases, copy `main.js`, `styles.css` and `manifest.json` into `[your-vault]/.obsidian/plugins/` ### Get your database ready.
2. Get your database. IBM Cloudant is preferred for testing. Or you can use your own server with CouchDB.
For more information, refer below: First, get your database ready. IBM Cloudant is preferred for testing. Or you can use your own server with CouchDB. For more information, refer below:
1. [Setup IBM Cloudant](docs/setup_cloudant.md) 1. [Setup IBM Cloudant](docs/setup_cloudant.md)
2. [Setup your CouchDB](docs/setup_own_server.md) 2. [Setup your CouchDB](docs/setup_own_server.md)
3. Enter connection information to Plugin's setting dialog. In details, refer [Settings of Self-hosted LiveSync](docs/settings.md)
4. Enable LiveSync or other Synchronize method as you like. ### First device
1. Install the plugin on your device.
2. Configure with the remote database.
1. Fill your server's information into the `Remote Database configuration` pane.
2. Enabling `End to End Encryption` is recommended. After inputting the passphrase, you have to press `Just apply`.
3. Hit `Test Database Connection` and make sure that the plugin says `Connected`.
4. Hit `Check database configuration` and make sure all tests have been passed.
3. Configure how to synchronize on `Sync setting`. (You can leave these configures later)
1. If you want to synchronize in real-time, enable `LiveSync`.
2. Or, set up the synchronization as you like.
3. Additional configuration is also here. I recommend enabling `Use Trash for deleted files, but you can leave all configurations disabled.
4. Configure miscellaneous features.
1. Enabling `Show staus inside editor` bring you information. While edit mode, you can see the status on the top-right of the editor. (Recommended)
2. Enabling `Use history` let you see the diffs between your edit and synchronization. (Recommended)
5. Back to the editor. I hope that initial scan is in the progress or done.
6. When status became stabilized (All ⏳ and 🧩 have been disappeared), you are ready to synchronize with the server.
7. Press the replicate icon on the Ribbon or run `Replicate now` from the Command pallet. You'll send all your data to the server.
8. Open the command palette, `Copy setup URI`, and set the passphrase to encrypt the information. Then your configuration will be copied to the clipboard. Please share copied URI with your other devices.
**IMPORTANT NOTICE: BE CAREFUL TO TREAT THIS URI. THE URI CONTAINS YOUR CREDENTIALS EVEN THOUGH NOBODY COULD READ WITHOUT THE PASSPHRASE.**
### Subsequent Devices
Strongly recommend using the vault in which all files are completely synchronized including timestamps. Otherwise, some files will be corrupted if failed to resolve conflicts. To simplify, I recommend using a new empty vault.
1. Install the plug-in.
2. Open the link that you had been copied to the other device.
3. The plug-in asks you that are you sure to apply the configurations. Please answer `Yes` and the following instruction below:
1. Answer `Yes` to `Keep local DB?`.
*Note: If you started with existed vault, you have to answer `No`. And `No` to `Rebuild the database?`.*
2. Answer `Yes` to `Keep remote DB?`.
3. Answer `Yes` to `Replicate once?`.
Yes, you have to answer `Yes` to everything.
Then, all your settings are copied from the first device.
4. Your notes will arrive soon.
## Something looks corrupted...
Please open the link again and Answer as below:
- If your local database looks corrupted
(in other words, when your Obsidian getting weird even standalone.)
- Answer `No` to `Keep local DB?`
- If your remote database looks corrupted
(in other words, when something happens while replicating)
- Answer `No` to `Keep remote DB?`
If you answered `No` to both, your databases will be rebuilt by the content on your device. And the remote database will lock out other devices. You have to synchronize all your devices again. (When this time, almost all your files should be synchronized including a timestamp. So you can use the existed vault).
## Test Server ## Test Server
Setting up an instance of Cloudant or local CouchDB is a little complicated, so I made the [Tasting server of self-hosted-livesync](https://olstaste.vrtmrz.net/) up. Try free! Setting up an instance of Cloudant or local CouchDB is a little complicated, so I made the [Tasting server of self-hosted-livesync](https://olstaste.vrtmrz.net/) up. Try free!
Note: Please read "Limitations" carefully. Do not send your private vault. Note: Please read "Limitations" carefully. Do not send your private vault.
## WebClipper is also available. ## Information in StatusBar
Available from on Chrome Web Store:[obsidian-livesync-webclip](https://chrome.google.com/webstore/detail/obsidian-livesync-webclip/jfpaflmpckblieefkegjncjoceapakdf)
Repo is here: [obsidian-livesync-webclip](https://github.com/vrtmrz/obsidian-livesync-webclip). (Docs are work in progress.)
# Information in StatusBar
Synchronization status is shown in statusbar. Synchronization status is shown in statusbar.
@@ -75,11 +101,16 @@ Synchronization status is shown in statusbar.
- ⚠ Error occurred. - ⚠ Error occurred.
- ↑ Uploaded pieces - ↑ Uploaded pieces
- ↓ Downloaded pieces - ↓ Downloaded pieces
- ⏳ Count of the pending process - ⏳ Number of the pending processes
If you have deleted or renamed files, please wait until this disappears. - 🧩 Number of the files that waiting for their chunks.
If you have deleted or renamed files, please wait until ⏳ disappeared.
# More supplements
## Hints
- When the folder became empty by replication, The folder will be deleted in the default setting. But you can change this behaivour. Check the [Settings](docs/settings.md).
- LiveSync mode drains many batteries in mobile devices. Periodic sync and some automatic sync is recommended.
- Mobile Obsidian can not connect to the non-secure(HTTP) or local CA-signed servers, even though the certificate is stored in the device store.
- There are no 'exclude_folders' like configurations.
- When synchronized, files are compared by their modified times and overwritten by the newer ones once. Then plugin checks the conflicts and if a merge is needed, the dialog will open. - When synchronized, files are compared by their modified times and overwritten by the newer ones once. Then plugin checks the conflicts and if a merge is needed, the dialog will open.
- Rarely, the file in the database would be broken. The plugin will not write storage when it looks broken, so some old files must be on your device. If you edit the file, it will be cured. But if the file does not exist on any device, can not rescue it. So you can delete these items from the setting dialog. - Rarely, the file in the database would be broken. The plugin will not write storage when it looks broken, so some old files must be on your device. If you edit the file, it will be cured. But if the file does not exist on any device, can not rescue it. So you can delete these items from the setting dialog.
- If your database looks corrupted, try "Drop History". Usually, It is the easiest way. - If your database looks corrupted, try "Drop History". Usually, It is the easiest way.
@@ -87,7 +118,11 @@ If you have deleted or renamed files, please wait until this disappears.
- Q: Database is growing, how can I shrink it up? - Q: Database is growing, how can I shrink it up?
A: each of the docs is saved with their old 100 revisions to detect and resolve confliction. Picture yourself that one device has been off the line for a while, and joined again. The device has to check his note and remote saved note. If exists in revision histories of remote notes even though the device's note is a little different from the latest one, it could be merged safely. Even if that is not in revision histories, we only have to check differences after the revision that both devices commonly have. This is like The git's conflict resolving method. So, We have to make the database again like an enlarged git repo if you want to solve the root of the problem. A: each of the docs is saved with their old 100 revisions to detect and resolve confliction. Picture yourself that one device has been off the line for a while, and joined again. The device has to check his note and remote saved note. If exists in revision histories of remote notes even though the device's note is a little different from the latest one, it could be merged safely. Even if that is not in revision histories, we only have to check differences after the revision that both devices commonly have. This is like The git's conflict resolving method. So, We have to make the database again like an enlarged git repo if you want to solve the root of the problem.
- And more technical Information are in the [Technical Information](docs/tech_info.md) - And more technical Information are in the [Technical Information](docs/tech_info.md)
- If you want to synchronize files without obsidian, you can use [filesystem-livesync](https://github.com/vrtmrz/filesystem-livesync).
- WebClipper is also available.
Available from on Chrome Web Store:[obsidian-livesync-webclip](https://chrome.google.com/webstore/detail/obsidian-livesync-webclip/jfpaflmpckblieefkegjncjoceapakdf)
Repo is here: [obsidian-livesync-webclip](https://github.com/vrtmrz/obsidian-livesync-webclip). (Docs are work in progress.)
# License ## License
The source code is licensed MIT. The source code is licensed MIT.

120
docs/settings.md
View File

@@ -2,6 +2,20 @@ NOTE: This document surely became outdated. I'll improve this doc in a while. bu
# Settings of this plugin # Settings of this plugin
The settings dialog has been quite long, so I split each configuration into tabs.
If you feel something, please feel free to inform me.
| icon | description |
| :---: | ----------------------------------------------------------------- |
| 🛰️ | [Remote Database Configurations](#remote-database-configurations) |
| 📦 | [Local Database Configurations](#local-database-configurations) |
| ⚙️ | [General Settings](#general-settings) |
| 🔁 | [Sync setting](#sync-setting) |
| 🔧 | [Miscellaneous](#miscellaneous) |
| 🧰 | [Hatch](#miscellaneous) |
| 🔌 | [Plugin and its settings](#plugin-and-its-settings) |
| 🚑 | [Corrupted data](#corrupted-data) |
## Remote Database Configurations ## Remote Database Configurations
Configure settings of synchronize server. If any synchronization is enabled, you can't edit this section. Please disable all synchronization to change. Configure settings of synchronize server. If any synchronization is enabled, you can't edit this section. Please disable all synchronization to change.
@@ -21,12 +35,6 @@ The Database name to synchronize.
⚠️If not exist, created automatically. ⚠️If not exist, created automatically.
### Use the old connecting method
This option has been removed at v0.10.0
### End to End Encryption ### End to End Encryption
Encrypt your database. It affects only the database, your files are left as plain. Encrypt your database. It affects only the database, your files are left as plain.
@@ -38,26 +46,16 @@ Note: If you want to use "Plugins and their settings", you have to enable this.
The passphrase to used as the key of encryption. Please use the long text. The passphrase to used as the key of encryption. Please use the long text.
### Apply ### Apply
To enable End-to-End encryption, there must be no items of the same content encrypted with different passphrases to avoid attackers guessing passphrases. Self-hosted LiveSync uses crc32 of the chunks, It is really a must. Set the End to End encryption enabled and its passphrase for use in replication.
If you change the passphrase with existen database, overwriting remote database is strongly recommended.
So, this plugin completely deletes everything from both local and remote databases before enabling it and then synchronizing again.
To enable, "Apply and send" from the most powerful device. ### Overwrite by local DB
If you want to synchronize an existing database, set passphrase and press "Just apply". Overwrite the remote database by the local database using the passphrase you applied.
- Apply and send
1. Initialize the Local Database and set (or clear) passphrase, put all files into the database again.
2. Initialize the Remote Database.
3. Lock the Remote Database.
4. Send it all.
This process is simply heavy. Using a PC or Mac is preferred. ### Rebuild
- Apply and receive Rebuild remote and local databases with local files. It will delete all document history and retained chunks, and shrink the database.
1. Initialize the Local Database and set (or clear) the passphrase.
2. Unlock the Remote Database.
3. Retrieve all and decrypt to file.
When running these operations, every synchronization settings is disabled.
### Test Database connection ### Test Database connection
You can check the connection by clicking this button. You can check the connection by clicking this button.
@@ -65,6 +63,9 @@ You can check the connection by clicking this button.
### Check database configuration ### Check database configuration
You can check and modify your CouchDB's configuration from here directly. You can check and modify your CouchDB's configuration from here directly.
### Lock remote database.
Other devices are banned from the database when you have locked the database.
If you have something troubled with other devices, you can protect the vault and remote database by your device.
## Local Database Configurations ## Local Database Configurations
"Local Database" is created inside your obsidian. "Local Database" is created inside your obsidian.
@@ -73,27 +74,17 @@ You can check and modify your CouchDB's configuration from here directly.
Delay database update until raise replication, open another file, window visibility changed, or file events except for file modification. Delay database update until raise replication, open another file, window visibility changed, or file events except for file modification.
This option can not be used with LiveSync at the same time. This option can not be used with LiveSync at the same time.
### Auto Garbage Collection delay ### Garbage check
When the note has been modified, Self-hosted LiveSync splits the note into some chunks by parsing the markdown structure. And saving only file information and modified chunks into the Local Database. At this time, do not delete old chunks. This plugin saves the file by splitting it into chunks to speed replication up and keep low bandwidth.
So, Self-hosted LiveSync has to delete old chunks somewhen.
However, the chunk is represented as the crc32 of their contents and shared between all notes. In this way, Self-hosted LiveSync dedupes the entries and keeps low bandwidth and low transfer amounts. They share the chunk if you use the same paragraph in some notes. And if you change the file, only the paragraph you changed is transferred with metadata of the file. And I know that editing notes are not so straight. Sometimes paragraphs will be back into an old phrase. In these cases, we do not have to transfer the chunk again if the chunk will not be deleted. So all chunks will be reused.
In addition to this, when we edit notes, sometimes back to the previous expression. So It cannot be said that it will be unnecessary immediately. As the side effect of this, you can see history the file.
Therefore, the plugin deletes unused chunks at once when you leave Obsidian for a while (after this setting seconds). The check will show the number of chunks used or retained. If there are so many retained chunks, you can rebuild the database.
This process is called "Garbage Collection" ### Fetch rebuilt DB.
If one device rebuilds or locks the remote database, every other device will be locked out from the remote database until it fetches rebuilt DB.
As a result, Obsidian's behavior is temporarily slowed down.
Default is 300 seconds.
If you are an early adopter, maybe this value is left as 30 seconds. Please change this value to larger values.
Note: If you want to use "Use history", this vault must be set to 0.
### Manual Garbage Collect
Run "Garbage Collection" manually.
### minimum chunk size and LongLine threshold ### minimum chunk size and LongLine threshold
The configuration of chunk splitting. The configuration of chunk splitting.
@@ -212,28 +203,6 @@ You can set synchronization method at once as these pattern:
- Sync on File Open : disabled - Sync on File Open : disabled
- Sync on Start : disabled - Sync on Start : disabled
### Use history
If you enable this option, you can keep document histories in your database.
(Not all intermediate changes are synchronized.)
You can check the changes caused by your edit and/or replication.
### Enable plugin synchronization
If you want to use this feature, you have to activate this feature by this switch.
### Sweep plugins automatically
Plugin sweep will run before replication automatically.
### Sweep plugins periodically
Plugin sweep will run each 1 minute.
### Notify updates
When replication is complete, a message will be notified if a newer version of the plugin applied to this device is configured on another device.
### Device and Vault name
To save the plugins, you have to set a unique name every each device.
### Open
Open the "Plugins and their settings" dialog.
## Hatch ## Hatch
From here, everything is under the hood. Please handle it with care. From here, everything is under the hood. Please handle it with care.
@@ -280,11 +249,6 @@ Same as a setting passphrase, database locking is also performed.
3. Retrieve all and decrypt to file. 3. Retrieve all and decrypt to file.
### Lock remote database
Lock the remote database to ban out other devices for synchronization. It is the same as the database lock that happened in dropping databases or applying passphrases.
Use it as an emergency evacuation method to protect local or remote databases when synchronization has been broken.
### Suspend file watching ### Suspend file watching
If enable this option, Self-hosted LiveSync dismisses every file change or deletes the event. If enable this option, Self-hosted LiveSync dismisses every file change or deletes the event.
@@ -301,9 +265,27 @@ Discard the data stored in the local database.
### Initialize local database again ### Initialize local database again
Discard the data stored in the local database and initialize and create the database from the files on storage. Discard the data stored in the local database and initialize and create the database from the files on storage.
### Corrupted data ## Plugins and settings (beta)
### Enable plugin synchronization
If you want to use this feature, you have to activate this feature by this switch.
### Sweep plugins automatically
Plugin sweep will run before replication automatically.
### Sweep plugins periodically
Plugin sweep will run each 1 minute.
### Notify updates
When replication is complete, a message will be notified if a newer version of the plugin applied to this device is configured on another device.
### Device and Vault name
To save the plugins, you have to set a unique name every each device.
### Open
Open the "Plugins and their settings" dialog.
### Corrupted or missing data
![CorruptedData](../images/corrupted_data.png) ![CorruptedData](../images/corrupted_data.png)
When Self-hosted LiveSync could not write to the file on the storage, the files are shown here. If you have the old data in your vault, change it once, it will be cured. Or you can use the "File History" plugin. When Self-hosted LiveSync could not write to the file on the storage, the files are shown here. If you have the old data in your vault, change it once, it will be cured. Or you can use the "File History" plugin.
But if you don't, sorry, we can't rescue the file, and error messages are shown frequently, and you have to delete the file from here.

4
manifest.json
View File

@@ -1,10 +1,10 @@
{ {
"id": "obsidian-livesync", "id": "obsidian-livesync",
"name": "Self-hosted LiveSync", "name": "Self-hosted LiveSync",
"version": "0.11.2", "version": "0.11.9",
"minAppVersion": "0.9.12", "minAppVersion": "0.9.12",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz", "author": "vorotamoroz",
"authorUrl": "https://github.com/vrtmrz", "authorUrl": "https://github.com/vrtmrz",
"isDesktopOnly": false "isDesktopOnly": false
} }

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{ {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.11.2", "version": "0.11.9",
"lockfileVersion": 2, "lockfileVersion": 2,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.11.2", "version": "0.11.9",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"diff-match-patch": "^1.0.5", "diff-match-patch": "^1.0.5",

6
package.json
View File

@@ -1,6 +1,6 @@
{ {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.11.2", "version": "0.11.9",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js", "main": "main.js",
"type": "module", "type": "module",
@@ -37,8 +37,8 @@
"diff-match-patch": "^1.0.5", "diff-match-patch": "^1.0.5",
"esbuild": "0.13.12", "esbuild": "0.13.12",
"esbuild-svelte": "^0.6.0", "esbuild-svelte": "^0.6.0",
"idb": "^7.0.1", "idb": "^7.0.1",
"svelte-preprocess": "^4.10.2", "svelte-preprocess": "^4.10.2",
"xxhash-wasm": "^0.4.2" "xxhash-wasm": "^0.4.2"
} }
} }

52
src/KeyValueDB.ts Normal file
View File

@@ -0,0 +1,52 @@
import { deleteDB, IDBPDatabase, openDB } from "idb";
export interface KeyValueDatabase {
get<T>(key: string): Promise<T>;
set<T>(key: string, value: T): Promise<IDBValidKey>;
del(key: string): Promise<void>;
clear(): Promise<void>;
keys(query?: IDBValidKey | IDBKeyRange, count?: number): Promise<IDBValidKey[]>;
close(): void;
destroy(): void;
}
const databaseCache: { [key: string]: IDBPDatabase<any> } = {};
export const OpenKeyValueDatabase = async (dbKey: string): Promise<KeyValueDatabase> => {
if (dbKey in databaseCache) {
databaseCache[dbKey].close();
delete databaseCache[dbKey];
}
const storeKey = dbKey;
const dbPromise = openDB(dbKey, 1, {
upgrade(db) {
db.createObjectStore(storeKey);
},
});
let db: IDBPDatabase<any> = null;
db = await dbPromise;
databaseCache[dbKey] = db;
return {
get<T>(key: string): Promise<T> {
return db.get(storeKey, key);
},
set<T>(key: string, value: T) {
return db.put(storeKey, value, key);
},
del(key: string) {
return db.delete(storeKey, key);
},
clear() {
return db.clear(storeKey);
},
keys(query?: IDBValidKey | IDBKeyRange, count?: number) {
return db.getAllKeys(storeKey, query, count);
},
close() {
delete databaseCache[dbKey];
return db.close();
},
async destroy() {
delete databaseCache[dbKey];
db.close();
await deleteDB(dbKey);
},
};
};

494
src/LocalPouchDB.ts
View File

@@ -19,75 +19,37 @@ import {
VER, VER,
MILSTONE_DOCID, MILSTONE_DOCID,
DatabaseConnectingStatus, DatabaseConnectingStatus,
ChunkVersionRange,
} from "./lib/src/types"; } from "./lib/src/types";
import { RemoteDBSettings } from "./lib/src/types"; import { RemoteDBSettings } from "./lib/src/types";
import { resolveWithIgnoreKnownError, delay, runWithLock, NewNotice, WrappedNotice, shouldSplitAsPlainText, splitPieces2, enableEncryption } from "./lib/src/utils"; import { resolveWithIgnoreKnownError, runWithLock, shouldSplitAsPlainText, splitPieces2, enableEncryption } from "./lib/src/utils";
import { path2id } from "./utils"; import { path2id } from "./utils";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { checkRemoteVersion, connectRemoteCouchDBWithSetting, getLastPostFailedBySize } from "./utils_couchdb"; import { checkRemoteVersion, connectRemoteCouchDBWithSetting, getLastPostFailedBySize } from "./utils_couchdb";
import { openDB, deleteDB, IDBPDatabase } from "idb"; import { KeyValueDatabase, OpenKeyValueDatabase } from "./KeyValueDB";
import { LRUCache } from "./lib/src/LRUCache";
// when replicated, LiveSync checks chunk versions that every node used.
// If all minumum version of every devices were up, that means we can convert database automatically.
const currentVersionRange: ChunkVersionRange = {
min: 0,
max: 1,
current: 1,
}
type ReplicationCallback = (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => Promise<void>; type ReplicationCallback = (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => Promise<void>;
class LRUCache {
cache = new Map<string, string>([]);
revCache = new Map<string, string>([]);
maxCache = 100;
constructor() {}
get(key: string) {
// debugger
const v = this.cache.get(key);
if (v) {
// update the key to recently used.
this.cache.delete(key);
this.revCache.delete(v);
this.cache.set(key, v);
this.revCache.set(v, key);
}
return v;
}
revGet(value: string) {
// debugger
const key = this.revCache.get(value);
if (value) {
// update the key to recently used.
this.cache.delete(key);
this.revCache.delete(value);
this.cache.set(key, value);
this.revCache.set(value, key);
}
return key;
}
set(key: string, value: string) {
this.cache.set(key, value);
this.revCache.set(value, key);
if (this.cache.size > this.maxCache) {
for (const kv of this.cache) {
this.revCache.delete(kv[1]);
this.cache.delete(kv[0]);
if (this.cache.size <= this.maxCache) break;
}
}
}
}
export class LocalPouchDB { export class LocalPouchDB {
auth: Credential; auth: Credential;
dbname: string; dbname: string;
settings: RemoteDBSettings; settings: RemoteDBSettings;
localDatabase: PouchDB.Database<EntryDoc>; localDatabase: PouchDB.Database<EntryDoc>;
kvDB: KeyValueDatabase;
nodeid = ""; nodeid = "";
isReady = false; isReady = false;
recentModifiedDocs: string[] = [];
h32: (input: string, seed?: number) => string; h32: (input: string, seed?: number) => string;
h64: (input: string, seedHigh?: number, seedLow?: number) => string;
h32Raw: (input: Uint8Array, seed?: number) => number; h32Raw: (input: Uint8Array, seed?: number) => number;
// hashCache: {
// [key: string]: string;
// } = {};
// hashCacheRev: {
// [key: string]: string;
// } = {};
hashCaches = new LRUCache(); hashCaches = new LRUCache();
corruptedEntries: { [key: string]: EntryDoc } = {}; corruptedEntries: { [key: string]: EntryDoc } = {};
@@ -106,6 +68,8 @@ export class LocalPouchDB {
isMobile = false; isMobile = false;
chunkVersion = 0;
cancelHandler<T extends PouchDB.Core.Changes<EntryDoc> | PouchDB.Replication.Sync<EntryDoc> | PouchDB.Replication.Replication<EntryDoc>>(handler: T): T { cancelHandler<T extends PouchDB.Core.Changes<EntryDoc> | PouchDB.Replication.Sync<EntryDoc> | PouchDB.Replication.Replication<EntryDoc>>(handler: T): T {
if (handler != null) { if (handler != null) {
handler.removeAllListeners(); handler.removeAllListeners();
@@ -115,7 +79,7 @@ export class LocalPouchDB {
return null; return null;
} }
onunload() { onunload() {
this.recentModifiedDocs = []; this.kvDB.close();
this.leafArrivedCallbacks; this.leafArrivedCallbacks;
this.changeHandler = this.cancelHandler(this.changeHandler); this.changeHandler = this.cancelHandler(this.changeHandler);
this.syncHandler = this.cancelHandler(this.syncHandler); this.syncHandler = this.cancelHandler(this.syncHandler);
@@ -139,6 +103,7 @@ export class LocalPouchDB {
if (this.localDatabase != null) { if (this.localDatabase != null) {
this.localDatabase.close(); this.localDatabase.close();
} }
this.kvDB.close();
} }
async isOldDatabaseExists() { async isOldDatabaseExists() {
@@ -167,6 +132,7 @@ export class LocalPouchDB {
revs_limit: 100, revs_limit: 100,
deterministic_revs: true, deterministic_revs: true,
}); });
this.kvDB = await OpenKeyValueDatabase(this.dbname + "-livesync-kv");
Logger("Database info", LOG_LEVEL.VERBOSE); Logger("Database info", LOG_LEVEL.VERBOSE);
Logger(await this.localDatabase.info(), LOG_LEVEL.VERBOSE); Logger(await this.localDatabase.info(), LOG_LEVEL.VERBOSE);
Logger("Open Database..."); Logger("Open Database...");
@@ -221,9 +187,8 @@ export class LocalPouchDB {
return nextSeq(); return nextSeq();
} }
// //
const progress = NewNotice("Converting..", 0); Logger("We have to upgrade database..", LOG_LEVEL.NOTICE, "conv");
try { try {
Logger("We have to upgrade database..", LOG_LEVEL.NOTICE);
// To debug , uncomment below. // To debug , uncomment below.
@@ -243,14 +208,12 @@ export class LocalPouchDB {
} }
const rep = old.replicate.to(this.localDatabase, { batch_size: 25, batches_limit: 10 }); const rep = old.replicate.to(this.localDatabase, { batch_size: 25, batches_limit: 10 });
rep.on("change", (e) => { rep.on("change", (e) => {
progress.setMessage(`Converting ${e.docs_written} docs...`); Logger(`Converting ${e.docs_written} docs...`, LOG_LEVEL.NOTICE, "conv");
Logger(`Converting ${e.docs_written} docs...`, LOG_LEVEL.VERBOSE);
}); });
const w = await rep; const w = await rep;
progress.hide();
if (w.ok) { if (w.ok) {
Logger("Conversion completed!", LOG_LEVEL.NOTICE); Logger("Conversion completed!", LOG_LEVEL.NOTICE, "conv");
old.destroy(); // delete the old database. old.destroy(); // delete the old database.
this.isReady = true; this.isReady = true;
return await nextSeq(); return await nextSeq();
@@ -258,8 +221,7 @@ export class LocalPouchDB {
throw new Error("Conversion failed!"); throw new Error("Conversion failed!");
} }
} catch (ex) { } catch (ex) {
progress.hide(); Logger("Conversion failed!, If you are fully synchronized, please drop the old database in the Hatch pane in setting dialog. or please make an issue on Github.", LOG_LEVEL.NOTICE, "conv");
Logger("Conversion failed!, If you are fully synchronized, please drop the old database in the Hatch pane in setting dialog. or please make an issue on Github.", LOG_LEVEL.NOTICE);
Logger(ex); Logger(ex);
this.isReady = false; this.isReady = false;
return false; return false;
@@ -271,9 +233,8 @@ export class LocalPouchDB {
async prepareHashFunctions() { async prepareHashFunctions() {
if (this.h32 != null) return; if (this.h32 != null) return;
const { h32, h64, h32Raw } = await xxhash(); const { h32, h32Raw } = await xxhash();
this.h32 = h32; this.h32 = h32;
this.h64 = h64;
this.h32Raw = h32Raw; this.h32Raw = h32Raw;
} }
@@ -303,7 +264,6 @@ export class LocalPouchDB {
} }
async getDBLeaf(id: string, waitForReady: boolean): Promise<string> { async getDBLeaf(id: string, waitForReady: boolean): Promise<string> {
await this.waitForGCComplete();
// when in cache, use that. // when in cache, use that.
const leaf = this.hashCaches.revGet(id); const leaf = this.hashCaches.revGet(id);
if (leaf) { if (leaf) {
@@ -336,7 +296,6 @@ export class LocalPouchDB {
} }
async getDBEntryMeta(path: string, opt?: PouchDB.Core.GetOptions): Promise<false | LoadedEntry> { async getDBEntryMeta(path: string, opt?: PouchDB.Core.GetOptions): Promise<false | LoadedEntry> {
await this.waitForGCComplete();
const id = path2id(path); const id = path2id(path);
try { try {
let obj: EntryDocResponse = null; let obj: EntryDocResponse = null;
@@ -355,8 +314,10 @@ export class LocalPouchDB {
if (!obj.type || (obj.type && obj.type == "notes") || obj.type == "newnote" || obj.type == "plain") { if (!obj.type || (obj.type && obj.type == "notes") || obj.type == "newnote" || obj.type == "plain") {
const note = obj as Entry; const note = obj as Entry;
let children: string[] = []; let children: string[] = [];
let type: "plain" | "newnote" = "plain";
if (obj.type == "newnote" || obj.type == "plain") { if (obj.type == "newnote" || obj.type == "plain") {
children = obj.children; children = obj.children;
type = obj.type;
} }
const doc: LoadedEntry & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = { const doc: LoadedEntry & PouchDB.Core.IdMeta & PouchDB.Core.GetMeta = {
data: "", data: "",
@@ -368,7 +329,7 @@ export class LocalPouchDB {
_rev: obj._rev, _rev: obj._rev,
_conflicts: obj._conflicts, _conflicts: obj._conflicts,
children: children, children: children,
datatype: "newnote", datatype: type,
}; };
return doc; return doc;
} }
@@ -381,7 +342,6 @@ export class LocalPouchDB {
return false; return false;
} }
async getDBEntry(path: string, opt?: PouchDB.Core.GetOptions, dump = false, waitForReady = true): Promise<false | LoadedEntry> { async getDBEntry(path: string, opt?: PouchDB.Core.GetOptions, dump = false, waitForReady = true): Promise<false | LoadedEntry> {
await this.waitForGCComplete();
const id = path2id(path); const id = path2id(path);
try { try {
let obj: EntryDocResponse = null; let obj: EntryDocResponse = null;
@@ -481,7 +441,6 @@ export class LocalPouchDB {
return false; return false;
} }
async deleteDBEntry(path: string, opt?: PouchDB.Core.GetOptions): Promise<boolean> { async deleteDBEntry(path: string, opt?: PouchDB.Core.GetOptions): Promise<boolean> {
await this.waitForGCComplete();
const id = path2id(path); const id = path2id(path);
try { try {
@@ -528,7 +487,6 @@ export class LocalPouchDB {
} }
} }
async deleteDBEntryPrefix(prefixSrc: string): Promise<boolean> { async deleteDBEntryPrefix(prefixSrc: string): Promise<boolean> {
await this.waitForGCComplete();
// delete database entries by prefix. // delete database entries by prefix.
// it called from folder deletion. // it called from folder deletion.
let c = 0; let c = 0;
@@ -579,8 +537,7 @@ export class LocalPouchDB {
Logger(`deleteDBEntryPrefix:deleted ${deleteCount} items, skipped ${notfound}`); Logger(`deleteDBEntryPrefix:deleted ${deleteCount} items, skipped ${notfound}`);
return true; return true;
} }
async putDBEntry(note: LoadedEntry) { async putDBEntry(note: LoadedEntry, saveAsBigChunk?: boolean) {
await this.waitForGCComplete();
// let leftData = note.data; // let leftData = note.data;
const savenNotes = []; const savenNotes = [];
let processed = 0; let processed = 0;
@@ -590,7 +547,7 @@ export class LocalPouchDB {
let plainSplit = false; let plainSplit = false;
let cacheUsed = 0; let cacheUsed = 0;
const userpasswordHash = this.h32Raw(new TextEncoder().encode(this.settings.passphrase)); const userpasswordHash = this.h32Raw(new TextEncoder().encode(this.settings.passphrase));
if (shouldSplitAsPlainText(note._id)) { if (!saveAsBigChunk && shouldSplitAsPlainText(note._id)) {
pieceSize = MAX_DOC_SIZE; pieceSize = MAX_DOC_SIZE;
plainSplit = true; plainSplit = true;
} }
@@ -696,9 +653,6 @@ export class LocalPouchDB {
} }
} }
} }
if (saved) {
Logger(`Chunk saved:${newLeafs.length} chunks`);
}
} catch (ex) { } catch (ex) {
Logger("Chunk save failed:", LOG_LEVEL.NOTICE); Logger("Chunk save failed:", LOG_LEVEL.NOTICE);
Logger(ex, LOG_LEVEL.NOTICE); Logger(ex, LOG_LEVEL.NOTICE);
@@ -706,15 +660,14 @@ export class LocalPouchDB {
} }
} }
if (saved) { if (saved) {
Logger(`note content saven, pieces:${processed} new:${made}, skip:${skiped}, cache:${cacheUsed}`); Logger(`Content saved:${note._id} ,pieces:${processed} (new:${made}, skip:${skiped}, cache:${cacheUsed})`);
const newDoc: PlainEntry | NewEntry = { const newDoc: PlainEntry | NewEntry = {
NewNote: true,
children: savenNotes, children: savenNotes,
_id: note._id, _id: note._id,
ctime: note.ctime, ctime: note.ctime,
mtime: note.mtime, mtime: note.mtime,
size: note.size, size: note.size,
type: plainSplit ? "plain" : "newnote", type: note.datatype,
}; };
// Here for upsert logic, // Here for upsert logic,
await runWithLock("file:" + newDoc._id, false, async () => { await runWithLock("file:" + newDoc._id, false, async () => {
@@ -760,19 +713,17 @@ export class LocalPouchDB {
return true; return true;
} }
replicateAllToServer(setting: RemoteDBSettings, showingNotice?: boolean) { replicateAllToServer(setting: RemoteDBSettings, showingNotice?: boolean) {
return new Promise(async (res, rej) => { return new Promise((res, rej) => {
await this.waitForGCComplete();
this.openOneshotReplication( this.openOneshotReplication(
setting, setting,
showingNotice, showingNotice,
async (e) => {}, async (e) => { },
false, false,
(e) => { (e) => {
if (e === true) res(e); if (e === true) res(e);
rej(e); rej(e);
}, },
true, "pushOnly"
false
); );
}); });
} }
@@ -783,9 +734,8 @@ export class LocalPouchDB {
return false; return false;
} }
await this.waitForGCComplete();
if (setting.versionUpFlash != "") { if (setting.versionUpFlash != "") {
NewNotice("Open settings and check message, please."); Logger("Open settings and check message, please.", LOG_LEVEL.NOTICE);
return false; return false;
} }
const uri = setting.couchDB_URI + (setting.couchDB_DBNAME == "" ? "" : "/" + setting.couchDB_DBNAME); const uri = setting.couchDB_URI + (setting.couchDB_DBNAME == "" ? "" : "/" + setting.couchDB_DBNAME);
@@ -812,19 +762,38 @@ export class LocalPouchDB {
created: (new Date() as any) / 1, created: (new Date() as any) / 1,
locked: false, locked: false,
accepted_nodes: [this.nodeid], accepted_nodes: [this.nodeid],
node_chunk_info: { [this.nodeid]: currentVersionRange }
}; };
const remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defMilestonePoint); const remoteMilestone: EntryMilestoneInfo = { ...defMilestonePoint, ...(await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defMilestonePoint)) };
remoteMilestone.node_chunk_info = { ...defMilestonePoint.node_chunk_info, ...remoteMilestone.node_chunk_info };
this.remoteLocked = remoteMilestone.locked; this.remoteLocked = remoteMilestone.locked;
this.remoteLockedAndDeviceNotAccepted = remoteMilestone.locked && remoteMilestone.accepted_nodes.indexOf(this.nodeid) == -1; this.remoteLockedAndDeviceNotAccepted = remoteMilestone.locked && remoteMilestone.accepted_nodes.indexOf(this.nodeid) == -1;
const writeMilestone = ((remoteMilestone.node_chunk_info[this.nodeid].min != currentVersionRange.min || remoteMilestone.node_chunk_info[this.nodeid].max != currentVersionRange.max)
|| typeof remoteMilestone._rev == "undefined");
if (writeMilestone) {
await dbret.db.put(remoteMilestone);
}
let globalMin = currentVersionRange.min;
let globalMax = currentVersionRange.max;
for (const nodeid of remoteMilestone.accepted_nodes) {
if (nodeid in remoteMilestone.node_chunk_info) {
const nodeinfo = remoteMilestone.node_chunk_info[nodeid];
globalMin = Math.max(nodeinfo.min, globalMin);
globalMax = Math.min(nodeinfo.max, globalMax);
} else {
globalMin = 0;
globalMax = 0;
}
}
//If globalMin and globalMax is suitable, we can upgrade.
if (remoteMilestone.locked && remoteMilestone.accepted_nodes.indexOf(this.nodeid) == -1) { if (remoteMilestone.locked && remoteMilestone.accepted_nodes.indexOf(this.nodeid) == -1) {
Logger("Remote database marked as 'Auto Sync Locked'. And this devide does not marked as resolved device. see settings dialog.", LOG_LEVEL.NOTICE); Logger("The remote database has been rebuilt or corrupted since we have synchronized last time. Fetch rebuilt DB or explicit unlocking is required. See the settings dialog.", LOG_LEVEL.NOTICE);
return false; return false;
} }
if (typeof remoteMilestone._rev == "undefined") {
await dbret.db.put(remoteMilestone);
}
} }
const syncOptionBase: PouchDB.Replication.SyncOptions = { const syncOptionBase: PouchDB.Replication.SyncOptions = {
batches_limit: setting.batches_limit, batches_limit: setting.batches_limit,
@@ -839,59 +808,54 @@ export class LocalPouchDB {
if (keepAlive) { if (keepAlive) {
this.openContinuousReplication(setting, showResult, callback, false); this.openContinuousReplication(setting, showResult, callback, false);
} else { } else {
this.openOneshotReplication(setting, showResult, callback, false, null, false, false); this.openOneshotReplication(setting, showResult, callback, false, null, "sync");
} }
} }
replicationActivated(notice: WrappedNotice) { replicationActivated(showResult: boolean) {
this.syncStatus = "CONNECTED"; this.syncStatus = "CONNECTED";
this.updateInfo(); this.updateInfo();
Logger("Replication activated"); Logger("Replication activated", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO, "sync");
if (notice != null) notice.setMessage(`Activated..`);
} }
async replicationChangeDetected(e: PouchDB.Replication.SyncResult<EntryDoc>, notice: WrappedNotice, docSentOnStart: number, docArrivedOnStart: number, callback: ReplicationCallback) { async replicationChangeDetected(e: PouchDB.Replication.SyncResult<EntryDoc>, showResult: boolean, docSentOnStart: number, docArrivedOnStart: number, callback: ReplicationCallback) {
try { try {
if (e.direction == "pull") { if (e.direction == "pull") {
await callback(e.change.docs); await callback(e.change.docs);
Logger(`replicated ${e.change.docs_read} doc(s)`);
this.docArrived += e.change.docs.length; this.docArrived += e.change.docs.length;
} else { } else {
this.docSent += e.change.docs.length; this.docSent += e.change.docs.length;
} }
if (notice != null) { if (showResult) {
notice.setMessage(`↑${this.docSent - docSentOnStart} ↓${this.docArrived - docArrivedOnStart}`); Logger(`↑${this.docSent - docSentOnStart} ↓${this.docArrived - docArrivedOnStart}`, LOG_LEVEL.NOTICE, "sync");
} }
this.updateInfo(); this.updateInfo();
} catch (ex) { } catch (ex) {
Logger("Replication callback error", LOG_LEVEL.NOTICE); Logger("Replication callback error", LOG_LEVEL.NOTICE, "sync");
Logger(ex, LOG_LEVEL.NOTICE); Logger(ex, LOG_LEVEL.NOTICE);
// //
} }
} }
replicationCompleted(notice: WrappedNotice, showResult: boolean) { replicationCompleted(showResult: boolean) {
this.syncStatus = "COMPLETED"; this.syncStatus = "COMPLETED";
this.updateInfo(); this.updateInfo();
Logger("Replication completed", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO); Logger("Replication completed", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO, showResult ? "sync" : "");
if (notice != null) notice.hide();
this.syncHandler = this.cancelHandler(this.syncHandler); this.syncHandler = this.cancelHandler(this.syncHandler);
} }
replicationDeniend(notice: WrappedNotice, e: any) { replicationDeniend(e: any) {
this.syncStatus = "ERRORED"; this.syncStatus = "ERRORED";
this.updateInfo(); this.updateInfo();
this.syncHandler = this.cancelHandler(this.syncHandler); this.syncHandler = this.cancelHandler(this.syncHandler);
if (notice != null) notice.hide(); Logger("Replication denied", LOG_LEVEL.NOTICE, "sync");
Logger("Replication denied", LOG_LEVEL.NOTICE);
Logger(e); Logger(e);
} }
replicationErrored(notice: WrappedNotice, e: any) { replicationErrored(e: any) {
this.syncStatus = "ERRORED"; this.syncStatus = "ERRORED";
this.syncHandler = this.cancelHandler(this.syncHandler); this.syncHandler = this.cancelHandler(this.syncHandler);
this.updateInfo(); this.updateInfo();
} }
replicationPaused(notice: WrappedNotice) { replicationPaused() {
this.syncStatus = "PAUSED"; this.syncStatus = "PAUSED";
this.updateInfo(); this.updateInfo();
if (notice != null) notice.hide(); Logger("replication paused", LOG_LEVEL.VERBOSE, "sync");
Logger("replication paused", LOG_LEVEL.VERBOSE);
} }
async openOneshotReplication( async openOneshotReplication(
@@ -900,23 +864,21 @@ export class LocalPouchDB {
callback: (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => Promise<void>, callback: (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => Promise<void>,
retrying: boolean, retrying: boolean,
callbackDone: (e: boolean | any) => void, callbackDone: (e: boolean | any) => void,
pushOnly: boolean, syncmode: "sync" | "pullOnly" | "pushOnly"
pullOnly: boolean
): Promise<boolean> { ): Promise<boolean> {
if (this.syncHandler != null) { if (this.syncHandler != null) {
Logger("Replication is already in progress.", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO); Logger("Replication is already in progress.", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO, "sync");
return; return;
} }
Logger("Oneshot Sync begin..."); Logger(`Oneshot Sync begin... (${syncmode})`);
let thisCallback = callbackDone; let thisCallback = callbackDone;
const ret = await this.checkReplicationConnectivity(setting, true, retrying, showResult); const ret = await this.checkReplicationConnectivity(setting, true, retrying, showResult);
let notice: WrappedNotice = null;
if (ret === false) { if (ret === false) {
Logger("Could not connect to server.", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO); Logger("Could not connect to server.", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO, "sync");
return; return;
} }
if (showResult) { if (showResult) {
notice = NewNotice("Looking for the point last synchronized point.", 0); Logger("Looking for the point last synchronized point.", LOG_LEVEL.NOTICE, "sync");
} }
const { db, syncOptionBase } = ret; const { db, syncOptionBase } = ret;
this.syncStatus = "STARTED"; this.syncStatus = "STARTED";
@@ -928,52 +890,61 @@ export class LocalPouchDB {
this.originalSetting = setting; this.originalSetting = setting;
} }
this.syncHandler = this.cancelHandler(this.syncHandler); this.syncHandler = this.cancelHandler(this.syncHandler);
if (!pushOnly && !pullOnly) { if (syncmode == "sync") {
this.syncHandler = this.localDatabase.sync(db, { checkpoint: "target", ...syncOptionBase }); this.syncHandler = this.localDatabase.sync(db, { checkpoint: "target", ...syncOptionBase });
this.syncHandler this.syncHandler
.on("change", async (e) => { .on("change", async (e) => {
await this.replicationChangeDetected(e, notice, docSentOnStart, docArrivedOnStart, callback); await this.replicationChangeDetected(e, showResult, docSentOnStart, docArrivedOnStart, callback);
if (retrying) { if (retrying) {
if (this.docSent - docSentOnStart + (this.docArrived - docArrivedOnStart) > this.originalSetting.batch_size * 2) { if (this.docSent - docSentOnStart + (this.docArrived - docArrivedOnStart) > this.originalSetting.batch_size * 2) {
// restore configration. // restore configration.
Logger("Back into original settings once."); Logger("Back into original settings once.");
if (notice != null) notice.hide();
this.syncHandler = this.cancelHandler(this.syncHandler); this.syncHandler = this.cancelHandler(this.syncHandler);
this.openOneshotReplication(this.originalSetting, showResult, callback, false, callbackDone, pushOnly, pullOnly); this.openOneshotReplication(this.originalSetting, showResult, callback, false, callbackDone, syncmode);
} }
} }
}) })
.on("complete", (e) => { .on("complete", (e) => {
this.replicationCompleted(notice, showResult); this.replicationCompleted(showResult);
if (thisCallback != null) { if (thisCallback != null) {
thisCallback(true); thisCallback(true);
} }
}); });
} else if (pullOnly) { } else if (syncmode == "pullOnly") {
this.syncHandler = this.localDatabase.replicate.to(db, { checkpoint: "target", ...syncOptionBase }); this.syncHandler = this.localDatabase.replicate.from(db, { checkpoint: "target", ...syncOptionBase });
this.syncHandler this.syncHandler
.on("change", async (e) => { .on("change", async (e) => {
await this.replicationChangeDetected({ direction: "pull", change: e }, notice, docSentOnStart, docArrivedOnStart, callback); await this.replicationChangeDetected({ direction: "pull", change: e }, showResult, docSentOnStart, docArrivedOnStart, callback);
if (retrying) { if (retrying) {
if (this.docSent - docSentOnStart + (this.docArrived - docArrivedOnStart) > this.originalSetting.batch_size * 2) { if (this.docSent - docSentOnStart + (this.docArrived - docArrivedOnStart) > this.originalSetting.batch_size * 2) {
// restore configration. // restore configration.
Logger("Back into original settings once."); Logger("Back into original settings once.");
if (notice != null) notice.hide();
this.syncHandler = this.cancelHandler(this.syncHandler); this.syncHandler = this.cancelHandler(this.syncHandler);
this.openOneshotReplication(this.originalSetting, showResult, callback, false, callbackDone, pushOnly, pullOnly); this.openOneshotReplication(this.originalSetting, showResult, callback, false, callbackDone, syncmode);
} }
} }
}) })
.on("complete", (e) => { .on("complete", (e) => {
this.replicationCompleted(notice, showResult); this.replicationCompleted(showResult);
if (thisCallback != null) { if (thisCallback != null) {
thisCallback(true); thisCallback(true);
} }
}); });
} else if (pushOnly) { } else if (syncmode == "pushOnly") {
this.syncHandler = this.localDatabase.replicate.to(db, { checkpoint: "target", ...syncOptionBase }); this.syncHandler = this.localDatabase.replicate.to(db, { checkpoint: "target", ...syncOptionBase });
this.syncHandler.on("change", async (e) => {
await this.replicationChangeDetected({ direction: "push", change: e }, showResult, docSentOnStart, docArrivedOnStart, callback);
if (retrying) {
if (this.docSent - docSentOnStart + (this.docArrived - docArrivedOnStart) > this.originalSetting.batch_size * 2) {
// restore configration.
Logger("Back into original settings once.");
this.syncHandler = this.cancelHandler(this.syncHandler);
this.openOneshotReplication(this.originalSetting, showResult, callback, false, callbackDone, syncmode);
}
}
})
this.syncHandler.on("complete", (e) => { this.syncHandler.on("complete", (e) => {
this.replicationCompleted(notice, showResult); this.replicationCompleted(showResult);
if (thisCallback != null) { if (thisCallback != null) {
thisCallback(true); thisCallback(true);
} }
@@ -981,17 +952,16 @@ export class LocalPouchDB {
} }
this.syncHandler this.syncHandler
.on("active", () => this.replicationActivated(notice)) .on("active", () => this.replicationActivated(showResult))
.on("denied", (e) => { .on("denied", (e) => {
this.replicationDeniend(notice, e); this.replicationDeniend(e);
if (thisCallback != null) { if (thisCallback != null) {
thisCallback(e); thisCallback(e);
} }
}) })
.on("error", (e) => { .on("error", (e) => {
this.replicationErrored(notice, e); this.replicationErrored(e);
Logger("Replication stopped.", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO); Logger("Replication stopped.", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO, "sync");
if (notice != null) notice.hide();
if (getLastPostFailedBySize()) { if (getLastPostFailedBySize()) {
// Duplicate settings for smaller batch. // Duplicate settings for smaller batch.
const xsetting: RemoteDBSettings = JSON.parse(JSON.stringify(setting)); const xsetting: RemoteDBSettings = JSON.parse(JSON.stringify(setting));
@@ -1002,17 +972,19 @@ export class LocalPouchDB {
} else { } else {
Logger(`Retry with lower batch size:${xsetting.batch_size}/${xsetting.batches_limit}`, showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO); Logger(`Retry with lower batch size:${xsetting.batch_size}/${xsetting.batches_limit}`, showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO);
thisCallback = null; thisCallback = null;
this.openOneshotReplication(xsetting, showResult, callback, true, callbackDone, pushOnly, pullOnly); this.openOneshotReplication(xsetting, showResult, callback, true, callbackDone, syncmode);
} }
} else { } else {
Logger("Replication error", LOG_LEVEL.NOTICE); Logger("Replication error", LOG_LEVEL.NOTICE, "sync");
Logger(e); Logger(e);
} }
if (thisCallback != null) { if (thisCallback != null) {
thisCallback(e); thisCallback(e);
} }
}) })
.on("paused", (e) => this.replicationPaused(notice)); .on("paused", (e) => this.replicationPaused());
await this.syncHandler;
} }
openContinuousReplication(setting: RemoteDBSettings, showResult: boolean, callback: (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => Promise<void>, retrying: boolean) { openContinuousReplication(setting: RemoteDBSettings, showResult: boolean, callback: (e: PouchDB.Core.ExistingDocument<EntryDoc>[]) => Promise<void>, retrying: boolean) {
@@ -1029,13 +1001,12 @@ export class LocalPouchDB {
async () => { async () => {
Logger("LiveSync begin..."); Logger("LiveSync begin...");
const ret = await this.checkReplicationConnectivity(setting, true, true, showResult); const ret = await this.checkReplicationConnectivity(setting, true, true, showResult);
let notice: WrappedNotice = null;
if (ret === false) { if (ret === false) {
Logger("Could not connect to server.", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO); Logger("Could not connect to server.", showResult ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO);
return; return;
} }
if (showResult) { if (showResult) {
notice = NewNotice("Looking for the point last synchronized point.", 0); Logger("Looking for the point last synchronized point.", LOG_LEVEL.NOTICE, "sync");
} }
const { db, syncOption } = ret; const { db, syncOption } = ret;
this.syncStatus = "STARTED"; this.syncStatus = "STARTED";
@@ -1057,29 +1028,27 @@ export class LocalPouchDB {
}, },
}); });
this.syncHandler this.syncHandler
.on("active", () => this.replicationActivated(notice)) .on("active", () => this.replicationActivated(showResult))
.on("change", async (e) => { .on("change", async (e) => {
await this.replicationChangeDetected(e, notice, docSentOnStart, docArrivedOnStart, callback); await this.replicationChangeDetected(e, showResult, docSentOnStart, docArrivedOnStart, callback);
if (retrying) { if (retrying) {
if (this.docSent - docSentOnStart + (this.docArrived - docArrivedOnStart) > this.originalSetting.batch_size * 2) { if (this.docSent - docSentOnStart + (this.docArrived - docArrivedOnStart) > this.originalSetting.batch_size * 2) {
// restore sync values // restore sync values
Logger("Back into original settings once."); Logger("Back into original settings once.");
if (notice != null) notice.hide();
this.syncHandler = this.cancelHandler(this.syncHandler); this.syncHandler = this.cancelHandler(this.syncHandler);
this.openContinuousReplication(this.originalSetting, showResult, callback, false); this.openContinuousReplication(this.originalSetting, showResult, callback, false);
} }
} }
}) })
.on("complete", (e) => this.replicationCompleted(notice, showResult)) .on("complete", (e) => this.replicationCompleted(showResult))
.on("denied", (e) => this.replicationDeniend(notice, e)) .on("denied", (e) => this.replicationDeniend(e))
.on("error", (e) => { .on("error", (e) => {
this.replicationErrored(notice, e); this.replicationErrored(e);
Logger("Replication stopped.", LOG_LEVEL.NOTICE); Logger("Replication stopped.", LOG_LEVEL.NOTICE, "sync");
}) })
.on("paused", (e) => this.replicationPaused(notice)); .on("paused", (e) => this.replicationPaused());
}, },
false, "pullOnly"
true
); );
} }
@@ -1096,24 +1065,24 @@ export class LocalPouchDB {
const oldDB = await this.isOldDatabaseExists(); const oldDB = await this.isOldDatabaseExists();
if (oldDB) { if (oldDB) {
oldDB.destroy(); oldDB.destroy();
NewNotice("Deleted! Please re-launch obsidian.", LOG_LEVEL.NOTICE); Logger("Deleted! Please re-launch obsidian.", LOG_LEVEL.NOTICE);
} else { } else {
NewNotice("Old database is not exist.", LOG_LEVEL.NOTICE); Logger("Old database is not exist.", LOG_LEVEL.NOTICE);
} }
} }
async resetDatabase() { async resetDatabase() {
await this.waitForGCComplete();
this.changeHandler = this.cancelHandler(this.changeHandler); this.changeHandler = this.cancelHandler(this.changeHandler);
await this.closeReplication(); this.closeReplication();
Logger("Database closed for reset Database."); Logger("Database closed for reset Database.");
this.isReady = false; this.isReady = false;
await this.localDatabase.destroy(); await this.localDatabase.destroy();
await this.kvDB.destroy();
this.localDatabase = null; this.localDatabase = null;
await this.initializeDatabase(); await this.initializeDatabase();
Logger("Local Database Reset", LOG_LEVEL.NOTICE); Logger("Local Database Reset", LOG_LEVEL.NOTICE);
} }
async tryResetRemoteDatabase(setting: RemoteDBSettings) { async tryResetRemoteDatabase(setting: RemoteDBSettings) {
await this.closeReplication(); this.closeReplication();
const con = await connectRemoteCouchDBWithSetting(setting, this.isMobile); const con = await connectRemoteCouchDBWithSetting(setting, this.isMobile);
if (typeof con == "string") return; if (typeof con == "string") return;
try { try {
@@ -1126,7 +1095,7 @@ export class LocalPouchDB {
} }
} }
async tryCreateRemoteDatabase(setting: RemoteDBSettings) { async tryCreateRemoteDatabase(setting: RemoteDBSettings) {
await this.closeReplication(); this.closeReplication();
const con2 = await connectRemoteCouchDBWithSetting(setting, this.isMobile); const con2 = await connectRemoteCouchDBWithSetting(setting, this.isMobile);
if (typeof con2 === "string") return; if (typeof con2 === "string") return;
@@ -1150,9 +1119,11 @@ export class LocalPouchDB {
created: (new Date() as any) / 1, created: (new Date() as any) / 1,
locked: locked, locked: locked,
accepted_nodes: [this.nodeid], accepted_nodes: [this.nodeid],
node_chunk_info: { [this.nodeid]: currentVersionRange }
}; };
const remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defInitPoint); const remoteMilestone: EntryMilestoneInfo = { ...defInitPoint, ...await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defInitPoint) };
remoteMilestone.node_chunk_info = { ...defInitPoint.node_chunk_info, ...remoteMilestone.node_chunk_info };
remoteMilestone.accepted_nodes = [this.nodeid]; remoteMilestone.accepted_nodes = [this.nodeid];
remoteMilestone.locked = locked; remoteMilestone.locked = locked;
if (locked) { if (locked) {
@@ -1180,22 +1151,15 @@ export class LocalPouchDB {
created: (new Date() as any) / 1, created: (new Date() as any) / 1,
locked: false, locked: false,
accepted_nodes: [this.nodeid], accepted_nodes: [this.nodeid],
node_chunk_info: { [this.nodeid]: currentVersionRange }
}; };
// check local database hash status and remote replicate hash status // check local database hash status and remote replicate hash status
const remoteMilestone: EntryMilestoneInfo = await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defInitPoint); const remoteMilestone: EntryMilestoneInfo = { ...defInitPoint, ...await resolveWithIgnoreKnownError(dbret.db.get(MILSTONE_DOCID), defInitPoint) };
// remoteMilestone.locked = false; remoteMilestone.node_chunk_info = { ...defInitPoint.node_chunk_info, ...remoteMilestone.node_chunk_info };
remoteMilestone.accepted_nodes = Array.from(new Set([...remoteMilestone.accepted_nodes, this.nodeid])); remoteMilestone.accepted_nodes = Array.from(new Set([...remoteMilestone.accepted_nodes, this.nodeid]));
// this.remoteLocked = false;
Logger("Mark this device as 'resolved'.", LOG_LEVEL.NOTICE); Logger("Mark this device as 'resolved'.", LOG_LEVEL.NOTICE);
await dbret.db.put(remoteMilestone); await dbret.db.put(remoteMilestone);
} }
gcRunning = false;
async waitForGCComplete() {
while (this.gcRunning) {
Logger("Waiting for Garbage Collection completed.");
await delay(1000);
}
}
async sanCheck(entry: EntryDoc): Promise<boolean> { async sanCheck(entry: EntryDoc): Promise<boolean> {
if (entry.type == "plain" || entry.type == "newnote") { if (entry.type == "plain" || entry.type == "newnote") {
const children = entry.children; const children = entry.children;
@@ -1215,159 +1179,59 @@ export class LocalPouchDB {
return false; return false;
} }
async garbageCollect() { garbageCheck() {
// if (this.settings.useHistory) { Logger(`Checking garbages`, LOG_LEVEL.NOTICE, "gc");
// Logger("GC skipped for using history", LOG_LEVEL.VERBOSE); let docNum = 0;
// return; const chunks = new Map<string, Set<string>>();
// } this.localDatabase
// NOTE:Garbage collection could break old revisions. .changes({
await runWithLock("replicate", true, async () => { since: 0,
if (this.gcRunning) return; include_docs: true,
this.gcRunning = true; return_docs: false,
let idbGC: IDBPDatabase<{ id: string }> = null; style: "all_docs",
const storeIDB = "gc"; // selector:
const idbname = "idb-" + this.dbname + "-idb-gcx"; })
try { .on("change", (e) => {
const procAllDocs = async (getLeaf: boolean, startkey: string, endkey: string, callback: (idordoc: string[]) => Promise<void>) => { if (e.id.startsWith("h:")) {
let c = 0; const chunk = e.id;
let readCount = 0; let c = chunks.get(chunk);
do { if (c == null) c = new Set<string>();
const result = await this.localDatabase.allDocs({ include_docs: false, skip: c, limit: 2000, conflicts: !getLeaf, startkey: startkey, endkey: endkey }); chunks.set(chunk, c);
readCount = result.rows.length; } else if ("children" in e.doc) {
if (readCount > 0) { docNum++;
await callback(result.rows.map((e) => e.id)); if (docNum % 100 == 0) Logger(`Processing ${docNum}`, LOG_LEVEL.NOTICE, "gc");
if (!e.deleted) {
for (const chunk of e.doc.children) {
let c = chunks.get(chunk);
if (c == null) c = new Set<string>();
c.add(e.id);
chunks.set(chunk, c);
} }
c += readCount; } else {
} while (readCount != 0); for (const chunk of e.doc.children) {
}; let c = chunks.get(chunk);
if (c == null) c = new Set<string>();
// Delete working indexedDB once. c.delete(e.id);
chunks.set(chunk, c);
await deleteDB(idbname);
idbGC = await openDB(idbname, 1, {
upgrade(db) {
db.createObjectStore(storeIDB, { keyPath: "id" });
},
});
// Mark all chunks once.
await procAllDocs(true, "h:", "h_", async (docs) => {
Logger(`Chunks marked - :${docs.length}`);
const tx = idbGC.transaction(storeIDB, "readwrite");
const store = tx.objectStore(storeIDB);
for (const docId of docs) {
await store.put({ id: docId });
}
await tx.done;
});
Logger("All chunks are marked once");
const unmarkUsedByHashId = async (doc: EntryDoc) => {
if ("children" in doc) {
const tx = idbGC.transaction(storeIDB, "readwrite");
const store = tx.objectStore(storeIDB);
for (const hashId of doc.children) {
await store.delete(hashId);
}
await tx.done;
}
};
Logger("Processing existen docs");
let procDocs = 0;
await procAllDocs(false, null, null, async (doc) => {
const docIds = (doc as string[]).filter((e) => !e.startsWith("h:") && !e.startsWith("ps:"));
for (const docId of docIds) {
procDocs++;
if (procDocs % 25 == 0) Logger(`${procDocs} Processed`);
const docT = await this.localDatabase.get(docId, { revs_info: true });
if (docT._deleted) continue;
// Unmark about latest doc.
unmarkUsedByHashId(docT);
const revs = docT._revs_info;
// Unmark old revisions
for (const rev of revs) {
if (rev.status != "available") continue;
const docRev = await this.localDatabase.get(docId, { rev: rev.rev });
unmarkUsedByHashId(docRev);
if (docRev._conflicts) {
// Unmark the conflicted chunks of old revisions.
for (const cid of docRev._conflicts) {
const docConflict = await this.localDatabase.get<EntryDoc>(docId, { rev: cid });
unmarkUsedByHashId(docConflict);
}
}
}
// Unmark the conflicted chunk.
if (docT._conflicts) {
for (const cid of docT._conflicts) {
const docConflict = await this.localDatabase.get<EntryDoc>(docId, { rev: cid });
unmarkUsedByHashId(docConflict);
}
} }
} }
});
// All marked chunks could be deleted.
Logger("Delete non-used chunks");
let dataLeft = false;
let chunkKeys: string[] = [];
let totalDelCount = 0;
do {
const tx = idbGC.transaction(storeIDB, "readonly");
const store = tx.objectStore(storeIDB);
let cursor = await store.openCursor();
if (cursor == null) break;
const maxconcurrentDocs = 10;
let delChunkCount = 0;
do {
// console.log(cursor.key, cursor.value);
if (cursor) {
chunkKeys.push(cursor.key as string);
delChunkCount++;
dataLeft = true;
} else {
dataLeft = false;
}
cursor = await cursor.continue();
} while (cursor && dataLeft && delChunkCount < maxconcurrentDocs);
// if (chunkKeys.length > 0) {
totalDelCount += delChunkCount;
const delDocResult = await this.localDatabase.allDocs({ keys: chunkKeys, include_docs: true });
const delDocs = delDocResult.rows.map((e) => ({ ...e.doc, _deleted: true }));
await this.localDatabase.bulkDocs(delDocs);
Logger(`deleted from pouchdb:${delDocs.length}`);
const tx2 = idbGC.transaction(storeIDB, "readwrite");
const store2 = tx2.objectStore(storeIDB);
for (const doc of chunkKeys) {
await store2.delete(doc);
}
Logger(`deleted from workspace:${chunkKeys.length}`);
await tx2.done;
// }
chunkKeys = [];
} while (dataLeft);
Logger(`Deleted ${totalDelCount} chunks`);
Logger("Teardown the database");
if (idbGC != null) {
idbGC.close();
idbGC = null;
} }
await deleteDB(idbname); })
this.gcRunning = false; .on("complete", (v) => {
Logger("Done"); // console.dir(chunks);
} catch (ex) {
Logger("Error on garbage collection"); let alive = 0;
Logger(ex); let nonref = 0;
} finally { for (const chunk of chunks) {
if (idbGC != null) { const items = chunk[1];
idbGC.close(); if (items.size == 0) {
nonref++;
} else {
alive++;
}
} }
await deleteDB(idbname); Logger(`Garbage checking completed, documents:${docNum}. Used chunks:${alive}, Retained chunks:${nonref}. Retained chunks will be reused, but you can rebuild database if you feel there are too much.`, LOG_LEVEL.NOTICE, "gc");
this.gcRunning = false; });
} return;
});
} }
} }

259
src/ObsidianLiveSyncSettingTab.ts
View File

@@ -1,10 +1,10 @@
import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, RequestUrlParam, requestUrl } from "obsidian"; import { App, PluginSettingTab, Setting, sanitizeHTMLToDom, RequestUrlParam, requestUrl } from "obsidian";
import { EntryDoc, LOG_LEVEL, RemoteDBSettings } from "./lib/src/types"; import { EntryDoc, LOG_LEVEL, RemoteDBSettings } from "./lib/src/types";
import { path2id, id2path } from "./utils"; import { path2id, id2path } from "./utils";
import { NewNotice, runWithLock } from "./lib/src/utils"; import { delay, runWithLock } from "./lib/src/utils";
import { Logger } from "./lib/src/logger"; import { Logger } from "./lib/src/logger";
import { checkSyncInfo, connectRemoteCouchDBWithSetting } from "./utils_couchdb"; import { checkSyncInfo, connectRemoteCouchDBWithSetting } from "./utils_couchdb";
import { testCrypt } from "./lib/src/e2ee"; import { testCrypt } from "./lib/src/e2ee_v2";
import ObsidianLiveSyncPlugin from "./main"; import ObsidianLiveSyncPlugin from "./main";
export class ObsidianLiveSyncSettingTab extends PluginSettingTab { export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
@@ -15,15 +15,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
this.plugin = plugin; this.plugin = plugin;
} }
async testConnection(): Promise<void> { async testConnection(): Promise<void> {
// const db = await connectRemoteCouchDB(
// this.plugin.settings.couchDB_URI + (this.plugin.settings.couchDB_DBNAME == "" ? "" : "/" + this.plugin.settings.couchDB_DBNAME),
// {
// username: this.plugin.settings.couchDB_USER,
// password: this.plugin.settings.couchDB_PASSWORD,
// },
// this.plugin.settings.disableRequestURI,
// this.plugin.settings.encrypt ? this.plugin.settings.passphrase : this.plugin.settings.encrypt
// );
const db = await connectRemoteCouchDBWithSetting(this.plugin.settings, this.plugin.localDatabase.isMobile); const db = await connectRemoteCouchDBWithSetting(this.plugin.settings, this.plugin.localDatabase.isMobile);
if (typeof db === "string") { if (typeof db === "string") {
this.plugin.addLog(`could not connect to ${this.plugin.settings.couchDB_URI} : ${this.plugin.settings.couchDB_DBNAME} \n(${db})`, LOG_LEVEL.NOTICE); this.plugin.addLog(`could not connect to ${this.plugin.settings.couchDB_URI} : ${this.plugin.settings.couchDB_DBNAME} \n(${db})`, LOG_LEVEL.NOTICE);
@@ -174,21 +165,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}) })
) )
// new Setting(containerRemoteDatabaseEl)
// .setDesc("This feature is locked in mobile")
// .setName("Use the old connecting method")
// .addToggle((toggle) => {
// toggle.setValue(this.plugin.settings.disableRequestURI).onChange(async (value) => {
// this.plugin.settings.disableRequestURI = value;
// await this.plugin.saveSettings();
// });
// toggle.setDisabled(this.plugin.isMobile);
// return toggle;
// })
); );
new Setting(containerRemoteDatabaseEl) new Setting(containerRemoteDatabaseEl)
.setName("End to End Encryption") .setName("End to End Encryption")
.setDesc("Encrypting contents on the database.") .setDesc("Encrypt contents on the remote database. If you use the plugins synchronizing feature, enabling this is recommend.")
.addToggle((toggle) => .addToggle((toggle) =>
toggle.setValue(this.plugin.settings.workingEncrypt).onChange(async (value) => { toggle.setValue(this.plugin.settings.workingEncrypt).onChange(async (value) => {
this.plugin.settings.workingEncrypt = value; this.plugin.settings.workingEncrypt = value;
@@ -198,7 +178,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
); );
const phasspharase = new Setting(containerRemoteDatabaseEl) const phasspharase = new Setting(containerRemoteDatabaseEl)
.setName("Passphrase") .setName("Passphrase")
.setDesc("Encrypting passphrase") .setDesc("Encrypting passphrase. If you change the passphrase with existen database, overwriting remote database is strongly recommended.")
.addText((text) => { .addText((text) => {
text.setPlaceholder("") text.setPlaceholder("")
.setValue(this.plugin.settings.workingPassphrase) .setValue(this.plugin.settings.workingPassphrase)
@@ -209,9 +189,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
text.inputEl.setAttribute("type", "password"); text.inputEl.setAttribute("type", "password");
}); });
phasspharase.setDisabled(!this.plugin.settings.workingEncrypt); phasspharase.setDisabled(!this.plugin.settings.workingEncrypt);
containerRemoteDatabaseEl.createEl("div", {
text: "If you change the passphrase, rebuilding the remote database is required. Please press 'Apply and send'. Or, If you have configured it to connect to an existing database, click 'Just apply'.",
});
const checkWorkingPassphrase = async (): Promise<boolean> => { const checkWorkingPassphrase = async (): Promise<boolean> => {
const settingForCheck: RemoteDBSettings = { const settingForCheck: RemoteDBSettings = {
...this.plugin.settings, ...this.plugin.settings,
@@ -271,20 +248,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}; };
new Setting(containerRemoteDatabaseEl) new Setting(containerRemoteDatabaseEl)
.setName("Apply") .setName("Apply")
.setDesc("apply encryption settinngs, and re-initialize remote database") .setDesc("Apply encryption settinngs")
.addButton((button) => .addButton((button) =>
button button
.setButtonText("Apply and send") .setButtonText("Apply")
.setWarning()
.setDisabled(false)
.setClass("sls-btn-left")
.onClick(async () => {
await applyEncryption(true);
})
)
.addButton((button) =>
button
.setButtonText("Just apply")
.setWarning() .setWarning()
.setDisabled(false) .setDisabled(false)
.setClass("sls-btn-right") .setClass("sls-btn-right")
@@ -293,6 +260,66 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}) })
); );
const rebuildDB = async (method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice") => {
this.plugin.settings.liveSync = false;
this.plugin.settings.periodicReplication = false;
this.plugin.settings.syncOnSave = false;
this.plugin.settings.syncOnStart = false;
this.plugin.settings.syncOnFileOpen = false;
await this.plugin.saveSettings();
applyDisplayEnabled();
await delay(2000);
if (method == "localOnly") {
await this.plugin.resetLocalDatabase();
await this.plugin.markRemoteResolved();
await this.plugin.replicate(true);
}
if (method == "remoteOnly") {
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await this.plugin.replicateAllToServer(true);
}
if (method == "rebuildBothByThisDevice") {
await this.plugin.resetLocalDatabase();
await this.plugin.initializeDatabase(true);
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await this.plugin.replicateAllToServer(true);
}
}
new Setting(containerRemoteDatabaseEl)
.setName("Overwrite by local DB")
.setDesc("Overwrite remote database with local DB and passphrase.")
.addButton((button) =>
button
.setButtonText("Send")
.setWarning()
.setDisabled(false)
.setClass("sls-btn-left")
.onClick(async () => {
await rebuildDB("remoteOnly");
})
)
new Setting(containerRemoteDatabaseEl)
.setName("Rebuild")
.setDesc("Rebuild local and remote database with local files.")
.addButton((button) =>
button
.setButtonText("Rebuild")
.setWarning()
.setDisabled(false)
.setClass("sls-btn-left")
.onClick(async () => {
await rebuildDB("rebuildBothByThisDevice");
})
)
new Setting(containerRemoteDatabaseEl) new Setting(containerRemoteDatabaseEl)
.setName("Test Database Connection") .setName("Test Database Connection")
.setDesc("Open database connection. If the remote database is not found and you have the privilege to create a database, the database will be created.") .setDesc("Open database connection. If the remote database is not found and you have the privilege to create a database, the database will be created.")
@@ -473,6 +500,18 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
text: "", text: "",
}); });
new Setting(containerRemoteDatabaseEl)
.setName("Lock remote database")
.setDesc("Lock remote database to prevent synchronization with other devices.")
.addButton((button) =>
button
.setButtonText("Lock")
.setDisabled(false)
.setWarning()
.onClick(async () => {
await this.plugin.markRemoteLocked();
})
);
addScreenElement("0", containerRemoteDatabaseEl); addScreenElement("0", containerRemoteDatabaseEl);
const containerLocalDatabaseEl = containerEl.createDiv(); const containerLocalDatabaseEl = containerEl.createDiv();
containerLocalDatabaseEl.createEl("h3", { text: "Local Database configuration" }); containerLocalDatabaseEl.createEl("h3", { text: "Local Database configuration" });
@@ -492,31 +531,29 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}) })
); );
new Setting(containerLocalDatabaseEl) new Setting(containerLocalDatabaseEl).setName("Garbage check").addButton((button) =>
.setName("Auto Garbage Collection delay")
.setDesc("(seconds), if you set zero, you have to run manually.")
.addText((text) => {
text.setPlaceholder("")
.setValue(this.plugin.settings.gcDelay + "")
.onChange(async (value) => {
let v = Number(value);
if (isNaN(v) || v > 5000) {
v = 0;
}
this.plugin.settings.gcDelay = v;
await this.plugin.saveSettings();
});
text.inputEl.setAttribute("type", "number");
});
new Setting(containerLocalDatabaseEl).setName("Manual Garbage Collect").addButton((button) =>
button button
.setButtonText("Collect now") .setButtonText("Check now")
.setDisabled(false) .setDisabled(false)
.onClick(async () => { .onClick(async () => {
await this.plugin.garbageCollect(); await this.plugin.garbageCheck();
}) })
); );
new Setting(containerLocalDatabaseEl)
.setName("Fetch rebuilt DB")
.setDesc("Restore or reconstruct local database from remote database.")
.addButton((button) =>
button
.setButtonText("Fetch")
.setWarning()
.setDisabled(false)
.setClass("sls-btn-left")
.onClick(async () => {
await rebuildDB("localOnly");
})
)
containerLocalDatabaseEl.createEl("div", { containerLocalDatabaseEl.createEl("div", {
text: sanitizeHTMLToDom(`Advanced settings<br> text: sanitizeHTMLToDom(`Advanced settings<br>
Configuration of how LiveSync makes chunks from the file.`), Configuration of how LiveSync makes chunks from the file.`),
@@ -710,15 +747,15 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}) })
); );
new Setting(containerSyncSettingEl) // new Setting(containerSyncSettingEl)
.setName("Skip old files on sync") // .setName("Skip old files on sync")
.setDesc("Skip old incoming if incoming changes older than storage.") // .setDesc("Skip old incoming if incoming changes older than storage.")
.addToggle((toggle) => // .addToggle((toggle) =>
toggle.setValue(this.plugin.settings.skipOlderFilesOnSync).onChange(async (value) => { // toggle.setValue(this.plugin.settings.skipOlderFilesOnSync).onChange(async (value) => {
this.plugin.settings.skipOlderFilesOnSync = value; // this.plugin.settings.skipOlderFilesOnSync = value;
await this.plugin.saveSettings(); // await this.plugin.saveSettings();
}) // })
); // );
new Setting(containerSyncSettingEl) new Setting(containerSyncSettingEl)
.setName("Check conflict only on opening file.") .setName("Check conflict only on opening file.")
.setDesc("Do not check conflict while replication") .setDesc("Do not check conflict while replication")
@@ -831,15 +868,6 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
}) })
); );
new Setting(containerMiscellaneousEl)
.setName("Use history")
.setDesc("Use history dialog (Restart required, auto compaction would be disabled, and more storage will be consumed)")
.addToggle((toggle) =>
toggle.setValue(this.plugin.settings.useHistory).onChange(async (value) => {
this.plugin.settings.useHistory = value;
await this.plugin.saveSettings();
})
);
addScreenElement("40", containerMiscellaneousEl); addScreenElement("40", containerMiscellaneousEl);
const containerHatchEl = containerEl.createDiv(); const containerHatchEl = containerEl.createDiv();
@@ -875,30 +903,10 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
} }
const hatchWarn = containerHatchEl.createEl("div", { text: `To stop the bootup sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` }); const hatchWarn = containerHatchEl.createEl("div", { text: `To stop the bootup sequence for fixing problems on databases, you can put redflag.md on top of your vault (Rebooting obsidian is required).` });
hatchWarn.addClass("op-warn-info"); hatchWarn.addClass("op-warn-info");
const dropHistory = async (sendToServer: boolean) => {
this.plugin.settings.liveSync = false;
this.plugin.settings.periodicReplication = false;
this.plugin.settings.syncOnSave = false;
this.plugin.settings.syncOnStart = false;
this.plugin.settings.syncOnFileOpen = false;
await this.plugin.saveSettings();
applyDisplayEnabled();
await this.plugin.resetLocalDatabase();
if (sendToServer) {
await this.plugin.initializeDatabase(true);
await this.plugin.markRemoteLocked();
await this.plugin.tryResetRemoteDatabase();
await this.plugin.markRemoteLocked();
await this.plugin.replicateAllToServer(true);
} else {
await this.plugin.markRemoteResolved();
await this.plugin.replicate(true);
}
};
new Setting(containerHatchEl) new Setting(containerHatchEl)
.setName("Verify and repair all files") .setName("Verify and repair all files")
.setDesc("Verify and repair all files and update database without dropping history") .setDesc("Verify and repair all files and update database without restoring")
.addButton((button) => .addButton((button) =>
button button
.setButtonText("Verify and repair") .setButtonText("Verify and repair")
@@ -906,13 +914,13 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.setWarning() .setWarning()
.onClick(async () => { .onClick(async () => {
const files = this.app.vault.getFiles(); const files = this.app.vault.getFiles();
Logger("Verify and repair all files started", LOG_LEVEL.NOTICE); Logger("Verify and repair all files started", LOG_LEVEL.NOTICE, "verify");
const notice = NewNotice("", 0); // const notice = NewNotice("", 0);
let i = 0; let i = 0;
for (const file of files) { for (const file of files) {
i++; i++;
Logger(`Update into ${file.path}`); Logger(`Update into ${file.path}`);
notice.setMessage(`${i}/${files.length}\n${file.path}`); Logger(`${i}/${files.length}\n${file.path}`, LOG_LEVEL.NOTICE, "verify");
try { try {
await this.plugin.updateIntoDB(file); await this.plugin.updateIntoDB(file);
} catch (ex) { } catch (ex) {
@@ -920,8 +928,7 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
Logger(ex); Logger(ex);
} }
} }
notice.hide(); Logger("done", LOG_LEVEL.NOTICE, "verify");
Logger("done", LOG_LEVEL.NOTICE);
}) })
); );
new Setting(containerHatchEl) new Setting(containerHatchEl)
@@ -933,9 +940,8 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
.setDisabled(false) .setDisabled(false)
.setWarning() .setWarning()
.onClick(async () => { .onClick(async () => {
const notice = NewNotice("", 0); // const notice = NewNotice("", 0);
Logger(`Begin sanity check`, LOG_LEVEL.INFO); Logger(`Begin sanity check`, LOG_LEVEL.NOTICE, "sancheck");
notice.setMessage(`Begin sanity check`);
await runWithLock("sancheck", true, async () => { await runWithLock("sancheck", true, async () => {
const db = this.plugin.localDatabase.localDatabase; const db = this.plugin.localDatabase.localDatabase;
const wf = await db.allDocs(); const wf = await db.allDocs();
@@ -943,59 +949,22 @@ export class ObsidianLiveSyncSettingTab extends PluginSettingTab {
let count = 0; let count = 0;
for (const id of filesDatabase) { for (const id of filesDatabase) {
count++; count++;
notice.setMessage(`${count}/${filesDatabase.length}\n${id2path(id)}`); Logger(`${count}/${filesDatabase.length}\n${id2path(id)}`, LOG_LEVEL.NOTICE, "sancheck");
const w = await db.get<EntryDoc>(id); const w = await db.get<EntryDoc>(id);
if (!(await this.plugin.localDatabase.sanCheck(w))) { if (!(await this.plugin.localDatabase.sanCheck(w))) {
Logger(`The file ${id2path(id)} missing child(ren)`, LOG_LEVEL.NOTICE); Logger(`The file ${id2path(id)} missing child(ren)`, LOG_LEVEL.NOTICE);
} }
} }
}); });
notice.hide(); Logger(`Done`, LOG_LEVEL.NOTICE, "sancheck");
Logger(`Done`, LOG_LEVEL.NOTICE);
// Logger("done", LOG_LEVEL.NOTICE); // Logger("done", LOG_LEVEL.NOTICE);
}) })
); );
new Setting(containerHatchEl)
.setName("Drop History")
.setDesc("Initialize local and remote database, and send all or retrieve all again.")
.addButton((button) =>
button
.setButtonText("Drop and send")
.setWarning()
.setDisabled(false)
.setClass("sls-btn-left")
.onClick(async () => {
await dropHistory(true);
})
)
.addButton((button) =>
button
.setButtonText("Drop and receive")
.setWarning()
.setDisabled(false)
.setClass("sls-btn-right")
.onClick(async () => {
await dropHistory(false);
})
);
new Setting(containerHatchEl)
.setName("Lock remote database")
.setDesc("Lock remote database for synchronize")
.addButton((button) =>
button
.setButtonText("Lock")
.setDisabled(false)
.setWarning()
.onClick(async () => {
await this.plugin.markRemoteLocked();
})
);
new Setting(containerHatchEl) new Setting(containerHatchEl)
.setName("Suspend file watching") .setName("Suspend file watching")
.setDesc("if enables it, all file operations are ignored.") .setDesc("If enables it, all file operations are ignored.")
.addToggle((toggle) => .addToggle((toggle) =>
toggle.setValue(this.plugin.settings.suspendFileWatching).onChange(async (value) => { toggle.setValue(this.plugin.settings.suspendFileWatching).onChange(async (value) => {
this.plugin.settings.suspendFileWatching = value; this.plugin.settings.suspendFileWatching = value;

2
src/lib

Submodule src/lib updated: 6451afd112...548265c701

455
src/main.ts
View File

@@ -1,4 +1,4 @@
import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, PluginManifest, Modal, App, FuzzySuggestModal } from "obsidian"; import { debounce, Notice, Plugin, TFile, addIcon, TFolder, normalizePath, TAbstractFile, Editor, MarkdownView, PluginManifest, Modal, App, FuzzySuggestModal, Setting } from "obsidian";
import { diff_match_patch } from "diff-match-patch"; import { diff_match_patch } from "diff-match-patch";
import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID } from "./lib/src/types"; import { EntryDoc, LoadedEntry, ObsidianLiveSyncSettings, diff_check_result, diff_result_leaf, EntryBody, LOG_LEVEL, VER, DEFAULT_SETTINGS, diff_result, FLAGMD_REDFLAG, SYNCINFO_ID } from "./lib/src/types";
@@ -16,8 +16,8 @@ import {
isPlainText, isPlainText,
setNoticeClass, setNoticeClass,
NewNotice, NewNotice,
allSettledWithConcurrencyLimit,
getLocks, getLocks,
Parallels,
} from "./lib/src/utils"; } from "./lib/src/utils";
import { Logger, setLogger } from "./lib/src/logger"; import { Logger, setLogger } from "./lib/src/logger";
import { LocalPouchDB } from "./LocalPouchDB"; import { LocalPouchDB } from "./LocalPouchDB";
@@ -26,10 +26,13 @@ import { ConflictResolveModal } from "./ConflictResolveModal";
import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab"; import { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab";
import { DocumentHistoryModal } from "./DocumentHistoryModal"; import { DocumentHistoryModal } from "./DocumentHistoryModal";
//@ts-ignore
import PluginPane from "./PluginPane.svelte"; import PluginPane from "./PluginPane.svelte";
import { id2path, path2id } from "./utils"; import { id2path, path2id } from "./utils";
import { decrypt, encrypt } from "./lib/src/e2ee"; import { decrypt, encrypt } from "./lib/src/e2ee_v2";
const isDebug = false; const isDebug = false;
setNoticeClass(Notice); setNoticeClass(Notice);
class PluginDialogModal extends Modal { class PluginDialogModal extends Modal {
plugin: ObsidianLiveSyncPlugin; plugin: ObsidianLiveSyncPlugin;
@@ -58,9 +61,65 @@ class PluginDialogModal extends Modal {
} }
} }
} }
class InputStringDialog extends Modal {
result: string | false = false;
onSubmit: (result: string | boolean) => void;
title: string;
key: string;
placeholder: string;
isManuallyClosed = false;
constructor(app: App, title: string, key: string, placeholder: string, onSubmit: (result: string | false) => void) {
super(app);
this.onSubmit = onSubmit;
this.title = title;
this.placeholder = placeholder;
this.key = key;
}
onOpen() {
const { contentEl } = this;
contentEl.createEl("h1", { text: this.title });
new Setting(contentEl).setName(this.key).addText((text) =>
text.onChange((value) => {
this.result = value;
})
);
new Setting(contentEl).addButton((btn) =>
btn
.setButtonText("Ok")
.setCta()
.onClick(() => {
this.isManuallyClosed = true;
this.close();
})
).addButton((btn) =>
btn
.setButtonText("Cancel")
.setCta()
.onClick(() => {
this.close();
})
);
}
onClose() {
const { contentEl } = this;
contentEl.empty();
if (this.isManuallyClosed) {
this.onSubmit(this.result);
} else {
this.onSubmit(false);
}
}
}
class PopoverYesNo extends FuzzySuggestModal<string> { class PopoverYesNo extends FuzzySuggestModal<string> {
app: App; app: App;
callback: (e: string) => void = () => {}; callback: (e: string) => void = () => { };
constructor(app: App, note: string, callback: (e: string) => void) { constructor(app: App, note: string, callback: (e: string) => void) {
super(app); super(app);
@@ -98,6 +157,27 @@ const askYesNo = (app: App, message: string): Promise<"yes" | "no"> => {
}); });
}; };
const askString = (app: App, title: string, key: string, placeholder: string): Promise<string | false> => {
return new Promise((res) => {
const dialog = new InputStringDialog(app, title, key, placeholder, (result) => res(result));
dialog.open();
});
};
let touchedFiles: string[] = [];
function touch(file: TFile | string) {
const f = file instanceof TFile ? file : app.vault.getAbstractFileByPath(file) as TFile;
const key = `${f.path}-${f.stat.mtime}-${f.stat.size}`;
touchedFiles.push(key);
touchedFiles = touchedFiles.slice(0, 100);
}
function recentlyTouched(file: TFile) {
const key = `${file.path}-${file.stat.mtime}-${file.stat.size}`;
if (touchedFiles.indexOf(key) == -1) return false;
return true;
}
function clearTouched() {
touchedFiles = [];
}
export default class ObsidianLiveSyncPlugin extends Plugin { export default class ObsidianLiveSyncPlugin extends Plugin {
settings: ObsidianLiveSyncSettings; settings: ObsidianLiveSyncSettings;
localDatabase: LocalPouchDB; localDatabase: LocalPouchDB;
@@ -240,20 +320,40 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
Logger(ex, LOG_LEVEL.VERBOSE); Logger(ex, LOG_LEVEL.VERBOSE);
} }
}); });
const configURIBase = "obsidian://setuplivesync?settings=";
this.addCommand({ this.addCommand({
id: "livesync-exportconfig", id: "livesync-copysetupuri",
name: "Copy setup uri (beta)", name: "Copy setup URI (beta)",
callback: async () => { callback: async () => {
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(this.settings), "---")); const encryptingPassphrase = await askString(this.app, "Encrypt your settings", "Passphrase", "");
const uri = `obsidian://setuplivesync?settings=${encryptedSetting}`; if (encryptingPassphrase === false) return;
const encryptedSetting = encodeURIComponent(await encrypt(JSON.stringify(this.settings), encryptingPassphrase));
const uri = `${configURIBase}${encryptedSetting}`;
await navigator.clipboard.writeText(uri); await navigator.clipboard.writeText(uri);
Logger("Setup uri copied to clipboard", LOG_LEVEL.NOTICE); Logger("Setup URI copied to clipboard", LOG_LEVEL.NOTICE);
}, },
}); });
this.registerObsidianProtocolHandler("setuplivesync", async (conf: any) => { this.addCommand({
id: "livesync-opensetupuri",
name: "Open setup URI (beta)",
callback: async () => {
const setupURI = await askString(this.app, "Set up manually", "Set up URI", `${configURIBase}aaaaa`);
if (setupURI === false) return;
if (!setupURI.startsWith(`${configURIBase}`)) {
Logger("Set up URI looks wrong.", LOG_LEVEL.NOTICE);
return;
}
const config = decodeURIComponent(setupURI.substring(configURIBase.length));
console.dir(config)
await setupwizard(config);
},
});
const setupwizard = async (confString: string) => {
try { try {
const oldConf = JSON.parse(JSON.stringify(this.settings)); const oldConf = JSON.parse(JSON.stringify(this.settings));
const newconf = await JSON.parse(await decrypt(conf.settings, "---")); const encryptingPassphrase = await askString(this.app, "Passphrase", "Passphrase for your settings", "");
if (encryptingPassphrase === false) return;
const newconf = await JSON.parse(await decrypt(confString, encryptingPassphrase));
if (newconf) { if (newconf) {
const result = await askYesNo(this.app, "Importing LiveSync's conf, OK?"); const result = await askYesNo(this.app, "Importing LiveSync's conf, OK?");
if (result == "yes") { if (result == "yes") {
@@ -268,6 +368,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
// nothing to do. so peaceful. // nothing to do. so peaceful.
this.settings = newSettingW; this.settings = newSettingW;
await this.saveSettings(); await this.saveSettings();
const replicate = await askYesNo(this.app, "Unlock and replicate?");
if (replicate == "yes") {
await this.replicate(true);
await this.markRemoteUnlocked();
}
Logger("Configuration loaded.", LOG_LEVEL.NOTICE); Logger("Configuration loaded.", LOG_LEVEL.NOTICE);
return; return;
} }
@@ -312,8 +417,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
Logger("Cancelled.", LOG_LEVEL.NOTICE); Logger("Cancelled.", LOG_LEVEL.NOTICE);
} }
} catch (ex) { } catch (ex) {
Logger("Couldn't parse configuration uri.", LOG_LEVEL.NOTICE); Logger("Couldn't parse or decrypt configuration uri.", LOG_LEVEL.NOTICE);
} }
};
this.registerObsidianProtocolHandler("setuplivesync", async (conf: any) => {
await setupwizard(conf.settings);
}); });
this.addCommand({ this.addCommand({
id: "livesync-replicate", id: "livesync-replicate",
@@ -338,9 +446,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}); });
this.addCommand({ this.addCommand({
id: "livesync-gc", id: "livesync-gc",
name: "garbage collect now", name: "Check garbages now",
callback: () => { callback: () => {
this.garbageCollect(); this.garbageCheck();
}, },
}); });
this.addCommand({ this.addCommand({
@@ -414,15 +522,19 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
onunload() { onunload() {
this.hidePluginSyncModal(); this.hidePluginSyncModal();
this.localDatabase.onunload(); if (this.localDatabase != null) {
this.localDatabase.onunload();
}
if (this.gcTimerHandler != null) { if (this.gcTimerHandler != null) {
clearTimeout(this.gcTimerHandler); clearTimeout(this.gcTimerHandler);
this.gcTimerHandler = null; this.gcTimerHandler = null;
} }
this.clearPeriodicSync(); this.clearPeriodicSync();
this.clearPluginSweep(); this.clearPluginSweep();
this.localDatabase.closeReplication(); if (this.localDatabase != null) {
this.localDatabase.close(); this.localDatabase.closeReplication();
this.localDatabase.close();
}
window.removeEventListener("visibilitychange", this.watchWindowVisiblity); window.removeEventListener("visibilitychange", this.watchWindowVisiblity);
Logger("unloading plugin"); Logger("unloading plugin");
} }
@@ -442,8 +554,8 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
return await this.localDatabase.initializeDatabase(); return await this.localDatabase.initializeDatabase();
} }
async garbageCollect() { async garbageCheck() {
await this.localDatabase.garbageCollect(); await this.localDatabase.garbageCheck();
} }
async loadSettings() { async loadSettings() {
@@ -452,6 +564,12 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.settings.workingPassphrase = this.settings.passphrase; this.settings.workingPassphrase = this.settings.passphrase;
// Delete this feature to avoid problems on mobile. // Delete this feature to avoid problems on mobile.
this.settings.disableRequestURI = true; this.settings.disableRequestURI = true;
// GC is disabled.
this.settings.gcDelay = 0;
// So, use history is always enabled.
this.settings.useHistory = true;
const lsname = "obsidian-live-sync-vaultanddevicename-" + this.app.vault.getName(); const lsname = "obsidian-live-sync-vaultanddevicename-" + this.app.vault.getName();
if (this.settings.deviceAndVaultName != "") { if (this.settings.deviceAndVaultName != "") {
if (!localStorage.getItem(lsname)) { if (!localStorage.getItem(lsname)) {
@@ -488,7 +606,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} }
this.gcTimerHandler = setTimeout(() => { this.gcTimerHandler = setTimeout(() => {
this.gcTimerHandler = null; this.gcTimerHandler = null;
this.garbageCollect(); this.garbageCheck();
}, GC_DELAY); }, GC_DELAY);
} }
@@ -551,6 +669,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
watchVaultCreate(file: TFile, ...args: any[]) { watchVaultCreate(file: TFile, ...args: any[]) {
if (this.settings.suspendFileWatching) return; if (this.settings.suspendFileWatching) return;
if (recentlyTouched(file)) {
return;
}
this.watchVaultChangeAsync(file, ...args); this.watchVaultChangeAsync(file, ...args);
} }
@@ -558,6 +679,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (!(file instanceof TFile)) { if (!(file instanceof TFile)) {
return; return;
} }
if (recentlyTouched(file)) {
return;
}
if (this.settings.suspendFileWatching) return; if (this.settings.suspendFileWatching) return;
// If batchsave is enabled, queue all changes and do nothing. // If batchsave is enabled, queue all changes and do nothing.
@@ -586,20 +710,28 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
return await runWithLock("batchSave", false, async () => { return await runWithLock("batchSave", false, async () => {
const batchItems = JSON.parse(JSON.stringify(this.batchFileChange)) as string[]; const batchItems = JSON.parse(JSON.stringify(this.batchFileChange)) as string[];
this.batchFileChange = []; this.batchFileChange = [];
const promises = batchItems.map(async (e) => { const limit = 3;
try { const p = Parallels();
const f = this.app.vault.getAbstractFileByPath(normalizePath(e));
if (f && f instanceof TFile) { for (const e of batchItems) {
await this.updateIntoDB(f); const w = (async () => {
Logger(`Batch save:${e}`); try {
const f = this.app.vault.getAbstractFileByPath(normalizePath(e));
if (f && f instanceof TFile) {
await this.updateIntoDB(f);
Logger(`Batch save:${e}`);
}
} catch (ex) {
Logger(`Batch save error:${e}`, LOG_LEVEL.NOTICE);
Logger(ex, LOG_LEVEL.VERBOSE);
} }
} catch (ex) { })();
Logger(`Batch save error:${e}`, LOG_LEVEL.NOTICE); p.add(w);
Logger(ex, LOG_LEVEL.VERBOSE); await p.wait(limit)
} }
}); this.refreshStatusText();
await p.all();
this.refreshStatusText(); this.refreshStatusText();
await allSettledWithConcurrencyLimit(promises, 3);
return; return;
}); });
} }
@@ -608,6 +740,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
async watchVaultChangeAsync(file: TFile, ...args: any[]) { async watchVaultChangeAsync(file: TFile, ...args: any[]) {
if (file instanceof TFile) { if (file instanceof TFile) {
if (recentlyTouched(file)) {
return;
}
await this.updateIntoDB(file); await this.updateIntoDB(file);
this.gcHook(); this.gcHook();
} }
@@ -615,9 +750,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
watchVaultDelete(file: TAbstractFile) { watchVaultDelete(file: TAbstractFile) {
// When save is delayed, it should be cancelled. // When save is delayed, it should be cancelled.
this.batchFileChange = this.batchFileChange.filter((e) => e == file.path); this.batchFileChange = this.batchFileChange.filter((e) => e != file.path);
if (this.settings.suspendFileWatching) return; if (this.settings.suspendFileWatching) return;
this.watchVaultDeleteAsync(file).then(() => {}); this.watchVaultDeleteAsync(file).then(() => { });
} }
async watchVaultDeleteAsync(file: TAbstractFile) { async watchVaultDeleteAsync(file: TAbstractFile) {
@@ -646,7 +781,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
watchVaultRename(file: TAbstractFile, oldFile: any) { watchVaultRename(file: TAbstractFile, oldFile: any) {
if (this.settings.suspendFileWatching) return; if (this.settings.suspendFileWatching) return;
this.watchVaultRenameAsync(file, oldFile).then(() => {}); this.watchVaultRenameAsync(file, oldFile).then(() => { });
} }
getFilePath(file: TAbstractFile): string { getFilePath(file: TAbstractFile): string {
@@ -704,7 +839,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
lastLog = ""; lastLog = "";
// eslint-disable-next-line require-await // eslint-disable-next-line require-await
async addLog(message: any, level: LOG_LEVEL = LOG_LEVEL.INFO) { async addLog(message: any, level: LOG_LEVEL = LOG_LEVEL.INFO, key = "") {
if (level == LOG_LEVEL.DEBUG && !isDebug) { if (level == LOG_LEVEL.DEBUG && !isDebug) {
return; return;
} }
@@ -727,13 +862,24 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
// } // }
if (level >= LOG_LEVEL.NOTICE) { if (level >= LOG_LEVEL.NOTICE) {
if (messagecontent in this.notifies) { if (!key) key = messagecontent;
clearTimeout(this.notifies[messagecontent].timer); if (key in this.notifies) {
this.notifies[messagecontent].count++; // @ts-ignore
this.notifies[messagecontent].notice.setMessage(`(${this.notifies[messagecontent].count}):${messagecontent}`); const isShown = this.notifies[key].notice.noticeEl?.isShown()
this.notifies[messagecontent].timer = setTimeout(() => { if (!isShown) {
const notify = this.notifies[messagecontent].notice; this.notifies[key].notice = new Notice(messagecontent, 0);
delete this.notifies[messagecontent]; }
clearTimeout(this.notifies[key].timer);
if (key == messagecontent) {
this.notifies[key].count++;
this.notifies[key].notice.setMessage(`(${this.notifies[key].count}):${messagecontent}`);
} else {
this.notifies[key].notice.setMessage(`${messagecontent}`);
}
this.notifies[key].timer = setTimeout(() => {
const notify = this.notifies[key].notice;
delete this.notifies[key];
try { try {
notify.hide(); notify.hide();
} catch (ex) { } catch (ex) {
@@ -742,11 +888,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}, 5000); }, 5000);
} else { } else {
const notify = new Notice(messagecontent, 0); const notify = new Notice(messagecontent, 0);
this.notifies[messagecontent] = { this.notifies[key] = {
count: 0, count: 0,
notice: notify, notice: notify,
timer: setTimeout(() => { timer: setTimeout(() => {
delete this.notifies[messagecontent]; delete this.notifies[key];
notify.hide(); notify.hide();
}, 5000), }, 5000),
}; };
@@ -783,12 +929,13 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} }
const doc = await this.localDatabase.getDBEntry(pathSrc, { rev: docEntry._rev }); const doc = await this.localDatabase.getDBEntry(pathSrc, { rev: docEntry._rev });
if (doc === false) return; if (doc === false) return;
const msg = `DB -> STORAGE (create${force ? ",force" : ""},${doc.datatype}) `;
const path = id2path(doc._id); const path = id2path(doc._id);
if (doc.datatype == "newnote") { if (doc.datatype == "newnote") {
const bin = base64ToArrayBuffer(doc.data); const bin = base64ToArrayBuffer(doc.data);
if (bin != null) { if (bin != null) {
if (!isValidPath(path)) { if (!isValidPath(path)) {
Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${path}`, LOG_LEVEL.NOTICE); Logger(msg + "ERROR, invalid path: " + path, LOG_LEVEL.NOTICE);
return; return;
} }
await this.ensureDirectory(path); await this.ensureDirectory(path);
@@ -797,16 +944,18 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
ctime: doc.ctime, ctime: doc.ctime,
mtime: doc.mtime, mtime: doc.mtime,
}); });
Logger("live : write to local (newfile:b) " + path); this.batchFileChange = this.batchFileChange.filter((e) => e != newfile.path);
Logger(msg + path);
touch(newfile);
this.app.vault.trigger("create", newfile); this.app.vault.trigger("create", newfile);
} catch (ex) { } catch (ex) {
Logger("could not write to local (newfile:bin) " + path, LOG_LEVEL.NOTICE); Logger(msg + "ERROR, Could not write: " + path, LOG_LEVEL.NOTICE);
Logger(ex, LOG_LEVEL.VERBOSE); Logger(ex, LOG_LEVEL.VERBOSE);
} }
} }
} else if (doc.datatype == "plain") { } else if (doc.datatype == "plain") {
if (!isValidPath(path)) { if (!isValidPath(path)) {
Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${path}`, LOG_LEVEL.NOTICE); Logger(msg + "ERROR, invalid path: " + path, LOG_LEVEL.NOTICE);
return; return;
} }
await this.ensureDirectory(path); await this.ensureDirectory(path);
@@ -815,14 +964,16 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
ctime: doc.ctime, ctime: doc.ctime,
mtime: doc.mtime, mtime: doc.mtime,
}); });
Logger("live : write to local (newfile:p) " + path); this.batchFileChange = this.batchFileChange.filter((e) => e != newfile.path);
Logger(msg + path);
touch(newfile);
this.app.vault.trigger("create", newfile); this.app.vault.trigger("create", newfile);
} catch (ex) { } catch (ex) {
Logger("could not write to local (newfile:plain) " + path, LOG_LEVEL.NOTICE); Logger(msg + "ERROR, Could not parse: " + path + "(" + doc.datatype + ")", LOG_LEVEL.NOTICE);
Logger(ex, LOG_LEVEL.VERBOSE); Logger(ex, LOG_LEVEL.VERBOSE);
} }
} else { } else {
Logger("live : New data imcoming, but we cound't parse that." + doc.datatype, LOG_LEVEL.NOTICE); Logger(msg + "ERROR, Could not parse: " + path + "(" + doc.datatype + ")", LOG_LEVEL.NOTICE);
} }
} }
@@ -866,41 +1017,46 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const docMtime = ~~(docEntry.mtime / 1000); const docMtime = ~~(docEntry.mtime / 1000);
if (localMtime < docMtime || force) { if (localMtime < docMtime || force) {
const doc = await this.localDatabase.getDBEntry(pathSrc); const doc = await this.localDatabase.getDBEntry(pathSrc);
let msg = "livesync : newer local files so write to local:" + file.path;
if (force) msg = "livesync : force write to local:" + file.path;
if (doc === false) return; if (doc === false) return;
const msg = `DB -> STORAGE (modify${force ? ",force" : ""},${doc.datatype}) `;
const path = id2path(doc._id); const path = id2path(doc._id);
if (doc.datatype == "newnote") { if (doc.datatype == "newnote") {
const bin = base64ToArrayBuffer(doc.data); const bin = base64ToArrayBuffer(doc.data);
if (bin != null) { if (bin != null) {
if (!isValidPath(path)) { if (!isValidPath(path)) {
Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${path}`, LOG_LEVEL.NOTICE); Logger(msg + "ERROR, invalid path: " + path, LOG_LEVEL.NOTICE);
return; return;
} }
await this.ensureDirectory(path); await this.ensureDirectory(path);
try { try {
await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime }); await this.app.vault.modifyBinary(file, bin, { ctime: doc.ctime, mtime: doc.mtime });
Logger(msg); this.batchFileChange = this.batchFileChange.filter((e) => e != file.path);
this.app.vault.trigger("modify", file); Logger(msg + path);
const xf = this.app.vault.getAbstractFileByPath(file.path) as TFile;
touch(xf);
this.app.vault.trigger("modify", xf);
} catch (ex) { } catch (ex) {
Logger("could not write to local (modify:bin) " + path, LOG_LEVEL.NOTICE); Logger(msg + "ERROR, Could not write: " + path, LOG_LEVEL.NOTICE);
} }
} }
} else if (doc.datatype == "plain") { } else if (doc.datatype == "plain") {
if (!isValidPath(path)) { if (!isValidPath(path)) {
Logger(`The file that having platform dependent name has been arrived. This file has skipped: ${path}`, LOG_LEVEL.NOTICE); Logger(msg + "ERROR, invalid path: " + path, LOG_LEVEL.NOTICE);
return; return;
} }
await this.ensureDirectory(path); await this.ensureDirectory(path);
try { try {
await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime }); await this.app.vault.modify(file, doc.data, { ctime: doc.ctime, mtime: doc.mtime });
Logger(msg); Logger(msg + path);
this.app.vault.trigger("modify", file); this.batchFileChange = this.batchFileChange.filter((e) => e != file.path);
const xf = this.app.vault.getAbstractFileByPath(file.path) as TFile;
touch(xf);
this.app.vault.trigger("modify", xf);
} catch (ex) { } catch (ex) {
Logger("could not write to local (modify:plain) " + path, LOG_LEVEL.NOTICE); Logger(msg + "ERROR, Could not write: " + path, LOG_LEVEL.NOTICE);
} }
} else { } else {
Logger("live : New data imcoming, but we cound't parse that.:" + doc.datatype + "-", LOG_LEVEL.NOTICE); Logger(msg + "ERROR, Could not parse: " + path + "(" + doc.datatype + ")", LOG_LEVEL.NOTICE);
} }
} else if (localMtime > docMtime) { } else if (localMtime > docMtime) {
// newer local file. // newer local file.
@@ -945,7 +1101,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
}[] = []; }[] = [];
chunkWaitTimeout = 60000; chunkWaitTimeout = 60000;
async saveQueuedFiles() { saveQueuedFiles() {
const saveData = JSON.stringify(this.queuedFiles.filter((e) => !e.done).map((e) => e.entry._id)); const saveData = JSON.stringify(this.queuedFiles.filter((e) => !e.done).map((e) => e.entry._id));
const lsname = "obsidian-livesync-queuefiles-" + this.app.vault.getName(); const lsname = "obsidian-livesync-queuefiles-" + this.app.vault.getName();
localStorage.setItem(lsname, saveData); localStorage.setItem(lsname, saveData);
@@ -961,7 +1117,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} }
} }
async procQueuedFiles() { async procQueuedFiles() {
await runWithLock("procQueue", true, async () => { await runWithLock("procQueue", false, async () => {
this.saveQueuedFiles(); this.saveQueuedFiles();
for (const queue of this.queuedFiles) { for (const queue of this.queuedFiles) {
if (queue.done) continue; if (queue.done) continue;
@@ -971,9 +1127,9 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (isValidPath(id2path(queue.entry._id))) { if (isValidPath(id2path(queue.entry._id))) {
Logger(`Applying ${queue.entry._id} (${queue.entry._rev}) change...`); Logger(`Applying ${queue.entry._id} (${queue.entry._rev}) change...`);
await this.handleDBChanged(queue.entry); await this.handleDBChanged(queue.entry);
Logger(`Applied ${queue.entry._id} (${queue.entry._rev})`);
} }
} } else if (now > queue.timeout) {
if (now > queue.timeout) {
if (!queue.warned) Logger(`Timed out: ${queue.entry._id} could not collect ${queue.missingChildren.length} chunks. plugin keeps watching, but you have to check the file after the replication.`, LOG_LEVEL.NOTICE); if (!queue.warned) Logger(`Timed out: ${queue.entry._id} could not collect ${queue.missingChildren.length} chunks. plugin keeps watching, but you have to check the file after the replication.`, LOG_LEVEL.NOTICE);
queue.warned = true; queue.warned = true;
continue; continue;
@@ -1005,13 +1161,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (isNewFileCompleted) this.procQueuedFiles(); if (isNewFileCompleted) this.procQueuedFiles();
} }
async parseIncomingDoc(doc: PouchDB.Core.ExistingDocument<EntryBody>) { async parseIncomingDoc(doc: PouchDB.Core.ExistingDocument<EntryBody>) {
const skipOldFile = this.settings.skipOlderFilesOnSync; const skipOldFile = this.settings.skipOlderFilesOnSync && false; //patched temporary.
if (skipOldFile) { if (skipOldFile) {
const info = this.app.vault.getAbstractFileByPath(id2path(doc._id)); const info = this.app.vault.getAbstractFileByPath(id2path(doc._id));
if (info && info instanceof TFile) { if (info && info instanceof TFile) {
const localMtime = ~~((info as TFile).stat.mtime / 1000); const localMtime = ~~((info as TFile).stat.mtime / 1000);
const docMtime = ~~(doc.mtime / 1000); const docMtime = ~~(doc.mtime / 1000);
//TODO: some margin required.
if (localMtime >= docMtime) { if (localMtime >= docMtime) {
Logger(`${doc._id} Skipped, older than storage.`, LOG_LEVEL.VERBOSE); Logger(`${doc._id} Skipped, older than storage.`, LOG_LEVEL.VERBOSE);
return; return;
@@ -1027,15 +1184,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if ("children" in doc) { if ("children" in doc) {
const c = await this.localDatabase.localDatabase.allDocs({ keys: doc.children, include_docs: false }); const c = await this.localDatabase.localDatabase.allDocs({ keys: doc.children, include_docs: false });
const missing = c.rows.filter((e) => "error" in e).map((e) => e.key); const missing = c.rows.filter((e) => "error" in e).map((e) => e.key);
if (missing.length) Logger(`${doc._id}(${doc._rev}) Queued (waiting ${missing.length} items)`, LOG_LEVEL.VERBOSE); if (missing.length > 0) Logger(`${doc._id}(${doc._rev}) Queued (waiting ${missing.length} items)`, LOG_LEVEL.VERBOSE);
newQueue.missingChildren = missing; newQueue.missingChildren = missing;
this.queuedFiles.push(newQueue); this.queuedFiles.push(newQueue);
this.saveQueuedFiles();
} else { } else {
this.queuedFiles.push(newQueue); this.queuedFiles.push(newQueue);
this.saveQueuedFiles();
this.procQueuedFiles();
} }
this.saveQueuedFiles();
this.procQueuedFiles();
} }
periodicSyncHandler: number = null; periodicSyncHandler: number = null;
@@ -1207,8 +1363,19 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
const procsDisp = procs == 0 ? "" : ` ⏳${procs}`; const procsDisp = procs == 0 ? "" : ` ⏳${procs}`;
const message = `Sync:${w} ↑${sent} ↓${arrived}${waiting}${procsDisp}${queued}`; const message = `Sync:${w} ↑${sent} ↓${arrived}${waiting}${procsDisp}${queued}`;
const locks = getLocks(); const locks = getLocks();
const pendingTask = locks.pending.length ? `\nPending:${locks.pending.join(", ")}` : ""; const pendingTask = locks.pending.length
const runningTask = locks.running.length ? `\nRunning:${locks.running.join(", ")}` : ""; ? "\nPending: " +
Object.entries(locks.pending.reduce((p, c) => ({ ...p, [c]: (p[c] ?? 0) + 1 }), {} as { [key: string]: number }))
.map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
.join(", ")
: "";
const runningTask = locks.running.length
? "\nRunning: " +
Object.entries(locks.running.reduce((p, c) => ({ ...p, [c]: (p[c] ?? 0) + 1 }), {} as { [key: string]: number }))
.map((e) => `${e[0]}${e[1] == 1 ? "" : `(${e[1]})`}`)
.join(", ")
: "";
this.setStatusBarText(message + pendingTask + runningTask); this.setStatusBarText(message + pendingTask + runningTask);
} }
@@ -1235,11 +1402,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
this.lastLog = newLog; this.lastLog = newLog;
} }
} }
updateStatusBarText() {} updateStatusBarText() { }
async replicate(showMessage?: boolean) { async replicate(showMessage?: boolean) {
if (this.settings.versionUpFlash != "") { if (this.settings.versionUpFlash != "") {
NewNotice("Open settings and check message, please."); Logger("Open settings and check message, please.", LOG_LEVEL.NOTICE);
return; return;
} }
await this.applyBatchChange(); await this.applyBatchChange();
@@ -1282,16 +1449,21 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
async syncAllFiles(showingNotice?: boolean) { async syncAllFiles(showingNotice?: boolean) {
// synchronize all files between database and storage. // synchronize all files between database and storage.
let notice: Notice = null; let initialScan = false;
if (showingNotice) { if (showingNotice) {
notice = NewNotice("Initializing", 0); Logger("Initializing", LOG_LEVEL.NOTICE, "syncAll");
} }
const filesStorage = this.app.vault.getFiles(); const filesStorage = this.app.vault.getFiles();
const filesStorageName = filesStorage.map((e) => e.path); const filesStorageName = filesStorage.map((e) => e.path);
const wf = await this.localDatabase.localDatabase.allDocs(); const wf = await this.localDatabase.localDatabase.allDocs();
const filesDatabase = wf.rows.filter((e) => !e.id.startsWith("h:") && !e.id.startsWith("ps:") && e.id != "obsydian_livesync_version").map((e) => id2path(e.id)); const filesDatabase = wf.rows.filter((e) => !e.id.startsWith("h:") && !e.id.startsWith("ps:") && e.id != "obsydian_livesync_version").filter(e => isValidPath(e.id)).map((e) => id2path(e.id));
const isInitialized = await (this.localDatabase.kvDB.get<boolean>("initialized")) || false;
// Make chunk bigger if it is the initial scan. There must be non-active docs.
if (filesDatabase.length == 0 && !isInitialized) {
initialScan = true;
Logger("Database looks empty, save files as initial sync data");
}
const onlyInStorage = filesStorage.filter((e) => filesDatabase.indexOf(e.path) == -1); const onlyInStorage = filesStorage.filter((e) => filesDatabase.indexOf(e.path) == -1);
const onlyInDatabase = filesDatabase.filter((e) => filesStorageName.indexOf(e) == -1); const onlyInDatabase = filesDatabase.filter((e) => filesStorageName.indexOf(e) == -1);
@@ -1307,44 +1479,71 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
Logger(procedurename); Logger(procedurename);
let i = 0; let i = 0;
// let lastTicks = performance.now() + 2000; // let lastTicks = performance.now() + 2000;
let workProcs = 0; // let workProcs = 0;
const procs = objects.map(async (e) => { const p = Parallels();
try { const limit = 10;
workProcs++;
await callback(e); Logger(`${procedurename} exec.`);
for (const v of objects) {
// workProcs++;
if (!this.localDatabase.isReady) throw Error("Database is not ready!");
p.add(callback(v).then(() => {
i++; i++;
if (i % 25 == 0) { if (i % 100 == 0) {
const notify = `${procedurename} : ${workProcs}/${count} (Pending:${workProcs})`; const notify = `${procedurename} : ${i}/${count}`;
if (notice != null) notice.setMessage(notify); if (showingNotice) {
Logger(notify); Logger(notify, LOG_LEVEL.NOTICE, "syncAll");
} else {
Logger(notify);
}
this.setStatusBarText(notify); this.setStatusBarText(notify);
} }
} catch (ex) { }).catch(ex => {
Logger(`Error while ${procedurename}`, LOG_LEVEL.NOTICE); Logger(`Error while ${procedurename}`, LOG_LEVEL.NOTICE);
Logger(ex); Logger(ex);
} finally { }).finally(() => {
workProcs--; // workProcs--;
} })
}); );
await p.wait(limit);
await allSettledWithConcurrencyLimit(procs, 10); }
await p.all();
Logger(`${procedurename} done.`);
}; };
await runAll("UPDATE DATABASE", onlyInStorage, async (e) => { await runAll("UPDATE DATABASE", onlyInStorage, async (e) => {
Logger(`Update into ${e.path}`); Logger(`Update into ${e.path}`);
await this.updateIntoDB(e);
}); await this.updateIntoDB(e, initialScan);
await runAll("UPDATE STORAGE", onlyInDatabase, async (e) => {
Logger(`Pull from db:${e}`);
await this.pullFile(e, filesStorage, false, null, false);
});
await runAll("CHECK FILE STATUS", syncFiles, async (e) => {
await this.syncFileBetweenDBandStorage(e, filesStorage);
}); });
if (!initialScan) {
await runAll("UPDATE STORAGE", onlyInDatabase, async (e) => {
Logger(`Pull from db:${e}`);
await this.pullFile(e, filesStorage, false, null, false);
});
}
if (!initialScan) {
let caches: { [key: string]: { storageMtime: number; docMtime: number } } = {};
caches = await this.localDatabase.kvDB.get<{ [key: string]: { storageMtime: number; docMtime: number } }>("diff-caches") || {};
const docsCount = syncFiles.length;
do {
const syncFilesX = syncFiles.splice(0, 100);
const docs = await this.localDatabase.localDatabase.allDocs({ keys: syncFilesX.map(e => path2id(e.path)), include_docs: true })
const syncFilesToSync = syncFilesX.map((e) => ({ ...e, doc: docs.rows.find(ee => ee.id == path2id(e.path)).doc as LoadedEntry }));
await runAll(`CHECK FILE STATUS:${syncFiles.length}/${docsCount}`, syncFilesToSync, async (e) => {
caches = await this.syncFileBetweenDBandStorage(e, initialScan, caches);
});
} while (syncFiles.length > 0);
await this.localDatabase.kvDB.set("diff-caches", caches);
}
this.setStatusBarText(`NOW TRACKING!`); this.setStatusBarText(`NOW TRACKING!`);
Logger("Initialized,NOW TRACKING!"); Logger("Initialized,NOW TRACKING!");
if (!isInitialized) {
await (this.localDatabase.kvDB.set("initialized", true))
}
if (showingNotice) { if (showingNotice) {
notice.hide(); Logger("Initialize done!", LOG_LEVEL.NOTICE, "syncAll");
Logger("Initialize done!", LOG_LEVEL.NOTICE);
} }
} }
@@ -1526,7 +1725,7 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} else if (toDelete == null) { } else if (toDelete == null) {
Logger("Leave it still conflicted"); Logger("Leave it still conflicted");
} else { } else {
Logger(`resolved conflict:${file.path}`); Logger(`Conflict resolved:${file.path}`);
await this.localDatabase.deleteDBEntry(file.path, { rev: toDelete }); await this.localDatabase.deleteDBEntry(file.path, { rev: toDelete });
await this.pullFile(file.path, null, true, toKeep); await this.pullFile(file.path, null, true, toKeep);
setTimeout(() => { setTimeout(() => {
@@ -1607,17 +1806,24 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
//when to opened file; //when to opened file;
} }
async syncFileBetweenDBandStorage(file: TFile, fileList?: TFile[]) { async syncFileBetweenDBandStorage(file: TFile & { doc?: LoadedEntry }, initialScan: boolean, caches: { [key: string]: { storageMtime: number; docMtime: number } }) {
const doc = await this.localDatabase.getDBEntryMeta(file.path); const doc = file.doc;
if (doc === false) return; if (!doc) return;
const storageMtime = ~~(file.stat.mtime / 1000); const storageMtime = ~~(file.stat.mtime / 1000);
const docMtime = ~~(doc.mtime / 1000); const docMtime = ~~(doc.mtime / 1000);
const dK = `${file.path}-diff`;
const isLastDiff = dK in caches ? caches[dK] : { storageMtime: 0, docMtime: 0 };
if (isLastDiff.docMtime == docMtime && isLastDiff.storageMtime == storageMtime) {
caches[dK] = { storageMtime, docMtime };
return caches;
}
if (storageMtime > docMtime) { if (storageMtime > docMtime) {
//newer local file. //newer local file.
Logger("STORAGE -> DB :" + file.path); Logger("STORAGE -> DB :" + file.path);
Logger(`${storageMtime} > ${docMtime}`); Logger(`${storageMtime} > ${docMtime}`);
await this.updateIntoDB(file); await this.updateIntoDB(file, initialScan);
caches[dK] = { storageMtime, docMtime };
return caches;
} else if (storageMtime < docMtime) { } else if (storageMtime < docMtime) {
//newer database file. //newer database file.
Logger("STORAGE <- DB :" + file.path); Logger("STORAGE <- DB :" + file.path);
@@ -1626,18 +1832,22 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (docx != false) { if (docx != false) {
await this.doc2storage_modify(docx, file); await this.doc2storage_modify(docx, file);
} }
caches[dK] = { storageMtime, docMtime };
return caches;
} else { } else {
// Logger("EVEN :" + file.path, LOG_LEVEL.VERBOSE); // Logger("EVEN :" + file.path, LOG_LEVEL.VERBOSE);
// Logger(`${storageMtime} = ${docMtime}`, LOG_LEVEL.VERBOSE); // Logger(`${storageMtime} = ${docMtime}`, LOG_LEVEL.VERBOSE);
//eq.case //eq.case
} }
caches[dK] = { storageMtime, docMtime };
return caches;
} }
async updateIntoDB(file: TFile) { async updateIntoDB(file: TFile, initialScan?: boolean) {
if (shouldBeIgnored(file.path)) { if (shouldBeIgnored(file.path)) {
return; return;
} }
await this.localDatabase.waitForGCComplete();
let content = ""; let content = "";
let datatype: "plain" | "newnote" = "newnote"; let datatype: "plain" | "newnote" = "newnote";
if (!isPlainText(file.name)) { if (!isPlainText(file.name)) {
@@ -1659,13 +1869,17 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
datatype: datatype, datatype: datatype,
}; };
//upsert should locked //upsert should locked
const msg = `DB <- STORAGE (${datatype}) `;
const isNotChanged = await runWithLock("file:" + fullpath, false, async () => { const isNotChanged = await runWithLock("file:" + fullpath, false, async () => {
if (recentlyTouched(file)) {
return true;
}
const old = await this.localDatabase.getDBEntry(fullpath, null, false, false); const old = await this.localDatabase.getDBEntry(fullpath, null, false, false);
if (old !== false) { if (old !== false) {
const oldData = { data: old.data, deleted: old._deleted }; const oldData = { data: old.data, deleted: old._deleted };
const newData = { data: d.data, deleted: d._deleted }; const newData = { data: d.data, deleted: d._deleted };
if (JSON.stringify(oldData) == JSON.stringify(newData)) { if (JSON.stringify(oldData) == JSON.stringify(newData)) {
Logger("not changed:" + fullpath + (d._deleted ? " (deleted)" : ""), LOG_LEVEL.VERBOSE); Logger(msg + "Skipped (not changed) " + fullpath + (d._deleted ? " (deleted)" : ""), LOG_LEVEL.VERBOSE);
return true; return true;
} }
// d._rev = old._rev; // d._rev = old._rev;
@@ -1673,10 +1887,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
return false; return false;
}); });
if (isNotChanged) return; if (isNotChanged) return;
await this.localDatabase.putDBEntry(d); await this.localDatabase.putDBEntry(d, initialScan);
this.queuedFiles = this.queuedFiles.map((e) => ({ ...e, ...(e.entry._id == d._id ? { done: true } : {}) })); this.queuedFiles = this.queuedFiles.map((e) => ({ ...e, ...(e.entry._id == d._id ? { done: true } : {}) }));
Logger("put database:" + fullpath + "(" + datatype + ") ");
Logger(msg + fullpath);
if (this.settings.syncOnSave && !this.suspended) { if (this.settings.syncOnSave && !this.suspended) {
await this.replicate(); await this.replicate();
} }
@@ -1699,9 +1914,11 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
} }
async resetLocalDatabase() { async resetLocalDatabase() {
clearTouched();
await this.localDatabase.resetDatabase(); await this.localDatabase.resetDatabase();
} }
async resetLocalOldDatabase() { async resetLocalOldDatabase() {
clearTouched();
await this.localDatabase.resetLocalOldDatabase(); await this.localDatabase.resetLocalOldDatabase();
} }
@@ -1738,10 +1955,6 @@ export default class ObsidianLiveSyncPlugin extends Plugin {
if (!this.localDatabase.isReady) return; if (!this.localDatabase.isReady) return;
await runWithLock("sweepplugin", true, async () => { await runWithLock("sweepplugin", true, async () => {
const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO; const logLevel = showMessage ? LOG_LEVEL.NOTICE : LOG_LEVEL.INFO;
if (!this.settings.encrypt) {
Logger("You have to encrypt the database to use plugin setting sync.", LOG_LEVEL.NOTICE);
return;
}
if (!this.deviceAndVaultName) { if (!this.deviceAndVaultName) {
Logger("You have to set your device and vault name.", LOG_LEVEL.NOTICE); Logger("You have to set your device and vault name.", LOG_LEVEL.NOTICE);
return; return;

2
src/utils_couchdb.ts
View File

@@ -40,6 +40,8 @@ export const connectRemoteCouchDBWithSetting = (settings: RemoteDBSettings, isMo
const connectRemoteCouchDB = async (uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> => { const connectRemoteCouchDB = async (uri: string, auth: { username: string; password: string }, disableRequestURI: boolean, passphrase: string | boolean): Promise<string | { db: PouchDB.Database<EntryDoc>; info: PouchDB.Core.DatabaseInfo }> => {
if (!isValidRemoteCouchDBURI(uri)) return "Remote URI is not valid"; if (!isValidRemoteCouchDBURI(uri)) return "Remote URI is not valid";
if (uri.toLowerCase() != uri) return "Remote URI and database name cound not contain capital letters.";
if (uri.indexOf(" ") !== -1) return "Remote URI and database name cound not contain spaces.";
let authHeader = ""; let authHeader = "";
if (auth.username && auth.password) { if (auth.username && auth.password) {
const utf8str = String.fromCharCode.apply(null, new TextEncoder().encode(`${auth.username}:${auth.password}`)); const utf8str = String.fromCharCode.apply(null, new TextEncoder().encode(`${auth.username}:${auth.password}`));

41
styles.css
View File

@@ -2,28 +2,33 @@
color: var(--text-on-accent); color: var(--text-on-accent);
background-color: var(--text-accent); background-color: var(--text-accent);
} }
.normal { .normal {
color: var(--text-normal); color: var(--text-normal);
} }
.deleted { .deleted {
color: var(--text-on-accent); color: var(--text-on-accent);
background-color: var(--text-muted); background-color: var(--text-muted);
text-decoration: line-through;
} }
.op-scrollable { .op-scrollable {
overflow-y: scroll; overflow-y: scroll;
/* min-height: 280px; */ /* min-height: 280px; */
max-height: 280px; max-height: 280px;
user-select: text; user-select: text;
} }
.op-pre { .op-pre {
white-space: pre-wrap; white-space: pre-wrap;
} }
.op-warn { .op-warn {
border: 1px solid salmon; border: 1px solid salmon;
padding: 2px; padding: 2px;
border-radius: 4px; border-radius: 4px;
} }
.op-warn::before { .op-warn::before {
content: "Warning"; content: "Warning";
font-weight: bold; font-weight: bold;
@@ -31,11 +36,13 @@
position: relative; position: relative;
display: block; display: block;
} }
.op-warn-info { .op-warn-info {
border: 1px solid rgb(255, 209, 81); border: 1px solid rgb(255, 209, 81);
padding: 2px; padding: 2px;
border-radius: 4px; border-radius: 4px;
} }
.op-warn-info::before { .op-warn-info::before {
content: "Notice"; content: "Notice";
font-weight: bold; font-weight: bold;
@@ -43,27 +50,33 @@
position: relative; position: relative;
display: block; display: block;
} }
.syncstatusbar { .syncstatusbar {
-webkit-filter: grayscale(100%); -webkit-filter: grayscale(100%);
filter: grayscale(100%); filter: grayscale(100%);
} }
.tcenter { .tcenter {
text-align: center; text-align: center;
} }
.sls-plugins-wrap { .sls-plugins-wrap {
display: flex; display: flex;
flex-grow: 1; flex-grow: 1;
max-height: 50vh; max-height: 50vh;
overflow-y: scroll; overflow-y: scroll;
} }
.sls-plugins-tbl { .sls-plugins-tbl {
border: 1px solid var(--background-modifier-border); border: 1px solid var(--background-modifier-border);
width: 100%; width: 100%;
max-height: 80%; max-height: 80%;
} }
.divider th { .divider th {
border-top: 1px solid var(--background-modifier-border); border-top: 1px solid var(--background-modifier-border);
} }
/* .sls-table-head{ /* .sls-table-head{
width:50%; width:50%;
} }
@@ -75,9 +88,11 @@
.sls-btn-left { .sls-btn-left {
padding-right: 4px; padding-right: 4px;
} }
.sls-btn-right { .sls-btn-right {
padding-left: 4px; padding-left: 4px;
} }
.sls-hidden { .sls-hidden {
display: none; display: none;
} }
@@ -85,8 +100,9 @@
:root { :root {
--slsmessage: ""; --slsmessage: "";
} }
.CodeMirror-wrap::before, .CodeMirror-wrap::before,
.cm-s-obsidian > .cm-editor::before { .cm-s-obsidian>.cm-editor::before {
content: var(--slsmessage); content: var(--slsmessage);
text-align: right; text-align: right;
white-space: pre-wrap; white-space: pre-wrap;
@@ -105,12 +121,15 @@
.CodeMirror-wrap::before { .CodeMirror-wrap::before {
right: 0px; right: 0px;
} }
.cm-s-obsidian > .cm-editor::before {
.cm-s-obsidian>.cm-editor::before {
right: 16px; right: 16px;
} }
.sls-setting-tab { .sls-setting-tab {
display: none; display: none;
} }
div.sls-setting-menu-btn { div.sls-setting-menu-btn {
color: var(--text-normal); color: var(--text-normal);
background-color: var(--background-secondary-alt); background-color: var(--background-secondary-alt);
@@ -131,8 +150,9 @@ div.sls-setting-menu-btn {
flex-grow: 1; flex-grow: 1;
/* width: 100%; */ /* width: 100%; */
} }
.sls-setting-tab:hover ~ div.sls-setting-menu-btn,
.sls-setting-tab:checked ~ div.sls-setting-menu-btn { .sls-setting-tab:hover~div.sls-setting-menu-btn,
.sls-setting-tab:checked~div.sls-setting-menu-btn {
background-color: var(--interactive-accent); background-color: var(--interactive-accent);
color: var(--text-on-accent); color: var(--text-on-accent);
} }
@@ -143,14 +163,17 @@ div.sls-setting-menu-btn {
/* flex-wrap: wrap; */ /* flex-wrap: wrap; */
overflow-x: auto; overflow-x: auto;
} }
.sls-setting-label { .sls-setting-label {
flex-grow: 1; flex-grow: 1;
display: inline-flex; display: inline-flex;
justify-content: center; justify-content: center;
} }
.setting-collapsed { .setting-collapsed {
display: none; display: none;
} }
.sls-plugins-tbl-buttons { .sls-plugins-tbl-buttons {
text-align: right; text-align: right;
} }
@@ -159,13 +182,16 @@ div.sls-setting-menu-btn {
flex-grow: 0; flex-grow: 0;
padding: 6px 10px; padding: 6px 10px;
} }
.sls-plugins-tbl-device-head { .sls-plugins-tbl-device-head {
background-color: var(--background-secondary-alt); background-color: var(--background-secondary-alt);
color: var(--text-accent); color: var(--text-accent);
} }
.op-flex { .op-flex {
display: flex; display: flex;
} }
.op-flex input { .op-flex input {
display: inline-flex; display: inline-flex;
flex-grow: 1; flex-grow: 1;
@@ -185,9 +211,11 @@ div.sls-setting-menu-btn {
color: var(--text-on-accent); color: var(--text-on-accent);
background-color: var(--text-accent); background-color: var(--text-accent);
} }
.history-normal { .history-normal {
color: var(--text-normal); color: var(--text-normal);
} }
.history-deleted { .history-deleted {
color: var(--text-on-accent); color: var(--text-on-accent);
background-color: var(--text-muted); background-color: var(--text-muted);
@@ -197,6 +225,7 @@ div.sls-setting-menu-btn {
.ob-btn-config-fix label { .ob-btn-config-fix label {
margin-right: 40px; margin-right: 40px;
} }
.ob-btn-config-info { .ob-btn-config-info {
border: 1px solid salmon; border: 1px solid salmon;
padding: 2px; padding: 2px;
@@ -208,4 +237,4 @@ div.sls-setting-menu-btn {
padding: 2px; padding: 2px;
margin: 1px; margin: 1px;
border-radius: 4px; border-radius: 4px;
} }