public/obsidian-livesync
1
0 Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-10-30 13:03:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main and refresh translation resources

Browse Source
This commit is contained in:
vorotamoroz committed 2026-10-29 11:28:41 +00:00
2 parents a9434895fb a2eecce239
commit 037e39c9ff
247 files changed
+23914 -2510
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/cli-deno-tests.yml
+3
View File
@@ -22,6 +22,7 @@ on:
- 'package.json'
- 'package-lock.json'
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
paths:
- '.github/workflows/cli-deno-tests.yml'
- 'src/apps/cli/**'
@@ -66,6 +67,7 @@ concurrency:
jobs:
prepare:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
@@ -125,6 +127,7 @@ jobs:
retention-days: 1
test:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
needs: prepare
runs-on: ubuntu-latest
timeout-minutes: 60
.github/workflows/cli-docker-validation.yml
+4 -1
View File
@@ -2,6 +2,7 @@ name: CLI Docker validation
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
paths:
- '.github/workflows/cli-docker.yml'
- '.github/workflows/cli-docker-validation.yml'
@@ -40,6 +41,7 @@ concurrency:
jobs:
bundle:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
name: Build shared Docker CLI bundle
runs-on: ubuntu-24.04
timeout-minutes: 15
@@ -72,6 +74,7 @@ jobs:
retention-days: 1
validate:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
name: Docker E2E (${{ matrix.arch }})
needs: bundle
runs-on: ${{ matrix.runner }}
@@ -171,7 +174,7 @@ jobs:
summary:
name: Collect both Docker E2E results
needs: validate
if: always()
if: ${{ always() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }}
runs-on: ubuntu-24.04
timeout-minutes: 5
outputs:
.github/workflows/cli-p2p-tests.yml
+2
View File
@@ -24,6 +24,7 @@ on:
- '.dockerignore'
- 'test/bench-network/**'
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
paths:
- '.github/workflows/cli-p2p-tests.yml'
- 'src/apps/cli/**'
@@ -53,6 +54,7 @@ concurrency:
jobs:
compose-p2p-e2e:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
.github/workflows/deploy-pages.yml
+2
View File
@@ -14,6 +14,7 @@ on:
- 'package-lock.json'
- '.github/workflows/deploy-pages.yml'
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
paths:
- 'aggregator.html'
- 'src/**'
@@ -32,6 +33,7 @@ concurrency:
jobs:
build:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
name: Validate and package Pages site
runs-on: ubuntu-latest
steps:
.github/workflows/unit-ci.yml
+6
View File
@@ -13,6 +13,7 @@ on:
- 'src/**'
- 'test/**'
- '_tools/**'
- 'scripts/**'
- 'package.json'
- 'package-lock.json'
- 'tsconfig.json'
@@ -44,12 +45,14 @@ on:
- '.github/workflows/cli-docker.yml'
- '.github/workflows/unit-ci.yml'
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
paths:
- 'manifest.json'
- 'updates.md'
- 'src/**'
- 'test/**'
- '_tools/**'
- 'scripts/**'
- 'package.json'
- 'package-lock.json'
- 'tsconfig.json'
@@ -86,6 +89,7 @@ permissions:
jobs:
setup-tools:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
name: Self-hosted setup tools
runs-on: ubuntu-latest
timeout-minutes: 15
@@ -134,6 +138,7 @@ jobs:
run: npm run test:docker-couchdb:stop || true
unit-test:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
name: Unit Tests
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -173,6 +178,7 @@ jobs:
path: coverage/**
integration-test:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
name: Integration Tests
runs-on: ubuntu-latest
timeout-minutes: 30
AGENTS.md
+22
View File
@@ -2,6 +2,12 @@
When working on this repository (writing code, comments, documentation, or commits), you MUST follow these guidelines to maintain consistency.
## Documentation Navigation and Maintenance
- Use the [documentation index](docs/README.md) to select the relevant implementation reference, ADR, proposal, or historical record. The required references below still apply.
- When implementation or publication changes a recorded status, update its publication note and supporting references in the same change. Preserve the original decision date, reviewed-version baseline, and remaining follow-ups.
- Mark superseded or frozen documents at their entry point and link to the maintained reference. Keep historical evidence distinguishable from current operating instructions, and preserve existing links when reorganising documentation.
## Required Reference Files
Before making changes to documentation, user-facing text, or settings:
@@ -39,6 +45,13 @@ Always adhere to the following stylistic and spelling rules:
---
## Release Notes
- Follow the release-note scope and history rules in [devs.md](devs.md#release-notes). Structure substantive entries as the user-visible outcome, the method that produces it, and supporting conditions, defaults, limitations, or required actions. Put the outcome first and use subordinate items when the explanation needs them.
- When a pull request explains only its implementation, consult the relevant issue or design decision to establish the problem it resolves. Verify the claimed outcome against the change and its evidence. Distinguish removing a cause from reducing occurrence, detecting a problem, or improving recovery.
---
## Technical & Architecture Rules
1. **Database Structure**:
@@ -56,6 +69,15 @@ Always adhere to the following stylistic and spelling rules:
---
## Failure Assessment and Recovery
- Establish the intended behaviour from [the conflict policy](devs.md#conflict-merge-policy) and [the revision-provenance contract](docs/specs_conflict_resolution.md) before classifying a failure. A visible conflict which preserves both versions can be the correct outcome; manual selection alone does not establish a serious defect.
- Assess preservation of user edits and required revision ancestry separately from extra work, temporary synchronisation delays, or duplicate internal records. A successful automatic merge or an empty queue is not proof of correct recovery. Check the resulting contents and history, safe convergence, and behaviour after restart where applicable.
- Distinguish data still present on a device or in readable history from irretrievable loss. Verify that a recovery source contains the affected edits before treating the failure as fully recoverable. Follow [the recovery guidance](docs/recovery.md) before recommending a rebuild or reset which could discard unique work.
- Give demonstrated irreversible information loss particular weight. Describe the actual burden of safe manual conflict resolution without treating automation as a correctness requirement. If preservation, recovery, impact, or the appropriate response remains unclear, review the evidence and alternatives with the collaborating user before changing the design or accepting the risk; continue independent investigation.
---
## Development & Verification Commands
Before submitting code, you should run verification scripts locally to ensure correct syntax and function.
CITATION.cff
+3 -3
View File
@@ -9,9 +9,9 @@ authors:
- name: "Self-hosted LiveSync Contributors"
repository-code: "https://github.com/vrtmrz/obsidian-livesync"
url: "https://github.com/vrtmrz/obsidian-livesync"
version: 1.0.23
doi: 10.5281/zenodo.22247183
date-released: "2026-09-05"
version: 1.0.38
doi: 10.5281/zenodo.23438598
date-released: "2026-10-17"
license: MIT
keywords:
- obsidian
README.md
+56 -58
View File
@@ -1,11 +1,12 @@
# Self-hosted LiveSync
[Japanese docs](./README_ja.md) - [Chinese docs](./README_cn.md).
Self-hosted LiveSync is a community-developed synchronisation plug-in available on all Obsidian-compatible platforms. It leverages robust server solutions such as CouchDB or object storage systems (e.g., MinIO, S3, R2, etc.) to ensure reliable data synchronisation.
Additionally, it supports peer-to-peer synchronisation using WebRTC, enabling devices to exchange notes without a central data-storage server. A signalling relay is still required for peer discovery. See [How peer-to-peer synchronisation works](./docs/p2p.md).
Find setup guides, troubleshooting, technical references, design decisions, and historical material in the [documentation index](docs/README.md).
![obsidian_live_sync_demo](https://user-images.githubusercontent.com/45774780/137355323-f57a8b09-abf2-4501-836c-8cb7d2ff24a3.gif)
>[!IMPORTANT]
@@ -18,12 +19,8 @@ Additionally, it supports peer-to-peer synchronisation using WebRTC, enabling de
- Use open-source solutions for the server.
- Compatible solutions are supported.
- Support end-to-end encryption.
- Synchronise settings, snippets, themes, and plug-ins via [Customisation Sync (Beta)](docs/settings.md#6-customisation-sync-advanced) or [Hidden File Sync](docs/tips/hidden-file-sync.md).
- Enable supported, opt-in WebRTC peer-to-peer synchronisation.
- No central data-storage server is required, but a signalling relay is still required for peer discovery.
- At least one device containing the required data must be online while another device synchronises.
- Follow the [Peer-to-Peer Setup](docs/setup_p2p.md) after reviewing the [P2P communication model](docs/p2p.md).
- If you would like to see how P2P setup and connectivity behave on your devices, try the guided connection check in the experimental [WebPeer browser utility](https://vrtmrz.github.io/obsidian-livesync/webpeer/). It uses an empty Vault and does not test note synchronisation. See the [WebPeer documentation](src/apps/webpeer/README.md) for its scope and limitations.
- Synchronise settings, snippets, themes, and plug-ins via [Customisation Sync (Advanced)](docs/settings.md#6-customisation-sync-advanced) or [Hidden File Sync](docs/tips/hidden-file-sync.md).
- Synchronise peer-to-peer using WebRTC.
This plug-in may be particularly useful for researchers, engineers, and developers who need to keep their notes fully self-hosted for security reasons. It is also suitable for anyone seeking the peace of mind that comes with knowing their notes remain entirely private.
@@ -34,75 +31,76 @@ This plug-in may be particularly useful for researchers, engineers, and develope
## How to Use
### 3-minute setup - CouchDB on fly.io
Self-hosting lets us choose a server we trust and manage it ourselves. Please set up and maintain your own server, and keep it secure.
**Recommended for beginners**
<a id="setup-workflows"></a>
[![LiveSync Setup onto Fly.io SpeedRun 2024 using Google Colab](https://img.youtube.com/vi/7sa_I1832Xc/0.jpg)](https://www.youtube.com/watch?v=7sa_I1832Xc)
### What you need
1. [Set up CouchDB on fly.io](docs/setup_flyio.md)
2. Configure plug-in in [Quick Setup](docs/quick_setup.md)
| Method | What you need | Nice to have |
| --- | --- | --- |
| CouchDB | A CouchDB-compatible server. | — |
| Object Storage | An S3-compatible service or server. | — |
| Peer-to-Peer | Nothing. | Your own signalling server. |
### Setup workflows
For a CouchDB-compatible server, choose a setup guide for your environment:
Choose a synchronisation method, prepare its server where required, then follow the corresponding client setup:
- [Your own server](docs/setup_own_server.md): run CouchDB on a Raspberry Pi or another server you manage.
- [Docker](docker/README.md): deploy CouchDB with the supplied Docker stack.
- [Fly.io](docs/setup_flyio.md): manage your own CouchDB instance on hosted infrastructure.
- [IBM Cloudant (managed)](docs/setup_cloudant.md): use a database service managed by IBM.
1. CouchDB
1. Prepare the server:
- [Set up your own CouchDB server](docs/setup_own_server.md).
- [Set up CouchDB on fly.io](docs/setup_flyio.md).
2. Configure the clients by following [CouchDB Quick Setup](docs/quick_setup.md).
2. Object Storage
1. Prepare the server. A maintained MinIO server installation guide is not currently available here, so set up an S3-compatible service or server of your choice.
2. Configure the clients by following [Object Storage Setup](docs/setup_object_storage.md).
3. Peer-to-Peer
1. No central data-storage server is required. The project's public signalling relay requires no server provisioning; controlled deployments can provide another compatible relay.
2. Configure the clients by following [Peer-to-Peer Setup](docs/setup_p2p.md).
The linked guides explain the setup and any provider-specific costs.
Each workflow establishes ordinary note synchronisation on the first device, generates a Setup URI for each additional device from that working device, and verifies synchronisation in both directions.
<a id="3-minute-setup---couchdb-on-flyio"></a>
> [!TIP]
> Fly.io is no longer free. Fortunately, we can still use IBM Cloudant despite some limitations. Refer to [Set up IBM Cloudant](docs/setup_cloudant.md).
> We can also use peer-to-peer synchronisation without a central data-storage server; a signalling relay is still used for peer discovery. Alternatively, cheap object storage like Cloudflare R2 can be used for free.
> However, most importantly, we can use a server that we trust. Therefore, please set up your own server.
> CouchDB can also be run on a Raspberry Pi (please be mindful of your server's security).
For the Fly.io video walkthrough, see [automated setup](docs/setup_flyio.md#a-very-automated-setup).
P2P does not require a central data-storage server, but a signalling relay is still required for peer discovery. You can use the project's public signalling relay, so there is no server to provision. At least one device containing the required data must be online while another device synchronises. Follow the [Peer-to-Peer Setup](docs/setup_p2p.md) after reviewing the [P2P communication model](docs/p2p.md).
### Third-party managed CouchDB hosting
If you would like to see how P2P setup and connectivity behave on your devices, try the guided connection check in the experimental [WebPeer browser utility](https://vrtmrz.github.io/obsidian-livesync/webpeer/). It uses an empty Vault and does not test note synchronisation. See the [WebPeer documentation](src/apps/webpeer/README.md) for its scope and limitations.
### Start synchronising
1. Install and configure Self-hosted LiveSync on your first device using [Quick Setup](docs/quick_setup.md) for CouchDB, [Object Storage Setup](docs/setup_object_storage.md), or [Peer-to-Peer Setup](docs/setup_p2p.md).
2. Generate a Setup URI from that working device and follow the same guide to add your other devices.
3. Create test notes and confirm that they synchronise in both directions.
<a id="third-party-managed-couchdb-hosting"></a>
### Hosting services proposed by the community
> [!NOTE]
> The following is a third-party hosting option proposed by Zenith Hosting. It is not an official Self-hosted LiveSync service, and it is neither endorsed nor recommended by this project.
> The services below are operated by third parties. They are not official Self-hosted LiveSync services, and this project neither endorses nor recommends them. The providers proposed the listings in the table.
If you would rather not set up and maintain a server yourself, Zenith Hosting offers a managed CouchDB server which this plug-in can be configured to connect to: [Zenith Hosting](https://zenith.hosting/host/obsidian-livesync). As with any hosted service, your data will reside on a server operated by a third party, so please consider whether that is acceptable for your vault before using it.
If you would rather not set up and maintain a server yourself, the following providers offer managed CouchDB servers which this plug-in can be configured to connect to:
| Provider | Offering |
| --- | --- |
| [Zenith Hosting](https://zenith.hosting/host/obsidian-livesync) | Managed CouchDB hosting. |
Your synchronisation data will reside on a server operated by the provider. Consider whether that is acceptable for your Vault, and review the provider's current terms, pricing, and data-handling policies before using the service.
## Information in the Status Bar
Synchronisation status is shown in the status bar with the following icons.
Synchronisation status is shown in the status bar and inside the editor by default. The main indicators are:
- Activity Indicator
- 📲 A finite remote operation is in progress
- 🌐N Approximate remote requests currently in progress
- Status
- ⏹️ Stopped
- 💤 LiveSync enabled. Waiting for changes
- ⚡️ Synchronisation in progress
- ⚠ An error occurred
- Statistical Indicators
- ↑ Uploaded chunks and metadata
- ↓ Downloaded chunks and metadata
- Progress Indicators
- 📥 Unprocessed transferred items
- 📄 Working database operation
- 💾 Working write storage processes
- ⏳ Working read storage processes
- 🛫 Pending read storage processes
- 📬 Batched read storage processes
- ⚙️ Working or pending storage processes for hidden files
- 🛄 Pending initial on-demand chunk requests
- 🔁 Pending chunk retries, including retry delays
- 🔌 Working customisation items (configuration, snippets, and plug-ins)
| Indicator | Meaning |
| --- | --- |
| ⏹️ | Stopped. |
| 🌀 | Starting synchronisation. |
| 💤 | LiveSync mode is waiting for changes. |
| ⚡️ | Synchronisation is in progress. |
| ⏩ | Comparing CouchDB change history, including entries already held locally. |
| ⚠ | An error occurred. |
| ↑ / ↓ | Uploaded / downloaded Chunks and Metadata. |
| 📲 / 🌐N | A finite remote operation is in progress / approximate remote requests in progress. |
To prevent file and database corruption, please avoid closing Obsidian until all progress indicators have disappeared as much as possible (although the plug-in will attempt to resume if interrupted). This is especially important if you have deleted or renamed files.
<a id="customising-status-items-with-css-snippets"></a>
See the [status display guide](docs/tips/status-display.md) for detailed progress counters and [CSS customisation](docs/tips/status-display.md#customising-status-items-with-css-snippets).
Allow pending file operations to finish before closing Obsidian, especially after deleting or renaming files.
## Tips and Troubleshooting
- If you want a faster and simpler initial replication when setting up subsequent devices, see the [Fast Setup Guide](docs/tips/fast-setup.md).
devs.md
+34 -5
View File
@@ -1,5 +1,7 @@
# Self-hosted LiveSync Development Guide
Use the [documentation index](docs/README.md) to find implementation references, architectural decisions, proposals, and historical material for the area being changed.
## Project Overview
Self-hosted LiveSync is an Obsidian plugin for synchronising vaults across devices using CouchDB, MinIO/S3, or peer-to-peer WebRTC. The codebase uses a modular architecture with TypeScript, Svelte, and PouchDB.
@@ -78,7 +80,7 @@ Regression tests remain in the suite owned by the implementation under test. Plu
- **CLI E2E** (`src/apps/cli/testdeno/`): Host-independent consumer workflows. The canonical Compose P2P suite covers ordinary two-peer synchronisation, replacement of the current Replicator followed by transfer with the same peer, and explicit relay disconnection followed by paused and resumed reconnection. Its lifecycle entry point is included only in the Docker test build and does not add a public CLI command. Run `npm run test:e2e:cli` for the ordinary suite or `npm run test:e2e:cli:p2p` for P2P validation.
- **Self-hosted setup tools** (`utils/couchdb/`, `utils/setup/`, and `utils/flyio/`): Deno contract tests consume the exact locked Commonlib registry package, verify current CouchDB, Object Storage, and random-room P2P Setup URI defaults and remote profiles, and keep CouchDB administration separate from package-owned LiveSync database-version negotiation. `unit-ci` also provisions a real temporary CouchDB database and verifies its version document against the installed Commonlib package. Run `npm run test:setup-tools` for the local contract gate.
- **Real Obsidian E2E** (`test/e2e-obsidian/`): Local-first scripts that launch real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in. Use these for boot-up sequence, vault reflection, RedFlag flows, Fast Setup (Simple Fetch), settings dialogues, restart-sensitive workflows, Object Storage regressions, and other behaviour that depends on Obsidian itself. Run focused scripts such as `npm run test:e2e:obsidian:two-vault-sync`, or use `npm run test:e2e:obsidian:local-suite:services` to run the broader local suite with CouchDB and RustFS fixtures managed by the wrapper.
- **Real Obsidian E2E** (`test/e2e-obsidian/`): Local-first scripts that launch real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in. Use these for boot-up sequence, vault reflection, RedFlag flows, Fast Setup (Simple Fetch), settings dialogues, restart-sensitive workflows, Object Storage regressions, and other behaviour that depends on Obsidian itself. After source changes, run the build-owning wrapper, for example `npm run test:e2e:obsidian:focused -- two-vault-sync`, with the required fixture already running. Direct scenario scripts reuse the existing bundle and are suitable for reruns only while that build remains unchanged. Use `npm run test:e2e:obsidian:local-suite:services` to run the broader local suite with CouchDB and RustFS fixtures managed by the wrapper.
- **Docker Services**: Service-backed tests use CouchDB and RustFS (S3). Canonical P2P validation owns its relay through the CLI Compose runner:
@@ -166,6 +168,14 @@ Legacy Modules remain grouped by directory:
- `features/` contains optional features; and
- `extras/` contains development and testing tools.
### UI commands and dialogue ownership
Declare shared UI requests with `declareUICommand<Request, Result>` from `@vrtmrz/livesync-commonlib/ui-commands`. The host's `services.UI.commands` registry owns one handler per declaration. A serviceFeature registers the handler, releases it on unload, and owns the associated dialogue lifetime. Callers receive a typed result or an explicit `unavailable` result; handler failures propagate.
`OPEN_JOINED_DEVICES` is shared by Remote Configuration, version Notices, and the command palette. `useJoinedDevicesFeature` registers it once and gives the dialogue narrow refresh, forget, and cancellation callbacks. Each remote operation uses a settings snapshot and releases its finite resource in `finally`. Results from an old remote configuration or a closed dialogue are discarded. A write already sent to the captured remote can still complete after the dialogue closes.
The registry belongs to one UIService instance. Reuse the same declaration object at each call site, and keep active UI cleanup in the feature and dialogue owner. `UIService` disposes the registry when the host's context emits the plug-in-unloaded event.
### Key Architectural Components
- **LiveSyncLocalDB** (`@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB`): Local PouchDB database wrapper
@@ -204,6 +214,9 @@ The [multiple-device conflict test procedure](test/README.md#multiple-device-con
### File Structure Conventions
- **Class modules**: Start a new or renamed TypeScript or JavaScript filename with an upper-case letter when the file contains a class with the same name.
- **Function modules**: Start the filename with a lower-case letter when the module primarily exports functions, such as `initialFetchFastForward.ts`. Follow the surrounding filename style for the rest of the basename. Corresponding test files use the same basename.
- **Existing files**: Retain established names for unrelated files and separate conventions for components or other file types.
- **Platform-specific code**: Use `.platform.ts` suffix (replaced with `.obsidian.ts` in production builds via esbuild)
- **Development code**: Use `.dev.ts` suffix (replaced with `.prod.ts` in production)
- **Path aliases**: `@/*` maps to `src/*`; Commonlib uses package exports rather than a source alias
@@ -245,6 +258,18 @@ Where `performance.memory` is available, the report includes approximate JavaScr
The developer-only adapter in `src/features/ReviewHarness/reviewHarnessIdBenchmarkRuntime.ts` imports `HashManager` from Commonlib's public `/hashing` entry. Compilation, packed-package checks, and runtime tests cover this boundary. The algorithms remain owned by Commonlib.
#### File content hash performance measurements
Commonlib owns the algorithm choice and provenance contract. Its [file-reflection content hash decision](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/file-reflection-content-hash.md) records the SHA-256 rationale, xxHash64 comparison, measured costs, and conditions for reconsideration. This section describes the host-owned measurement procedure.
Choose **Run** beside **File content hash performance** in the Review Harness. **Automatic** does not run this measurement; **Full review** includes it.
The measurement compares the Self-hosted LiveSync SHA-256 file content hash helper with `xxHash64` for fixed in-memory `Blob` inputs of 4 KiB, 64 KiB, 1 MiB, and 50 MiB. Each input begins with the same deterministic byte sequence. Input construction is excluded from the timed samples; `xxHash64` engine initialisation is timed and reported separately. One 4-KiB warm-up call per algorithm is also measured separately and excluded from samples. SHA-256 uses the same helper as the product's file content hash path; `xxHash64` timing includes `Blob.arrayBuffer()` and `h64Raw`. Calls run one at a time, alternate algorithm order between samples, and yield to the interface between calls. The 50-MiB input uses the same three samples as the other sizes.
The report gives the median and range for each algorithm and input size. Where `performance.memory` is available, it also records approximate JavaScript heap samples before, during, and after measurement. These samples may include other Obsidian activity and garbage collection; they are neither total process RAM nor a guaranteed peak. Unsupported devices explicitly report that heap measurements are unavailable.
The scenario uses only fixed in-memory inputs. It does not read or write the Vault, database, settings, remote storage, or user content. It does not expose a hash-method setting or use a fixed performance threshold. The developer-only adapter in `src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts` imports `computeFileContentHash` from Commonlib's public `/hashing` entry and `xxhashNew()` from `octagonal-wheels`. The benchmark compares the product helper with that package implementation; it does not change the hash method used by LiveSync.
#### Logs
- Use `this._log(msg, LOG_LEVEL_INFO)` in modules (automatically prefixes with module name)
@@ -297,6 +322,8 @@ Existing legacy Modules continue to register their handlers in `onBindFunction()
database and Replicator readiness. See
[the declarative settings adapter ADR](docs/adr/2026_08_declarative_settings_adapter.md).
- Use `this.services.setting.saveSettingData()` instead of using plugin methods directly
- Inject `@vrtmrz/value-store` at host composition. `setting.deviceValues` uses exact device-local keys; `setting.config` and the Service Hub `config` share the existing Vault and database-suffix scope. Await reads, writes, and deletions before advancing the workflow. Browser and Obsidian hosts retain their Web Storage keys, and the CLI retains its existing flat JSON string map.
- Seed the settings tab with `await settingTab.loadSettings()` before registration. Its editing buffer remains synchronous for Obsidian rendering; storage failures must not replace a previously loaded value with an empty default.
### Database Operations
@@ -328,6 +355,7 @@ This policy governs plug-in pre-releases and their accompanying CLI images. For
- Add only user-facing changes that help users understand what they gain, what has changed, or what they may need to do after updating.
- Avoid listing purely internal refactors, maintenance chores, generated-file changes, and dependency updates unless they affect users; group and label them when they are included.
- When preparing a release, replace `## Unreleased` with the target version heading (for example, `## 0.25.81`) and add a fresh empty `## Unreleased` section above it for the next cycle.
- For a stable release, review the complete user-facing changes since the previous stable release. Classify new features and changed behaviour relative to that stable release. Include changes from intervening pre-releases which never became stable, even when their original notes have already moved into a release history.
- Review and polish the released section in the release PR before tagging, because the content is embedded into the plug-in and may be reused as the GitHub Release notes.
- Keep approximately the five most recent published plug-in versions in the embedded `updates.md`. Move older published sections unchanged into the appropriate release-line archive under `docs/releases/`, and update the history references when rotating them.
@@ -344,11 +372,12 @@ For a fix needed only by the CLI, use [CLI-only patch releases](#cli-only-patch-
- For a hyphenated pre-release, run finalisation with `prerelease=true`; CLI publication remains optional. For a stable version awaiting BRAT validation, use `prerelease=true` and `publish_cli=false`.
- Approve the `Release Obsidian Plugin` workflow for the `release` environment, then inspect the generated draft GitHub Release. When a hyphenated pre-release includes the CLI, confirm that it received only its immutable version and SHA-qualified image tags.
- Publish the draft as a GitHub pre-release without replacing the latest stable release. Keep its release pull request in draft and leave its base branch unchanged throughout BRAT validation. Record that state in the pull request.
- Validate the published release through BRAT. Confirm start-up, ordinary bidirectional synchronisation, and any regression scenario relevant to the release.
- Validate installation or update through BRAT, and cold-start Obsidian to confirm the plug-in loads. Verify ordinary bidirectional synchronisation and relevant regression scenarios in real Obsidian against the published artefacts, or a build with matching hashes. When automated scenarios establish that coverage, BRAT acceptance need only cover installation or update and start-up; manually check any remaining release-specific behaviour. Record the artefact hashes and validation evidence as described in the [real Obsidian test guide](test/e2e-obsidian/README.md#combined-setup-and-security-regression-checks).
- After a hyphenated pre-release passes, keep its release pull request unmerged. Add a reviewed metadata-only commit to the selected base branch which records the published version in `versions.json` and moves its exact tagged release notes out of `## Unreleased`, then close the release pull request only through a separate maintainer action.
- After a stable version passes, remove the GitHub pre-release designation and make that exact release the latest stable release. Confirm that the release is no longer a pre-release, then mark its release pull request ready, merge the exact release commit into the selected base branch with a merge commit, and integrate that exact commit through the reviewed branch chain into the repository's default branch. Confirm that the default branch contains the matching stable metadata. Create the stable CLI tag and publish its `latest` and major-minor image tags, if selected, through a separate maintainer gate.
- If BRAT validation fails, keep the release PR in draft and do not move published tags. Before preparing the next version, add a reviewed metadata-only commit to the selected base branch which records the published version in `versions.json` and moves its exact tagged release notes out of `## Unreleased`. Keep only changes made after that tag under `## Unreleased`. Compare the historical section with `git show <tag>:updates.md`; do not merge the failed release PR or describe it as validated. The next release PR can then rotate only the correction notes while preserving the immutable release history.
- Prepare and publish the next patch or pre-release version from that reconciled base. Leave the failed release PR draft until it is deliberately closed as superseded under a separate maintainer action.
- If BRAT validation fails, keep the release PR in draft and leave its published tag and artefacts unchanged. Prepare the correction on a new branch descended from that exact release commit, retaining its existing changes and `versions.json` entry. Prepare the next patch or pre-release version on that branch, with its own version, tag, and release PR. The selected base branch remains unchanged until a successor stable release passes validation and is promoted.
- For the successor stable release, consolidate all user-facing changes since the last stable release, including changes from intervening pre-releases which never became stable. Preserve each published pre-release's exact tagged notes in the appropriate history under `docs/releases/`, and compare the archived section with `git show <tag>:updates.md`. The new release notes must describe the whole stable upgrade, including the correction.
- After the successor stable release is validated and promoted, merge its exact release commit and ancestry through the reviewed branch chain. Leave the failed release PR in draft until it is deliberately closed as superseded under a separate maintainer action.
- A hyphenated version is rejected unless `prerelease=true`. A stable version staged with `prerelease=true` is rejected unless `publish_cli=false`.
### Release Cheat Sheet
@@ -378,7 +407,7 @@ For a fix needed only by the CLI, use [CLI-only patch releases](#cli-only-patch-
9. Validate the published release through BRAT, including start-up, ordinary bidirectional synchronisation, and any release-specific regression scenario.
10. After a hyphenated pre-release passes, keep its release PR unmerged, reconcile its `versions.json` entry and exact release-note section into the selected base branch as metadata only, then close the PR through a separate maintainer action.
11. After a stable version passes, remove the pre-release designation and make the exact release the latest stable release. Confirm that promotion, then mark the release PR ready, merge the exact release commit into the selected base branch, and integrate it through the reviewed branch chain into the repository's default branch. Confirm that the default branch contains the matching stable metadata, then publish stable CLI tags through a separate maintainer gate if selected.
12. If validation fails, leave the PR in draft and do not move the published tags. Reconcile the published version's `updates.md` section and `versions.json` entry into the base branch as metadata only, then prepare the next patch or pre-release version from the remaining `## Unreleased` entries.
12. If validation fails, leave the PR in draft and keep its published tag and artefacts unchanged. Start a new branch from that release commit, add the correction, and prepare the next patch or pre-release version. Retain the published version's `versions.json` entry and exact archived notes, and consolidate the successor stable release notes from the last stable release. Keep the base branch unchanged until the successor stable release is validated and promoted.
### CLI-only patch releases
docker/.env.example
+4 -4
View File
@@ -34,9 +34,9 @@ ACME_EMAIL=you@yourdomain.com
# PROFILE: tailscale (--profile tailscale)
# =============================================================================
# Tailscale OAuth key (not a regular auth key — must be OAuth for persistent use)
# Generate at: https://login.tailscale.com/admin/settings/oauth
# Scopes needed: devices:write
# Tailscale authentication key for this container.
# OAuth clients require their own scopes and tag configuration; see:
# https://tailscale.com/docs/features/containers/docker/docker-params#ts_authkey
TS_AUTHKEY=tskey-auth-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
# Hostname this node will have on your tailnet (becomes <hostname>.<tailnet>.ts.net)
@@ -47,6 +47,6 @@ TS_HOSTNAME=livesync
# =============================================================================
# Tunnel token from Cloudflare Zero Trust dashboard
# Create at: https://one.dash.cloudflare.com/ → Networks → Tunnels → Create tunnel
# Current setup guide: https://developers.cloudflare.com/tunnel/get-started/
# Copy the token from the "Install connector" step
CF_TUNNEL_TOKEN=eyJhIjoixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
docker/README.md
+127 -85
View File
@@ -61,11 +61,11 @@ docker compose --profile cloudflare up -d
```bash
# Should return {"status":"ok"}
curl -u admin:yourpassword http://localhost:5984/_up
curl -u '<COUCHDB_USER>:<COUCHDB_PASSWORD>' http://localhost:5984/_up
# Check CORS headers
curl -v -H "Origin: app://obsidian.md" \
-u admin:yourpassword \
-u '<COUCHDB_USER>:<COUCHDB_PASSWORD>' \
http://localhost:5984/
```
@@ -75,7 +75,7 @@ In the Obsidian plugin settings (**Self-hosted LiveSync**):
| Field | Value |
|---|---|
| URI | `https://your-domain-or-ts-hostname:5984` (or `http://localhost:5984` for LAN-only) |
| URI | `https://your-domain-or-ts-hostname` for an HTTPS profile, or `http://localhost:5984` for local HTTP |
| Username | value of `COUCHDB_USER` |
| Password | value of `COUCHDB_PASSWORD` |
| Database name | value of `COUCHDB_DATABASE` (default: `obsidiannotes`) |
@@ -85,10 +85,13 @@ In the Obsidian plugin settings (**Self-hosted LiveSync**):
## Profile Details
### Default (no profile) — LAN / Localhost only
<a id="default-no-profile--lan--localhost-only"></a>
CouchDB is exposed on `http://localhost:5984` (or LAN IP).
**Desktop Obsidian works over HTTP.** Mobile Obsidian requires HTTPS — use a tunnel profile.
### Default (no profile) — Host HTTP access
The Compose file publishes CouchDB on host port 5984 by default, on all host interfaces. Its reachability depends on the host firewall and network; selecting an HTTPS profile does not close that direct HTTP port. Restrict the host port to the intended network, bind it to loopback, or remove the mapping when all access uses the proxy.
Desktop Obsidian can use HTTP. Use HTTPS for mobile clients and connections outside a trusted local network. The supplied HTTPS profiles use port 443, so their client URL normally has no `:5984` suffix.
### `--profile caddy` — Public Domain + Auto TLS
@@ -104,13 +107,14 @@ ACME_EMAIL=you@example.com
Caddy automatically issues a Let's Encrypt certificate. No manual cert management.
### `--profile tailscale` — No Domain Required ✅ Recommended for privacy
<a id="--profile-tailscale--no-domain-required--recommended-for-privacy"></a>
### `--profile tailscale` — Tailnet or public HTTPS
**Requires**:
- Free [Tailscale account](https://login.tailscale.com/)
- Install the Tailscale app on all your Obsidian devices
- Generate an **OAuth key** at: https://login.tailscale.com/admin/settings/oauth
(Scopes: `devices:write`)
- A [Tailscale account](https://login.tailscale.com/).
- The Tailscale app on devices using private tailnet access.
- An authentication key for `TS_AUTHKEY`. Follow the [Docker authentication parameters](https://tailscale.com/docs/features/containers/docker/docker-params#ts_authkey) if using an OAuth client instead.
**Set in `.env`**:
```
@@ -119,33 +123,29 @@ TS_HOSTNAME=livesync
```
**Two sub-modes**:
- **VPN mode** (default): CouchDB accessible only to devices on your Tailnet at
`https://livesync.<tailnet>.ts.net` — completely private
- **Funnel mode**: public HTTPS at `https://livesync.<tailnet>.ts.net` — no domain purchase
Enable in your [Tailscale ACL](https://login.tailscale.com/admin/acls):
```json
"nodeAttrs": [{"target": ["tag:container"], "attr": ["funnel"]}]
```
> **Note on Windows Docker Desktop**: If `/dev/net/tun` is unavailable, add `TS_USERSPACE=true`
> to the tailscale service environment in `docker-compose.yml`.
- **Private tailnet access (VPN / Serve)**: choose this for access from your own devices on your tailnet, subject to its access rules. Set `AllowFunnel` to `false` in `config/ts-serve.json`, and use the Tailscale app on those devices.
- **Public HTTPS (Funnel)**: choose this when devices need to connect over the Internet without joining your tailnet. Keep `AllowFunnel` set to `true`, and allow Funnel for this server in your tailnet's access controls.
Both modes use a Tailscale HTTPS name, such as `https://livesync.<tailnet>.ts.net`, so you do not need to buy a domain. Use the actual DNS name assigned to your server in Tailscale. See the [Tailscale Docker guide](https://tailscale.com/blog/docker-tailscale-guide) for HTTPS and Funnel prerequisites.
Choose the mode before starting the profile. The bundled `config/ts-serve.json` sets `AllowFunnel` to `true`, requesting public HTTPS when the tailnet permits Funnel; change that entry to `false` to keep the Tailscale endpoint private. After starting or restarting the Tailscale service, check the effective configuration with `docker exec livesync-tailscale tailscale serve status`. The separate host HTTP port described above is unaffected by this choice.
The bundled service expects `/dev/net/tun`. On hosts without that device, follow Tailscale's userspace-networking instructions and adjust the device mount as well as `TS_USERSPACE`; changing the environment variable alone leaves the required bind mount in place.
### `--profile cloudflare` — Cloudflare Tunnel
**Requires**:
- Free [Cloudflare account](https://www.cloudflare.com/)
- A domain managed by Cloudflare DNS (can transfer existing domain for free)
- Cloudflare Zero Trust account (free)
- A [Cloudflare account](https://www.cloudflare.com/).
- A domain whose DNS is managed by Cloudflare.
#### Step 1: Create a Cloudflare Tunnel
1. Log in to [Cloudflare Zero Trust](https://one.dash.cloudflare.com/)
2. Navigate to **Networks → Tunnels**
3. Click **Create a tunnel**
4. Choose **Cloudflared** as tunnel type
5. Name your tunnel (e.g., `obsidian-livesync`)
6. Click **Save tunnel**
7. **Copy the tunnel token** — it looks like `eyJhIjoiZX...` (very long, ~400 characters)
1. Open the [Cloudflare dashboard](https://dash.cloudflare.com/) and go to **Networking → Tunnels**.
2. Select **Create Tunnel**, give it a name such as `obsidian-livesync`, and create it.
3. On the connector setup screen, copy the installation command into a text editor without running it. Extract only the tunnel token, the string beginning with `eyJ`, and put it in `CF_TUNNEL_TOKEN` in the next step. Keep the token private.
For an existing tunnel, select it and use **Add a replica** to display the installation command again. This Compose profile starts the connector in step 4. Until then, the dashboard can show the tunnel as disconnected. See the official [tunnel setup](https://developers.cloudflare.com/tunnel/get-started/) and [token instructions](https://developers.cloudflare.com/tunnel/reference/tunnel-tokens/) for the current screens.
#### Step 2: Configure Environment
@@ -157,27 +157,14 @@ COUCHDB_DOMAIN=sync.yourdomain.com # Must be a domain managed by Cloudflare
#### Step 3: Add Public Hostname Route
🚨 **CRITICAL**: Token-based tunnels ignore the local `cloudflared.yml` config file. All routing is controlled from the dashboard.
Configure the route in the Cloudflare dashboard for this remotely managed tunnel. The local `config/cloudflared.yml` is not the routing source for the token-managed profile.
Back in the Zero Trust dashboard, **in the same tunnel creation flow** (or edit your tunnel later):
1. Open the tunnel under **Networking → Tunnels**.
2. Under **Routes**, add a **Published application**.
3. Choose the public hostname, for example `sync.yourdomain.com`.
4. Set the service URL to `http://couchdb:5984` and save the route.
1. Go to the **Public Hostname** tab
2. Click **Add a public hostname**
3. Configure:
- **Subdomain**: `sync` (or your preferred subdomain)
- **Domain**: Select your Cloudflare domain from dropdown
- **Type**: `HTTP`
- **URL**: `couchdb:5984` ← **Do NOT use `localhost`!**
**Why `couchdb:5984` not `localhost:5984`?**
- The `cloudflared` container runs inside Docker on the same network as `couchdb`
- Docker's internal DNS resolves `couchdb` to the correct container
- Using `localhost` would look inside the `cloudflared` container (nothing there)
4. Under **Additional application settings** (expand):
- **No TLS Verify**: Leave **OFF** (CouchDB uses plain HTTP internally, that's fine)
- Leave other settings at defaults
5. Click **Save hostname**
The `cloudflared` and `couchdb` services share a Docker network. The service name reaches CouchDB; `localhost` would address the `cloudflared` container itself. The local connection is HTTP, while the public client URL is HTTPS. Keep TLS certificate verification enabled for any HTTPS origin you configure later. The [current setup guide](https://developers.cloudflare.com/tunnel/get-started/) describes the dashboard flow.
#### Step 4: Start the Stack
@@ -203,7 +190,7 @@ You should see:
curl -I https://sync.yourdomain.com
# Should return {"couchdb":"Welcome",...}
curl -u admin:yourpassword https://sync.yourdomain.com
curl -u '<COUCHDB_USER>:<COUCHDB_PASSWORD>' https://sync.yourdomain.com
```
If you get **404**, see Troubleshooting below.
@@ -220,9 +207,7 @@ In Obsidian → Settings → **Self-hosted LiveSync**:
| Database name | value of `COUCHDB_DATABASE` from `.env` (default: `obsidiannotes`) |
| End-to-end passphrase | *Choose your own* — never stored server-side |
Under **Remote Database Configuration → Advanced**:
- Enable: ✅ **Use Request API to avoid inevitable CORS problem**
(See "Known Issue" below for why this is critical)
If browser requests encounter a CORS failure, consult [Use Request API](../docs/settings.md#use-request-api-to-avoid-inevitable-cors-problem). That setting selects Obsidian's HTTP request adapter. It does not select the synchronisation mode or remove Cloudflare's proxy limits.
---
@@ -241,13 +226,11 @@ docker logs livesync-cloudflared --tail 20
**Fix**: The public hostname rule is missing or incorrect.
1. Go to Zero Trust → Networks → Tunnels → your tunnel → **Edit**
2. Click **Public Hostname** tab
3. Verify a hostname exists with:
- Service Type: `HTTP`
- URL: `couchdb:5984` (NOT `localhost:5984`)
4. If no hostname exists, add it (see Step 3 above)
5. Wait 30 seconds for changes to propagate, then test again
1. Open the tunnel's published application routes in the Cloudflare dashboard.
2. Verify that the requested hostname maps to `http://couchdb:5984`.
3. Add or correct the route if required, then retry after the configuration has propagated.
A connected tunnel alone does not establish that the hostname route or origin is correct.
**Problem: Connection immediately closes / 502 Bad Gateway**
@@ -258,10 +241,10 @@ docker ps --filter "name=livesync-couchdb"
# Status should be: Up (healthy)
docker inspect livesync-couchdb -f '{{.NetworkSettings.Networks}}'
# Should show: livesync-net
# Both services should list the same Compose network.
docker inspect livesync-cloudflared -f '{{.NetworkSettings.Networks}}'
# Should also show: livesync-net
# Compare with the CouchDB network above.
```
**Fix**: If CouchDB is unhealthy, check logs:
@@ -271,41 +254,100 @@ docker logs livesync-couchdb --tail 50
**Problem: 524 Timeout Errors During Sync**
**Root cause**: Cloudflare's proxy has a **100-second idle timeout**. CouchDB's replication protocol uses long-polling on the `_changes` feed, which can idle for longer during quiet periods.
A `524` can indicate that the origin did not respond or accept request data within Cloudflare's proxy timeout. Inspect the failing request, CouchDB response times, and connector logs before selecting a remedy. See [Cloudflare's current 524 guidance](https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-5xx-errors/error-524/) for the applicable limits and diagnostic steps.
**Fix**: Switch to short-polling mode in the Obsidian plugin:
1. Obsidian → Settings → Self-hosted LiveSync
2. **Remote Database Configuration → Advanced**
3. Enable: ✅ **Use Request API to avoid inevitable CORS problem**
4. Save and restart sync
This keeps all requests under 100 seconds.
**Alternative**: Use Tailscale or Caddy profiles instead — neither has aggressive timeouts.
**Use Request API** changes the HTTP adapter; it does not turn LiveSync mode into short polling or guarantee that a request completes within a proxy timeout. Review the actual synchronisation mode and [heartbeat/timeout setting](../docs/settings.md#use-timeouts-instead-of-heartbeats), then verify transfers on the intended devices. If the required traffic does not fit the proxy limits, consider an appropriately secured direct HTTPS route through Caddy or private Tailscale access.
---
## Data & Backup
All vault data lives in the `couchdb-data` Docker named volume.
The server database files live in the `couchdb-data` Docker named volume. Keep Vault backups, encryption passphrases, and ID recovery information separately. Back up the Compose configuration and CouchDB configuration as well; the database volume alone does not contain all of them.
The following example creates an archive while CouchDB is stopped. Verify the volume name before running it, especially if you have overridden the Compose project name:
```bash
# Backup
docker run --rm -v obsidian-livesync_couchdb-data:/data \
-v $(pwd)/backup:/backup alpine \
tar czf /backup/couchdb-backup-$(date +%Y%m%d).tar.gz -C /data .
# Restore
docker run --rm -v obsidian-livesync_couchdb-data:/data \
-v $(pwd)/backup:/backup alpine \
tar xzf /backup/couchdb-backup-20260218.tar.gz -C /data
docker volume inspect obsidian-livesync_couchdb-data
mkdir -p backup
docker compose stop couchdb
docker run --rm --network none -v obsidian-livesync_couchdb-data:/data:ro \
-v "$(pwd)/backup:/backup" alpine \
tar czf "/backup/couchdb-backup-$(date +%Y%m%d-%H%M%S).tar.gz" -C /data .
docker compose start couchdb
```
### Restore a backup
Restore into a new volume so that the current database remains available. The following Bash commands run from the `docker/` directory. Use a trusted archive from the backup above, and the same CouchDB image version used to create it. Keep the saved CouchDB configuration available too. Run each block only after the previous step succeeds.
First, choose the archive and create a new recovery volume. Docker generates an unused volume name:
```bash
backupArchive='<BACKUP ARCHIVE FILENAME>'
recoveryImage='<COUCHDB IMAGE TAG OR DIGEST FROM THE BACKUP>'
recoveryVolume=$(docker volume create) &&
docker run --rm --network none \
--mount "type=volume,src=$recoveryVolume,dst=/data" \
--mount "type=bind,src=$(pwd)/backup,dst=/backup,readonly" \
alpine:3.22 tar xzf "/backup/$backupArchive" -C /data
```
If extraction fails, stop here. The new recovery volume may be empty or incomplete; remove it with `docker volume rm "$recoveryVolume"` before retrying. The original database volume is unaffected.
Start a temporary CouchDB with the restored data. It has no external network connection or published ports, so synchronising clients cannot reach it:
```bash
recoveryUser='<COUCHDB USERNAME>'
recoveryPassword='<COUCHDB PASSWORD>'
recoveryContainer=$(docker run -d --network none \
-e COUCHDB_USER="$recoveryUser" \
-e COUCHDB_PASSWORD="$recoveryPassword" \
--mount "type=volume,src=$recoveryVolume,dst=/opt/couchdb/data" \
"$recoveryImage")
```
Wait for CouchDB to start, then inspect its health, database list, and your restored database. Replace the database name with the one you backed up:
```bash
recoveryDatabase='<DATABASE NAME>'
docker exec "$recoveryContainer" curl -fsS \
-u "$recoveryUser:$recoveryPassword" http://127.0.0.1:5984/_up
docker exec "$recoveryContainer" curl -fsS \
-u "$recoveryUser:$recoveryPassword" http://127.0.0.1:5984/_all_dbs
docker exec "$recoveryContainer" curl -fsS \
-u "$recoveryUser:$recoveryPassword" "http://127.0.0.1:5984/$recoveryDatabase"
docker exec "$recoveryContainer" curl -fsS \
-u "$recoveryUser:$recoveryPassword" \
"http://127.0.0.1:5984/$recoveryDatabase/_all_docs?limit=5"
```
Compare the database names, document counts, and expected document IDs with your backup. These checks establish that CouchDB can read the restored files; they do not establish that every required note is present. If a check fails, inspect `docker logs "$recoveryContainer"` and keep the original volume in place.
Once you have verified the restored database, stop and remove only the temporary container, and note the recovery volume name:
```bash
docker stop "$recoveryContainer" && docker rm "$recoveryContainer"
docker volume inspect "$recoveryVolume" --format '{{.Name}}'
```
To use the restored copy, first [choose the authoritative copy](../docs/recovery.md#first-choose-the-authoritative-copy) and suspend synchronisation on your devices. Stop the stack with `docker compose down`, using the same profile options as usual and without `-v`. Save a copy of `docker-compose.yml`, then replace only its `couchdb-data` volume declaration with the following, using the printed recovery volume name. Keep the other volume declarations:
```yaml
volumes:
couchdb-data:
external: true
name: <RECOVERY VOLUME NAME>
```
Use the same CouchDB image version in the `couchdb` service, restore any saved configuration changes, and start the stack with your usual `docker compose up -d` profile command. The original volume and archive remain available. Verify the required notes from a separate test Vault. When the restored remote is your chosen source of truth, follow [Reset Synchronisation on This Device](../docs/recovery.md#reset-synchronisation-on-this-device) on your existing devices before resuming them, preserving any newer local work first.
See [Backing up CouchDB](https://docs.couchdb.org/en/stable/maintenance/backups.html) for database, configuration, and replication-based backup options.
---
## Security Notes
- CouchDB requires authentication for **all** requests (configured by `livesync.ini`)
- The init service enables authenticated CouchDB access through the REST API. Verify that protected database endpoints reject unauthenticated requests before exposing the service.
- Enable **End-to-End Encryption** passphrase in the Obsidian plugin — vault data is
encrypted before it ever leaves your device
- The init container runs once and exits — it has no persistent access
@@ -344,8 +386,8 @@ open http://localhost:5984/_utils
| Init container keeps restarting | CouchDB not healthy yet — wait 30s, check `docker compose logs couchdb` |
| `curl: (52) Empty reply` | CouchDB not fully started — the healthcheck should gate this |
| Mobile can't connect | Needs HTTPS — use tailscale or caddy profile |
| 524 errors with Cloudflare | Enable "Use Request API" toggle in Obsidian plugin |
| `Permission denied` on volumes | Run `docker compose down -v` and retry — first-run volume ownership issue |
| 524 errors with Cloudflare | Inspect the failing request and origin response time; follow the timeout guidance above. |
| `Permission denied` on volumes | Preserve the volume, inspect the CouchDB logs and mount ownership, and correct the specific permission problem. `docker compose down -v` deletes the database volume and is not a repair step. |
| CORS errors in browser | Confirm CouchDB headers: `curl -v -H "Origin: app://obsidian.md" http://localhost:5984/` |
| CouchDB exits immediately, zero logs (Windows) | **Do not add `:ro`** to the `livesync.ini` volume mount. CouchDB's entrypoint runs `chmod 0644` on all files in `/opt/couchdb/etc` — read-only bind mounts cause a silent EPERM crash on Docker Desktop for Windows (WSL2). The compose file is already correct; do not modify it. |
| Settings in `livesync.ini` seem ignored | Settings requiring restart (e.g. bind_address) load at start. Runtime-only settings (require_valid_user, enable_cors) are set by the init container via REST API and take effect immediately without restart. |
docker/config/cloudflared.yml
+7 -8
View File
@@ -2,17 +2,16 @@
# =============================================================================
#
# Prerequisites:
# 1. Create a tunnel in Cloudflare Zero Trust → Networks → Tunnels
# 1. Create a remotely managed tunnel using the current Cloudflare dashboard guide.
# 2. Copy the tunnel token to CF_TUNNEL_TOKEN in your .env
# 3. Add a public hostname in the tunnel config:
# 3. Add a published application route in the dashboard:
# Hostname : couchdb.yourdomain.com (or whatever you set COUCHDB_DOMAIN to)
# Service : http://couchdb:5984
#
# Known issue: Cloudflare's 100-second proxy timeout can interrupt CouchDB's
# long-polling replication change feed, causing 524 errors.
# MITIGATION: In the Obsidian plugin settings, enable:
# "Use Request API to avoid inevitable CORS problem"
# This switches from long-poll to short-poll mode.
# The token-managed Compose profile uses dashboard routing. The local ingress
# example below describes locally managed routing and is not that profile's
# source of routes. Diagnose 524 errors using current Cloudflare guidance.
# The Obsidian Request API setting changes the HTTP adapter, not the sync mode.
#
# =============================================================================
@@ -23,7 +22,7 @@ ingress:
- hostname: ${COUCHDB_DOMAIN}
service: http://couchdb:5984
originRequest:
# Increase timeouts for CouchDB replication streams
# Origin connection settings for this locally managed example
connectTimeout: 30s
keepAliveTimeout: 90s
keepAliveConnections: 100
docker/config/livesync.ini
+4 -5
View File
@@ -1,10 +1,9 @@
; CouchDB local configuration for Self-hosted LiveSync
; This file is volume-mounted into /opt/couchdb/etc/local.d/livesync.ini
; This file is volume-mounted into /opt/couchdb/etc/local.ini
;
; IMPORTANT: Do NOT set require_valid_user here.
; CouchDB needs to start without auth to complete its first-run cluster setup
; (_users, _replicator databases must be created first).
; The couchdb-init service applies auth lockdown via REST API after first-run.
; The couchdb-init service configures require_valid_user and CORS through the
; REST API during provisioning. Verify that the init service completes before
; exposing the database to clients.
[couchdb]
; Max size per document (50MB). Large enough for binary attachments.
docker/coturn/README.md
+1 -1
View File
@@ -51,7 +51,7 @@ Enter both client paths in the P2P profile's TURN server list:
turn:turn.example.com:3478?transport=udp,turn:turn.example.com:3478?transport=tcp
```
Use `TURN_USERNAME` and `TURN_PASSWORD` as the TURN username and credential. Keep the normal `Automatic` ICE policy unless a future LiveSync release offers `TURN relay only` and the direct path needs to be excluded deliberately.
Use `TURN_USERNAME` and `TURN_PASSWORD` as the TURN username and credential. Keep the normal `Automatic` connection path unless the direct path needs to be excluded deliberately. Select `TURN relay only` under `Connection compatibility` to require the configured TURN route.
Both synchronising devices must be able to reach the server. Prove an explicit two-way `Replicate now` round trip on the intended networks before relying on the configuration.
docker/docker-compose.yml
+4 -4
View File
@@ -2,7 +2,7 @@
# =============================================================================
# PROFILES
# --------
# (default) CouchDB only — LAN/localhost access, no TLS
# (default) CouchDB only — host HTTP port, no TLS
# Suitable for desktop-only use or testing.
#
# --profile caddy CouchDB + Caddy reverse proxy
@@ -14,7 +14,7 @@
#
# --profile cloudflare CouchDB + cloudflared tunnel daemon
# Free public HTTPS via Cloudflare. Needs a CF account + tunnel token.
# NOTE: Enable "Use Request API" in the Obsidian plugin to avoid 524 timeouts.
# Review proxy timeouts separately from the Obsidian HTTP adapter setting.
#
# QUICK START (local test):
# cp .env.example .env && edit .env
@@ -147,8 +147,8 @@ services:
# PROFILE: cloudflare — cloudflared tunnel daemon
# Requirements: Cloudflare account, tunnel token from CF Zero Trust dashboard
# Usage: docker compose --profile cloudflare up -d
# NOTE: Enable "Use Request API" toggle in the Obsidian LiveSync plugin settings
# to avoid Cloudflare's 100-second proxy timeout (524 errors).
# NOTE: The Request API setting changes the HTTP adapter, not the sync mode.
# Diagnose proxy timeouts using the Docker README and current provider limits.
# ---------------------------------------------------------------------------
cloudflared:
image: cloudflare/cloudflared:latest
docs/README.md
+94
View File
@@ -0,0 +1,94 @@
# Documentation index
Choose a document by the task and its status. User guides describe supported operations; implementation references describe contracts and their limits; ADRs record why a decision was made. Proposals and historical designs are grouped separately below.
A document's original date and reviewed-version fields describe its baseline. Read its implementation or publication note before treating that baseline as the current release. A published feature does not by itself establish that every proposal or validation step in its design is complete.
## Setup and everyday use
| Task | Start here |
|---|---|
| Get started | [What you need](../README.md#what-you-need), then [Start synchronising](../README.md#start-synchronising) |
| Choose a CouchDB setup | [Own server](setup_own_server.md), [Docker stack](../docker/README.md), [Fly.io](setup_flyio.md), or [IBM Cloudant (managed)](setup_cloudant.md) |
| Configure CouchDB clients | [Quick setup](quick_setup.md) |
| Find hosting services proposed by the community | [Third-party hosting listings](../README.md#hosting-services-proposed-by-the-community) |
| Set up S3-compatible storage | [Object Storage Setup](setup_object_storage.md) |
| Set up peer-to-peer synchronisation | [P2P communication model](p2p.md), then [Peer-to-Peer Setup](setup_p2p.md) |
| Diagnose P2P connectivity | [Browser-assisted check](../src/apps/webpeer/README.md#p2p-connection-check), [network tips](tips/p2p-sync-tips.md), and [self-hosted TURN](../docker/coturn/README.md) |
| Understand a setting or command | [Settings reference](settings.md) |
| Read or customise synchronisation status | [Status indicators and CSS snippets](tips/status-display.md) |
| Investigate or recover from a problem | [Troubleshooting](troubleshooting.md), then the relevant [recovery procedure](recovery.md) |
| Prepare an issue report | [Issue reporting](to_issue_reporting.md) |
The project overview links to Japanese and Chinese documentation.
## Implementation and technical references
Use these references for the area being changed, together with its relevant ADR. Check the document's stated scope and validation limits before relying on a guarantee.
| Area | References |
|---|---|
| Overall synchronisation and terminology | [Technical overview](tech_info.md), [glossary](glossary.md), and [terminology conventions](terms.md) |
| Metadata, Chunks, and database representations | [Data structures](datastructure.md) |
| Conflict resolution, provenance, and history restoration | [Conflict and provenance contract](specs_conflict_resolution.md), [Document History restoration](design_docs/document_history_revision_restoration.md), and [local save completion](adr/2026_10_local_file_save_completion_and_batching.md) |
| Replicator and host boundaries | [Replicator architecture snapshot](design_docs/replicator_architecture.md) and [service feature and legacy Module boundaries](design_docs/service_feature_and_legacy_module_boundaries.md) |
| Chunk retrieval and maintenance | [Chunk retrieval and waiting](design_docs/chunk_retrieval_and_waiting.md), [Garbage Collection V3](specs_garbage_collection.md), and [Data Compression](specs_data_compression.md) |
| Document identity and validation | [Unicode path identity](design_docs/unicode_path_identity.md), [configurable ID derivation](design_docs/configurable_id_derivation.md), [Metadata ID validation and repair](design_docs/metadata_document_id_validation_and_repair.md), and [path component compatibility](design_docs/path_component_length_compatibility.md) |
| Settings compatibility and profiles | [Tweak compatibility and recovery](design_docs/tweak_compatibility.md) and [remote configuration management](design_docs_of_remote_configurations.md) |
| Encryption and sharing | [Metadata encryption](design_docs/internal_metadata_encryption.md) and [time-bound Setup URIs](design_docs/time_bound_setup_uri.md) |
| P2P connection preparation | [TURN connection settings](design_docs/renewable_turn_credentials.md) |
| Journal storage internals | [Journal Replicator 2nd Edition](design_docs_of_journalsync_2nd.md) |
The installed `@vrtmrz/livesync-commonlib` package is authoritative for shared runtime contracts. Use the exact dependency selected by [package.json](../package.json) and its lockfile when checking behaviour. The [Commonlib documentation](https://github.com/vrtmrz/livesync-commonlib/tree/main/docs) is maintained in that repository; this index does not replace its package contracts.
## Architectural decisions
Read the decisions relevant to the change, rather than treating the whole ADR directory as a prerequisite. A decision's acceptance, its implementation, its release, and its remaining verification work are separate facts.
| Decision area | Records |
|---|---|
| Shared package boundary | [Common library package boundary](adr/2026_07_common_library_package_boundary.md) |
| Replicator capabilities and migration | [Core contract](adr/2026_08_replicator_capabilities_01_core_contract.md), [P2P service lifecycle](adr/2026_08_replicator_capabilities_02_p2p_service_lifecycle.md), and [migration plan](adr/2026_08_replicator_capabilities_03_migration_plan.md) |
| Remote resources and Fast Setup | [CouchDB connection ownership](adr/2026_08_couchdb_remote_connection_ownership.md), [Fast Fetch eligibility](adr/2026_08_fast_fetch_transport_eligibility.md), and [persistence and completion](adr/2026_08_fast_fetch_persistence_and_completion.md) |
| P2P lifecycle and setup | [Transport lifecycle](adr/2026_07_p2p_transport_lifecycle.md), [connection check](adr/2026_07_p2p_connection_check.md), [profile-aware onboarding](adr/2026_07_multiple_remote_onboarding.md), and [compatibility controls](adr/2026_08_p2p_transport_compatibility.md) |
| Settings interface integration | [Declarative settings adapter](adr/2026_08_declarative_settings_adapter.md) |
| Saving and path identity | [Local file save completion and batching](adr/2026_10_local_file_save_completion_and_batching.md) and [Unicode path identity](adr/2026_10_unicode_path_identity.md) |
| Activity and Chunk arrival | [Bounded remote activity](adr/2026_07_bounded_remote_activity.md) and [Chunk arrival quiescence](adr/2026_07_chunk_arrival_quiescence.md) |
| Compatibility and validation | [Release notes and database compatibility gates](adr/2026_07_release_notes_and_database_compatibility.md) and [real Obsidian E2E](adr/2026_06_real_obsidian_e2e.md) |
## Proposals and incomplete explanations
These records are useful for design work. Their proposed representation or behaviour is not a current runtime contract.
| Document | Recorded state |
|---|---|
| [Tiered Chunk Storage with Live Compaction](design_docs/tired_chunk_pack.md) | In design |
| [Tiered Chunk Storage for Bucket Sync](design_docs/tired_chunk_pack_bucket.md) | In design |
| [Design intent for Metadata and Chunks](design_docs/intention_of_chunks.md) | Incomplete explanation; use the data-structure reference for current representations |
| [Feature maturity for 1.0](adr/2026_07_feature_maturity_for_1_0.md) | Proposed; publication of 1.0 alone does not establish formal acceptance of this record |
## Historical and retired designs
These documents preserve earlier motivations, approaches, or results. Follow their maintained replacement for current operations.
| Historical material | Current starting point |
|---|---|
| [Original Tweak-sharing proposal](design_docs_of_sharing_tweak_value.md) | [Tweak compatibility and recovery](design_docs/tweak_compatibility.md) |
| [First Journal Sync design](design_docs_of_journalsync.md) | [Journal Replicator 2nd Edition](design_docs_of_journalsync_2nd.md) |
| [Eden design and operational results](design_docs_of_keep_newborn_chunks.md#results-from-actual-operation) | The document records its deprecated, frozen status; use [Data structures](datastructure.md) for representations still accepted by current code |
| [Withdrawn Chunk Aggregation by Prefix](design_docs/chunk_aggregation_by_prefix.md) | Retained as a withdrawn proposal, not an implemented storage layout |
Dated measurements, including the [Data Compression benchmark](specs_data_compression.md#reproducible-benchmark), remain evidence for their recorded revision and environment. A current rerun is a separate measurement.
## Development, validation, and release history
- [Development guide](../devs.md): build, module ownership, and release workflow.
- [Contributing](../CONTRIBUTING.md): contribution and documentation checks.
- Independent applications: [CLI](../src/apps/cli/README.md), [WebApp](../src/apps/webapp/README.md), and [WebPeer](../src/apps/webpeer/README.md).
- [Setup utilities](../utils/readme.md): provisioning and Setup URI generation.
- [Review Harness](review_harness.md): bounded real-device observations and performance measurements.
- [Test procedures](../test/README.md): choose the suite which owns the boundary being changed.
- [Change notes](../updates.md): current release and unreleased changes.
- Published release archives: [1.0](releases/1.0.md), [1.0 previews](releases/1.0-previews.md), [0.25](releases/0.25.md), and [earlier releases](releases/legacy.md).
When a feature ships or a design is replaced, update its implementation note and this index if its role changes. Preserve the original decision and measurement history, state the maintained replacement, and keep existing links working.
docs/adr/2026_08_p2p_transport_compatibility.md
+21 -11
View File
@@ -2,7 +2,10 @@
## Status
Accepted — the user-facing controls will be introduced in stages. This record defines their boundaries before Commonlib settings and LiveSync interfaces are changed.
Accepted. The P2P message-size presets and connection-path choice are present
in current Self-hosted LiveSync 1.0.42 with Commonlib 0.2.2. The proposed
`Test TURN server` action and concise route diagnostics in the Obsidian
interface remain future work.
## Context
@@ -11,10 +14,12 @@ WebRTC connectivity depends on both devices, their browsers or embedded WebViews
The current P2P transport has several relevant properties:
- Trystero supplies the Nostr signalling strategy and the browser-owned WebRTC connection.
- Commonlib limits one RPC wire payload to 15,360 bytes so it remains below Trystero's own action-chunk boundary.
- Trystero supplies ordinary STUN servers and accepts an optional TURN server list with one username and credential.
- ICE chooses a direct, server-reflexive, or TURN-relayed path automatically.
- Commonlib can collect raw WebRTC statistics, but LiveSync does not yet present the selected candidate route in a concise diagnostic result.
- Commonlib defaults to a 15,360-byte RPC wire payload limit, below Trystero's own action-chunk boundary; the presets below can lower it.
- Manual TURN settings use one URL list, username, and credential. Managed preparation can supply multiple ICE server entries with their own credentials.
- ICE chooses a direct, server-reflexive, or TURN-relayed path automatically unless the profile selects `TURN relay only`.
- Commonlib can collect raw WebRTC statistics. The CLI reports the selected
candidate route, but the Obsidian interface does not yet present it in a
concise diagnostic result.
Issue reports suggest that reducing the application payload may improve some mobile and constrained-network paths. A VPN such as Tailscale may also turn an unreliable route into a reliable one. These observations are consistent with NAT, path-MTU, fragmentation, or intermediary behaviour, but they do not prove one universal cause. Browser WebRTC implementations retain responsibility for SCTP, DTLS, ICE, packetisation, congestion control, and retransmission.
@@ -24,7 +29,7 @@ One low-level number cannot represent all of these concerns. Users need a small
### Message-size presets
LiveSync will expose a `P2P message size` choice with four presets:
LiveSync exposes a `P2P message size` choice with four presets:
| Label | Maximum RPC wire payload | Intended use |
| ----------------------- | -----------------------: | ------------------------------------------------------------------------------- |
@@ -37,11 +42,11 @@ This value limits Commonlib RPC wire payloads before Trystero applies its own fr
The bound applies to outgoing messages. A device which only lowers its own value still receives messages produced under the sender's value. The selected preset therefore belongs to the P2P profile and is included in an encrypted Setup URI for additional devices. A device which was configured earlier must be changed separately; the interface and troubleshooting guidance must state that the same conservative preset should be selected on every participating device. An absent key preserves the current 15,360-byte default.
Automatic negotiation or fallback between presets is deferred. A failed ordered data channel may require connection replacement before a smaller retry can prove anything, and changing transport parameters during a replication session would broaden the lifecycle contract considerably. The first implementation remains explicit, stable for one room lifetime, and inspectable.
Automatic negotiation or fallback between presets is deferred. A failed ordered data channel may require connection replacement before a smaller retry can prove anything, and changing transport parameters during a replication session would broaden the lifecycle contract considerably. The implementation remains explicit, stable for one room lifetime, and inspectable.
### Connection path
LiveSync will expose a separate `Connection path` choice:
LiveSync exposes a separate `Connection path` choice:
- `Automatic` retains normal ICE selection and is the default.
- `TURN relay only` supplies `iceTransportPolicy: 'relay'` and prevents direct or server-reflexive candidates from being selected.
@@ -81,11 +86,15 @@ Where the browser exposes the evidence, the result should report:
- UDP, TCP, or TLS transport; and
- a bounded failure or inconclusive result.
Ordinary P2P diagnostics should later summarise the selected candidate pair as direct, server-reflexive, or relayed, with its transport. Raw `getStats()` output remains supporting evidence rather than the primary interface.
The proposed Obsidian P2P diagnostics should summarise the selected candidate
pair as direct, server-reflexive, or relayed, with its transport. The CLI's
candidate-route output does not provide this user-facing summary. Raw
`getStats()` output remains supporting evidence rather than the primary
interface.
### Placement and defaults
These controls belong inside `P2P Configuration` under a `Connection compatibility` section. They do not require the repository-wide Advanced, Power User, or Edge Case modes. P2P itself remains a supported opt-in feature.
These controls appear inside `P2P Configuration` under a `Connection compatibility` section. They do not require the repository-wide Advanced, Power User, or Edge Case modes. P2P itself remains a supported opt-in feature.
Existing profiles retain the following defaults:
@@ -161,7 +170,8 @@ A layer-4 TLS router could share one public address between distinct CouchDB and
## Verification
The first implementation stage must add focused tests before production changes:
The following list records the original focused-test plan for the message-size
and connection-path controls. It remains useful as regression coverage:
- settings-schema defaults for absent keys;
- P2P connection-string and Setup URI round trips which retain both transport compatibility settings;
docs/adr/2026_10_local_file_save_completion_and_batching.md
+448
View File
@@ -0,0 +1,448 @@
---
date: 2026-10-24
commonlib-version: "0.2.0"
self-hosted-livesync-version: "1.0.40"
status: accepted
---
# Architectural Decision Record: Local File Save Completion and Batching
## Status
Accepted on 2026-10-25. The recovery contracts and LiveSync mode batching in
implementation-order steps 1–3 are present in current Self-hosted LiveSync
1.0.42 with Commonlib 0.2.2. The version fields above retain the investigated
baseline. Further same-path coalescing in step 4 remains a follow-up.
## Context
[Issue #1207](https://github.com/vrtmrz/obsidian-livesync/issues/1207) includes
conflicts around automated note creation, editing, and renaming. Batching local
file updates can reduce the intermediate revisions which another device receives.
It also extends the period during which the Vault contains an edit which the
local database has not saved. Recovery and incoming-file protection must remain
correct throughout that period.
Two-device Obsidian E2E tests using Self-hosted LiveSync 1.0.40 and Commonlib
0.2.0 found that a test-only removal of the LiveSync mode restriction reduced
six consecutive edits from six additional revisions to one, in both directions.
Incoming identical content produced no extra conflict, while genuinely different
edits or a deletion preserved the necessary branches. A synthetic
create–edit–rename sequence still delivered intermediate content. These tests
used Vault operations, without Templater or Modal Forms themselves.
Restart tests exposed a separate recovery gap:
1. The event manager removes an event from its waiting map when its timer is
released. It also removes a buffered event before dispatching it. Its saved
snapshot covers buffered and waiting events, but omits work in progress.
2. `waitForIdle()` waits for the waiting gates to open, rather than for the file
handlers to finish. `ServiceFileAccessBase` reports that pending events have
been committed after this gate-only wait. Some handler failures also settle
without propagating a failure to that caller.
3. If the process stops before the database write, the file can retain the new
content while the database retains the old content and the saved event
snapshot is empty.
4. At start-up, the Offline Scanner's `EVEN` branch tries to record missing
provenance, then reports completion. It ignores that helper's Boolean result.
Different content can consequently remain unsaved when both modification
times fall in the same two-second comparison bucket.
The last condition was reproduced by pausing the target operation immediately
before database processing, then restarting the same isolated profile. A
different timestamp bucket recovered the edit; the same bucket did not. The
changed-condition success demonstrates the boundary, rather than a fix. This
does not establish the cause of the reporter's historical conflicts.
The runtime evidence covers Linux aarch64, Obsidian 1.12.7, and CouchDB 3.5.0.
Windows, mobile lifecycles, and the reporter's full automation remain outside
that evidence. The inspected Commonlib source is commit
`aa54282937c20d3db7f43778d240feeb1586cced`.
## Decision
Make local file processing recoverable until its result is established, retain
strict content and provenance reconciliation for manual and recovery scans, and
enable batching in LiveSync mode only through an explicit choice after those
contracts pass verification. Ordinary Obsidian start-up may skip a body read
only under the exact-match conditions below. Apply the recovery contract with
batching enabled or disabled.
The content-reconciliation change targets selected ordinary files in the
Offline Scanner's `NEWER_WINS` / `sync-newer` action. Keep the existing action
matrix, explicit `DB_APPLY` policy, selection and size limits, conflict guards,
and recovery suspensions. Hidden and internal files retain their owning
feature's recovery contract; an ordinary file scan cannot recover those paths
on its behalf.
### 1. Distinguish local persistence from replication
The following boundaries have different meanings:
| Boundary | Meaning |
| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Batch wait released | The file operation may start. Its content can still be absent from the local database. |
| Local processing completed | The intended change is represented durably by the local database and its required content, or has been proved already represented or safely superseded. |
| Replication completed | A finite replication reports its own outcome. A local save alone does not prove remote receipt or reflection into another Vault. |
Ordinary file saving by Obsidian remains independent of database saving. Pending
and processing indicators must include work in progress and unresolved failures,
so an empty timer map cannot imply that local processing is complete.
### Optional file ownership and completion
Hidden File Sync and Customisation Sync register the new
`processOptionalFileEventWithResult` contract together. `firstHandled` tries
providers in order until one returns `completed` or `veto`; only
`not-applicable` advances to the next provider. Exceptions propagate unchanged.
`HandlerOutcome<TValue, TVetoReason>` gives completion data and refusal reasons
independent types. These saves use `void` completion data and string reasons.
A provider returns `completed` after its database operation succeeds or the
current state is already represented. Failed writes and deletions throw, and
the manager retains the event. Hidden File Sync acknowledges its processed
fingerprint after successful processing. Customisation Sync waits for the real
save after its existing 100ms quiet period. All notifications coalesced into a
V1 group receive that save's result; V2 files retain separate save groups.
A finite flush resolves each captured event's `whenFlushRequested` signal. This
releases its customisation delay and freezes that group; later groups do not
extend its completion. During start-up replay or suspension, providers use
physical database readiness rather than waiting for application readiness or
resume. Hidden File Sync loads its existing per-database caches before replay
and reuses them during the following initialisation hook.
The existing boolean optional API remains a legacy notification contract for
other callers and older hosts. Once a host registers typed providers, the
manager dispatches through that chain exclusively. Without a typed provider,
a resolved legacy notification completes that dispatch only; its boolean does
not establish a database save. Snapshot-write failures remain visible under
both contracts. No persistent journal, new snapshot schema, or automatic retry
loop is introduced.
### 2. Keep a recoverable record through processing
The event manager owns each operation from queue admission through completion.
Its existing persistent snapshot must represent buffered, waiting, and
in-progress work in an order which preserves deletion and rename dependencies.
Moving work between those states must not create a persistent gap.
An event can leave the recoverable set only after its effect is durable, current
state proves that the effect is already represented, or a durably recorded
successor safely subsumes it. Finishing an older operation must not remove a
newer operation for the same path. Cancellation of a batch timer alone is not
evidence that its content is safe to forget.
Snapshot writes must preserve mutation order. An older asynchronous write must
not replace a newer snapshot, and a failed snapshot write must remain visible
to the lifecycle caller. Reuse the existing device-local snapshot store; a
second journal and a transaction spanning the file system and database are not
part of this proposal.
The first implementation should retain the existing event snapshot format,
including event keys and rename intent, and add pending ownership within the
event manager. The persistence adapter remains responsible for storing the
ordered snapshots. Handler outcomes must distinguish a failed attempt from a
policy-approved skip before the manager can acknowledge either safely.
A crash after the database write but before snapshot removal can replay an
operation. Replay must check current storage and revision provenance, retain
the existing protection against stale restored events, and never delete a
newer same-path file. Once both the database write and provenance write have
completed, replay must not add another revision. If the database write
succeeded before provenance was recorded, replay may first create an identical
sibling even when no remote descendant has arrived. In the tested case
without a remote edit, replay creates that sibling. Duplicate handling then
tombstones one of the identical branches and converges to one live leaf with
the exact saved content.
Another restart must add no revision. Replay must remain safe to interrupt;
existing saved snapshots remain readable.
The database write and the device-local provenance write are separate,
sequential operations. A stop between them can leave the previous provenance
record in place. This pre-provenance gap is the known exception to the
no-extra-revision result after both writes complete: replay can first create a
duplicate sibling; one of the identical branches is then tombstoned while
one live leaf retains the saved bytes when no remote descendant has arrived. If continuous
replication advances the local database during the gap, replay can instead
expose the local content as a sibling of that descendant. The [conflict
resolution specification](../specs_conflict_resolution.md#known-potentially-problematic-behaviour)
records the required mergeable and manual-resolution outcomes. The per-file
lock does not fence replication writes. The interruption window is expected
to be very rare, but its field frequency is unmeasured; this decision adds no
transaction or second journal to close the interval.
Boolean failure, an exception, and an unavailable comparison are not successful
completion. Keep unresolved work recoverable and report the failure through the
existing service and diagnostic boundaries. Explicit exclusions or a verified
superseding change may finish work without a new revision; a generic handler
failure may not. Recovery may continue with independent paths, while retaining
the failed path and its dependencies. This proposal adds no unbounded retry loop.
The recoverable snapshot is an operation record, not a backup of every transient
editor buffer. File content already saved by the host remains available for
start-up reconciliation if the process stops before the first snapshot write.
Crash guarantees cannot include bytes or events which the host never persisted
or delivered, or repair arbitrary storage failure.
### 3. Give flush a finite, truthful completion boundary
An explicit commit of pending file events captures the work accepted before
that request, releases its batch delays, and waits for the actual processing
results, including buffered and already-running work. Later edits cannot extend
that boundary indefinitely. A newer coalesced operation may satisfy earlier work
only when its successful result covers that work.
The internal ordering barrier before a delete or rename waits only for its
predecessors. It must not wait for itself or later events in the same dispatch
loop. Replacing every gate wait with a wait for global idleness would risk a
dependency cycle or starvation.
Manual synchronisation and replication readiness use the completed local
boundary. Settings application must observe its result before reporting success
or resuming work which depends on it. Persisting a settings file is a separate
operation: `saveSettingData()` returning is not a file-processing completion
signal, and making every settings-saved notification synchronous is unnecessary.
When a host permits orderly shutdown, stop admitting watcher work before
draining accepted operations or preserving their pending records. Keep their
storage available until that work settles. Suspension, forced termination, and
Obsidian's synchronous unload callback cannot guarantee that asynchronous work
finishes; correctness must also follow from the persistent record and start-up
reconciliation. Remote availability must not determine whether local pending
work survives shutdown.
### 4. Preserve strict content reconciliation
Strict content checking retains the timestamp and provenance reconciliation
contract for platforms with coarse modification times. Timestamp equality,
including exactly equal raw timestamps, does not prove content equality. With
strict checking enabled, the Offline Scanner must establish one of the
following outcomes through the ordinary non-forced file handler and its
provenance rules:
| Evidence | Required outcome |
| ----------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
| Vault content already matches an eligible current revision | Record or retain valid provenance as appropriate; create no duplicate revision. |
| Vault content matches its recorded revision or saved content evidence, while the current database revision has advanced | Apply the database state through the existing protected reflection path. Do not resurrect the unchanged older content as an edit. |
| Vault content differs from its recorded revision | Preserve the edit on that recorded branch. Keep independent concurrent branches available for conflict resolution. |
| Vault content has no provable origin and matches no current live leaf | Preserve it according to the existing independent-branch rules. Do not attach it to an arbitrary database winner. |
| Required content or provenance cannot be read, or state changes during comparison | Retain unresolved work and report or re-evaluate it. Do not mark it reconciled merely because the timestamp comparison is `EVEN`. |
The `strictContentCheck` scan option is strict when omitted. Ordinary Obsidian
start-up can opt out through **Strict content check on startup**, which is
disabled by default. With the option disabled, a body read may be skipped only
when the file's raw modification time, byte size, and valid device-local
provenance exactly match a conflict-free current revision and no file
processing is pending. All other files retain strict content reconciliation.
An unrecorded edit that preserves its exact size and modification time can
therefore remain undetected by ordinary start-up while the option is disabled.
Manual scans, Fetch, Rebuild, and other explicit recovery operations remain
strict. Pending-event replay, incoming-change protection, and deletion guards
remain active in both modes.
The table describes stable-state strict reconciliation, not atomic database and
provenance persistence. Replay in the pre-provenance gap may create an
identical sibling; with no remote descendant, one of the identical branches
is tombstoned, leaving one live leaf with the exact saved content. If replication has
already added a descendant, preserve its exact content through an existing
safe merge, or retain both branches for manual resolution. See [Known
potentially problematic behaviour](../specs_conflict_resolution.md#known-potentially-problematic-behaviour).
Revalidate the relevant file and database state before acting on a comparison.
Reuse the existing content and provenance checks; avoid a separate conflict
algorithm in the scanner. Metadata continues to reference Chunks containing the
file content.
`tryRecordUntrackedFileRevision()` answers whether it could safely record
provenance. Its `false` also covers an already-populated record, conflicting
history, unavailable content, and a concurrent change. It is not an 'edited
file' predicate. A fallback which treats every `false` as permission for a
forced save would violate the required outcomes.
Propagate the file handler's actual outcome into the pair result, including for
storage-to-database writes. Preserve the scanner's `completed`, `skipped`, and
`failed` distinctions and its existing `continueOnFileFailure` policy. Ordinary
Obsidian start-up can report `completed-with-file-failures` and allow unaffected
files to proceed; an unresolved pair must not become `completed` to achieve
that result. Keep the stricter callers' existing completion requirements.
Keep the original unchanged-file, edited-file, untracked-provenance,
incoming-protection, and concurrent-recheck fixtures under strict content
checking, including identical timestamps and distinct raw timestamps within
one comparison bucket. Do not weaken those strict-mode assertions to describe
the ordinary start-up opt-out. Test the opt-out separately: only the exact
stat, provenance, conflict-free, and no-pending-processing match may skip a
body read. Pending-event replay and incoming or deletion guards remain covered
with the opt-out enabled.
### 5. Make batch timing and activation explicit
Batching is a bounded delay before local database processing. The quiet-period
timer follows the latest eligible content update; the maximum intentional delay
starts with the first uncommitted update and is not reset by later edits or time
spent awaiting a processing slot. Once due, work must become eligible to run.
The maximum delay is not a guarantee of database completion, network delivery,
or another device's reflection time. Waiting timers should not consume the
capacity needed to process unrelated ready files.
For ordinary local content updates, queue the changed path and operation intent
without reading the file body merely to wait. Acquire the current content when
the operation becomes due, after coalescing eligible events. Reuse that content
snapshot within the operation for comparison, provenance evidence, and Chunk
creation. This retains the current ordinary-save boundary; it does not require
reading and hashing the file for every keystroke or queued notification.
Content reuse within one operation is distinct from keeping a body cache across
edits. A cached body would need a proven invalidation and freshness contract;
mtime and size alone cannot establish that it is current. A saved provenance
hash records evidence about earlier content, rather than replacing acquisition
of the current bytes. Restart recovery, an incoming update, or a comparison
which needs revalidation can still require additional reads. Preserve those
checks rather than making a global promise of one read per path.
Batching therefore aims to reduce repeated content acquisition, comparison,
Chunk processing, and local database updates, as well as intermediate revisions
and subsequent replication and reflection work. Event filtering, bookkeeping,
and pending-operation persistence still occur while waiting. The revision
reduction measured by the existing E2E does not quantify file reads, CPU cost,
physical disk I/O, or network traffic; measure those costs separately. Host or
operating-system caching also means that a read API call is not a physical disk
read count.
Manual flush and required deletion or rename ordering can end a batch early.
Ordinary incoming replication continues, using the existing protection for
unsaved local content. It must not require all unrelated local batches to flush
for every incoming document.
Add an independent `batchSaveInLiveSync` Boolean, defaulting to `false`
for both existing settings and new Vaults. In LiveSync mode, this value selects
batching; other modes continue to use `batchSave`. Both choices use the existing
minimum and maximum delay settings. The interface presents the choice for the
current mode. An ordinary mode switch preserves each mode's choice.
This adds one persisted setting to distinguish a new explicit choice from old
`liveSync: true, batchSave: true` configurations, where batching was ignored.
Neither upgrade nor import of that legacy pair may activate batching. An older
client continues to apply its existing LiveSync restriction. Include the new
property in Commonlib's setting policy and import/export tests; do not infer
consent from the presence of the old `batchSave` property.
The setting and scheduling changes follow the recovery work. Recovery in the
currently supported modes must pass verification before the new mode is enabled.
### 6. Keep batching separate from automation completion
The intended benefit is fewer short-lived intermediate revisions. Batching
cannot discover when a third-party template has finished, particularly while a
dialogue is open or another device also runs automation. A maximum delay or a
rename boundary can legitimately publish intermediate content.
Coalescing `CREATE` followed by `CHANGED` for the same still-existing file is a
useful follow-up, but it needs its own ordering and interruption tests. Preserve
the existing cross-path rename rule: store the destination before deleting the
proven source branch. Do not retarget or discard pending source operations
without proving the effect is already covered.
A guarantee that a multi-step template appears remotely only after completion
would need an explicit producer completion contract and its failure behaviour.
That is a separate design. Enabling batching alone must not be described as
eliminating genuine concurrent-edit conflicts or as a confirmed fix for #1207.
## Ownership and implementation order
The list below records the implementation sequence. Steps 1–3 are published
in the current version line; the further coalescing in step 4 remains planned.
1. **Commonlib recovery:** event ownership, ordered snapshot persistence,
successful completion and failure propagation, finite flush boundaries, and
same-time content reconciliation. Add focused failing regressions before
changing production code. Reuse the existing event manager, file handler,
scanner, and provenance store.
2. **Host integration:** keep the Obsidian persistence adapter and lifecycle
wiring aligned with those contracts. Verify callers of
`commitPendingFileEvents`, including settings application and database
preparation, and preserve their failure outcomes. Validate the packed
Commonlib artefact and update the consumer's exact dependency.
3. **LiveSync mode batching:** add the explicit setting and its primary-language
presentation only after the recovery regressions pass. Establish the stated
timing semantics and repeat the two-device comparisons with the actual
package, including start-up with batching already enabled.
4. **Further coalescing:** assess same-path creation/editing and any proposed
rename optimisation separately, against the original automation problem.
Keep these changes independently reviewable. Shared behaviour belongs in
Commonlib; the plug-in must not acquire a source mirror or patch installed
package files. Snapshot format changes, a second persistent store, or a new
automation transaction API require a further design decision.
## Verification required
The matrix below records the acceptance criteria for implementation steps 1–3
and remains useful for regression work. Publication in the current version line
does not extend the historical test results above to untested platforms or
automation workflows.
Use controlled promises at the manager and persistence boundaries for focused
regressions, and real PouchDB tests for content and ancestry. The acceptance
matrix is:
| Scenario | Required observation |
| -------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- |
| Stop after dequeue, timer release, or during a database write | Pending work remains recoverable; restart eventually represents the saved Vault edit. |
| Database and provenance writes are complete before replay; restart twice | Replay adds no extra revision. |
| Stop after the database write but before provenance; no remote descendant; replay twice | The tested replay creates an identical sibling and tombstones one of the two branches; one live leaf retains exact saved content, and the second restart adds no revision. |
| A remote descendant arrives during the pre-provenance gap | The mergeable case preserves its exact content and ancestry; the overlapping case retains both bodies for manual resolution. |
| Old and new events share a path; snapshot writes overlap | Completing old work cannot remove new work or regress the durable snapshot. |
| Handler returns false, throws, or snapshot persistence fails | Flush reports failure; pending work is retained; unrelated recoverable work can continue. |
| Buffered or in-progress work exists with no waiting timer | Flush still waits for actual completion. |
| A flush overlaps new typing or a rename/delete barrier | The captured work completes without waiting for later work or deadlocking. |
| Strict check: equal timestamps, unchanged or edited files, untracked or pruned provenance, unavailable Chunks, or concurrent recheck changes | Preserve correct ancestry and bytes, avoid duplicate revisions, and distinguish unresolved comparisons from successful reconciliation. |
| Start-up check disabled: exact raw time, size, and provenance match with no conflict or pending processing | The body read may be skipped; an unrecorded edit which preserves exact size and modification time remains an accepted limitation. |
| Pending-event replay and incoming or deletion guards, with either start-up setting | Keep replay and protection active; do not skip work or remove a file based only on matching statistics. |
| Settings save followed by stop; ordinary unload; interruption during replay | No operation is forgotten because a settings callback or shutdown released its timer. |
| Sustained edits and more paths than available processing slots | Observe the intentional-delay deadline and progress of unrelated ready files. |
| Old settings, mode changes, explicit opt-in, and settings import | Existing LiveSync mode users retain immediate saving until they choose batching. |
| Incoming identical content, an edit, a deletion, or delete/recreate while waiting | Preserve genuine branches without extra conflicts or loss of local work. |
The original E2E plan compared batching disabled, an ignored legacy setting, and
explicitly enabled batching in both directions, with manual synchronisation,
saved-snapshot restart, and controlled in-progress restart. Record the original
failure separately from repaired results; a pass with a different timestamp
bucket or a test-only bundle is not verification of the production fix. Keep
platform-specific lifecycle evidence bounded to the tested environments before
extending conclusions to Windows or mobile.
## Alternatives and consequences
- **Remove only the LiveSync guard:** leaves recovery gaps and can activate a
previously ignored setting. The explicit choice follows correctness work.
- **Wait only at unload:** cannot cover abrupt termination and hosts which do
not await asynchronous unload. Continuous recoverability is still required.
- **Repair only the scanner:** improves surviving-file recovery, but does not
establish event ordering, deletion intent, or truthful flush completion.
- **Retain only in-progress events:** closes a persistence gap, but still needs
success/failure handling, replay checks, and recovery before the first snapshot.
- **Remove timestamp rounding or force the Vault copy into the current winner:**
fails for exactly equal timestamps and can destroy branch intent. Content and
provenance determine the safe action.
Keeping work recoverable adds snapshot bookkeeping and can make local flush
wait longer because it now waits for real processing. Same-time reconciliation
can require additional content reads. Batching trades prompt remote visibility
for fewer database updates and may increase the window for genuine concurrent
edits. Measure these costs without weakening the content-preservation contract.
## References
- [Conflict resolution and revision provenance](../specs_conflict_resolution.md)
- [Batch database update settings](../settings.md#batch-database-update)
- [Common library package boundary](2026_07_common_library_package_boundary.md)
- [Fast Fetch persistence and completion](2026_08_fast_fetch_persistence_and_completion.md)
- [Two-device Batch save reproduction](../../test/e2e-obsidian/scripts/issue1207-livesync-batch.ts)
- [Commonlib event manager at the investigated revision](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/managers/StorageEventManager.ts)
- [Commonlib Offline Scanner at the investigated revision](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/serviceFeatures/offlineScanner.ts)
- [Commonlib file handler at the investigated revision](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/serviceModules/ServiceFileHandlerBase.ts)
- [Commonlib scanner policy and stored event interpretation](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/docs/offline-scanner.md)
- [Commonlib event admission and reflection contract](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/docs/storage-events-and-reflection.md)
- [Existing equal-time provenance regression](https://github.com/vrtmrz/livesync-commonlib/blob/aa54282937c20d3db7f43778d240feeb1586cced/src/serviceFeatures/offlineScanner.provenance.unit.spec.ts)
docs/adr/2026_10_unicode_path_identity.md
+138
View File
@@ -0,0 +1,138 @@
---
date: 2026-10-17
commonlib-version: "0.1.42"
self-hosted-livesync-version: "1.0.37"
status: released
---
# Architectural Decision Record: Unicode Path Identity for CouchDB
## Status
Accepted on 2026-10-17. Unicode NFC path identity was released in Self-hosted
LiveSync 1.0.38 with Commonlib 0.1.42. Local validation evidence and platform
limits are recorded in the design document.
## Context
Self-hosted LiveSync derives normal-file Metadata document IDs from Vault paths.
Unicode permits visually identical text to use different code-point sequences.
For example, `é` can be represented as one precomposed character or as `e`
followed by a combining acute accent. Filesystems and applications can expose
these forms differently. Without a shared canonical form, two clients can
derive different document IDs for the same logical path, or fail to resolve an
existing document when another client reports an equivalent spelling.
Changing only a lookup performed when receiving a document is insufficient.
New documents, edits, deletions, local provenance, and resumable Fetch
checkpoints must all use the same path identity. Applying the change to a
populated database in place would
also leave old document IDs and revision histories behind. Distinct files or
Metadata documents can collapse to one identity under NFC, so automatic
selection or merging could lose user data.
The Metadata path and file content have separate roles: Metadata stores file
properties and references to Chunks, while Chunks store the content. A path
identity change must not require physical files to be renamed or content to be
rewritten in Metadata.
## Decision
Add the opt-in `pathNormalizationVersion` setting. An absent value and `0`
retain the existing path-to-ID behaviour. Version `1` applies Unicode
Normalization Form C (NFC) to the logical path used for Metadata IDs and path
comparison keys. Existing case handling remains an independent rule; CLI path
resolution uses NFC equivalence regardless of the `handleFilenameCaseSensitive`
setting. NFC changes logical identity only: it does not rewrite the stored
`Metadata.path`, rename a physical file, or change Chunk IDs or content. It is
independent of `idDerivationVersion` and does not require encryption.
When the CLI resolves a path, it uses the actual spelling of each existing
file and directory. A missing component is created under the actual resolved
parent using the spelling received for that component. Obsidian resolves paths
through its `TFile` and Vault APIs. Its collision checks are limited to the
paths those APIs expose and do not promise to find raw physical aliases hidden
by the Obsidian index.
The initial support boundary is CLI and Obsidian clients using CouchDB.
Journal and P2P reject version `1`; Direct file access refuses a remote which
declares the NFC feature. Other host adapters are not supported for version `1`
until they provide equivalent path resolution and ambiguity handling within
their supported storage listing.
Version `1` is represented by the required `nfc-path-identity-v1` feature in
the existing generation-13 database version document. Every client must agree
on the path identity before it can write Metadata. A mode `0` client cannot
join a database requiring this feature. An existing database without the
feature cannot acquire it through an ordinary Metadata write while it already
contains Metadata. These admission checks remain active when ordinary
configuration-mismatch warnings are disabled.
`pathNormalizationVersion` is a required Tweak even when encryption is
disabled. It does not replace or modify `idDerivationVersion`.
The supported migration is reconstruction, not an in-place ID rewrite:
1. Update every participating client and stop synchronisation.
2. Select a complete, authoritative Vault and resolve NFC-equivalent files or
Metadata documents which would produce the same ID.
3. Set version `1` on the source client and Rebuild both databases from that
Vault.
4. Set version `1` on each remaining client and Fetch the rebuilt remote.
5. Resume synchronisation only after every client has the same path identity.
In Obsidian, the setting remains pending through ordinary settings saves, the
settings-only Apply fallback, and reload. It becomes active only when the
existing Rebuild or Fetch operation is successfully reserved. A `remoteOnly`
operation cannot apply a pending identity change, and the settings-only
fallback is not offered. The Setup URI shares
`pathNormalizationVersion` so a new client receives the intended value;
importing that value does not replace the required Rebuild or Fetch operation.
The CLI exposes explicit `rebuild` and `fetch` commands. Both require
`--force` before application start-up because they replace local or remote
database state. `--force` confirms the requested operation; it does not bypass
identity checks, collision preflight, compatibility checks, or required human
remediation. The commands use the existing Commonlib Rebuilder; this change
does not introduce a second migration engine.
The migration procedure also uses `--write-settings` to retain the selected
local database and settings across subsequent CLI invocations. Collision
preflight uses the supported host listing and local Metadata. It refuses
ambiguous file or parent-directory candidates before database reconstruction;
it does not claim to detect physical aliases which a host does not expose.
## Alternatives considered
- **Use NFC by default:** rejected because existing clients and databases must
retain their current IDs until their owners deliberately migrate them.
- **Normalise only lookups:** rejected because new Metadata writes and deletes
could continue using legacy IDs and create duplicate documents.
- **Change `idDerivationVersion`:** rejected because it controls cryptographic
ID derivation, not Unicode canonicalisation, and must remain meaningful when
encryption is disabled.
- **Rename files to NFC:** rejected because the decision concerns logical
identity, while physical spelling belongs to the Vault and its filesystem.
- **Rewrite remote IDs automatically or merge collisions:** rejected because
changing document IDs cannot preserve CouchDB revision ancestry, and the
system cannot choose safely between distinct file histories.
## Consequences
Legacy mode remains unchanged, and NFC identity requires a coordinated,
destructive Rebuild and Fetch. An ambiguity found in the supported storage
listing or local Metadata stops reconstruction before either database is
reset; the user must resolve the names or synchronisation scope. The check is
bounded by each host's listing contract: Obsidian uses its `TFile` and Vault
APIs and does not scan the physical Vault for aliases hidden by that index.
No claim is made that raw physical aliases outside a host's supported listing
are detected.
## References
- [Unicode path identity design](../design_docs/unicode_path_identity.md)
- [Setting details](../settings.md#unicode-path-identity)
- [User migration procedure](../recovery.md#migrate-to-unicode-nfc-path-identity)
- [Device-local file provenance](../specs_conflict_resolution.md#device-local-file-provenance)
- [Commonlib Unicode path identity contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md)
- [Project glossary](../glossary.md)
docs/design_docs/configurable_id_derivation.md
+5 -5
View File
@@ -2,7 +2,7 @@
date: 2026-09-29
commonlib-version: "0.1.33"
self-hosted-livesync-version: "1.0.32"
status: unreleased
status: released
---
# Configurable ID derivation
@@ -21,12 +21,12 @@ including its passphrase-persistence correction at commit
the `next` tag, provides the construction described here. It replaces the
independent Chunk algorithm from the `0.1.32` prerelease without a
compatibility branch or a new settings version. Legacy ID generation remains
unchanged. LiveSync pins the published `0.1.33` package and its registry
integrity in the lockfile.
unchanged. At that implementation baseline, LiveSync pinned the published
`0.1.33` package and its registry integrity in the lockfile.
Its [Internal Metadata encryption design](https://github.com/vrtmrz/obsidian-livesync/blob/126d6eadb858a79a08ad7f600061e54fc8d31196/docs/design_docs/internal_metadata_encryption.md)
remains the basis for Properties encryption and CouchDB feature admission.
This document records an unreleased LiveSync feature. It does not select a
plug-in release version.
Independent ID derivation shipped in Self-hosted LiveSync 1.0.33. This document
retains the design and package baseline recorded above.
## Feasibility
docs/design_docs/document_history_revision_restoration.md
+25 -20
View File
@@ -10,40 +10,45 @@ This document uses the independent revision properties defined under
[Revision](../glossary.md#revision) and the general state model in
[Conflict resolution and revision provenance](../specs_conflict_resolution.md).
Document History can reconstruct an available historical revision from its
Chunks and write that content to the Vault through **Back to this revision**.
The current action writes directly through the storage adapter. It does not
create a new Metadata revision, clear a logical deletion through a successor,
or record which database revision produced the restored Vault file.
Before the current restoration implementation, Document History could
reconstruct an available historical revision from its Chunks and write that
content to the Vault through **Back to this revision**. That action wrote
directly through the storage adapter. It did not create a new Metadata revision,
clear a logical deletion through a successor, or record which database revision
produced the restored Vault file.
This leaves a logically deleted Metadata document at `deleted: true` after the
file has returned to the Vault. A later ordinary Vault save may create a
non-deleted successor, but restoration must not depend on an unrelated later
file event.
That left a logically deleted Metadata document at `deleted: true` after the
file had returned to the Vault. A later ordinary Vault save could create a
non-deleted successor, but restoration needed to succeed without depending on
an unrelated later file event.
The same action does not inspect or preserve revision-tree intent explicitly
when conflicts exist. Document History currently displays the ancestry of the
PouchDB winner. It does not display the complete revision tree or the ancestry
of every conflict leaf.
Restoration does not resolve conflicts or provide a branch-aware history view.
Document History displays the ancestry of the PouchDB winner; it does not
display the complete revision tree or the ancestry of every conflict leaf.
This design covers restoration of one readable historical revision of a normal
Vault file. It creates a new non-deleted successor revision, reflects that exact
revision to the Vault, and leaves every other conflict branch available for the
existing conflict workflow.
The current **Back to this revision** implementation writes that successor
revision, reflects the exact created revision to the Vault, and records it as
device-local provenance. The storage-only behaviour described above is the
pre-fix history.
## Evidence
A real-Obsidian exercise created a Markdown document, removed it through the
Vault API, and waited for LiveSync to store a logical-deletion successor. The
Metadata retained all referenced Chunks, and Document History could reconstruct
the deleted content.
A real-Obsidian exercise before the current restoration implementation created
a Markdown document, removed it through the Vault API, and waited for LiveSync
to store a logical-deletion successor. The Metadata retained all referenced
Chunks, and Document History could reconstruct the deleted content.
Selecting **Back to this revision** restored the file and its exact bytes to the
Vault. The local database nevertheless retained the same deleted current
revision after file processing had settled. An additional ordinary Vault save
then created a new non-deleted successor. This demonstrates that content
reconstruction works and that the missing operation is the database-aware
restoration step.
then created a new non-deleted successor. This historical reproduction showed
that content reconstruction worked while the database-aware restoration step
was missing.
## Revision-tree decision
docs/design_docs/internal_metadata_encryption.md
+8 -7
View File
@@ -2,14 +2,15 @@
date: 2026-09-27
commonlib-version: "0.1.30"
self-hosted-livesync-version: "1.0.32"
status: unreleased
status: released
---
# Internal Metadata encryption and remote feature changes
This document defines the LiveSync integration of Commonlib's remote feature
contract and encrypted Metadata for Hidden File Sync and Customisation Sync.
It describes unreleased behaviour being implemented in this branch.
The implementation shipped in Self-hosted LiveSync 1.0.33; this document
retains its original design and implementation context.
Commonlib's companion `docs/remote-feature-compatibility.md` is the
source of truth for the wire document, identifiers, validation, and shared
@@ -159,11 +160,11 @@ after alignment. With the preference subsequently OFF, verify that Fast Fetch
still decodes encrypted Metadata and preserves the remote feature declaration.
Keep the primary-language settings and troubleshooting guides, the
database-compatibility ADR, and Unreleased notes aligned with this behaviour.
Keep the detailed shared protocol in Commonlib and link to it after publication;
do not maintain another copy of its wire schema here. Translations are a separate
change. Update tested-version evidence when the implementation and its
validation have been accepted.
database-compatibility ADR, and release notes consistent with this behaviour.
Commonlib's companion document remains the source of truth for the shared
protocol; this document does not repeat its wire schema. Translations remain a
separate change. The version fields above retain the original design baseline;
add later tested-version evidence only when validation is recorded.
Related application contracts: [Replicator architecture](replicator_architecture.md),
[Tweak compatibility](tweak_compatibility.md), and
docs/design_docs/path_component_length_compatibility.md
+4 -4
View File
@@ -2,7 +2,7 @@
date: 2026-09-04
commonlib-version: "0.1.21"
self-hosted-livesync-version: "1.0.24"
status: unreleased
status: released
---
# Path component length compatibility
@@ -11,9 +11,9 @@ status: unreleased
File systems place limits on each file or folder name, rather than applying one
common limit to an entire Vault-relative path. Those limits are also expressed
in different units. Self-hosted LiveSync therefore treats 255 UTF-8 bytes as a
focused Android and Linux compatibility warning, not as a universal definition
of a valid path.
in different units. The warning shipped in Self-hosted LiveSync 1.0.26. It
treats 255 UTF-8 bytes as a focused Android and Linux compatibility warning,
not as a universal definition of a valid path.
## Basis for the 255-byte warning
docs/design_docs/renewable_turn_credentials.md
+4 -1
View File
@@ -2,7 +2,7 @@
date: 2026-09-16
commonlib-version: "0.1.25"
self-hosted-livesync-version: "1.0.28"
status: unreleased
status: released
---
# TURN credentials in P2P connection settings
@@ -14,6 +14,9 @@ by acquiring temporary TURN credentials on the device before opening a P2P room.
The [P2P transport compatibility ADR](../adr/2026_08_p2p_transport_compatibility.md)
records the connection and persistence policy.
Managed Cloudflare TURN credentials shipped in Self-hosted LiveSync 1.0.29.
This document records the design and its remaining runtime verification limits.
LiveSync prepares a connection copy of `P2PSyncSetting`. Commonlib owns the room
lifecycle and consumes the resulting ICE settings. Service-specific HTTP and
validation remain under `src/integrations/`; Commonlib has no provider catalogue
docs/design_docs/time_bound_setup_uri.md
+104 -96
View File
@@ -3,40 +3,42 @@ date: 2026-09-28
commonlib-version: "0.1.33"
feasibility-probe-version: "0.1.27"
self-hosted-livesync-version: "1.0.32"
status: implementing
status: released
---
# Time-bound Setup URIs
## Purpose and status
Offer two modes when generating a [Setup URI](../glossary.md):
The published Obsidian generator offers two choices when creating a [Setup URI](../glossary.md):
- **Ephemeral**, selected by default, derives an effective passphrase from the
entered passphrase and the current fixed time window.
- **Persistent** uses the entered passphrase and the existing URI format
unchanged. It also serves as the compatibility mode for older readers.
- **Time-bound** (the `ephemeral` derivation mode), selected by default,
derives an effective passphrase from the entered passphrase and the current
fixed time window.
- **Compatible (no time limit)** (the `persistent` derivation mode) uses the
entered passphrase and the existing URI format unchanged. It also serves as
the compatibility mode for older readers.
The generation mode choice shows the exact time until which an Ephemeral URI
can be opened through the ordinary reader. Import still requires only the URI and
the entered passphrase. Neither the mode nor a timestamp is stored in the URI.
The Time-bound choice shows the exact time until which the URI can be opened
through the ordinary reader. Import still requires only the URI and the entered
passphrase. Neither the mode nor a timestamp is stored in the URI.
The design is technically feasible with the existing encryption primitives.
An executable protocol probe passes 22 cases against Commonlib 0.1.27 and
octagonal-wheels 0.1.54. The implementation was first published as Commonlib 0.1.32
on the npm `next` tag. LiveSync now pins Commonlib 0.1.33 from that tag in its npm
dependency and independent Deno tool imports. A mobile performance bound remains
outstanding.
Both URI availability choices shipped in Self-hosted LiveSync 1.0.33. The
initial executable protocol probe passed 22 cases against Commonlib 0.1.27 and
octagonal-wheels 0.1.54. The implementation was first published as Commonlib
0.1.32 on the npm `next` tag. Current Self-hosted LiveSync 1.0.42 uses Commonlib
0.2.2. A mobile performance bound remains outstanding.
This document records the proposed key derivation and integration contract. The
companion probe demonstrates them without changing an application entry point.
This document records the protocol and the integration plan used for the
published feature. The companion probe remains a standalone design check; it
does not exercise an application entry point.
## Scope
The first version offers Ephemeral and Persistent only. Ephemeral uses one
fixed seven-day window shared by every implementation. Arbitrary start dates,
custom end dates, selectable window lengths, and rolling seven-day lifetimes
are outside this version.
The published protocol offers the Ephemeral and Persistent modes only.
Ephemeral uses one fixed seven-day window shared by every implementation.
Arbitrary start dates, custom end dates, selectable window lengths, and rolling
seven-day lifetimes are outside this version.
Keep the current settings filtering, remote profiles, main and P2P selections,
and short/full export variants. Time binding affects opening the exported
@@ -255,15 +257,16 @@ separately shared message.
## Ownership and compatibility
Commonlib owns preparation of exported settings, the time-window calculation,
passphrase derivation, format parsing, and dual-candidate decoding. Make these
changes in the Commonlib repository. LiveSync must consume a validated packed
artefact and then an exact reviewed package version, following
passphrase derivation, format parsing, and dual-candidate decoding. These
changes were made in the Commonlib repository. LiveSync consumes the reviewed
Commonlib package, following
[the Commonlib dependency workflow](../../devs.md#commonlib-dependency).
Preserve the existing positional encoder API. Its third and fourth arguments
already mean properties to remove and whether to omit default values. Add a
separate encoder, provisionally `encodeTimeBoundSetupURI`, with an options
object containing the mode and the existing export options. Its result is:
The implementation preserves the existing positional encoder API. Its third
and fourth arguments retain their meanings: properties to remove and whether
to omit default values. It adds a separate `encodeTimeBoundSetupURI` encoder
with an options object containing the mode and the existing export options.
Its result is:
```typescript
type TimeBoundSetupURIResult = {
@@ -272,27 +275,27 @@ type TimeBoundSetupURIResult = {
};
```
The new encoder defaults to Ephemeral. Preserve the old encoder's legacy
behaviour for callers which have not migrated. Extend
`decodeSettingsFromSetupURI` to try both candidates for `%$` while retaining its
The new encoder defaults to Ephemeral. The old encoder retains its legacy
behaviour for callers which have not migrated. The updated
`decodeSettingsFromSetupURI` tries both candidates for `%$` while retaining its
settings return contract and historical format handling. Persistent generation
through the new API delegates to the old encoder. Time injection belongs in
through the new API delegates to the old encoder. Time injection remains in
internal test seams, not an import option exposed to users.
| Surface | Required work |
| Surface | Published integration |
| ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
| [Obsidian generation](../../src/serviceFeatures/setupObsidian/setupUri.ts) | Reuse the passphrase and copy dialogues, add a mode choice with the exact current window end, and use the new encoder for all three copy variants and the copy event. Preserve each variant's export filters. |
| [Obsidian import](../../src/modules/features/SetupWizard/dialogs/UseSetupURI.svelte) | Consume the updated decoder and retain one authentication failure message. The existing prefix check indicates URI shape, not successful authentication. |
| [Protocol handler](../../src/serviceFeatures/setupObsidian/setupProtocol.ts) | Keep the existing `settings` reconstruction; test encoded delimiters for both modes. |
| [CLI setup](../../src/apps/cli/commands/runCommand.ts) | Consume the updated decoder and leave settings unchanged after any rejection. |
| [WebPeer generator](../../src/apps/webpeer/src/P2PCheckSetup.ts) | Choose a mode explicitly and return availability to its result UI; keep Ephemeral as the default. |
| [Setup utility](../../utils/setup/generate_setup_uri.ts) | Add an explicit mode input, default to Ephemeral, and return/print the end time. Persistent is an explicit choice for durable provisioning output. |
| [Setup utility package facade](../../utils/setup/livesync-commonlib.ts) | Update its independent Commonlib pin and associated lockfiles; updating the root npm dependency alone is insufficient. |
| [Obsidian generation](../../src/serviceFeatures/setupObsidian/setupUri.ts) | Reuses the passphrase and copy dialogues, presents the mode choice and exact current window end, and uses the encoder for all copy variants and the copy event while preserving each variant's export filters. |
| [Obsidian import](../../src/modules/features/SetupWizard/dialogs/UseSetupURI.svelte) | Consumes the updated decoder and retains one authentication failure message. The existing prefix check indicates URI shape, not successful authentication. |
| [Protocol handler](../../src/serviceFeatures/setupObsidian/setupProtocol.ts) | Retains the existing `settings` reconstruction; encoded delimiters remain part of protocol validation for both modes. |
| [CLI setup](../../src/apps/cli/commands/runCommand.ts) | Uses the updated decoder and leaves settings unchanged after a rejection. |
| [WebPeer generator](../../src/apps/webpeer/src/P2PCheckSetup.ts) | Explicitly selects Time-bound mode and returns availability to its result UI; this generator does not offer a Persistent choice. |
| [Setup utility](../../utils/setup/generate_setup_uri.ts) | Accepts an explicit mode, defaults to Time-bound, and returns or prints the end time. Persistent is selected for durable provisioning output. |
| [Setup utility package facade](../../utils/setup/livesync-commonlib.ts) | Retains its independent Commonlib pin and associated lockfiles, separate from the root npm dependency. |
The Fly.io wrapper delegates to the setup utility and should inherit the same
contract. All maintained generators must migrate explicitly before this
feature is described as the default across applications. Non-interactive
automation which needs a durable URI must select Persistent.
The Fly.io wrapper delegates to the setup utility and inherits the same
contract. The initial plan required each maintained generator to select a mode
explicitly. Non-interactive automation which needs a durable URI selects
Persistent.
WebPeer hides an Ephemeral URI and disables its copy and additional-device
actions after the window ends. Its connection monitor remains available for a
@@ -309,8 +312,9 @@ The assessed, unchanged Commonlib 0.1.27 decoder opens Persistent with the
entered passphrase and rejects Ephemeral with that same passphrase. Persistent
compatibility means compatibility with readers which already support the
ordinary generated `%$` representation; it does not add that representation
to still older readers. Historical supported versions and their settings
schemas still need compatibility fixtures before rollout.
to still older readers. This design does not establish compatibility with every
historical reader or settings schema; compatibility fixtures are needed before
making broader claims about them.
Keep the original raw-passphrase behaviour of existing URIs. They are
indistinguishable from newly generated Persistent URIs to the updated reader.
@@ -358,68 +362,72 @@ All 22 cases passed:
and raw-passphrase candidates;
- the deliberate limitation that clock rollback reproduces an old key.
This demonstrates the proposed mechanism and baseline interoperability, not
completed application support, formal mode indistinguishability, or a mobile
latency bound. The present encryption API may perform two PBKDF2 derivations
on a cold import. Do not infer production performance from this small probe,
whose operations can reuse the dependency's in-memory key cache.
The probe demonstrates feasibility and baseline interoperability. Application
integration is recorded above; the probe does not establish formal mode
indistinguishability or a mobile latency bound. The encryption API may perform
two PBKDF2 derivations on a cold import. Do not infer production performance
from this small probe, whose operations can reuse the dependency's in-memory
key cache.
Implementation validation must cover Commonlib unit and packed-package tests,
plus shared fixed-input protocol vectors across consumers. Add targeted LiveSync
tests for the generation metadata, cancellation, a window change during mode
selection, encoded deep links, and rejected imports without settings writes. Extend CLI, WebPeer,
and Deno setup-tool round trips, including their Persistent selection.
The original implementation validation plan covered Commonlib unit and
packed-package tests, plus shared fixed-input protocol vectors across consumers.
It called for targeted LiveSync tests of generation metadata, cancellation, a
window change during mode selection, encoded deep links, and rejected imports
without settings writes, alongside CLI, WebPeer, and Deno setup-tool round trips
for both modes.
Then run the required LiveSync checks, unit suite, and production build, one
broad process at a time with `NODE_OPTIONS=--max-old-space-size=3072` and bounded
test workers. Use the existing focused real-Obsidian Setup URI workflow for
copy/paste, deep-link import, cancellation, and successful setup. Verify
selection-boundary handling with an injected test clock and mobile behaviour on a
supported runtime; do not change the host's system clock. No remote database
service is needed for the protocol tests themselves.
The plan also called for the LiveSync checks, unit suite, and production build,
one broad process at a time with `NODE_OPTIONS=--max-old-space-size=3072` and
bounded test workers. It used the focused real-Obsidian Setup URI workflow for
copy/paste, deep-link import, cancellation, and successful setup. Selection-
boundary handling was to use an injected test clock; mobile behaviour requires
a supported runtime and must not change the host's system clock. Protocol tests
do not require a remote database service.
There is no identified blocker in the Time-bound URI protocol. The Commonlib
candidate based on 0.1.31 passed its type, unit, boundary, release-process,
and packed-package gates before the 0.1.32 release. A clean `npm ci` against
the published 0.1.32 artefact and the frozen Deno tool lock resolve the same
registry integrity. LiveSync passes its source checks, production build, and
1,034 unit tests on a local stack with [PR #1222](https://github.com/vrtmrz/obsidian-livesync/pull/1222).
The two CLI installer tests need a runner which permits child processes; both
pass under normal process permissions. The Deno setup-tool suite, CLI setup
The original protocol assessment found no blocker in the Time-bound URI
protocol. The Commonlib candidate based on 0.1.31 passed its type, unit,
boundary, release-process, and packed-package gates before the 0.1.32 release. A
clean `npm ci` against the published 0.1.32 artefact and the frozen Deno tool
lock resolved the same registry integrity. In the original implementation
baseline, LiveSync passed its source checks, production build, and 1,034 unit
tests on a local stack with [PR #1222](https://github.com/vrtmrz/obsidian-livesync/pull/1222).
The two CLI installer tests needed a runner which permits child processes; both
passed under normal process permissions. The Deno setup-tool suite, CLI setup
and file-operation contract, and WebPeer browser tests passed before stacking.
Focused real-Obsidian generation checks pass on desktop and emulated mobile,
The recorded real-Obsidian generation checks passed on desktop and emulated mobile,
including the displayed Time-bound end, Compatible output, and mobile touch
targets. A two-device CouchDB workflow now passes on the local stack: the
provisioning tool creates a generation 12 database, the first device generates
targets. A two-device CouchDB workflow passed on the local stack: the
provisioning tool created a generation 12 database, the first device generated
a Time-bound Setup URI after declaring its required generation 13 feature, and
the second device imports it. An ordinary note completes a round trip, and a
hidden snippet synchronises. The separate real-Obsidian tests for live remote
the second device imported it. An ordinary note completed a round trip, and a
hidden snippet synchronised. The separate real-Obsidian tests for live remote
feature changes, internal Metadata migration, and a percent-prefixed E2EE
passphrase also pass with published Commonlib 0.1.32. Unit and protocol tests
cover window boundaries and preserve the existing distinction between an empty
passphrase and cancellation. The WebPeer browser test confirms that its monitor
remains available after the URI window ends for a device which has already
imported the URI.
passphrase also passed with published Commonlib 0.1.32. Unit and protocol tests
covered window boundaries and preserved the existing distinction between an
empty passphrase and cancellation. The WebPeer browser test confirmed that its
monitor remained available after the URI window ended for a device which had
already imported the URI.
The current Commonlib 0.1.33 integration includes the merged host changes from
PR #1222 and PR #1225. npm 10 and normal clean installations preserve the lockfile;
source checks, the production build, 1,064 unit tests, and ten Deno setup-tool
tests pass. Both URI modes preserve ID recovery and explicit legacy ID selection.
The Commonlib 0.1.33 integration included the merged host changes from
PR #1222 and PR #1225. At that baseline, npm 10 and normal clean installations
preserved the lockfile; source checks, the production build, 1,064 unit tests,
and ten Deno setup-tool tests passed. Both URI modes preserved ID recovery and
explicit legacy ID selection.
Real Obsidian tests cover independent ID keys through Time-bound, Compatible,
and QR setup, encrypted local persistence, natural restarts of both devices,
and bidirectional note transfers with stable document and Chunk IDs. Incorrect
passphrases and past-window URIs leave runtime and persisted settings unchanged.
The past clock belongs to an isolated fixture worker, not Obsidian or the host.
CouchDB tests also cover a custom ID source and recovery code, rejection of
incompatible document keys before remote writes, and the Doctor's decline,
reminder, dismissal, and later acceptance paths. The detailed commands and the
remaining published-artefact and physical-device checks are recorded in the
Recorded real-Obsidian tests covered independent ID keys through Time-bound,
Compatible, and QR setup, encrypted local persistence, natural restarts of both
devices, and bidirectional note transfers with stable document and Chunk IDs.
Incorrect passphrases and past-window URIs left runtime and persisted settings
unchanged. The past clock belonged to an isolated fixture worker, not Obsidian
or the host. CouchDB tests also covered a custom ID source and recovery code,
rejection of incompatible document keys before remote writes, and the Doctor's
decline, reminder, dismissal, and later acceptance paths. The detailed commands,
remaining published-artefact checks, and physical-device checks are recorded in the
[real Obsidian test guide](../../test/e2e-obsidian/README.md#combined-setup-and-security-regression-checks).
Compatible preserves the URI encryption format; each receiving client must
still support the shared settings and the remote's declared features. The host
compatibility checks from PR #1222 remain necessary. These results do not
establish a mobile performance bound. The first implementation changes only
primary-language resources; translation changes require separate scope.
establish a mobile performance bound. The initial implementation changed only
primary-language resources; translation changes remain a separate scope.
docs/design_docs/tweak_compatibility.md
+32 -8
View File
@@ -1,15 +1,17 @@
---
date: 2026-09-08
commonlib-version: "0.1.24"
self-hosted-livesync-version: "1.0.27"
status: unreleased
date: 2026-10-17
commonlib-version: "0.1.42"
self-hosted-livesync-version: "1.0.37"
status: released
---
# Tweak compatibility and recovery
This document describes the integration of Commonlib 0.1.24 with
Self-hosted LiveSync 1.0.27. Commonlib owns the interpretation of synchronisation
settings; LiveSync owns the dialogues and operations which consume that result.
This document describes the compatibility and recovery contract introduced
after the Self-hosted LiveSync 1.0.37 and Commonlib 0.1.42 baseline recorded
above. The current assessment is included in Self-hosted LiveSync 1.0.42 with
Commonlib 0.2.2. Commonlib owns the interpretation of synchronisation settings;
LiveSync owns the dialogues and operations which consume that result.
## Shared assessment
@@ -25,11 +27,16 @@ document IDs. An explicitly enabled value is therefore different from either an
explicitly disabled value or a missing value. Settings whose historical missing
value has not been established remain unadvertised; the evaluator does not
invent defaults or turn every falsy value into an absent value.
`pathNormalizationVersion` is another explicit compatibility rule: missing and
`0` select legacy identity, while `1` requires coordinated reconstruction.
The [Unicode path identity design](unicode_path_identity.md) defines its
admission checks and supported transports.
The central replication gate, mismatch dialogues, and RedFlag Fetch preparation
consume the same effective differences. The P2P transport retains its separate
policy: ordinary representation differences warn without rejecting transfer,
while its existing passphrase and peer checks still apply. Sharing assessment
while its existing passphrase and peer checks still apply. NFC path identity
is outside that warning-only policy: P2P refuses version `1`. Sharing assessment
does not make the central replication policy appropriate for every transport.
## Host responsibilities
@@ -65,6 +72,15 @@ for a decision and before applying it. A changed target or changed settings
discard that decision. The signature used for this check can contain sensitive
configuration and must not be logged, persisted, or included in diagnostics.
Central replication can assess an effective value which differs from a saved
preference, such as internal Metadata encryption whose prerequisites are inactive.
Recovery accepts the current saved or effective snapshot without treating that
representation difference as a settings change. Applying assessed changes retains
saved preferences only when they leave the adopted effective values unchanged.
If adoption activates encryption prerequisites, the adopted encryption value
takes precedence. The full settings signature still detects a preference changed
while awaiting a decision.
The active publication reservation is not held while waiting for the dialogue.
Remote writes use the failed attempt's publication guard. Fetch and Rebuild use
their existing owners and propagate failure without claiming a successful
@@ -91,3 +107,11 @@ replication, explicit Fetch, and replication after restart. A unit test which
mocks Fetch does not establish those behaviours, and a corrected mismatch
dialogue alone does not establish the cause of a reported persistent automatic
synchronisation failure.
The `tweak-compatibility` real-Obsidian scenario covers automatic and manual
adoption, stale target rejection, and adoption which activates either E2EE or
Property Encryption. Those prerequisite changes must retain the adopted
effective Metadata encryption value through Fetch, restart, and two-way file
transfer. See the [test guide](../../test/e2e-obsidian/README.md) for the scenario
and explicit artefact selection. The cause of an earlier saved chunk-size
change is separate from these recovery guarantees.
docs/design_docs/unicode_path_identity.md
+275
View File
@@ -0,0 +1,275 @@
---
date: 2026-10-19
commonlib-version: "0.1.43"
self-hosted-livesync-version: "1.0.38"
status: released
---
# Unicode path identity
## Purpose and baseline
This design defines how Self-hosted LiveSync opts into Unicode NFC for normal
file Metadata document IDs, how hosts preserve the Vault's physical names, and
how existing databases move to the new identity. The accepted rationale is in
the [Unicode path identity ADR](../adr/2026_10_unicode_path_identity.md).
Commonlib's [consumer contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md)
defines the shared package boundary.
NFC identity shipped in Self-hosted LiveSync `1.0.38` with Commonlib `0.1.42`.
Conflict-aware deletion handling during reconstruction was added separately
and is included in current Commonlib `0.2.2`, used by Self-hosted LiveSync
`1.0.42`. This later handling is distinct from the initial NFC identity
release.
## Scope
| Area | Contract |
| --- | --- |
| Logical identity | Version `1` uses NFC for Metadata ID and path comparison keys. Existing case handling remains a separate rule. Missing and `0` retain legacy behaviour. |
| Stored paths and content | Keep the spelling in `Metadata.path` and existing physical Vault names. NFC does not change Chunk IDs or content. |
| Initial hosts and remote | Support CLI and Obsidian with CouchDB. |
| Other transports | Journal and P2P reject version `1`; Direct file access refuses a remote declaring the NFC feature. Other host adapters remain outside the support matrix. |
| Migration | Rebuild both databases from one complete authoritative Vault, then Fetch each remaining client. Do not rewrite IDs or merge histories in place. |
## Path identity contract
`pathNormalizationVersion` selects the logical path used by Commonlib's
`PathService` for normal-file Metadata IDs and path comparison keys:
| Value | Path identity |
| --- | --- |
| Missing or `0` | Preserve the legacy path and ID behaviour exactly. |
| `1` | Apply Unicode NFC to the logical path, then apply the existing case handling and ID derivation rules. |
| Any other value | Reject it; do not fall back to legacy mode. |
NFC changes only logical identity and comparison keys. It is not a physical
filename conversion, a rewrite of `Metadata.path`, or a Unicode compatibility
transformation such as NFKC. Existing case handling remains governed by
`handleFilenameCaseSensitive`; it is independent of NFC equivalence resolution
in the CLI. `idDerivationVersion` continues to select its separate
cryptographic ID construction. NFC path identity works with or without
encryption and does not change Chunk IDs or content.
The version is a required Tweak, even when encryption is disabled, represented
by `nfc-path-identity-v1` in the existing generation-13 database version
document's `used_features` list. The Setup URI includes
`pathNormalizationVersion`, decodes an absent value as `0`, and rejects
unsupported values. Sharing the setting communicates the target identity;
setup still requires an explicit Rebuild or Fetch choice.
Fast Fetch checkpoints include the path identity version. A checkpoint created
under legacy identity cannot be resumed under version `1`, or vice versa. This
reuses the existing checkpoint identity and does not add a separate recovery
store.
## Admission and database compatibility
Clients validate local and remote database identity before accepting ordinary
replication, including when configuration-mismatch warnings are disabled. The
checks cover both directions:
| Client setting | Database feature | Result |
| --- | --- | --- |
| Missing or `0` | No NFC feature | Continue with legacy identity. |
| `1` | NFC feature present | Continue with NFC identity. |
| `1` | Neither database has the feature or Metadata | A fresh database may declare the feature before its first Metadata mutation. |
| `1` | A database has Metadata without the feature | Reject ordinary writes; require Rebuild. |
| Missing or `0` | A local or remote database has the NFC feature | Reject; the database requires a version-`1` client. |
The feature is independent of encryption. Receiving an incompatible version
document records the incompatibility and stops the active synchronisation
session. This is a compatibility fence, not a rollback of operations already
admitted elsewhere; every device must be stopped before migration begins.
## Host path resolution and collision preflight
Commonlib owns logical ID derivation and the Rebuilder's preflight contract.
Each host maps that logical path to entries in its supported storage listing.
The CLI resolves existing files and directories using NFC comparison keys,
regardless of `handleFilenameCaseSensitive`. It carries the actual spelling
reported for each existing component through reads, writes, deletes, stat,
directory creation, and rename operations. If a path has missing components,
the CLI uses the actual existing parent and preserves the received spelling for
each new component. A scan enumerates each directory once and stats the raw
paths returned by that enumeration; it does not repeat canonical lookup for
each listed entry. No persistent index or recovery store is added.
If more than one entry resolves to the same logical comparison key after NFC
and the configured case rule at any component, including a parent directory,
the path is ambiguous. The CLI warns and refuses the operation; it must not
select a candidate based on directory enumeration order, merge the entries, or
rename them. The same rule applies during Rebuild preflight.
Obsidian resolution remains within its `TFile` and Vault API boundary. Its
collision preflight checks paths exposed through those APIs; it does not walk
the physical Vault independently and cannot promise to detect raw filesystem
aliases hidden by Obsidian's index. Other host adapters remain outside the
version-`1` support matrix until their path resolution and ambiguity handling
are implemented and verified within their own supported listings.
Before resetting either database, Rebuild checks the supported storage
listing and local Metadata for paths which would resolve to the same identity
under NFC and the configured case rule. When a local database is not open,
preflight checks the storage paths returned by the host listing. If multiple
candidate files or Metadata documents collide, or a parent component is
ambiguous, the operation stops before local or remote mutation. It does not
choose a winner, merge revision histories, or rename a Vault file. The user
must resolve the ambiguous names or exclude one file from the synchronisation
scope, then retry the reconstruction. This check is bounded by the host's
listing contract and does not imply that every raw physical alias is visible.
For full Rebuild, Standard Fetch, and Fast Fetch, which replace the local
database, the Metadata enumeration requests conflict information. It excludes
`deleted: true` documents only when they have no unresolved conflict leaves.
A deleted winner with any unresolved conflict remains a collision candidate,
including when the other leaves are also logically deleted. Remote-only
Rebuild retains logically deleted Metadata and requires stored IDs to match
the selected identity. Fully tombstoned documents are already absent from
ordinary enumeration; a still non-deleted local copy remains subject to the
check until a remote deletion has been replicated.
## Reconstruction and client operations
Mode changes are never applied by an ordinary Metadata write. In Obsidian, the
Advanced page offers **Legacy path identity** (`0`) and **Unicode NFC path
identity** (`1`) for CouchDB. A changed `pathNormalizationVersion` remains
pending through ordinary settings saves, the settings-only Apply fallback, and
reload. It becomes active only after the existing Rebuild or Fetch operation
has been successfully reserved; cancellation or failed reservation leaves it
pending. A `remoteOnly` operation cannot apply a pending identity change, and
the settings-only fallback is not offered.
The [user migration procedure](../recovery.md#migrate-to-unicode-nfc-path-identity)
owns the operational sequence: updated and stopped clients, a complete backed-up
authoritative Vault, source Rebuild, then Fetch on each remaining client. The
source operation reconstructs both databases; a receiving client's Fetch
replaces only its local database. The [CLI guide](../../src/apps/cli/README.md)
provides the equivalent `rebuild` and `fetch` commands.
The CLI checks `--force` before core start-up so an omitted confirmation cannot
trigger database access or destructive preparation. Neither `--force` nor a
headless default choice overrides an NFC collision, a feature mismatch, or a
preflight failure. If a required setup decision cannot be made in a headless
CLI invocation, the command must fail with a clear remediation message rather
than report success after cancellation.
`--write-settings` retains the Rebuilder's selected local database name and
lasting settings changes for subsequent invocations. Without this option, the
existing CLI policy leaves the settings file unchanged, so a later command can
reopen the previously selected database.
## State which follows the selected identity
The path identity is also used by two existing resume mechanisms:
- **File-reflection provenance:** the existing device-local provenance storage
interface remains an exact-key store. The private
`ServiceFileHandlerBase.getProvenancePath()` method canonicalises provenance
keys with NFC in mode `1` for reads, writes, deletes, raw snapshot lookup,
and untracked revision recording. Mode `0` retains the old keys. Rebuild and
Fetch use the existing local-state reconstruction and replacement lifecycle;
no NFC-specific persistent field or in-place key migration is introduced.
Commonlib 0.1.42 separately adds content hashes to the same records. The
[file-provenance contract](../specs_conflict_resolution.md#device-local-file-provenance)
defines that content evidence; changing a path key neither changes file
bytes nor supplies missing revision ancestry.
- **Fast Fetch checkpoints:** the path version participates in checkpoint
identity so an interrupted transfer cannot resume using a different
path-to-ID rule. Existing checkpoint persistence and retry handling remain
in control.
These rules keep a physical NFD filename addressable when an NFC-equivalent
Metadata path arrives, while using one canonical identity for subsequent
updates and deletions.
## Compatibility matrix
| Client and remote | Version `0` | Version `1` |
| --- | --- | --- |
| CLI with CouchDB | Supported with legacy identity. | Supported with NFC path resolution. |
| Obsidian with CouchDB | Supported with legacy identity. | Supported with NFC path resolution. |
| Journal or P2P | Existing behaviour. | Rejected. |
| Direct file access | Existing behaviour. | Refuses a remote requiring `nfc-path-identity-v1`. |
| Other host adapters, including WebApp | Existing behaviour, subject to their existing support. | Outside the initial support matrix until physical-path handling is verified. |
| Older client without this feature | Can use a legacy database. | Cannot join a database declaring `nfc-path-identity-v1`. |
## Verification contract and limits
The implementation acceptance matrix covers:
- unchanged legacy IDs for missing and explicit version `0`, plus matching
ASCII IDs in both modes;
- identical normal-file Metadata IDs for NFC and NFD spellings in version `1`,
with and without path obfuscation, and with the existing case rule applied;
- Metadata creation, update, deletion, and file-reflection provenance using the
same canonical identity without renaming the physical file;
- CLI reads, writes, deletes, stat, directory creation, and rename through the
actual spelling of existing files and parents; preservation of the received
spelling for new components; and resolution independent of the case setting;
- refusal to write when an existing legacy database has Metadata, refusal of
version-`0` clients against an NFC database, and enforcement when ordinary
mismatch warnings are disabled;
- URI round trips, checkpoint invalidation across path versions, and transport
rejection for Journal, P2P, and Direct file access;
- refusal for ambiguous file or parent-directory candidates before local or
remote reset, preserving both databases on rejection, within the paths
returned by each host's supported listing; and
- CLI `rebuild` and `fetch` command dispatch, pre-start `--force` enforcement,
and end-to-end creation, update, and deletion through CouchDB between CLI and
Obsidian.
Focused unit and package checks do not substitute for the real CouchDB and
Obsidian round trip using the exact published Commonlib artefact. Obsidian checks
are limited to paths exposed through its `TFile` and Vault APIs; they do not
establish detection of physical aliases hidden by the Obsidian index. No result
for raw physical filesystem aliases outside a host's supported listing is
claimed.
Local validation on Linux with real Obsidian `1.12.7` and the published
Commonlib `0.1.42` package passed after the CLI path resolution, case-aware
collision scan, and pending-setting changes. It covered:
- NFC identity with unencrypted Metadata;
- NFC identity with E2EE and Path Obfuscation;
- two legacy CLI databases migrated through `rebuild` and `fetch`, followed by
the encrypted Obsidian round trip; and
- the existing encrypted ASCII scenario in legacy mode.
In those NFC scenarios, an NFD directory and mixed-form filename were retained
while checking content, Metadata creation time, remote updates, absence of an
NFC duplicate, and remote deletion through the polling CLI daemon. The `mirror`
command retains tombstoned local files by default; its update behaviour was
tested separately from daemon deletion reflection.
The published Commonlib `0.1.43` package also passed plain and encrypted NFC
migration with obsolete logical deletions, plus the legacy encrypted ASCII
scenario, on Linux with Obsidian `1.12.7` and CouchDB `3.5.0`. Both migration
variants checked CLI Rebuild, second-client Standard Fetch, and real Obsidian
Streaming Fast Fetch. Fast Fetch removed the obsolete Metadata and retained
the current NFC document, its active state, and its stored path. The existing
content, creation-time, physical-name, update, and deletion checks also passed.
## Non-goals
This design does not:
- change Chunk IDs, E2EE, Properties encryption, or the Metadata and Chunk
document formats;
- convert or rename existing physical files;
- migrate remote document IDs or revision histories in place;
- choose among ambiguous NFC-equivalent files or parents, or merge their
content;
- change the default away from legacy path identity;
- enable version `1` for Journal, P2P, Direct file access, WebApp, or an
unverified host adapter; or
- claim native iOS or macOS filesystem compatibility.
## References
- [Unicode path identity ADR](../adr/2026_10_unicode_path_identity.md)
- [Setting details](../settings.md#unicode-path-identity)
- [User migration procedure](../recovery.md#migrate-to-unicode-nfc-path-identity)
- [Commonlib Unicode path identity contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/unicode-path-identity.md)
- [Metadata and Chunks glossary](../glossary.md#user-facing-and-operational-terms)
docs/design_docs_of_journalsync.md
+3 -1
View File
@@ -1,5 +1,7 @@
## The design document of the journal sync
Historical design: this is the first Journal Sync design. Its original transfer sequence and discussion of possible storage backends are retained as history. For the maintained pipeline and storage-adapter design, read [Journal Replicator 2nd Edition](design_docs_of_journalsync_2nd.md) and the [Replicator architecture](design_docs/replicator_architecture.md). Use [Object Storage Setup](setup_object_storage.md) for supported client setup. A possible backend discussed here is not evidence that an adapter is available in the current release.
Original title: Synchronise without CouchDB
### Goal
@@ -47,4 +49,4 @@ This implementation replaces the synchronisation performed by CouchDB. Therefore
### Consideration and Conclusion
This design offers a novel approach to journal synchronisation without relying on CouchDB. It leverages PouchDB's journaling capabilities and leverages simple server-side storage for efficient data exchange. Hence, the new design could be said to have gotten a broader outlook.
This design offers a novel approach to journal synchronisation without relying on CouchDB. It leverages PouchDB's journaling capabilities and leverages simple server-side storage for efficient data exchange. Hence, the new design could be said to have gotten a broader outlook.
docs/design_docs_of_remote_configurations.md
+16 -5
View File
@@ -1,5 +1,9 @@
# The design document of remote configuration management
## Implementation status
This document describes the profile catalogue and its runtime projections as used by Self-hosted LiveSync 1.0.42 with Commonlib 0.2.2. Commonlib owns profile migration, activation, and protected settings persistence. The [Replicator architecture](design_docs/replicator_architecture.md) and [P2P service lifecycle decision](adr/2026_08_replicator_capabilities_02_p2p_service_lifecycle.md) describe the separate P2P service boundary.
## Goal
- Allow us to manage multiple remote connections in a single vault.
@@ -33,7 +37,9 @@ Once multiple remotes of the same type became desirable, the previous model no l
The current settings now have two layers for remote configuration.
1. A stored collection of named remotes.
2. One active remote projected into the legacy flat settings fields.
2. One active main remote projected into the legacy flat settings fields.
The P2P service also has its own selected profile, recorded by `P2P_ActiveRemoteConfigurationId`. This permits the P2P service to use a selected P2P profile alongside a CouchDB or Object Storage main remote; it does not introduce a second main remote.
This means the replication and database layers can continue to read the effective remote from the existing settings fields, while the settings dialogue and migration logic can manage many stored remotes.
@@ -54,6 +60,7 @@ type RemoteConfiguration = {
type RemoteConfigurations = {
remoteConfigurations: Record<string, RemoteConfiguration>;
activeConfigurationId: string;
P2P_ActiveRemoteConfigurationId: string;
};
```
@@ -116,7 +123,11 @@ Therefore:
- Import trusts the supplied connection string as-is after parsing.
- We should regard exported connection strings as sensitive information, much like Setup URI or a credentials-bearing configuration file.
The `isEncrypted` field is currently reserved for future expansion. At present, the connection string itself is stored plainly inside the settings data, in the same sense that the effective runtime configuration can contain usable remote credentials.
The `isEncrypted` field identifies an encrypted profile URI in persisted settings. When settings are saved with configuration encryption enabled, Commonlib encrypts the profile URIs and records `isEncrypted: true`. Loading protected settings decrypts the URIs into the runtime settings and clears that runtime flag before profile activation. The runtime settings and the persisted representation therefore have different purposes.
Protection depends on the configured settings-passphrase policy and a usable key. A plaintext configuration is not made confidential by the presence of the field alone. Consult [Encrypting sensitive configuration items](settings.md#encrypting-sensitive-configuration-items) for that policy.
Clipboard export serialises the usable connection string. Treat that exported value as sensitive even when its saved profile URI is encrypted at rest; storage protection does not encrypt the clipboard representation.
### Migration strategy
@@ -129,7 +140,7 @@ Rules:
- If no remote list exists and the legacy fields contain a Peer-to-Peer configuration, create `legacy-p2p`.
- If more than one legacy remote is populated, create all possible entries and select the active one according to `remoteType`.
This migration is intentionally additive. It does not remove the flat fields because they remain necessary as the active runtime projection.
This migration is intentionally additive. It does not remove the flat fields because they remain necessary as the active runtime projection. A migrated legacy P2P profile also initialises the separate P2P selection. Loading an existing profile catalogue does not unconditionally replace an intentionally empty P2P selection.
### Normalisation and application paths
@@ -190,7 +201,7 @@ The feature should be tested from four viewpoints.
## Outlook
Import/export configuration strings should also be encrypted in the future, but this is a separate feature that can be added on top of the current design.
A separately encrypted representation for clipboard import and export remains a possible extension. Encryption of profile URIs in persisted settings is already implemented as described above.
## Consideration and conclusion
@@ -203,4 +214,4 @@ Its core idea is modest but effective.
- Project it into the legacy settings for runtime use.
- Normalise every imported settings object through the same path.
This keeps the implementation understandable and migration-friendly. It also opens the door for future work, such as encrypted per-remote storage, richer remote metadata, or remote-scoped options, without forcing another large redesign of how remotes are represented.
This keeps the implementation understandable and migration-friendly. It also opens the door for future work, such as richer remote metadata or remote-scoped options, without forcing another large redesign of how remotes are represented.
docs/design_docs_of_sharing_tweak_value.md
+2 -2
View File
@@ -1,6 +1,6 @@
# Sharing `Tweak values`
NOTE: This is the planned feature design document. This is planned, but not be implemented now (v0.23.3). This has not reached the design freeze and will be added to from time to time.
Historical design: this document records the original proposal written for v0.23.3. Tweak sharing and compatibility assessment are implemented in published releases. Use [Tweak compatibility and recovery](design_docs/tweak_compatibility.md) for the maintained contract, and [Settings](settings.md) for the available controls. The proposal and test plan below are retained as design history; they do not define the current behaviour.
## Goal
@@ -52,4 +52,4 @@ B: On the setting dialogue.
- However, this might be an essential feature. Further efforts are desired.
### Consideration and Conclusion
To be described after implemented, tested, and, released.
The published implementation has evolved beyond this proposal, including classification of compatible differences and automatic alignment where configured. Its current decisions, recovery requirements, and remaining limitations are recorded in [Tweak compatibility and recovery](design_docs/tweak_compatibility.md). The original proposal does not establish that every item in its test plan was implemented unchanged.
docs/glossary.md
+4
View File
@@ -98,6 +98,10 @@ recovery guidance, or diagnostics intended for users.
- **ID recovery code:** A versioned text form of the saved ID key which can be
shown on the current device and imported without deriving a different key.
Treat it as a secret.
- **Joined devices:** The dialogue showing device records held by the active
CouchDB or Object Storage remote, with the current device first. Records
describe participation, not current online activity. **Forget device** removes
an obsolete device record; it does not delete Vault files or revoke credentials.
- **JWT Authentication:** An experimental CouchDB authentication option which
uses a JSON Web Token instead of standard credentials. It requires a private
key or secret, algorithm, expiry duration, subject, and key ID.
docs/recovery.md
+20
View File
@@ -18,6 +18,26 @@ Use the least destructive operation which matches the evidence:
Do not switch transport, enable P2P, or run Garbage Collection as a substitute for diagnosing a stopped CouchDB or Object Storage setup.
## Migrate to Unicode NFC path identity
Use this coordinated migration when CLI and Obsidian clients share a CouchDB database and composed and decomposed Unicode spellings should identify the same path. Version `0` remains the default and preserves legacy identity. Version `1` is supported with CouchDB only; it is unsupported with Object Storage and P2P.
This migration reconstructs the databases from one authoritative Vault. It does not rewrite document IDs in place or rename files.
1. Update every participating CLI and Obsidian client to a version which supports Unicode NFC path identity. Back up a complete authoritative Vault and the CouchDB database. Stop synchronisation and editing on every device until the migration is complete.
2. Choose the complete Vault which contains the intended files. Review any composed and decomposed names which would map to the same path identity, and resolve them before reconstruction. If Rebuild or Fetch reports an ambiguous path or Metadata ID, stop and resolve that collision before retrying; LiveSync does not choose or merge the files for you.
3. On the source client, select **Unicode NFC path identity** (`pathNormalizationVersion: 1`). In Obsidian, choose **Overwrite Server Data with This Device's Files**. In the CLI, set the same value in its settings and run `rebuild` with `--force --write-settings`.
4. On each remaining client, select **Unicode NFC path identity** (`pathNormalizationVersion: 1`) before fetching. In Obsidian, choose **Reset Synchronisation on This Device**. In the CLI, set the same value in its settings and run `fetch` with `--force --write-settings`.
5. Confirm that every client uses version `1` and that the expected files are present before resuming synchronisation. Do not reconnect a version `0` client to the migrated database.
Rebuild and both Fetch paths ignore local logical deletions without unresolved conflicts when replacing the local database. Deletion or rename history alone therefore does not block this migration. Fully tombstoned documents are already absent from the Metadata check. Unresolved conflicts and ambiguous current files still require review, even when a conflict's winning revision is logically deleted. Remote-only Rebuild retains its existing ID checks and is not a substitute for the source Rebuild above.
A remote deletion which has not reached this device does not establish that its local Metadata is deleted. Inspect the local state and preserve unsynchronised content before resolving a reported collision. Manual deletion of every database is not a prerequisite for this procedure. Clients already using NFC identity do not need to repeat the migration for this change.
The Inspector's **Repair this Metadata document ID** action repairs one local entry; it is not a bulk migration tool. Use the Rebuild and Fetch procedure above for path identity changes.
This procedure does not establish native macOS or iOS filesystem behaviour. Obsidian's collision checks use the paths exposed by its Vault APIs; see [Unicode path identity](settings.md#unicode-path-identity) for that visibility limit.
## Suspend before diagnosis
Close Obsidian completely, then create an empty file or directory named `redflag.md` at the root of the Vault. On the next start, LiveSync enters its emergency suspension state before ordinary database, file-watching, and synchronisation work continues.
docs/releases/1.0-previews.md
+62 -1
View File
@@ -1,9 +1,70 @@
# 1.0 preview release history
This document records the opt-in beta and release-candidate builds published before 1.0.0. Most users upgrading from 0.25.83 only need the consolidated [1.0.0 release notes](1.0.md#100).
This document records opt-in pre-release builds in the 1.0 release line. Consolidated notes for stable upgrades are available in the [current change log](../../updates.md) and the [1.0 release history](1.0.md).
The prepared `1.0.0-rc.0` tag was not published as a plug-in release and is therefore omitted.
Version 1.0.41 was published for validation and remains a GitHub pre-release. Its changes and the subsequent start-up correction are consolidated in the [1.0.42 release notes](../../updates.md#1042).
## 1.0.41
26th October, 2026
### Synchronisation
#### New Feature
- Rapid edits can produce fewer intermediate versions on other devices while LiveSync mode remains active.
- Enable **Batch database updates in LiveSync mode** to group successive local edits.
- Disabled by default; each synchronisation mode keeps its own choice.
- Updates reach other devices later. Batching does not detect template completion or prevent genuine concurrent-edit conflicts.
- CouchDB synchronisation no longer looks idle while it is comparing change history.
- A ⏩ icon and comparison count show activity before documents are transferred.
- This count represents history entries, not files remaining to synchronise.
- Status counts are easier to follow, with less movement as values change.
- CSS snippets can hide unwanted items or style individual `livesync-stat-` items, including inactive ones.
#### Fixed
- Recent Vault edits remain recoverable after an interrupted local database save.
- This includes edits whose modification times compare equal to the database copy, and pending Hidden File Sync and Customisation Sync changes.
- Manual synchronisation no longer starts before already accepted local changes finish saving to the database.
- Settings application also waits for those saves. Failed saves stay pending and report a failure.
- Text from third-party tools is handled more robustly.
- Characters split across Chunks no longer trigger false corruption warnings.
- Start-up integrity checks use less memory for healthy Vault files. (#1283)
- Start-up checks avoid false repair warnings for files that change during inspection. (#1283)
- This applies when the final size matches the database and there are no conflicts.
### Customisation Sync
#### Fixed
- Applying a multi-file customisation no longer skips files after the list is refreshed. (#1271)
- Choosing **Automatic** now explains a missing Hidden File Sync prerequisite instead of proceeding into a failed initialisation. (#1270)
- Enable and initialise Hidden File Sync on this device first. The current mode is kept when the prerequisite is missing.
- Initialising a whole plug-in in **Automatic** mode no longer omits its selected **MAIN** or **DATA** files.
### Setup tools
#### Fixed
- Setup URI generation can avoid shell-variable collisions and unintended use of the operating-system username. (#817, #1274)
- Use `LIVESYNC_HOST`, `LIVESYNC_USER`, `LIVESYNC_PASS`, `LIVESYNC_DB`, and `LIVESYNC_PASSPHRASE` for connection and encryption values.
- Existing scripts remain supported: the previous names are used when the corresponding prefixed value is unset or blank.
### Miscellaneous
- Dependencies have been updated, including security fixes.
### Acknowledgements
Thank you for your contributions!
- @rwbru, for the original Setup URI environment-variable fix ([#817](https://github.com/vrtmrz/obsidian-livesync/pull/817)).
- @junkerderprovinz, for adapting that fix to the current generator while preserving compatibility with existing scripts ([#1274](https://github.com/vrtmrz/obsidian-livesync/pull/1274)).
- @s-qin, for reducing memory use during start-up checks and avoiding repeated content decoding in Commonlib ([#1283](https://github.com/vrtmrz/obsidian-livesync/pull/1283), [Commonlib #184](https://github.com/vrtmrz/livesync-commonlib/pull/184)).
## 1.0.0-rc.1
27th July, 2026
docs/releases/1.0.md
+117
View File
@@ -2,6 +2,123 @@
This document contains earlier published releases from the 1.0 line of the [current Self-hosted LiveSync release history](../../updates.md). Beta and release-candidate builds published before 1.0.0 are recorded in the [1.0 preview history](1.0-previews.md). Earlier release lines continue in the [0.25 history](0.25.md) and the [legacy history](legacy.md).
## 1.0.35
5th October, 2026
### Synchronisation
#### Fixed
- We can continue synchronising after rebuilding an encrypted CouchDB remote, including when Hidden File Sync and Customisation Sync are enabled. (#1234)
- This plug-in reads the rebuilt data without reusing HTTP responses encrypted with the previous Security Seed.
### Customisation Sync
#### Fixed
- Customisation Sync now waits until this plug-in is ready before handling configuration file events, and applying settings while synchronisation is suspended no longer scans configuration files. (#1220)
- Resuming synchronisation scans once and restarts the periodic scan when enabled.
### Acknowledgements
Thank you for your contributions!
- @kimjansheden (#1220)
## 1.0.34
2nd October, 2026
### Security
- Fixed an issue where sensitive information could be included in exported configuration data under certain conditions. Updating is recommended. (refs #1218)
- Improved configuration persistence and JSON conflict handling.
### Acknowledgements
Thank you for your contributions!
- @kimjansheden (#1218)
## 1.0.33
1st October, 2026
This has turned into quite a substantial release, and I think it brings meaningful improvements to the core of Self-hosted LiveSync. If you notice anything, please feel free to let me know.
### Privacy and compatibility
#### New Feature
- An optional saved ID key can generate encrypted Chunk IDs and obfuscated Metadata document IDs independently of the current E2EE passphrase.
- New Vaults use a random key by default; existing Vaults keep their current ID configuration by default. You can also derive a key from the current E2EE passphrase, enter a separate source, or import a recovery code. The source is not retained; the saved key can be revealed locally as a recovery code.
- The saved key stays in place when the E2EE passphrase changes or E2EE is turned off. Share it with another device through a protected Setup URI. Changing document IDs on an existing remote requires the usual Rebuild and Fetch procedure.
- We can now keep the file properties used by Hidden File Sync and Customisation Sync private in CouchDB.
- **Encrypt internal file Properties** extends E2EE V2 and Property Encryption to their paths, times, sizes, and Chunk references.
- Existing configurations keep this preference disabled. New Vaults enable it for use when the required encryption settings are active.
- Update every synchronising device before enabling it. It protects future writes; a manual remote Rebuild is strongly recommended to protect existing properties.
- We can now see which unsupported feature prevents a client from synchronising with CouchDB.
- Clients check the features required by the remote before transferring data or resetting the local database for Fast Fetch. Receiving an unsupported requirement also stops active replication.
- We can now compare ID generation performance on a desktop or mobile device through **Open review harness**, available with the developers' debug tools enabled.
- The copied report includes legacy and independent ID timings and, where available, approximate JavaScript heap samples. The measurement uses fixed test data and keeps our Vault and settings unchanged.
#### Fixed
- Updated `brace-expansion` and `markdown-it` to include upstream security fixes.
- We can now keep using an E2EE passphrase beginning with `%` after restarting Obsidian. (#1221)
- This plug-in encrypts it before saving the settings. If an earlier version saved it in plain text, re-enter the passphrase used to encrypt the existing data after updating. Treat that passphrase as exposed if the affected `data.json` was shared.
- A receiving device now retries an unavailable CouchDB Chunk when file Metadata arrives before that Chunk is visible, helping rapid edits reach the Vault after an initial on-demand lookup misses it. (#1224)
- Retries start after two seconds and continue with increasing delays while finite replication is active. When it ends, this plug-in checks locally and makes a final lookup if needed, without waiting out the remaining retry delay.
- We can now distinguish initial on-demand Chunk requests (`🛄`) from retries (`🔁`) in the status bar. These replace `🧩`; each pending Chunk appears in one category, including while a retry is waiting.
### Synchronisation and storage
#### Fixed
- Received changes held during start-up or a fetch are applied when this plug-in becomes ready, without waiting for another change or a settings save. **Suspend database reflecting** continues to hold changes (#1200).
- Customisation Sync now compares full millisecond timestamps, so the freshness labels and **Select All Shiny** no longer mistake an older copy for a newer one because of timestamp truncation. (#1194)
- **Hide not applicable items** now hides identical Customisation Sync items and refreshes the list when toggled. Items with applicable differences stay visible. (#1193)
### Interface and translation
#### Improved
- More settings and messages are now available in Russian. (#1187)
- Dialogues show generated QR codes, key pairs, and database sizes again.
- We can now use updated Spanish translations for settings and messages. (#1212)
### Setup
#### New Feature
- We can now share a Setup URI with a displayed time limit, or choose **Compatible** for reuse without a time limit.
- **Time-bound** uses the current fixed seven-day UTC window, so the displayed end may be less than seven days away. Compatible retains the existing URI format; receiving devices still need to support the shared settings.
- The time condition applies when opening the URI. It does not revoke imported credentials or prevent reuse after rolling the device clock back.
#### Improved
- We can now distinguish the three Setup URI and QR code choices by their short labels and icons: initialise or overwrite the remote, join this device, or apply settings only.
#### Fixed
- We can now add a device or open a copied Vault without a compatibility pause solely because its device-local version record is absent.
- Existing version or settings incompatibilities still require review. A pause already saved by an earlier release still needs one explicit resume action.
### Acknowledgements
Thank you for your contributions!
- @kimjansheden (#1219)
- @Immick (#1195, #1196)
- @bolikcraft (#1187)
- @speedy-axolotl (#1212)
### Issue replies
I am a little behind on replying to issues, but I am reading them and will respond as I work through them. I have had little uninterrupted time recently, and that should improve soon.
## 1.0.32
27th September, 2026
docs/review_harness.md
+3 -2
View File
@@ -19,10 +19,11 @@ The command and view are registered only when developer tools are enabled. Disab
| --- | --- | --- | --- |
| Settings lifecycle | Automatic | Read-only | Exposes the seven synchronisation choices as typed observations and, for a genuinely new Vault, compares the selected recommendations with Commonlib's new-Vault contract. Existing-Vault setting preservation remains an automated migration and `settings-ui` E2E responsibility. |
| Compatibility review boundary | Guided | Device-local state | Observes the current dedicated compatibility controller and opens its actual review action. The Harness does not restore Change Log acknowledgement, `lastReadUpdates`, or a separate manual Pass or Fail result. |
| P2P composition | Automatic | Read-only | Confirms that the live P2P result resolves one replicator bound to the active Obsidian services. Peer discovery, replacement, reconnection, and transfer remain unit, CLI, and Compose E2E responsibilities. |
| ID generation performance | Automatic, individual | Read-only | Measures legacy and independent IDs using fixed in-memory inputs, including key derivation and heap samples where available. Keep Obsidian in the foreground. |
| File content hash performance | Manual | Read-only | Compares SHA-256 and xxHash64 on fixed in-memory Blob inputs. Input creation is excluded from hash timings; engine initialisation and warm-up are reported separately. |
| Vault fixture round trip | Automatic, explicit | Dedicated Vault fixtures | After explicit confirmation, creates, reads, modifies, renames, and removes one owned fixture tree. General Vault reflection remains covered by its separate real-Obsidian E2E workflow. |
**Automatic** runs only the two read-only local observations. **Full review** also starts the guided compatibility observation and asks before the Vault fixture scenario writes anything. A scenario can also be run individually.
**Automatic** runs only Settings lifecycle. **Full review** visits all listed scenarios, including the guided compatibility review and the two performance measurements, and asks before the Vault fixture scenario writes anything. Each scenario can also be run individually; a scenario's mode label does not by itself include it in the Automatic action.
These observations deliberately do not repeat stronger automated workflows. They exist to record which contracts the current real-device composition exposes while a maintainer reviews an immutable artefact.
docs/settings.md
+121 -13
View File
@@ -161,6 +161,8 @@ Following panes will be shown when you enable this setting.
### 1. Appearance
For icon meanings, progress counters, and CSS snippets, see the [status display guide](tips/status-display.md).
#### Display Language
Setting key: displayLanguage
@@ -234,6 +236,16 @@ Setting key: remoteType
The active connection type. This is automatically projected to the legacy configuration when you activate a connection profile.
### Joined devices
Choose **Review devices** under **Joined devices** to read the device records on the active CouchDB or Object Storage connection. The same **Review devices** action is available from the command palette and version Notices. **Refresh** reads the remote again. This device appears first, including when its record is absent from the remote. Opening or refreshing this dialogue does not create a missing CouchDB database.
The list shows each device ID, recorded platform, plug-in version, app version, and last recorded connection time. Older clients may have incomplete information. These records do not establish whether a device is online. On a locked remote, a device outside the accepted list is shown as **Locked out**.
Before synchronisation starts, and when this dialogue refreshes, Self-hosted LiveSync checks for recorded plug-in versions newer or older than its own. These Notices recommend reviewing and aligning plug-in versions across devices; they are advisory and do not stop synchronisation. The dialogue explains that synchronisation between different versions is generally safe unless the connection is explicitly blocked, but may be less efficient or use more storage. P2P synchronisation may also be unavailable. Unknown development version labels are left unclassified. Automatic checks run at most once per minute when synchronisation starts; they do not poll a continuously running connection.
**Forget device** removes another device's participation records after confirmation. It preserves the remote lock state and every other device. It does not delete Vault files or revoke credentials. A device can register again on an unlocked remote; on a locked remote, the existing recovery process applies. The result is checked after saving to CouchDB; if it cannot be confirmed, refresh the list and try again. CouchDB stores these records together in a local document which does not reject simultaneous updates, so changes from other devices can be overwritten. Pausing synchronisation on the other devices avoids that overlap. Object Storage uses conditional updates; if the server cannot support them, the record is retained.
### 2. Notification
#### Notify when the estimated remote storage size exceeds on start up
@@ -659,17 +671,28 @@ Setup URIs preserve both sleep preferences. Older Setup URIs which do not contai
#### Batch database update
Setting key: batchSave
Reducing the frequency with which on-disk changes are reflected into the DB
Combine repeated local file updates before saving them to the local database in modes other than LiveSync mode. This reduces intermediate database revisions and delays their availability to other devices.
#### Batch database updates in LiveSync mode
Setting key: batchSaveInLiveSync
Apply the same bounded delay in LiveSync mode. This separate choice is disabled by default for both existing settings and new Vaults. An older configuration containing `liveSync: true` and `batchSave: true` does not enable it. The settings interface shows the choice for the current mode, and switching modes preserves both choices. Older clients continue to ignore batching in LiveSync mode.
Both choices share the delays below. Manual synchronisation, deletion, or renaming can end a batch early. Batching reduces intermediate updates; it does not identify when a template has finished or prevent genuine concurrent edits on different devices.
#### Minimum delay for batch database updating
Setting key: batchSaveMinimumDelay
Seconds. Saving to the local database will be delayed until this value after we stop typing or saving.
Seconds of quiet after the latest eligible edit before saving to the local database, subject to the maximum delay below.
#### Maximum delay for batch database updating
Setting key: batchSaveMaximumDelay
Saving will be performed forcefully after this number of seconds.
Maximum intentional delay in seconds from the first pending update to a file. Later edits do not restart this limit, and waiting for a processing slot does not add another batch delay. This limit makes the work eligible to run; it does not guarantee database completion or receipt by another device within that time.
### 4. Deletion Propagation (Advanced)
@@ -750,11 +773,13 @@ If enabled, the notification of hidden files change will be suppressed.
#### Synchronising files
(RegExp) Empty to sync all files. Set filter as a regular expression to limit synchronising files.
Setting key: syncOnlyRegEx
(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns.
#### Non-Synchronising files
(RegExp) If this is set, any changes to local and remote files that match this will be skipped.
Setting key: syncIgnoreRegEx
(RegExp) Skip changes to matching normal and hidden files, locally and remotely.
#### Maximum file size
@@ -783,6 +808,8 @@ When a saved setting changes whether a normal file can be reflected, LiveSync re
Customisation Sync is a supported, advanced opt-in feature. Its current per-file implementation is covered by a two-Vault real-Obsidian workflow for snippets, configuration files, and plug-in files. Hidden File Sync is a separate feature with different setup, selection, and conflict behaviour; do not use both features to manage the same files.
**Automatic** delegates an item's files to Hidden File Sync. Enable and initialise Hidden File Sync in **Sync Settings** on this device first, after reviewing its [target and ignore patterns](./tips/hidden-file-sync.md#review-the-file-selection). Choosing Automatic while Hidden File Sync is disabled displays a Notice and leaves the current mode unchanged. Existing Automatic assignments, including those imported from another device, remain saved while Hidden File Sync is disabled.
### 1. Customisation Sync
#### Device name
@@ -820,9 +847,13 @@ Open the dialogue
## 7. Hatch
### 1. Reporting Issue
### 1. Reporting Tools
#### Make report to inform the issue
This panel groups the five actions that collect reports or inspect database state. These actions provide diagnostic information; they do not repair or reset the database.
#### Prepare a report to inform an issue
Select **Copy Report to clipboard** to collect diagnostic information and recent logs for a support report. Review the report before sharing it because it can contain sensitive information.
#### Copy database information for a file
@@ -830,12 +861,48 @@ Select a file to copy its local database information. The command **Copy databas
The report includes the Vault-relative path, document and chunk identifiers, local database revisions, conflicts, and local chunk availability. It does not query the remote or include file contents. Paths and identifiers can still be private metadata, so review the report before sharing it.
#### Find a file from a report pseudonym
Paste a path pseudonym from a diagnostic report, then select **Find file**. The lookup checks the current Vault and local normal-file Metadata only; it does not contact the remote database. It returns every matching local candidate, and shows each candidate's real Vault path and document ID only in this local dialogue. Multiple local document IDs for one path remain separate candidates. The follow-up for a Vault-only candidate marks Metadata as not inspected, because no local document was selected. No match can mean that the file was renamed or deleted, or that the matching Metadata exists only remotely.
For a selected candidate, open an existing Vault file or prepare a shareable local report using the original report's pseudonym salt and the exact selected document ID. The report includes current, conflict, and history status, Chunk availability, merge bases, and Vault size facts; it does not include file contents or credentials. Its history status listing does not scan every historical body.
Path pseudonyms use the report's public eight-character salt. Someone with the report can test a guessed path against a pseudonym. The same salt reproduces path and identifier pseudonyms in a targeted follow-up; older ordinal pseudonyms cannot be looked up with this tool.
#### Inspect database integrity
Select **Inspect** to choose original names or temporary pseudonyms and inspect normal-file Metadata and current conflicts. **Include remote database** is selected by default; clear it to inspect only the local database and Vault. This choice applies to the current run only, and the report marks remote coverage as not checked when excluded. The command **Inspect database integrity** opens the same dialogue. The inspection is read-only; its findings do not authorise deletion or perform repairs. See [Collect a report](troubleshooting.md#collect-a-report) for coverage, report formats, privacy, and cancellation, and the [compact Markdown reading guide](troubleshooting.md#read-the-compact-markdown-integrity-report) for interpreting its state and review index.
#### Analyse database usage
Select **Analyse** to choose a name policy and report local revision and Chunk usage. The command **Analyse Database Usage (advanced)** opens the same dialogue when [advanced features](#enable-advanced-features) are enabled. Complete runs offer TSV and JSON; incomplete runs retain their status and reasons in JSON. Existing on-demand Chunk fetching stays unchanged. See [Collect a report](troubleshooting.md#collect-a-report) for accounting limits and export instructions.
### 2. Checks and Recovery
This panel groups checks that may lead to changes to settings or database contents.
#### Setting Doctor
Run Setting Doctor to check for configuration mismatches or suboptimal settings. Applying its recommendations can change settings or request a database rebuild, so review the recommendations before proceeding.
#### Scan for broken files
Scan for files which are not stored correctly in the database. The scan can offer repair actions that change database contents; review each proposed action before confirming it.
### 3. TroubleShooting
This panel keeps the remote storage notification check and file logging controls separate from report collection and recovery checks.
#### Reset notification threshold and check the remote database usage
Reset the remote storage size threshold and check the remote storage size again. This changes the saved notification threshold.
#### Write logs into the file
Setting key: writeLogToTheFile
Warning! This will have a serious impact on performance. And the logs will not be synchronised under the default name. Please be careful with logs; they often contain your confidential information.
### 2. Scram Switches
### 4. Scram Switches
Emergency controls to suspend synchronisation processes in order to prevent database corruption. If a critical mismatch or sync error occurs, the plug-in may automatically enter a Scram state and suspend operations.
@@ -851,7 +918,7 @@ Setting key: suspendParseReplicationResult
Stop reflecting database changes to storage files.
### 3. Recovery and Repair
### 5. Recovery and Repair
#### Recreate chunks for current Vault files
@@ -871,7 +938,7 @@ After confirmation, resolve every conflict by modification time. This logically
#### Check and convert non-path-obfuscated files
### 4. Reset
### 6. Reset
#### Back to non-configured
@@ -896,6 +963,18 @@ Setting key: hashCacheMaxAmount
Setting key: customChunkSize
New setup uses 0 for every remote type. With V3 Rabin–Karp, this gives binary
Chunks a 100 KiB maximum, a 25 KiB minimum, and an approximate 25 KiB candidate
gap. The candidate gap is not the mean Chunk length. Text below 4 MiB retains
adaptive smaller boundaries; large text uses the binary thresholds with
UTF-8-safe cuts.
Larger values can reduce document counts for mostly immutable binary files,
but can increase the new content stored after an edit. Existing explicit and
agreed values are retained. Updating the plug-in does not re-chunk the Vault;
editing, forced writes, Chunk recreation, or a full Rebuild can create new
boundaries while existing content remains readable.
#### Chunk Splitter
Setting key: chunkSplitterVersion
@@ -912,12 +991,21 @@ If enabled, chunks will be split into no more than 100 items. However, dedupe is
Setting key: useSegmenter
If this enabled, chunks will be split into semantically meaningful segments. Not all platforms support this feature.
#### Strict content check on startup
Setting key: strictContentCheckOnStartup
When enabled, ordinary start-up compares file content with the local database even when the file's exact modification time and size match. When disabled (the default), this comparison can be skipped only when the modification time, size, and recorded device-local provenance match a conflict-free current revision, and no file processing is pending. An edit made outside the normal file-event path which preserves its exact modification time and size can therefore go undetected by ordinary start-up. Manual scans and explicit Fetch or Rebuild operations keep strict content checking.
### 3. Transfer Tweak
#### Fetch chunks on demand
Setting key: readChunksOnline
(ex. Read chunks online) If this option is enabled, LiveSync reads chunks online directly instead of replicating them locally. Increasing Custom chunk size is recommended.
If enabled, this plug-in obtains missing Chunks when they are needed instead
of including all Chunks in ordinary pull replication. New setup retains the
same Chunk-size recommendation of 0. This setting does not re-chunk content
already stored by another device.
#### Batch size of on-demand fetching
@@ -970,12 +1058,18 @@ The setting remains disabled by default because the measured storage and transfe
#### Batch size
Setting key: batch_size
Number of changes to sync at a time. Defaults to 50. Minimum is 2.
Number of changes in an ordinary CouchDB replication batch. The schema default
is 25, and the minimum is 2. Changing this value alone preserves the
replication checkpoint. Larger counts can reduce round trips, but do not bound
the final request size or memory use.
#### Batch limit
Setting key: batches_limit
Number of batches to process at a time. Defaults to 40. Minimum is 2. This along with batch size controls how many docs are kept in memory at a time.
Number of changes batches which PouchDB can read ahead. The schema default is
25, and the minimum is 2. This does not create parallel body transfers;
PouchDB processes one current replication batch at a time. It does not control
the separate P2P or Object Storage transfer units.
#### Use timeouts instead of heartbeats
@@ -1012,6 +1106,8 @@ Requires restart of Obsidian
Setting key: deleteMetadataOfDeletedFiles
For an unconflicted file, disabling this setting records a logical deletion in the Metadata document; enabling it removes that document through a PouchDB tombstone instead. Both behaviours propagate file deletions and the source side of a rename. Before removing a Vault file, LiveSync revalidates the current winning revision; if this final check observes a concurrent restoration, it keeps the restored file. This setting changes how the deletion is represented; file content remains stored separately in Chunks. Deletion during an unresolved conflict follows the separate [displayed-branch rules](specs_conflict_resolution.md#operations-while-a-conflict-exists).
#### Delete old metadata of deleted files on start-up
Setting key: automaticallyDeleteMetadataOfDeletedFiles
@@ -1050,6 +1146,18 @@ When this setting is disabled, changing only the letter case of a file name with
New Vaults use case-insensitive handling for cross-platform compatibility. Existing settings with an explicit value preserve that choice. Earlier releases also followed the case-insensitive branch when this value was absent, so 1.0 saves a missing legacy value as `false` without requiring a compatibility review or database rebuild.
#### Unicode path identity
Setting key: pathNormalizationVersion
Version `0` preserves legacy file identity and is the default for existing and new settings. Version `1` uses Unicode NFC when deriving document IDs, so equivalent composed and decomposed paths identify the same file across the CLI and Obsidian. Existing physical file names are retained. This mode currently supports CouchDB only.
Version `1` requires a coordinated migration: set it on the source client before Rebuild, then set it on each remaining client before Fetch. Changing the setting alone does not migrate existing Metadata. Follow [Migrate to Unicode NFC path identity](recovery.md#migrate-to-unicode-nfc-path-identity) for the complete procedure. Do not resume a version `0` client against the migrated remote.
The selected value stays pending through ordinary settings saves and reloads. It becomes active only when the Rebuild or Fetch operation is successfully reserved; cancellation leaves it pending.
Distinct files or retained Metadata documents which derive the same identity stop reconstruction before either database is reset. When replacing the local database, Rebuild and Fetch exclude logical deletions without unresolved conflicts; a deleted winner with unresolved conflicts still participates in the check. This check uses the host's supported file listing; Obsidian checks the paths exposed by its Vault APIs and cannot detect physical aliases hidden by its index. Resolve the duplicate names or synchronisation scope first; Self-hosted LiveSync does not merge them automatically. Case-insensitive handling also applies the existing letter-case rule.
### 4. Compatibility (Internal API Usage)
#### Scan changes on customisation sync
docs/setup_flyio.md
+50 -23
View File
@@ -3,16 +3,16 @@
This is how to configure fly.io and CouchDB on it for Self-hosted LiveSync.
The CouchDB instance is yours to manage. You can configure it yourself or set up another instance and switch to it, following the preservation and recovery guidance below.
> [!WARNING]
> It is **your** instance. In Obsidian, we have files locally. Hence, do not hesitate to destroy the remote database if you feel something have got weird. We can launch and switch to the new CouchDB instance anytime[^1].
>
[^1]: Actually, I am always building the database for reproduction of the issue like so.
> Before replacing or deleting an existing CouchDB instance, back up every available Vault and the remote database, then [choose the authoritative copy](recovery.md#first-choose-the-authoritative-copy). Stop or suspend synchronisation on other devices while reviewing recovery options. Local files alone do not establish that every needed edit or revision has been preserved.
> [!NOTE]
> **What and why is the Fly.io?**
> At some point, we started to experience problems related to our IBM Cloudant account. At the same time, Self-hosted LiveSync started to improve its functionality, requiring CouchDB in a more natural state to use all its features.
>
> Then we found Fly.io. Fly.io is the PaaS Platform, which can be useable for a very reasonable price. It generally falls within the `Free Allowances` range in most cases.
> Fly.io charges according to current account and resource usage, including Machines, persistent volumes, and network usage. Check [Fly.io pricing](https://fly.io/pricing/) and the [pricing documentation](https://fly.io/docs/about/pricing/) before deploying. A time-limited trial does not provide ongoing free hosting.
## Required materials
@@ -31,10 +31,10 @@ This is how to configure fly.io and CouchDB on it for Self-hosted LiveSync.
4. Copy the Setup-URI and Use it in the Obsidian.
5. You have been synchronised. Use the Setup-URI in subsequent devices.
Steps 4 and 5 are detailed in the [Quick Setup](./quick_setup.md#1-using-setup-uris).
For steps 4 and 5, see [Set up the first device](./quick_setup.md#set-up-the-first-device), then [Add another device](./quick_setup.md#add-another-device) for subsequent devices.
> [!NOTE]
> Your automatically configured configurations will be shown on the result in the Colab note like below, and **it will not be saved**. Please make a note of it somewhere.
> Save your own generated connection details and Setup URI securely. This is a historical Colab output example from 2024; its values are illustrative and will differ for each deployment. Replace all example credentials before deployment.
> ```
> -- YOUR CONFIGURATION --
> URL : https://billowing-dawn-6619.fly.dev
@@ -45,16 +45,18 @@ Steps 4 and 5 are detailed in the [Quick Setup](./quick_setup.md#1-using-setup-u
### B. Scripted Setup
Please refer to the document of [deploy-server.sh](../utils/readme.md#deploy-serversh).
Please refer to the document of [deploy-server.sh](../utils/readme.md#flyio-deployment).
### C. Manual Setup
| Used in the text | Meaning and where to use | Memo |
| ---------------- | --------------------------- | ------------------------------------------------------------------------ |
| campanella | Username | It is less likely to fail if it consists only of letters and numbers. |
| dfusiuada9suy | Password | |
| `<YOUR UNIQUE COUCHDB USERNAME>` | Username | Choose a unique value using only letters and numbers. |
| `<YOUR UNIQUE COUCHDB PASSWORD>` | Password | Generate a strong, unique password for this instance. |
| nrt | Region to make the instance | We can use any [region](https://fly.io/docs/reference/regions/) near us. |
Choose an available, unique Fly.io application name and replace both credential placeholders before deployment. Historical output below illustrates the command responses; use your own application name, hostname, and credentials.
#### 1. Install flyctl
- Mac or Linux
@@ -97,8 +99,8 @@ For more information, please refer to [Sign up](https://fly.io/docs/hands-on/sig
```bash
$ fly launch --name=billowing-dawn-6619 --env="COUCHDB_USER=campanella" --copy-config=true --detach --no-deploy --region nrt --yes
$ fly secrets set COUCHDB_PASSWORD=dfusiuada9suy
$ fly launch --name='<YOUR UNIQUE APP NAME>' --env="COUCHDB_USER=<YOUR UNIQUE COUCHDB USERNAME>" --copy-config=true --detach --no-deploy --region nrt --yes
$ fly secrets set COUCHDB_PASSWORD='<YOUR UNIQUE COUCHDB PASSWORD>'
```
#### 4. Deploy
@@ -170,24 +172,24 @@ Visit your newly deployed app at https://billowing-dawn-6619.fly.dev/
After the initial setup, CouchDB needs some more customisations to be used from Self-hosted LiveSync. It can be configured in browsers or by HTTP-REST APIs.
This section is set up using the REST API.
This section uses the REST API. The macOS and Linux commands require [jq](https://jqlang.org/download/) to encode credentials as JSON. Windows uses PowerShell's built-in `ConvertTo-Json`.
1. Prepare environment variables.
- Mac or Linux:
```bash
export couchHost=https://billowing-dawn-6619.fly.dev
export couchUser=campanella
export couchPwd=dfusiuada9suy
export couchHost='https://<YOUR APP HOSTNAME>'
export couchUser='<YOUR UNIQUE COUCHDB USERNAME>'
export couchPwd='<YOUR UNIQUE COUCHDB PASSWORD>'
```
- Windows
```powershell
set couchHost https://billowing-dawn-6619.fly.dev
set couchUser campanella
set couchPwd dfusiuada9suy
set couchHost 'https://<YOUR APP HOSTNAME>'
set couchUser '<YOUR UNIQUE COUCHDB USERNAME>'
set couchPwd '<YOUR UNIQUE COUCHDB PASSWORD>'
$creds = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("${couchUser}:${couchPwd}"))
```
@@ -196,13 +198,36 @@ $creds = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes(
- Mac or Linux
```bash
curl -X POST "${couchHost}/_cluster_setup" -H "Content-Type: application/json" -d "{\"action\":\"enable_single_node\",\"username\":\"${couchUser}\",\"password\":\"${couchPwd}\",\"bind_address\":\"0.0.0.0\",\"port\":5984,\"singlenode\":true}" --user "${couchUser}:${couchPwd}"
clusterSetupJSON=$(jq -n \
--arg username "$couchUser" \
--arg password "$couchPwd" \
'{
action: "enable_single_node",
username: $username,
password: $password,
bind_address: "0.0.0.0",
port: 5984,
singlenode: true
}') &&
curl -X POST "${couchHost}/_cluster_setup" \
-H "Content-Type: application/json" \
--data-binary "$clusterSetupJSON" \
--user "${couchUser}:${couchPwd}"
```
- Windows
```powershell
iwr -UseBasicParsing -Method 'POST' -ContentType 'application/json; charset=utf-8' -Headers @{ 'Authorization' = 'Basic ' + $creds } "${couchHost}/_cluster_setup" -Body "{""action"":""enable_single_node"",""username"":""${couchUser}"",""password"":""${couchPwd}"",""bind_address"":""0.0.0.0"",""port"":5984,""singlenode"":true}"
$clusterSetupJSON = @{
action = 'enable_single_node'
username = $couchUser
password = $couchPwd
bind_address = '0.0.0.0'
port = 5984
singlenode = $true
} | ConvertTo-Json -Compress
iwr -UseBasicParsing -Method 'POST' -ContentType 'application/json; charset=utf-8' -Headers @{ 'Authorization' = 'Basic ' + $creds } "${couchHost}/_cluster_setup" -Body $clusterSetupJSON
```
Note: if the response code is not 200. We have to retry the request once again.
@@ -242,11 +267,13 @@ Note: Each of these should also be repeated until finished in 200.
#### 6. Use it from Self-hosted LiveSync
Now the CouchDB is ready to use from Self-hosted LiveSync. We can use `https://billowing-dawn-6619.fly.dev` in URI, `campanella` in `Username` and `dfusiuada9suy` in `Password` on Self-hosted LiveSync. The `Database name` could be anything you want.
Please refer to the [Minimal Setup of the Quick Setup](./quick_setup.md#2-minimal-setup).
Now the CouchDB is ready to use from Self-hosted LiveSync. Use your Fly.io app's hostname in `URI`, and the unique credentials configured above in `Username` and `Password`. The `Database name` could be anything you want.
Please refer to [Configure CouchDB manually on the first device](./quick_setup.md#configure-couchdb-manually-on-the-first-device).
## Delete the Instance
Before deleting the instance, preserve the data you still need and follow [Recovery and flag files](recovery.md#first-choose-the-authoritative-copy).
If you want to delete the CouchDB instance, you can do that in [fly.io Dashboard](https://fly.io/dashboard/personal)
If you have done with [B. Scripted Setup](#b-scripted-setup), we can use [delete-server.sh](../utils/readme.md#delete-serversh).
If you have done with [B. Scripted Setup](#b-scripted-setup), we can use [delete-server.sh](../utils/readme.md#flyio-deployment).
docs/setup_object_storage.md
+5 -5
View File
@@ -18,14 +18,14 @@ The public generator applies the Object Storage preset and records the connectio
```sh
export remote_type=s3
export endpoint=https://objects.example.com
export access_key=<ACCESS KEY>
export secret_key=<SECRET KEY>
export endpoint='https://objects.example.com'
export access_key='<ACCESS KEY>'
export secret_key='<SECRET KEY>'
export bucket=vault-data
export region=auto
export bucket_prefix=my-vault
export passphrase=<A STRONG VAULT ENCRYPTION PASSPHRASE>
export uri_passphrase=<A SEPARATE SETUP URI PASSPHRASE>
export LIVESYNC_PASSPHRASE='<A STRONG VAULT ENCRYPTION PASSPHRASE>'
export uri_passphrase='<A SEPARATE SETUP URI PASSPHRASE>'
deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts
```
docs/setup_own_server.md
+29 -20
View File
@@ -15,7 +15,7 @@
- [4. Starting Docker Compose in background](#4-run-the-docker-compose-file-in-the-background)
- [C. Install CouchDB directly](#c-install-couchdb-directly)
- [2. Run couchdb-init.sh for initialise](#2-run-couchdb-initsh-for-initialise)
- [3. Expose CouchDB to the Internet](#3-expose-couchdb-to-the-internet)
- [3. Make CouchDB reachable from your devices](#3-make-couchdb-reachable-from-your-devices)
- [4. Client Setup](#4-client-setup)
- [1. Generate the setup URI on a desktop device or server](#1-generate-the-setup-uri-on-a-desktop-device-or-server)
- [2. Setup Self-hosted LiveSync to Obsidian](#2-setup-self-hosted-livesync-to-obsidian)
@@ -122,12 +122,15 @@ The `username` and `password` in this step are CouchDB administrator credentials
```
export hostname=http://localhost:5984
export username=<INSERT COUCHDB ADMINISTRATOR USERNAME HERE>
export password=<INSERT COUCHDB ADMINISTRATOR PASSWORD HERE>
export username='<INSERT COUCHDB ADMINISTRATOR USERNAME HERE>'
export password='<INSERT COUCHDB ADMINISTRATOR PASSWORD HERE>'
export database=obsidiannotes
curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | bash
```
> [!TIP]
> In fish, `hostname` is read-only and cannot be exported. Pass the values through `env` instead: `curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | env hostname='http://localhost:5984' username='<INSERT COUCHDB ADMINISTRATOR USERNAME HERE>' password='<INSERT COUCHDB ADMINISTRATOR PASSWORD HERE>' database=obsidiannotes bash`.
If it results like the following:
```
CouchDB provisioning completed.
@@ -137,21 +140,25 @@ The wrapper runs the exact registry-pinned Commonlib consumer. When `database` i
If you are using Docker Compose and the above command does not work or displays `ERROR: Hostname missing`, you can try running the following command, replacing the placeholders with your own values:
```
curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | hostname=http://<YOUR SERVER IP>:5984 username=<INSERT COUCHDB ADMINISTRATOR USERNAME HERE> password=<INSERT COUCHDB ADMINISTRATOR PASSWORD HERE> database=obsidiannotes bash
curl -s https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/couchdb/couchdb-init.sh | hostname='http://<YOUR SERVER IP>:5984' username='<INSERT COUCHDB ADMINISTRATOR USERNAME HERE>' password='<INSERT COUCHDB ADMINISTRATOR PASSWORD HERE>' database=obsidiannotes bash
```
## 3. Expose CouchDB to the Internet
<a id="3-expose-couchdb-to-the-internet"></a>
- You can skip this instruction if you using only in intranet and only with desktop devices.
- For mobile devices, Obsidian requires a valid SSL certificate. Usually, it needs exposing the internet.
## 3. Make CouchDB reachable from your devices
Whatever solutions we can use. For simplicity, the following sample uses Cloudflare Zero Trust for testing.
- You can skip this step if you use CouchDB only on a trusted local network with desktop devices.
- For mobile devices, Obsidian requires HTTPS with a valid certificate. HTTPS does not require making CouchDB publicly accessible.
Choose how your devices reach your CouchDB server. You can use a [reverse proxy you operate](#reverse-proxies), private-network access, or a tunnel service of your choice. The [Docker profiles](../docker/README.md#profile-details) provide examples.
Cloudflare is one optional choice. The following example uses a [Quick Tunnel](https://developers.cloudflare.com/tunnel/get-started/quick-tunnels/) for temporary testing. It prints a random hostname which changes each time you create a tunnel, and the URL stops working when the `cloudflared` process stops. Quick Tunnels are intended for testing and development, not production. For ongoing use, configure your chosen connection method; the [Cloudflare Tunnel guide](https://developers.cloudflare.com/tunnel/get-started/) applies if you choose that provider.
```
cloudflared tunnel --url http://localhost:5984
```
You will then get the following output:
The following 2024 output is an example of the format. Its hostname is illustrative only; use the URL printed by your own tunnel instead.
```
2024-02-14T10:35:25Z INF Thank you for trying Cloudflare Tunnel. Doing so, without a Cloudflare account, is a quick way to experiment and try it out. However, be aware that these account-less Tunnels have no uptime guarantee. If you intend to use Tunnels in production you should use a pre-created named tunnel by following: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
@@ -164,7 +171,7 @@ You will then get the following output:
:
:
```
Now `https://tiles-photograph-routine-groundwater.trycloudflare.com` is our server. Make it into the background once, please.
Use the URL printed by your tunnel as the server address. Keep the tunnel process running while you configure and test the client.
## 4. Client Setup
@@ -173,20 +180,22 @@ Now `https://tiles-photograph-routine-groundwater.trycloudflare.com` is our serv
### 1. Generate the setup URI on a desktop device or server
The `username` and `password` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account.
The `LIVESYNC_USER` and `LIVESYNC_PASS` here are the credentials which Self-hosted LiveSync will store for routine access to this database. They may be the administrator credentials from step 2. If you have separately configured a CouchDB account with the required access to this database, use that account instead. Neither the provisioning wrapper nor the Setup URI generator creates that separate account.
Use the server URL through which your devices reach CouchDB. For the temporary Quick Tunnel example above, use the URL printed by your own running tunnel.
```bash
export hostname=https://tiles-photograph-routine-groundwater.trycloudflare.com
export database=obsidiannotes
export username=<INSERT COUCHDB USERNAME FOR LIVESYNC>
export password=<INSERT THE COUCHDB PASSWORD>
export passphrase=<INSERT A STRONG VAULT ENCRYPTION PASSPHRASE>
export uri_passphrase=<INSERT A SEPARATE SETUP URI PASSPHRASE> # Optional
export LIVESYNC_HOST='<YOUR COUCHDB SERVER URL>'
export LIVESYNC_DB=obsidiannotes
export LIVESYNC_USER='<INSERT COUCHDB USERNAME FOR LIVESYNC>'
export LIVESYNC_PASS='<INSERT THE COUCHDB PASSWORD>'
export LIVESYNC_PASSPHRASE='<INSERT A STRONG VAULT ENCRYPTION PASSPHRASE>'
export uri_passphrase='<INSERT A SEPARATE SETUP URI PASSPHRASE>' # Optional
deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts
```
> [!TIP]
> `passphrase` protects the synchronised Vault data with end-to-end encryption. `uri_passphrase` protects only the Setup URI. Use different values, store both securely, and do not send the Setup URI and its passphrase through the same channel.
> `LIVESYNC_PASSPHRASE` protects the synchronised Vault data with end-to-end encryption. `uri_passphrase` protects only the Setup URI. Use different values, store both securely, and do not send the Setup URI and its passphrase through the same channel.
>
> If `uri_passphrase` is omitted, the generator creates a cryptographically random value and prints it once.
@@ -196,9 +205,9 @@ The generator consumes the exact registry-pinned Commonlib release used by the p
By default, the URI is Ephemeral and can be opened only until the exact UTC time printed by the generator. This is the end of a fixed seven-day window, not seven days after creation. Set `uri_mode=persistent` before running the command if the URI needs no time condition or must be readable by an older client that supports the existing encrypted format.
You will then get the following output:
The output has the following form. The timestamp and values below are illustrative:
```bash
```text
Generated couchdb Setup URI.
Ephemeral: usable until 2026-10-01T00:00:00.000Z (UTC).
Your passphrase for the Setup URI is: H7vX...a-random-32-character-value
docs/setup_p2p.md
+5 -5
View File
@@ -127,11 +127,11 @@ The public Setup URI generator is also available when configuration must be crea
```sh
export remote_type=p2p
export p2p_relays=wss://relay.example.com
export p2p_room_id=<A PRIVATE ROOM ID> # Optional; generated when omitted
export p2p_passphrase=<A PRIVATE P2P PASSPHRASE> # Optional; generated when omitted
export passphrase=<A STRONG VAULT ENCRYPTION PASSPHRASE>
export uri_passphrase=<A SEPARATE SETUP URI PASSPHRASE>
export p2p_relays='wss://relay.example.com'
export p2p_room_id='<A PRIVATE ROOM ID>' # Optional; generated when omitted
export p2p_passphrase='<A PRIVATE P2P PASSPHRASE>' # Optional; generated when omitted
export LIVESYNC_PASSPHRASE='<A STRONG VAULT ENCRYPTION PASSPHRASE>'
export uri_passphrase='<A SEPARATE SETUP URI PASSPHRASE>'
deno run --minimum-dependency-age=0 --allow-env https://raw.githubusercontent.com/vrtmrz/obsidian-livesync/main/utils/setup/generate_setup_uri.ts
```
docs/specs_conflict_resolution.md
+66 -10
View File
@@ -38,15 +38,17 @@ At most one revision is the winner, more than one revision can be Vault-matching
- Automatic text and structured-data merge uses the nearest `available` revision ID which is present in both leaf histories.
- Missing or compacted history stops conservative automatic merge instead of guessing a base.
- A Vault file which still matches its exact recorded revision is unchanged. An ordinary save does not append those stale bytes to a newer database revision; a newer, unconflicted database result is reflected through the existing file-reflection path.
- A Vault file which still matches its exact recorded content is unchanged. The comparison uses the readable recorded revision or its saved SHA-256 hash and byte length, including after that revision has been pruned. An ordinary save does not append those stale bytes to a newer database revision; a newer, unconflicted database result is reflected through the existing file-reflection path.
- A file which differs from its readable recorded revision is an edit of that revision, even if its bytes match another historical revision. Saving and incoming overwrite protection use the same rule.
- Without a readable recorded revision, current non-deleted leaves are checked for duplicate content. If none matches, the file is preserved as a fresh independent root under the same document ID. Its unknown ancestry cannot supply a three-way merge base.
- Without a readable recorded revision or matching saved content evidence, current non-deleted leaves are checked for duplicate content. If none matches, the file is preserved as a fresh independent root under the same document ID. Its unknown ancestry cannot supply a three-way merge base.
- File bytes, rather than path, size, modification time, or revision generation, determine whether content is known.
- Three or more current versions are reviewed one pair at a time in a deterministic order, with each completed pair committed before the next pair is read.
- Each device records the exact revision most recently reflected in each Vault file. An ordinary edit extends that displayed branch even before a conflict exists. Conflict-time deletion and case-only rename retain their separate displayed-branch contracts.
- A cross-path rename stores the target before logically deleting only the displayed source branch.
The recorded revision can belong to a deleted losing branch. If its readable body still matches the Vault, the propagated resolution can be reflected without recreating the conflict. A historical byte match without that record does not establish that the file is unchanged: it may be an intentional revert. Existing Vaults can lack records, so an upgrade, reset, or unavailable old body can expose additional conflicts requiring review.
The recorded revision can belong to a deleted losing branch. If its readable body or saved content evidence still matches the Vault, the propagated resolution can be reflected without recreating the conflict. A historical byte match without that record does not establish that the file is unchanged: it may be an intentional revert. Existing Vaults can lack records or contain legacy records without content evidence, so an upgrade, reset, or unavailable old body can still expose conflicts requiring review. Content evidence neither reconstructs a missing merge base nor resolves conflicts which have already been created.
Ordinary deletion reflection accepts both logical deletion Metadata and PouchDB tombstones produced when `deleteMetadataOfDeletedFiles` is enabled. Commonlib revalidates the winning deletion before removing the Vault file. A restoration observed during that final check prevents deletion and allows the restored revision to be reflected. This does not change the separate conflict-time deletion rules below.
The explicit **Always overwrite with a newer file** option retains its existing modification-time policy. An independent branch prevents an inferred three-way merge; it does not disable the user's selected conflict-resolution option. Metadata and Chunks retain their existing format, and matching chunks can be shared between branches.
@@ -59,10 +61,11 @@ The explicit **Always overwrite with a newer file** option retains its existing
| One side deletes content which the other leaves unchanged | Preserve the deletion. |
| One side deletes content which the other modifies | Ask the user. |
| A receiving file matches its exact readable recorded revision | Apply the propagated database result under the existing conflict policy. |
| A receiving file matches its recorded SHA-256 hash and byte length | Apply the propagated result under the same policy even if the recorded revision has been pruned. |
| A receiving file differs from its readable recorded revision | Preserve the edit as a child of that exact revision. |
| Provenance is unknown and no current non-deleted leaf matches the file | Preserve a fresh independent branch for conflict resolution. |
| Provenance is unknown and current non-deleted leaves already hold the file bytes | Avoid duplicate storage; infer provenance only for a unique match. |
| A required body or shared ancestor is missing or compacted | Ask the user. |
| A body or shared ancestor required for merging is missing or compacted | Ask the user. |
| Binary contents differ | Prefer an explicit user selection; semantic merge is unavailable. |
The compatibility implementation currently selects the newer modification time for differing binary conflicts even when the general **Always overwrite with a newer file** option is disabled. This is existing behaviour, not a new 1.0 guarantee. Changing it to explicit selection only is a separate compatibility decision.
@@ -140,10 +143,16 @@ Consequently, choosing **Not now** or closing Obsidian cannot undo a completed p
LiveSync composes Commonlib's injected `FileReflectionProvenance` with its local key-value database. Each device stores:
```text
path -> { revision, observedStorageMtime? }
path -> { revision, observedStorageMtime?, contentHash?, contentSize? }
```
`revision` identifies the exact database revision most recently saved from or reflected in this device's Vault. It is the base for subsequent local edits, rather than a certificate that the current file still contains those bytes. `observedStorageMtime` is the raw local modification time of the saved snapshot or the file observed after reflection. It is not rounded, combined with another device's value, or used as proof of branch identity. No content hash is persisted.
`revision` identifies the exact database revision most recently saved from or reflected in this device's Vault. It is the base for subsequent local edits, rather than a certificate that the current file still contains those bytes. `observedStorageMtime` is the raw local modification time of the saved snapshot or the file observed after reflection. It is not rounded, combined with another device's value, or used as proof of branch identity.
Commonlib 0.1.42 adds optional `contentHash` and `contentSize` fields for the exact immutable content successfully saved or reflected. The hash uses `sha256:<lowercase hexadecimal>`; the size is the byte length. LiveSync persists these fields in its existing `file-reflection-provenance-v1` store. They are device-local evidence, not replicated Metadata, a replacement revision ID, or a way to reconstruct missing ancestry. Matching evidence can establish unchanged content, but cannot attach a genuine edit to an unavailable base.
Legacy records remain readable. An equal-time scan can add evidence when the recorded revision is still the conflict-free current revision and its bytes match storage. An ordinary unchanged save can also upgrade its verified current record. Hashing is followed by the required storage and provenance revalidation; an unreadable or unrelated revision cannot be certified from the current file. Unsupported evidence or a hashing failure retains the existing revision-based comparison and conservative fallback.
Commonlib owns the [content-hash decision and compatibility contract](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/file-reflection-content-hash.md). It serialises hashing before reading the Blob to bound simultaneous whole-file buffers, and reuses results only for the same immutable Blob. Whole-file allocation remains necessary. LiveSync's [Review Harness](../devs.md#file-content-hash-performance-measurements) compares that production helper with xxHash64; it does not select the production algorithm.
The record changes after a successful database-to-Vault reflection or Vault-to-database write. An equal-time scan may also fill a missing record while the current database revision uniquely matches the file's bytes and no conflict leaf exists. It rereads storage and the current database revision before recording; it does not change file bytes or create a database revision. Ordinary file reads do not change the record. An existing recorded revision remains authoritative even if the user edits the file to bytes which equal another branch; otherwise content equality could silently move the edit to a branch which was not displayed.
@@ -153,7 +162,52 @@ The common lock does not stop Obsidian edits, external filesystem writes, or rep
LiveSync creates the namespaced store handle during service composition, before the key-value database is open. The sequential `onSettingLoaded` lifecycle opens that database before Vault scanning, watching, or replication starts. Store operations do not wait for implicit readiness: a lifecycle violation fails promptly, avoiding an indefinite or self-referential initialisation wait. Local database reset is a transient unavailable boundary, after which scanning reconstructs derived state.
For ordinary saves and incoming reflection, a missing or unreadable recorded base permits reconstruction only from exactly one matching current non-deleted leaf. Matching several current leaves avoids duplicate storage but does not identify a displayed branch. No current match creates an independent branch, even when an older ancestor has the same bytes. Deletion and rename retain their existing provenance-recovery contracts.
For ordinary saves and incoming reflection, a missing or unreadable recorded base without matching saved content evidence permits reconstruction only from exactly one matching current non-deleted leaf. Matching several current leaves avoids duplicate storage but does not identify a displayed branch. No current match creates an independent branch, even when an older ancestor has the same bytes. Deletion and rename retain their existing provenance-recovery contracts.
Provenance keys follow the selected [Unicode path identity](design_docs/unicode_path_identity.md#state-which-follows-the-selected-identity). In NFC mode, equivalent path spellings address the same record; the content evidence continues to describe file bytes, independently of that path spelling.
## Known potentially problematic behaviour
Saving a local file writes its revision to the local database, then writes
device-local provenance in a separate operation. The writes are not atomic.
If the process stops between them, provenance can still identify the revision
which preceded the local edit. Continuous replication which was already
running can add a remote descendant to the local database during this
interval. The per-file lock covers file handling and Vault reflection; it
does not block raw replication writes to the database.
For example, local content `2-B` may have been saved as a child of
displayed revision `1`, while provenance still identifies `1`.
On restart, replay of the pending save can treat `2-B` as an edit of
`1` and write sibling `2-D`. The tested case demonstrates that this
pre-provenance gap alone can produce the duplicate. Without a remote
descendant, replay creates this sibling; duplicate handling tombstones one
of the identical branches and converges to one live leaf whose exact content
matches the saved local file. A second restart must add no revision.
If replication has already added descendant `3-C` during the gap, replay
can expose an additional branch. When the revisions have a complete shared
base and their text changes can be combined unambiguously, sensible merge can retain the
exact content of the incoming descendant. In the tested fixture, merge
produced revision `4` as a child of `3-C`; its file bytes were exactly
those of `3-C`, `3-C` remained in its ancestry, and sibling
`2-D` was tombstoned. The acceptance condition is retaining
`3-C`'s exact content; it need not remain the current leaf. Both Vaults
converged to those bytes. This observed outcome is limited to that mergeable
fixture and is not a universal no-data-loss guarantee.
When changes cannot be combined safely, as in the tested competing whole-line
replacements, sensible merge does not combine the bodies. Both versions remain available for the existing
manual conflict dialogue, even when sensible merge is enabled.
An interruption in the database/provenance interval is expected to be very
rare, especially when combined with a remote descendant arriving in that
interval; the field frequency is unmeasured. Start-up replays restored work,
scans, and commits pending local file work before marking the plug-in ready;
ordinary replication proceeds after that ready boundary. This does not claim
remote synchronisation before start-up readiness. The accepted design adds no
cross-store transaction, second journal, or new persistent state to close the
interval.
## Operations while a conflict exists
@@ -240,7 +294,7 @@ If the user renames `draft.md` to `published.md`, LiveSync stores `published.md`
### A remote resolution reaches a device which still shows the losing content
Android may resolve a conflict and continue editing while Mac still shows the losing revision. When Mac receives the resolved tree, LiveSync compares Mac's bytes with the exact revision recorded for its Vault. If that body remains readable and matches, it can apply Android's resolution without asking Mac to resolve the same unchanged conflict again.
Android may resolve a conflict and continue editing while Mac still shows the losing revision. When Mac receives the resolved tree, LiveSync compares Mac's bytes with the exact revision recorded for its Vault or its saved content evidence. If they match, it can apply Android's resolution without asking Mac to resolve the same unchanged conflict again, including when the recorded revision has been pruned.
If the user edited the file on Mac before the resolution arrived, the bytes no longer match that historical revision. LiveSync preserves the Mac edit as an unsynchronised conflict instead of overwriting it.
@@ -277,9 +331,11 @@ Do not:
## Verification
LiveSync also exercises three and four independently editing devices through real CouchDB, using the installed Commonlib package and the CLI conflict-resolution command dispatcher. These tests check unchanged losing files before and after resolution, genuine edits on a losing branch, missing provenance, compacted bases, independent-root deduplication, and propagation of the selected result. See the [multiple-device regression procedure and coverage boundaries](../test/README.md#multiple-device-conflict-regression-tests).
LiveSync also exercises three and four independently editing devices through real CouchDB, using the installed Commonlib package and the CLI conflict-resolution command dispatcher. The compaction cases distinguish unchanged files with saved content evidence, legacy records without it, and genuine edits which retain the same byte length and modification time. Repeated saves must create no duplicate revisions, and the selected result must propagate to every replica. See the [multiple-device regression procedure and coverage boundaries](../test/README.md#multiple-device-conflict-regression-tests).
Commonlib owns the real-PouchDB and injected-boundary tests for revision ancestry, content preservation, provenance, independent branches, and repeated file events. LiveSync owns persistent host composition and actual Obsidian restart coverage. The focused `test:e2e:obsidian:stale-file-restart` scenario advances the local DB while old Vault bytes remain, persists pending file events, and restarts the same isolated profile. It requires an unchanged recorded file to reflect the DB without a new revision, an unknown file to remain on an independent branch alongside the DB content, and repeated processing after provenance loss to leave those branches unchanged. It also removes the record for a file whose bytes still match the current database revision, checks that start-up restores the record without a new database revision, and then checks that a later incoming revision reflects without a conflict. It uses real local storage and startup processing; transport replication and mobile lifecycle coverage are separate.
Commonlib owns the real-PouchDB and injected-boundary tests for revision ancestry, content preservation, provenance, independent branches, and repeated file events. LiveSync owns persistent host composition and actual Obsidian restart coverage. The focused `test:e2e:obsidian:stale-file-restart` scenario prunes a recorded revision while its old Vault bytes remain, persists the content evidence and pending file event, and restarts the same isolated profile. The unchanged file must reflect the newer content without a database write. Other fixtures check independent preservation of unknown content, deduplication after provenance loss, reconstruction of a missing record from the current revision, and verified upgrade of a legacy record. It uses real local storage and start-up processing; transport replication and mobile lifecycle coverage are separate.
`E2E_OBSIDIAN_ONLY_METADATA_DELETION=true` selects two-Vault creation, update, deletion, and rename checks with `deleteMetadataOfDeletedFiles` disabled and enabled. It also delivers a real CouchDB restoration during the final deletion check, requiring the receiving file bytes and restored winning revision to remain, with no deletion event.
LiveSync's optional real-Obsidian two-Vault checks have two scopes. `E2E_OBSIDIAN_INCLUDE_MARKDOWN_CONFLICT=true` resolves and edits a Markdown conflict, propagates it to a Vault which still displays the deleted losing content, and requires one current result to remain. `E2E_OBSIDIAN_INCLUDE_CONFLICT_OPERATIONS=true` edits, deletes, case-renames, and cross-path-renames files while conflicts remain active; it verifies the parent revision of each resulting branch, replicates those exact trees, and confirms that the other conflict branches remain intact.
docs/specs_data_compression.md
+5 -1
View File
@@ -33,7 +33,11 @@ The current benchmark measures the Node CLI process. It does not establish Obsid
## Reproducible benchmark
The benchmark is implemented under `src/apps/cli/testdeno` and packaged by `test/bench-network`. It uses the real CLI mirror and synchronisation path, Commonlib 0.1.0-rc.4, CouchDB 3.5.0, the V3 Rabin–Karp splitter, optional Data Compression, and E2EE V2. It runs each of these conditions three times in rotating order:
The benchmark is implemented under `src/apps/cli/testdeno` and packaged by `test/bench-network`. It exercises the real CLI mirror and synchronisation path. The recorded three-repeat result below used Commonlib 0.1.0-rc.4, CouchDB 3.5.0, the V3 Rabin–Karp splitter, optional Data Compression, and E2EE V2.
The current Compose runner installs the checked-out repository's package and lockfile with `npm ci`; Self-hosted LiveSync 1.0.42 selects Commonlib 0.2.2. Running the command below measures that checkout, rather than recreating the older package environment automatically. Keep the recorded results associated with their original measurement date and dependency version.
With the repeat count shown below, it runs each of these conditions three times in rotating order:
- E2EE off, compression off;
- E2EE off, compression on;
docs/tips/fast-setup.md
+11 -11
View File
@@ -1,8 +1,8 @@
# Fast Setup (Simple Fetch)
Fast Setup is a streamlined, user-friendly data retrieval and initialisation flow designed to simplify setting up secondary devices or recovering databases.
Fast Setup (Simple Fetch) is the preferred flow for initial replication to a new or empty secondary device. It can also be used for recovery, but a retrieval choice may replace local files. Preserve valuable local and remote copies before choosing how to handle a Vault that already contains work. See [Recovery and flag files](../recovery.md#first-choose-the-authoritative-copy) for guidance on identifying the authoritative copy.
Instead of guiding the user through the detailed multi-step setup wizard dialogues, Fast Setup prompts the user with high-level sync decisions and automates database download and local storage scanning in one continuous process.
Instead of guiding the user through the detailed multi-step setup wizard dialogues, Fast Setup prompts the user with high-level synchronisation decisions and automates data retrieval and local storage scanning in one process. The retrieval path depends on the selected remote.
---
@@ -16,14 +16,14 @@ On the next startup, the plug-in boots in scheduled fetch mode and opens a simpl
## Technical Characteristics
Fast Setup leverages several backend optimisations to make the retrieval fast, safe, and clean:
Fast Setup uses several retrieval and reflection behaviours:
1. **Stream-based Replication for Speed**
- It fetches all remote metadata via stream reception, which is significantly faster than traditional chunk-by-chunk retrieval.
1. **Stream-based Fast Fetch for CouchDB**
- The stream-based Fast Fetch path is CouchDB-specific. It retrieves remote Metadata incrementally; Object Storage and P2P use their own retrieval paths.
2. **Delayed File Reflection to Prevent Corrupted Warnings**
- By suspending file reflection during the download phase, it prevents the plug-in from raising temporary or false "corrupted data synchronisation" or "size mismatch" warnings that can occur during the chunk download process.
3. **Time-Based Comparison is Generally Sufficient**
- Since the vault is entering a fresh synchronisation or recovery state, comparing files based on their modification timestamps (newer-wins) is highly reliable and sufficient to reconcile files without needing complex manual conflict resolution.
- Delaying local file reflection during retrieval suppresses temporary synchronisation warnings while remote data is being fetched.
3. **Timestamp Comparison Selects a Whole File**
- **Compare time and take newer** compares modification timestamps and selects one whole-file version. It does not merge file contents. Before choosing this policy for a Vault with existing work, preserve valuable local and remote edits and review [Recovery and flag files](../recovery.md#first-choose-the-authoritative-copy).
---
@@ -33,11 +33,11 @@ Fast Setup leverages several backend optimisations to make the retrieval fast, s
You will be prompted to choose how the retrieved remote data will interact with your existing local files:
1. **Compare time and take newer (newer-wins)**
- Compares the modified time of files and accepts the newer version.
- **Recommended if:** You have been using Self-hosted LiveSync and have made changes on multiple devices that you want to merge.
- Compares the modified time of files and selects the newer whole-file version; it does not combine edits from both versions.
- **Recommended if:** You have preserved both versions and deliberately want to select one according to its modification time.
2. **Overwrite all with remote files (remote-wins)**
- Remote data is treated as the source of truth.
- **Recommended if:** You are setting up a brand new device with an empty or clean vault.
- **Recommended if:** You are setting up a new secondary device with an empty Vault.
- *Warning: This will overwrite local files with remote files. Please ensure you have a backup of your local vault before proceeding.*
3. **Use the detailed flow (legacy)**
- Switches back to the detailed, traditional setup wizard dialogues.
docs/tips/hidden-file-sync.md
+10
View File
@@ -32,6 +32,10 @@ Enabling Hidden File Sync requires an initialisation direction:
Prefer a narrow target list. Device-specific workspace state and another plug-in's credentials are usually poor candidates for cross-device synchronisation.
`Normal Files` → `Synchronising files` limits ordinary files only. Hidden File Sync uses its own `Target patterns`. The `Non-Synchronising files` patterns under `Normal Files` also exclude matching hidden files, so review them alongside the hidden-file selectors.
If an earlier version skipped hidden files because of the normal-file allow-list, repeat Hidden File Sync initialisation after updating. A routine scan can skip unchanged files which were already checked. Back up the Vault and choose the intended source direction as described above.
Target patterns also control directory traversal. A pattern must therefore match each parent directory as well as the intended files. For example, the following pattern admits the `.obsidian` parent and only its `snippets` subtree:
```text
@@ -68,6 +72,12 @@ Hidden File Sync must be enabled independently after ordinary setup on each devi
If you later run Rebuild or Fetch as a recovery operation, expect optional features to be disabled again. Complete ordinary recovery first, then repeat this guide deliberately.
## Automatic in Customisation Sync
**Automatic** delegates an item's files to Hidden File Sync. Enable Hidden File Sync on this device through the steps above before choosing Automatic. If Hidden File Sync is disabled, the choice displays a Notice without changing the item's mode or starting initialisation. Previously saved Automatic assignments remain saved, so this prerequisite also applies after importing settings from another device.
Once Hidden File Sync is enabled, choose the item's initial action in Customisation Sync. Selecting a whole plug-in applies that action to both its **MAIN** and **DATA** selections.
## Related settings
The [settings reference](../settings.md#7-hidden-files-advanced) describes periodic scanning, scanning before replication, notification suppression, selectors, and overwrite patterns. Change those controls only after the basic Hidden File Sync path is working.
docs/tips/status-display.md
+55
View File
@@ -0,0 +1,55 @@
# Synchronisation status and CSS customisation
Use [General Settings](../settings.md#2-general-settings) to choose whether synchronisation status appears in the status bar, inside the editor, or both. The following sections explain the indicators and how to customise their appearance.
## Status indicators
- Activity Indicator
- 📲 A finite remote operation is in progress
- 🌐N Approximate remote requests currently in progress
- Status
- ⏹️ Stopped
- 🌀 Starting synchronisation
- ⏩ Comparing CouchDB change history, including entries already held locally
- 💤 LiveSync enabled. Waiting for changes
- ⚡️ Synchronisation in progress
- ⚠ An error occurred
- Statistical Indicators
- ↑ Uploaded chunks and metadata
- ↓ Downloaded chunks and metadata
- Progress Indicators
- 📥 Unprocessed transferred items
- 📄 Working database operation
- 💾 Working write storage processes
- ⏳ Working read storage processes
- 🛫 Pending read storage processes
- 📬 Batched read storage processes
- ⚙️ Working or pending storage processes for hidden files
- 🛄 Pending initial on-demand chunk requests
- 🔁 Pending chunk retries, including retry delays
- 🔌 Working customisation items (configuration, snippets, and plug-ins)
To prevent file and database corruption, please avoid closing Obsidian until all progress indicators have disappeared as much as possible (although the plug-in will attempt to resume if interrupted). This is especially important if you have deleted or renamed files.
## CouchDB comparison count
When CouchDB change history is being compared, `(⏩ 1200) ↑ 0 ↓ 0` can appear before any new documents arrive. The count is the number of candidate changes-feed entries compared during the current replication attempt, including repeated entries on retries. It is separate from downloaded Chunks and Metadata, and is neither a Vault file count nor a count of entries already present locally. It appears before the upload count, disappears after three seconds without a change, and reappears when the count changes. The ⏩ status icon follows comparison activity independently of that timer.
## Customising status items with CSS snippets
The status bar and editor overlay use individual spans with the common `livesync-stat-` class prefix. CSS snippets can hide or style individual items. For example, this hides the comparison count and P2P summary:
```css
.livesync-stat-ff,
.livesync-stat-p2p {
display: none;
}
```
The item classes include `livesync-stat-network`, `livesync-stat-sync`, `livesync-stat-sent`, `livesync-stat-push-remaining`, `livesync-stat-arrived`, `livesync-stat-pull-remaining`, `livesync-stat-ff`, and `livesync-stat-p2p`. File-processing items use `livesync-stat-file-processing`, `livesync-stat-file-pending`, and `livesync-stat-file-batched`; queue items use `livesync-stat-queue-` followed by `replication`, `database`, `storage`, `chunk-initial`, `chunk-retrying`, `plugin-scan`, `hidden-files`, or `conflicts`. The skipped-file indicator uses `livesync-stat-file-skip`.
All items also have `livesync-stat-item`. Numeric children use `livesync-stat-value`, tabular digits, and right alignment. Each numeric item retains its widest displayed digit count during an active period, reducing movement as counts decrease. The editor overlay uses wider gaps between items, while remaining counts stay beside their transfer totals.
Inactive items receive `livesync-stat-inactive` instead of the HTML `hidden` attribute. The default CSS hides them immediately. They retain their last text, numeric width, and position amongst the other items so that CSS snippets can customise how they disappear. The numeric width starts afresh when an item becomes active again.
See the [documentation index](../README.md) for setup and troubleshooting guides.
docs/to_issue_reporting.md
+97
View File
@@ -4,6 +4,16 @@ Thank you for helping improve Self-hosted LiveSync. A concise report with the ri
Use the [issue report template](https://github.com/vrtmrz/obsidian-livesync/issues/new?template=issue-report.md) for the report itself. Use [Troubleshooting](troubleshooting.md) to diagnose a symptom or choose a recovery action.
If a maintainer requests additional evidence, follow the relevant procedure below. These reports supplement the required information in this guide.
| Requested evidence | Procedure |
| --------------------------------------------------------------------- | ------------------------------------- |
| Database integrity, current conflicts, or differences between sources | [Integrity report](#integrity-report) |
| Local revision and Chunk usage | [Usage report](#usage-report) |
| More information about a path pseudonym in an existing report | [File follow-up](#file-follow-up) |
Maintainers can link directly to each procedure and adapt the [example requests](#requesting-diagnostics).
## Preserve the original symptom
Do not reset a database, rebuild a remote, change transport, or enable P2P merely to see whether the problem disappears. These actions can change the evidence and may make the original cause harder to identify.
@@ -44,12 +54,18 @@ For a large report, you may share a GitHub Gist after reviewing and redacting it
## Additional evidence when relevant
### A problem involving database integrity or usage
Follow the [Integrity report](#integrity-report) procedure for database-wide consistency findings, or the [Usage report](#usage-report) procedure for local revision and Chunk accounting. [Collect a report](troubleshooting.md#collect-a-report) explains the inspection limits and how to read the results.
### A problem involving one file
Run `Copy database information for the active file`, or use **Hatch** → **Copy database information for a file** to select another file.
This report describes only the local database on that device. It includes the Vault-relative path, document and chunk identifiers, local revisions, conflicts, and local chunk availability. It does not query the remote or include file contents. Review paths and identifiers as private metadata before sharing them.
If the request identifies a path pseudonym from an existing report, use [File follow-up](#file-follow-up) to retain that report's pseudonyms.
### A problem which crosses a restart
Use `Write logs into the file` under **Hatch** only when the in-memory report cannot cover the restart. Persistent logging affects performance and can record private information. Disable it after reproducing the problem, review the log before sharing it, and remove the log file when it is no longer needed.
@@ -58,8 +74,89 @@ Use `Write logs into the file` under **Hatch** only when the in-memory report ca
Include network evidence only when the ordinary LiveSync log cannot show the rejected response. Follow [Inspect a network failure](troubleshooting.md#inspect-a-network-failure), and remove request paths, remote addresses, authority and authorisation values, cookies, credentials, payload identifiers, and response secrets before sharing screenshots or copied data.
## Requested diagnostic reports
Use the device and Vault named in the request. Label each result with a device role such as 'Device A: edits originate here', and include when it was collected. If an action below is unavailable, report the installed plug-in version and the missing action.
These reporting tools inspect existing state without repairing or resetting it. Preserve the original symptom as described above.
### Integrity report
1. Open Self-hosted LiveSync settings → **Hatch** → **Reporting Tools** → **Inspect database integrity**, then select **Inspect**.
2. Keep **Pseudonyms (recommended)** selected.
3. Leave **Include remote database** selected unless the request specifies a local-only inspection. For a local-only inspection, clear it. This controls diagnostic requests for this run; it does not change normal synchronisation settings.
4. Select **Begin inspection**. If you cancel, retain the resulting report and its cancellation status.
5. Choose **MARKDOWN**, then **Copy report** or **Save report**. Share the complete output after reviewing it. Use **JSON** instead if the maintainer requests it.
Keep the status, versions, settings, scope, coverage, counts, findings, and errors together. A partial or cancelled result is useful evidence. If the report is too large for a comment, share the saved report or a reviewed Gist, and identify its device role in the issue.
The remote inspection currently supports CouchDB. An excluded or unavailable remote remains visible in the report; changing remote settings is not part of this procedure. Content checks cover suspect groups, so `not-checked` is meaningful. A completed inspection does not establish that every file is healthy. See [Read the compact Markdown integrity report](troubleshooting.md#read-the-compact-markdown-integrity-report) for the state notation and Review index.
### Usage report
1. Open Self-hosted LiveSync settings → **Hatch** → **Reporting Tools** → **Analyse database usage**, then select **Analyse**.
2. Keep **Pseudonyms (recommended)** selected, then select **Begin inspection**.
3. For a complete result, choose **TSV**, unless the maintainer requests **JSON**. Partial or cancelled results use **JSON** to retain their status and reasons.
4. Select **Copy report** or **Save report**, review the output, and share it with the device role and collection time.
This analysis reads the local database. Its revision-reading total can grow during discovery, followed by a separate accounting stage. Reported sizes describe stored string units and do not measure the physical database file. Cancelling during accounting omits unfinished statistics.
### File follow-up
Use this procedure when a maintainer identifies a path token from an Integrity or Usage report. The complete token has the form `<salt>$<hash>`. Copy the token itself, including the salt and `$`, without surrounding Markdown backticks. An observation reference such as `local:12` or `remote:12`, a document pseudonym, or a revision pseudonym is not a path token.
1. On the requested device, open Self-hosted LiveSync settings → **Hatch** → **Reporting Tools** → **Find a file from a report pseudonym**, then select **Find file**.
2. Paste the complete token into **Report path pseudonym**, then select **Find file** in the dialogue.
3. For a matching candidate, select **Prepare follow-up report**.
4. Choose **MARKDOWN** or **JSON**, then **Copy report** or **Save report**. Review the generated output before sharing it, and identify the original report, token, and device role.
5. If several candidates match the path, use **Back to matches** and prepare a separate report for each, unless the maintainer has specified a particular candidate. Keep the reports separate so their document IDs can be compared.
Use the token from the original request throughout. The follow-up reuses its salt, which keeps pseudonyms comparable with the original report. A new database-wide report generates a new salt, so its pseudonyms will differ.
The candidate list displays real paths and document IDs locally. Share the generated pseudonymised report rather than a screenshot of that list. A Metadata candidate can still be inspected when its current Vault file is missing. A Vault-only candidate reports that local Metadata was not inspected.
If there are no matches, report that outcome and the device role. The lookup searches current Vault paths and current local normal-file Metadata; it does not search the remote or older history. A rename, deletion, or remote-only path can therefore have no local match. An older ordinal pseudonym cannot be used for this lookup.
The follow-up report inspects the selected local document, current conflicts, and selected merge bases, and lists available revision-history status. It omits file contents and credentials, does not query the remote, and does not scan every historical file body. Pseudonyms use a public salt, so someone with the report can test guessed paths against a token.
## Sharing the report
Paste reports into the matching collapsible sections in the issue template, or provide a link to an already-redacted Gist. A separate plug-in log is normally unnecessary because the full LiveSync report already contains the recent verbose log history.
Integrity and file follow-up Markdown already include a collapsible section and a code block. Paste that output directly into an issue comment, outside any existing code block. Keep each device's report separate, and preserve the original token when adding a follow-up.
If a maintainer asks for a more specialised diagnostic, collect only that additional evidence and review it again before publishing it.
## Requesting diagnostics
For maintainers: choose the procedure which answers the current question, specify the device role, and replace the placeholders before posting a request. The examples use direct links to the relevant steps. For Integrity, choose one remote setting explicitly; inclusion is the default. If the requested action requires a newer build, identify the version to use.
### Request an integrity report
```markdown
Would you be willing to collect an [Integrity report](https://github.com/vrtmrz/obsidian-livesync/blob/main/docs/to_issue_reporting.md#integrity-report) on **[DEVICE ROLE]**, using **Pseudonyms (recommended)** with **Include remote database [selected / cleared]**?
Please share the complete Markdown output after reviewing it, including its status, scope, coverage, and errors. A partial or cancelled result is useful too. Include the device role and collection time.
```
### Request a usage report
```markdown
Would you be willing to collect a [Usage report](https://github.com/vrtmrz/obsidian-livesync/blob/main/docs/to_issue_reporting.md#usage-report) on **[DEVICE ROLE]**, using **Pseudonyms (recommended)**?
Please share the reviewed TSV output for a complete result, or JSON for a partial or cancelled result, with the device role and collection time.
```
### Request a file follow-up
Include the original report link or label, its observation reference where available, and the complete path token. Keep the revision generation and document pseudonym available when discussing a particular branch. A reference such as `remote:12` locates an observation within that report; the path token is what the user enters into the lookup.
```markdown
Would you be willing to follow the [File follow-up steps](https://github.com/vrtmrz/obsidian-livesync/blob/main/docs/to_issue_reporting.md#file-follow-up) on **[DEVICE ROLE]** for this entry?
- Original report: [REPORT LINK OR LABEL]
- Observation: `[REFERENCE, IF AVAILABLE]`
- Path token: `[COMPLETE SALT$HASH TOKEN]`
Please share the generated pseudonymised Markdown or JSON report. If several candidates match, share a separate report for each. If none match, tell us that outcome. Include the device role and collection time; the generated reports are sufficient without sharing real paths or a screenshot of the candidate list.
```
docs/troubleshooting.md
+44 -2
View File
@@ -51,7 +51,9 @@ If the remote is healthy but one device's local database is not, use [Reset Sync
If the log says `Preserved unsynchronised local changes as a conflict`, keep both versions available until you have checked their contents. The message means that LiveSync could not prove that the Vault file was unchanged; it does not establish that you edited the file. Use **Inspect conflicts and file/database differences** in Hatch to review the current branches before selecting a version.
At start-up, LiveSync can restore a missing device-local revision record when the file still exactly matches the current local database revision. If a newer revision has already arrived, that match may no longer exist. LiveSync then preserves the old file for review because it cannot distinguish an unchanged copy from an intentional edit back to older content. A new release cannot resolve a conflict which was already created merely by recognising a historical content match. If the problem recurs, include the first relevant log entries, plug-in versions on both devices, and a redacted full report with the issue steps.
At start-up, LiveSync can restore a missing device-local revision record when the file exactly matches the conflict-free current local database revision. For a saved record, it compares the Vault file with that exact revision when the revision body is readable. It also saves a device-local content hash and byte length. Evidence saved before pruning lets LiveSync recognise an unchanged Vault file even after its recorded revision has been pruned.
If the recorded body is unavailable and no matching content evidence exists, LiveSync cannot prove that the file is unchanged and preserves it for review. Genuine edits and files with unknown origins remain protected; content evidence does not recreate missing revision history. A conflict which was already recorded is not automatically removed or resolved. Use **Inspect conflicts and file/database differences** to review its current branches. If the problem recurs, include the first relevant log entries, plug-in versions on both devices, and a redacted full report with the issue steps.
## Synchronisation is paused for compatibility review
@@ -96,7 +98,9 @@ Current releases automatically align compatible settings which control how new c
A missing legacy file-name case setting means case-insensitive handling. It matches an explicit disabled setting and does not require a rebuild for that difference. An explicitly enabled setting can use different document IDs and still requires a compatibility decision against either value. Other configuration differences shown in the dialogue must still be resolved.
If the mismatch names **Encrypt internal file Properties**, update every device before accepting that preference. It affects subsequent Metadata writes for Hidden File Sync and Customisation Sync; it does not automatically protect existing Metadata. A manual remote Rebuild is strongly recommended if you need to protect existing paths, times, sizes, and Chunk references.
The saved **Encrypt internal file Properties** preference is effective only for CouchDB when E2EE V2 and **Property Encryption** are also enabled. An inactive preference difference alone does not change Metadata behaviour or require a Rebuild. Applying settings retains that preference only when doing so leaves the adopted effective encryption settings unchanged.
When active, this option affects subsequent Metadata writes for Hidden File Sync and Customisation Sync; it does not automatically protect existing Metadata. To protect existing paths, times, sizes, and Chunk references, update every device and manually Rebuild the remote database.
The `Sync now` command keeps routine replication progress quiet so that it is convenient to assign to a keyboard shortcut; assign one in Obsidian if that suits your workflow. A quiet command may still open this dialogue when a mismatch or another decision requires your attention.
@@ -159,12 +163,50 @@ Enable Obsidian's `Detect all file extensions`, then check LiveSync selectors, i
## Collect a report
For a database-wide inspection, use **Hatch** → **Inspect database integrity**. Choose **Pseudonyms (recommended)** or **Original names** before beginning. Both choices are visible as radio buttons, with pseudonyms selected initially. **Include remote database** is selected by default; clear it to inspect the local database and current Vault files without making diagnostic requests to the remote. This choice applies to one run, and the report marks the remote as excluded with coverage not checked. When included, the inspection reads normal-file Metadata and current conflict branches locally and in CouchDB, then compares bytes within suspect groups and against the current Vault files. It reports unavailable sources, unreadable content, and detected changes during the run. Older revision history and other remote types are outside this inspection. A completed run means the stated inspection finished; it does not mean every entry is healthy or safe to delete.
The result dialogue provides Markdown and JSON reports with the same name policy. Available formats are displayed as radio buttons. Pseudonyms preserve exact path equality across sources while keeping different document IDs distinct. Each report uses a random eight-character public salt; path tokens are derived from the exact stored Vault-relative path without lower-casing or Unicode NFC normalisation. The same salt reproduces path, document, revision, and Chunk pseudonyms for a targeted follow-up. Because the salt is public, a guessed path can be checked against its token. Earlier ordinal pseudonyms cannot be looked up this way. Pseudonyms are temporary and are not saved as a global setting. Counts and relationships remain visible. Both modes omit file contents and credentials; review the report before sharing it. Cancelling keeps the evidence collected so far and marks the result as cancelled.
Integrity observations include recorded Metadata type, size, and total, unique, and embedded Chunk-reference counts. When suspect content has already been read, they also include measured decoded and Vault sizes, their differences from recorded or reconstructed sizes, and the Vault-to-Metadata modification-time difference. Case-only and NFC-equivalent path groups are listed once. The inspection uses content already needed for its comparisons and does not add database, HTTP, or Vault reads. Values which were not inspected remain absent or `not-checked`.
### Read the compact Markdown integrity report
Markdown keeps its status, settings, scope, coverage, and count summary, and adds separate counts for content and Vault checks which were not performed. **State counts** count observations, not unique files; each local or remote observation contributes to its state total. The **Review index** lists references to observations which match review conditions, before **Findings**. Those conditions include identity mismatches, current conflicts, case- or NFC-equivalent paths, non-NFC paths, a missing group in the other inspected database, missing or unreadable content or Vault files, unavailable Chunks, different or unavailable comparisons, non-zero measured differences, and absent Metadata. These are review candidates, not proof of corruption. A report with no review-index entry still does not confirm that unchecked content is healthy. JSON data and its schema are unchanged.
Each observation is shown in a compact form, while retaining every observation and its full paths, IDs, revisions, and references for follow-up:
```text
### `remote:4410` source=remote
path: `<salt>$<hash>`
doc: `<document-id>` rev=`<revision>` expected=self
state: consistent-winner-logicallyLive-NFC-newnote conflicts=0
paths: case=none nfcPeers=none | otherDB=present
data: recorded=47584B chunks=1/1/0 read=0
check: content=not-checked vault=not-checked@`vault:remote:4410`
```
The `state` values always appear in this order: identity, current branch, logical deletion, exact stored path normalisation, and Metadata type. Identity is `consistent`, `documentIdMismatch`, or `namespaceMismatch`; JSON retains `document-id-mismatch` and `namespace-mismatch`. `winner` is the selected current branch and can coexist with conflicts, shown by `conflicts`. `logicallyLive` and `logicallyDeleted` describe the deletion marker. `NFC` means the stored path is NFC-normalised; `nonNFC` means it is not. The Metadata type is `plain`, `newnote`, `notes`, or `unknown`; `notRecorded` means Metadata is absent. A `consistent` identity says only that the ID matches its expected form; it does not show that content is readable or equal across sources.
In `paths`, `case` and `nfcPeers` describe case-equivalent and NFC-equivalent paths: `none` means no equivalent path was found, `yes` means one was found, and `yes@` followed by a reference identifies the available group. `otherDB` says whether the same identity and path group occurs in the other inspected source. It does not compare content. If that source was excluded or unavailable, the field is `not-checked`.
In `data`, `chunks` is total/unique/embedded Chunk references, and `read` counts Chunk reads, not HTTP requests. A `?` means a value was not recorded; it is distinct from zero, missing, unreadable, and `not-checked`. If Metadata is absent, its recorded size and Chunk counts are `?`. In `doc`, `expected=self` means the actual document ID exactly matches the expected ID. Otherwise, the report shows the expected ID or `?` when it was not recorded.
The `check` statuses distinguish `available`, `missing`, `unreadable`, and `not-checked`; a recorded reason follows the status in parentheses. An optional `measure` line includes only available values: decoded content size and Vault size in bytes, `deltaContent` (decoded content minus recorded Metadata size), `deltaVault` (Vault minus decoded content), and `deltaMtime` (Vault minus Metadata modification time) in milliseconds. Positive differences have a `+` prefix, negative differences retain `-`, and zero is `0`. These measurements describe the evidence collected and do not select an authoritative version. Unavailable Chunk names, comparison results, path-equivalence groups, and errors retain their reasons and references. All observations remain in the report's collapsible section and fenced copy.
Usage progress first follows available-revision discovery, so its `n/m` total can grow and remains provisional until discovery ends. It then shows separate progress for accounting. Cancellation during revision reading keeps the collected evidence; cancellation during accounting omits incomplete statistics.
For storage accounting, use **Hatch** → **Analyse database usage** → **Analyse**. It offers the same name choice and reads existing local revisions without changing on-demand Chunk fetching. Complete runs retain the existing TSV columns and first-reference unique/shared Chunk accounting. Sizes count stored string units, not physical database size. Incomplete runs provide JSON so their status and reasons remain attached to the rows. **Save report** requests a download; **Copy report** remains available on devices which do not support downloads.
Follow [How to report an issue](to_issue_reporting.md) for the complete reporting checklist, including Obsidian debug information and the privacy review required before sharing evidence.
Run `Generate full report for opening the issue with debug info` to copy the current settings summary and recent verbose log lines. Remove credentials, remote URLs, Vault names, file contents, and other private information before sharing it.
When a problem concerns one file, run **Copy database information for the active file**, or use **Hatch** → **Copy database information for a file** to select another file. The report describes this device's local database view, including the Vault-relative path, document and chunk identifiers, local database revisions, conflicts, and local chunk availability. It does not query the remote server or include file contents. Treat paths and identifiers as private metadata before sharing.
To follow up on a path pseudonym from a report, use **Hatch** → **Find a file from a report pseudonym**, paste the token, and select **Find file**. The lookup checks the current Vault and local normal-file Metadata only; it does not contact the remote database. It lists every local candidate, with real paths and document IDs visible only in the local dialogue. Multiple IDs for the same path remain distinct. A Vault-only candidate means local Metadata was not inspected. No match may mean the file was renamed or deleted, or that the matching Metadata exists only remotely.
For a selected candidate, the tool can open an existing Vault file or prepare a shareable local report using the original salt and selected document ID. That report shows current, conflict, and history status, Chunk availability, merge bases, and Vault size facts, without file contents or credentials. Its history status is a listing; it does not inspect every historical body.
Use `Show log` for live inspection. Logs are intentionally kept in memory for a limited time to reduce accidental disclosure. Enable `Write logs into the file` only while reproducing a problem, then disable it and remove the file after review because persistent logging affects performance and may contain private data.
![Write logs into the file](../images/write_logs_into_the_file.png)
manifest.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"id": "obsidian-livesync",
"name": "Self-hosted LiveSync",
"version": "1.0.37",
"version": "1.0.42",
"minAppVersion": "1.7.2",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz",
package-lock.json
Generated
+206 -699
View File
File diff suppressed because it is too large. Load diff
package.json
+21 -7
View File
@@ -1,6 +1,6 @@
{
"name": "obsidian-livesync",
"version": "1.0.37",
"version": "1.0.42",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js",
"type": "module",
@@ -57,6 +57,10 @@
"test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts",
"test:e2e:obsidian:onboarding-invitation": "tsx test/e2e-obsidian/scripts/onboarding-invitation.ts",
"test:e2e:obsidian:dialog-mounts": "tsx test/e2e-obsidian/scripts/dialog-mounts.ts",
"test:e2e:obsidian:joined-devices": "tsx test/e2e-obsidian/scripts/joined-devices.ts",
"test:e2e:obsidian:joined-devices-object-storage": "tsx test/e2e-obsidian/scripts/joined-devices.ts --object-storage",
"test:e2e:obsidian:database-diagnostics": "tsx test/e2e-obsidian/scripts/database-diagnostics.ts",
"test:e2e:obsidian:database-diagnostics-couchdb": "tsx test/e2e-obsidian/scripts/database-diagnostics-couchdb.ts",
"test:e2e:obsidian:conflict-dialog-policy": "tsx test/e2e-obsidian/scripts/conflict-dialog-policy.ts",
"test:e2e:obsidian:revision-repair": "tsx test/e2e-obsidian/scripts/revision-repair.ts",
"test:e2e:obsidian:document-history-nav": "tsx test/e2e-obsidian/scripts/document-history-nav.ts",
@@ -66,6 +70,7 @@
"test:e2e:obsidian:configuration-encryption-sync": "tsx test/e2e-obsidian/scripts/configuration-encryption-sync.ts",
"test:e2e:obsidian:review-harness": "tsx test/e2e-obsidian/scripts/review-harness.ts",
"test:e2e:obsidian:p2p-pane": "tsx test/e2e-obsidian/scripts/p2p-pane.ts",
"test:e2e:obsidian:replication-status": "tsx test/e2e-obsidian/scripts/replication-status.ts",
"test:e2e:obsidian:vault-reflection": "tsx test/e2e-obsidian/scripts/vault-reflection.ts",
"test:e2e:obsidian:couchdb-upload": "tsx test/e2e-obsidian/scripts/couchdb-upload.ts",
"test:e2e:obsidian:couchdb-http-cache": "tsx test/e2e-obsidian/scripts/couchdb-http-cache.ts",
@@ -83,6 +88,7 @@
"test:e2e:obsidian:p2p-connection-check": "tsx test/e2e-obsidian/scripts/p2p-connection-check.ts",
"test:e2e:obsidian:p2p-connection-check:services": "npm run test:e2e:obsidian:p2p-connection-check -- --manage-p2p",
"test:e2e:obsidian:partial-startup-file-failure": "tsx test/e2e-obsidian/scripts/partial-startup-file-failure.ts",
"test:e2e:obsidian:startup-content-check": "tsx test/e2e-obsidian/scripts/startup-content-check.ts",
"test:e2e:obsidian:startup-scan": "tsx test/e2e-obsidian/scripts/startup-scan.ts",
"test:e2e:obsidian:stale-file-restart": "tsx test/e2e-obsidian/scripts/stale-file-restart.ts",
"test:e2e:obsidian:folder-batch": "tsx test/e2e-obsidian/scripts/folder-batch.ts",
@@ -90,7 +96,11 @@
"test:e2e:obsidian:two-vault-sync": "tsx test/e2e-obsidian/scripts/two-vault-sync.ts",
"test:e2e:obsidian:security-seed-reconnect": "tsx test/e2e-obsidian/scripts/security-seed-reconnect.ts",
"test:e2e:obsidian:hidden-file-snippet-sync": "tsx test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts",
"test:e2e:obsidian:hidden-file-selectors": "tsx test/e2e-obsidian/scripts/hidden-file-selectors.ts",
"test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts",
"test:e2e:obsidian:customisation-catalogue-reload": "tsx test/e2e-obsidian/scripts/customisation-catalogue-reload.ts",
"test:e2e:obsidian:optional-file-save-completion": "tsx test/e2e-obsidian/scripts/optional-file-save-completion.ts",
"test:e2e:obsidian:livesync-batch": "tsx test/e2e-obsidian/scripts/issue1207-livesync-batch.ts",
"test:e2e:obsidian:received-change-readiness": "tsx test/e2e-obsidian/scripts/received-change-readiness.ts",
"test:e2e:obsidian:remote-feature-change": "tsx test/e2e-obsidian/scripts/remote-feature-change.ts",
"test:e2e:obsidian:internal-metadata-migration": "tsx test/e2e-obsidian/scripts/internal-metadata-migration.ts",
@@ -118,7 +128,10 @@
"test:docker-all:start": "npm run test:docker-all:up && sleep 5 && npm run test:docker-all:init",
"test:docker-all:stop": "npm run test:docker-all:down",
"update-workspaces": "node update-workspaces.mjs",
"version": "node version-bump.mjs && node update-workspaces.mjs && npm run pretty:json && git add manifest.json versions.json src/apps/cli/package.json src/apps/webpeer/package.json src/apps/webapp/package.json"
"version": "node version-bump.mjs && node update-workspaces.mjs && npm run pretty:json && git add manifest.json versions.json src/apps/cli/package.json src/apps/webpeer/package.json src/apps/webapp/package.json",
"test:e2e:obsidian:startup-integrity": "tsx test/e2e-obsidian/scripts/startup-integrity.ts",
"test:e2e:obsidian:third-party-chunks": "tsx test/e2e-obsidian/scripts/third-party-chunks.ts",
"test:e2e:obsidian:upgrade-livesync-batching": "tsx test/e2e-obsidian/scripts/upgrade-livesync-batching.ts"
},
"keywords": [],
"author": "vorotamoroz",
@@ -142,7 +155,7 @@
"@types/pouchdb-replication": "^6.4.7",
"@types/transform-pouch": "^1.0.6",
"@typescript-eslint/parser": "8.69.0",
"@vitest/coverage-v8": "^4.1.11",
"@vitest/coverage-v8": "^4.1.12",
"@vrtmrz/obsidian-test-session": "0.2.6",
"dotenv-cli": "^11.0.0",
"esbuild": "0.28.1",
@@ -177,8 +190,8 @@
"tsx": "^4.21.0",
"typescript": "6.0.3",
"typescript-eslint": "8.69.0",
"vite": "^8.0.16",
"vitest": "^4.1.11",
"vite": "^8.1.6",
"vitest": "^4.1.12",
"yaml": "^2.8.2"
},
"dependencies": {
@@ -191,12 +204,13 @@
"@smithy/types": "^4.14.3",
"@smithy/util-retry": "^4.4.5",
"@vrtmrz/browser-ui-kit": "0.1.0",
"@vrtmrz/livesync-commonlib": "0.1.40",
"@vrtmrz/livesync-commonlib": "0.2.2",
"@vrtmrz/obsidian-plugin-kit": "0.1.4",
"@vrtmrz/ui-interactions": "0.1.2",
"@vrtmrz/value-store": "0.1.0",
"diff-match-patch": "^1.0.5",
"idb": "^8.0.3",
"markdown-it": "^14.3.2",
"markdown-it": "^14.3.3",
"minimatch": "^10.2.5",
"obsidian": "^1.13.1",
"octagonal-wheels": "^0.1.55",
utils/copy-build-files.ts → scripts/copy-build-files.ts
View File
File renamed without changes.
utils/copy-build-files.unit.spec.ts → scripts/copy-build-files.unit.spec.ts
View File
File renamed without changes.
src/apps/browser/LiveSyncBrowserUIService.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import type { AppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService";
import type { ConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/ConfigService";
import type { IConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
import type { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService";
import type { ReplicatorService } from "@vrtmrz/livesync-commonlib/compat/services/base/ReplicatorService";
import type { InjectableAPIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAPIService";
@@ -11,7 +11,7 @@ import { BrowserSvelteDialogManager } from "./BrowserSvelteDialogManager";
export interface LiveSyncBrowserUIServiceDependencies<T extends ServiceContext> {
API: InjectableAPIService<T>;
appLifecycle: AppLifecycleService<T>;
config: ConfigService<T>;
config: IConfigService;
control: ControlService<T>;
replicator: ReplicatorService<T>;
}
src/apps/browser/createLiveSyncBrowserServiceHub.ts
+10 -40
View File
@@ -1,11 +1,10 @@
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { KeyValueDatabaseFactory } from "@vrtmrz/livesync-commonlib/compat/interfaces/KeyValueDatabase";
import { PouchDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/pouchdb-browser";
import { ConfigService } from "@vrtmrz/livesync-commonlib/compat/services/base/ConfigService";
import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService";
import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService";
import { KeyValueDBService } from "@vrtmrz/livesync-commonlib/compat/services/base/KeyValueDBService";
import type { ISettingService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
import type { ValueDeleter, ValueStore } from "@vrtmrz/value-store";
import { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAppLifecycleService";
import { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService";
@@ -24,10 +23,7 @@ import { InjectableVaultServiceCompat } from "@vrtmrz/livesync-commonlib/compat/
import { setLang, translateLiveSyncMessage } from "@/common/translation";
import { BrowserConfirm } from "./BrowserConfirm";
import { createBrowserKeyValueDatabaseFactory } from "./BrowserKeyValueDatabase";
import {
LiveSyncBrowserAPIService,
type LiveSyncBrowserAPIServiceOptions,
} from "./LiveSyncBrowserAPIService";
import { LiveSyncBrowserAPIService, type LiveSyncBrowserAPIServiceOptions } from "./LiveSyncBrowserAPIService";
import { LiveSyncBrowserUIService } from "./LiveSyncBrowserUIService";
export interface LiveSyncBrowserSettingsPersistence {
@@ -43,6 +39,7 @@ export interface LiveSyncBrowserRestartPolicy {
}
export interface LiveSyncBrowserServiceHubOptions<T extends ServiceContext> {
deviceValues: ValueStore & ValueDeleter;
context?: T;
getSystemVaultName?: () => string;
settings?: LiveSyncBrowserSettingsPersistence;
@@ -51,35 +48,12 @@ export interface LiveSyncBrowserServiceHubOptions<T extends ServiceContext> {
API?: Omit<LiveSyncBrowserAPIServiceOptions, "confirm" | "getSystemVaultName">;
}
class LiveSyncBrowserAppLifecycleService<
T extends ServiceContext,
> extends InjectableAppLifecycleService<T> {}
class LiveSyncBrowserAppLifecycleService<T extends ServiceContext> extends InjectableAppLifecycleService<T> {}
class LiveSyncBrowserDatabaseService<T extends ServiceContext> extends DatabaseService<T> {}
class LiveSyncBrowserKeyValueDBService<T extends ServiceContext> extends KeyValueDBService<T> {}
class LiveSyncBrowserConfigService<T extends ServiceContext> extends ConfigService<T> {
constructor(
context: T,
private readonly setting: ISettingService
) {
super(context);
}
getSmallConfig(key: string): string | null {
return this.setting.getSmallConfig(key);
}
setSmallConfig(key: string, value: string): void {
this.setting.setSmallConfig(key, value);
}
deleteSmallConfig(key: string): void {
this.setting.deleteSmallConfig(key);
}
}
/** LiveSync-owned service composition shared by WebApp and WebPeer. */
export class LiveSyncBrowserServiceHub<T extends ServiceContext> extends InjectableServiceHub<T> {
constructor(options: LiveSyncBrowserServiceHubOptions<T>) {
@@ -98,17 +72,14 @@ export class LiveSyncBrowserServiceHub<T extends ServiceContext> extends Injecta
const setting = new InjectableSettingService(context, {
APIService: API,
onDisplayLanguageChanged: setLang,
deviceValues: options.deviceValues,
});
const settingsPersistence = options.settings;
setting.loadData.setHandler(
settingsPersistence
? () => settingsPersistence.load()
: () => Promise.resolve(undefined)
settingsPersistence ? () => settingsPersistence.load() : () => Promise.resolve(undefined)
);
setting.saveData.setHandler(
settingsPersistence
? (settings) => settingsPersistence.save(settings)
: () => Promise.resolve()
settingsPersistence ? (settings) => settingsPersistence.save(settings) : () => Promise.resolve()
);
const appLifecycle = new LiveSyncBrowserAppLifecycleService(context, {
@@ -142,7 +113,7 @@ export class LiveSyncBrowserServiceHub<T extends ServiceContext> extends Injecta
setting,
API,
});
const config = new LiveSyncBrowserConfigService(context, setting);
const config = setting.config;
const replicator = new InjectableReplicatorService(context, {
settingService: setting,
appLifecycleService: appLifecycle,
@@ -163,8 +134,7 @@ export class LiveSyncBrowserServiceHub<T extends ServiceContext> extends Injecta
databaseService: database,
});
const keyValueDB = new LiveSyncBrowserKeyValueDBService(context, {
openKeyValueDatabase:
options.openKeyValueDatabase ?? createBrowserKeyValueDatabaseFactory(),
openKeyValueDatabase: options.openKeyValueDatabase ?? createBrowserKeyValueDatabaseFactory(),
appLifecycle,
databaseEvents,
vault,
@@ -209,7 +179,7 @@ export class LiveSyncBrowserServiceHub<T extends ServiceContext> extends Injecta
}
export function createLiveSyncBrowserServiceHub<T extends ServiceContext>(
options: LiveSyncBrowserServiceHubOptions<T> = {}
options: LiveSyncBrowserServiceHubOptions<T>
): LiveSyncBrowserServiceHub<T> {
return new LiveSyncBrowserServiceHub(options);
}
src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts
+35 -8
View File
@@ -1,5 +1,6 @@
import { createServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { MemoryValueStore } from "@vrtmrz/value-store";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
@@ -21,7 +22,8 @@ describe("LiveSync browser service context contract", () => {
const context = createServiceContext({
translate: (key) => `webapp:${key}`,
});
const hub = createLiveSyncBrowserServiceHub({ context });
const deviceValues = new MemoryValueStore();
const hub = createLiveSyncBrowserServiceHub({ context, deviceValues });
expect(observeServiceContext(context, "moduleLocalDatabase.logWaitingForReady")).toEqual({
translation: "webapp:moduleLocalDatabase.logWaitingForReady",
@@ -32,20 +34,17 @@ describe("LiveSync browser service context contract", () => {
expect(SERVICE_CONTEXT_MEMBERS.filter((member) => !composition.servicesUsingExpectedContext[member])).toEqual(
[]
);
expect(hub.setting.deviceValues).toBe(deviceValues);
expect(hub.config).toBe(hub.setting.config);
});
it("binds browser identity, persistence, restart policy, and native Fetch while composing the hub", async () => {
const deviceLocalSettings = new Map<string, string>();
vi.stubGlobal("localStorage", {
getItem: (key: string) => deviceLocalSettings.get(key) ?? null,
setItem: (key: string, value: string) => deviceLocalSettings.set(key, value),
removeItem: (key: string) => deviceLocalSettings.delete(key),
clear: () => deviceLocalSettings.clear(),
});
const deviceValues = new MemoryValueStore();
const savedSettings: (typeof DEFAULT_SETTINGS)[] = [];
const restartCalls: string[] = [];
const nativeFetch = vi.fn(async () => new Response("ok"));
const options: LiveSyncBrowserServiceHubOptions<ReturnType<typeof createServiceContext>> = {
deviceValues,
getSystemVaultName: () => "browser-vault",
settings: {
load: async () => ({
@@ -69,6 +68,8 @@ describe("LiveSync browser service context contract", () => {
const hub = createLiveSyncBrowserServiceHub(options);
expect(hub.setting.deviceValues).toBe(deviceValues);
expect(hub.config).toBe(hub.setting.config);
expect(hub.API.getSystemVaultName()).toBe("browser-vault");
expect(hub.API.getPlatform()).toBe("browser");
const response = await hub.API.nativeFetch("https://example.invalid/");
@@ -86,4 +87,30 @@ describe("LiveSync browser service context contract", () => {
expect(hub.appLifecycle.isReloadingScheduled()).toBe(true);
expect(restartCalls).toEqual(["schedule", "perform", "ask:restart now"]);
});
it("keeps each browser composition bound to its own device value store", async () => {
const firstValues = new MemoryValueStore();
const secondValues = new MemoryValueStore();
const firstHub = createLiveSyncBrowserServiceHub({
deviceValues: firstValues,
getSystemVaultName: () => "first-vault",
});
const secondHub = createLiveSyncBrowserServiceHub({
deviceValues: secondValues,
getSystemVaultName: () => "second-vault",
});
await firstHub.config.setValue("probe", "first");
await secondHub.config.setValue("probe", "second");
expect(firstHub.setting.deviceValues).toBe(firstValues);
expect(secondHub.setting.deviceValues).toBe(secondValues);
expect(firstHub.config).toBe(firstHub.setting.config);
expect(secondHub.config).toBe(secondHub.setting.config);
expect(firstHub.config).not.toBe(secondHub.config);
await expect(firstHub.config.getValue("probe")).resolves.toBe("first");
await expect(secondHub.config.getValue("probe")).resolves.toBe("second");
await expect(firstValues.getValue("first-vault-probe")).resolves.toBe("first");
await expect(secondValues.getValue("second-vault-probe")).resolves.toBe("second");
});
});
src/apps/cli/README.md
+47 -5
View File
@@ -64,9 +64,10 @@ livesync-cli [database-path] [command] [args...]
- `database-path`: Path to the directory where `.livesync` folder and `settings.json` are (or will be) located.
- Note: In previous versions, this was referred to as the "vault" path. Now it is clearly distinguished from the actual vault (the directory containing your `.md` files).
- `--vault <path>` / `-V <path>`: (daemon/mirror only) Path to the vault directory containing `.md` files.
- `--vault <path>` / `-V <path>`: (daemon/mirror/rebuild/fetch) Path to the Vault directory containing `.md` files.
- Allows the PouchDB database directory and the actual vault directory to be different locations.
- For `mirror` command, the positional `[vault-path]` argument takes precedence over `--vault`.
- `--force` / `-f`: Required by `rebuild` and `fetch` because they replace a database. It also retains its existing `init-settings` overwrite behaviour.
- `--write-settings`: Write setting migrations and other lasting changes after the command succeeds.
- `init-settings` writes its target file. `setup`, `remote-add`, `remote-rm`, `remote-set`, and `remote-activate` write their settings changes without this option.
- All remaining commands leave the settings file unchanged by default.
@@ -82,6 +83,9 @@ livesync-cli [database-path] [command] [args...]
- If `vault-path` is provided, the CLI will synchronise the database with files in the vault directory.
- If `vault-path` is omitted, it defaults to `database-path` (compatibility mode).
- Use this command to keep your local `.md` files in sync with the database.
- `rebuild`: Rebuild the local and remote databases from the selected Vault. This replaces existing local and remote database state and requires `--force`.
- This command may select a new local database suffix. Add `--write-settings` if later CLI commands should use the resulting database selection; otherwise the existing settings file remains unchanged.
- `fetch`: Replace the local database with the configured remote state, then reflect fetched files to the selected Vault. Existing local database state is discarded, so this command requires `--force`.
- `ls [prefix]`: List files currently stored in the local database.
- `push <src> <dst>`: Push a local file `<src>` into the database at path `<dst>`.
- `pull <src> <dst>`: Pull a file `<src>` from the database into local file `<dst>`.
@@ -110,6 +114,12 @@ livesync-cli ./my-db sync
# Mirroring to your actual Obsidian vault
livesync-cli ./my-db mirror /path/to/obsidian-vault
# Rebuild local and remote databases from an authoritative Vault
livesync-cli ./my-db --vault /path/to/obsidian-vault rebuild --force --write-settings
# Replace the local database and Vault files from the configured remote
livesync-cli ./my-db --vault /path/to/obsidian-vault fetch --force --write-settings
# Manual file operations
livesync-cli ./my-db push ./note.md folder/note.md
livesync-cli ./my-db pull folder/note.md ./note.md
@@ -369,6 +379,36 @@ The CLI uses the same settings format as the Obsidian plug-in. Create a `.livesy
- `couchDB_DBNAME`: Database name
- `isConfigured`: Set to `true` after configuration
#### Unicode path identity
The `pathNormalizationVersion` setting controls how file paths identify
Metadata documents when using CouchDB. An absent value or `0` keeps the legacy
identity. Version `1` uses Unicode Normalization Form C (NFC) for Metadata IDs
and path comparison keys. It does not rewrite `Metadata.path`, rename existing
files, or change Chunks or file content. `handleFilenameCaseSensitive` remains
an independent case-handling setting; the CLI resolves NFC-equivalent names
even when case-sensitive handling is enabled.
When the CLI resolves an existing path, it uses the actual spelling of each
matching file or directory for reads, writes, deletes, `stat` calls, directory
creation, and renames. If a path component is missing, it uses the actual
existing parent and preserves the spelling received for the new component. If
a file or any parent directory has multiple candidates for the same logical
key after NFC and the existing case rule, the CLI warns and refuses the
operation. It does not choose a candidate based on directory order, merge
entries, or rename them.
Version `1` requires a coordinated Rebuild and Fetch; changing the setting by
itself does not migrate a database. Update every participating client, stop
synchronisation, and back up a complete authoritative Vault. Set
`pathNormalizationVersion` to `1` on the source client, then run `rebuild`
with `--force --write-settings`. Set the same value on each remaining CLI
client, then run `fetch` with `--force --write-settings`. Use **Overwrite
Server Data with This Device's Files** and **Reset Synchronisation on This
Device** for the corresponding Obsidian Rebuild and Fetch operations. Resolve
any ambiguous names before reconstruction. Version `1` is supported with
CouchDB only; do not resume a legacy client against the rebuilt database.
### Command-line Reference
```
@@ -381,11 +421,11 @@ Arguments:
Options:
--settings, -s <path> Path to settings file (default: .livesync/settings.json in local database directory)
--force, -f Overwrite existing file on init-settings
--force, -f Required by rebuild/fetch; overwrite existing file on init-settings
--verbose, -v Enable verbose logging
--debug, -d Enable debug logging (includes verbose)
--interval <N>, -i <N> (daemon only) Poll CouchDB every N seconds instead of using the _changes feed
--vault <path>, -V <path> (daemon/mirror) Path to vault directory, decoupled from database-path
--vault <path>, -V <path> (daemon/mirror/rebuild/fetch) Path to vault directory, decoupled from database-path
--write-settings Write setting changes after a successful command
--compat-remote-admin-exit-zero
Preserve the former zero exit code when remote-administration verification fails
@@ -398,6 +438,8 @@ Commands:
p2p-peers <timeout> Show discovered peers as [peer]<TAB><peer-id><TAB><peer-name>
p2p-sync <peer> <timeout> Synchronise with specified peer-id or peer-name
p2p-host Start P2P host mode and wait until interrupted (Ctrl+C)
rebuild Rebuild local and remote databases from the selected Vault (requires --force)
fetch Replace the local database with remote state and reflect it to the Vault (requires --force)
push <src> <dst> Push local file <src> into local database path <dst>
pull <src> <dst> Pull file <src> from local database into local file <dst>
pull-rev <src> <dst> <rev> Pull specific revision <rev> into local file <dst>
@@ -677,8 +719,8 @@ src/apps/cli/
│ └── ServiceFileAccessImpl.ts
├── services/
│ ├── NodeKeyValueDBService.ts
│ ├── NodeServiceHub.ts
│ └── NodeSettingService.ts
│ ├── NodeLocalStorage.ts
│ └── NodeServiceHub.ts
├── test/
│ ├── test-e2e-two-vaults-common.sh
│ ├── test-e2e-two-vaults-matrix.sh
src/apps/cli/adapters/NodeFileSystemAdapter.ts
+56 -15
View File
@@ -3,7 +3,7 @@ import type { IFileSystemAdapter } from "@vrtmrz/livesync-commonlib/compat/servi
import { NodePathAdapter } from "./NodePathAdapter";
import { NodeTypeGuardAdapter } from "./NodeTypeGuardAdapter";
import { NodeConversionAdapter } from "./NodeConversionAdapter";
import { NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node";
import { AmbiguousNormalisedPathError, UnicodeResolvingNodeStorageAdapter } from "./UnicodeResolvingNodeStorageAdapter";
import { NodeVaultAdapter } from "./NodeVaultAdapter";
import type { NodeFile, NodeFolder, NodeStat } from "./NodeTypes";
import { path } from "@vrtmrz/livesync-commonlib/node";
@@ -16,19 +16,21 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
readonly path: NodePathAdapter;
readonly typeGuard: NodeTypeGuardAdapter;
readonly conversion: NodeConversionAdapter;
readonly storage: NodeStorageAdapter;
readonly storage: UnicodeResolvingNodeStorageAdapter;
readonly vault: NodeVaultAdapter;
private fileCache = new Map<string, NodeFile>();
constructor(
private basePath: string,
private reportDiagnostic: CliDiagnosticReporter = () => undefined
private reportDiagnostic: CliDiagnosticReporter = () => undefined,
getPathNormalizationVersion: () => number = () => 0,
isCaseInsensitive: () => boolean = () => false
) {
this.path = new NodePathAdapter();
this.typeGuard = new NodeTypeGuardAdapter();
this.conversion = new NodeConversionAdapter();
this.storage = new NodeStorageAdapter(basePath);
this.storage = new UnicodeResolvingNodeStorageAdapter(basePath, getPathNormalizationVersion, isCaseInsensitive);
this.vault = new NodeVaultAdapter(this.storage);
}
@@ -58,6 +60,14 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
async getAbstractFileByPath(p: FilePath | string): Promise<NodeFile | null> {
const pathStr = this.normalisePath(p);
if (this.storage.isUnicodePathNormalizationEnabled) {
const resolvedPath = await this.storage.resolveLogicalPath(pathStr, false, false);
if (resolvedPath === null) {
this.fileCache.delete(pathStr);
return null;
}
return await this.refreshResolvedFile(resolvedPath);
}
if (!this.fileCache.has(pathStr) && !(await this.hasExactPathCase(pathStr))) {
this.fileCache.delete(pathStr);
return null;
@@ -68,6 +78,15 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
async getAbstractFileByPathInsensitive(p: FilePath | string): Promise<NodeFile | null> {
const pathStr = this.normalisePath(p);
if (this.storage.isUnicodePathNormalizationEnabled) {
const resolvedPath = await this.storage.resolveLogicalPath(pathStr, true, false);
if (resolvedPath === null) {
this.fileCache.delete(pathStr);
return null;
}
return await this.refreshResolvedFile(resolvedPath);
}
const exact = await this.getAbstractFileByPath(pathStr);
if (exact) {
return exact;
@@ -115,17 +134,34 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
// This is used by Obsidian to sync internal file metadata
}
async refreshFile(p: string): Promise<NodeFile | null> {
async refreshFile(p: string, caseInsensitive = false): Promise<NodeFile | null> {
const pathStr = this.normalisePath(p);
try {
const stat = await this.storage.stat(pathStr);
if (stat?.type !== "file") {
const resolvedPath = this.storage.isUnicodePathNormalizationEnabled
? await this.storage.resolveLogicalPath(pathStr, caseInsensitive, false)
: pathStr;
if (resolvedPath === null) {
this.fileCache.delete(pathStr);
return null;
}
return await this.refreshResolvedFile(resolvedPath);
} catch (error) {
if (error instanceof AmbiguousNormalisedPathError) throw error;
this.fileCache.delete(pathStr);
return null;
}
}
private async refreshResolvedFile(resolvedPath: string): Promise<NodeFile | null> {
try {
const stat = await this.storage.statExact(resolvedPath);
if (stat?.type !== "file") {
this.fileCache.delete(resolvedPath);
return null;
}
const file: NodeFile = {
path: pathStr as FilePath,
path: resolvedPath as FilePath,
stat: {
size: stat.size,
mtime: stat.mtime,
@@ -133,11 +169,12 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
type: "file",
},
};
this.fileCache.set(pathStr, file);
this.fileCache.set(resolvedPath, file);
return file;
} catch {
} catch (error) {
if (error instanceof AmbiguousNormalisedPathError) throw error;
// Evict so a deleted file is not returned by subsequent cache scans.
this.fileCache.delete(pathStr);
this.fileCache.delete(resolvedPath);
return null;
}
}
@@ -146,18 +183,21 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
* Helper method to recursively scan directory and populate file cache
*/
async scanDirectory(relativePath: string = ""): Promise<void> {
await this.scanDirectoryInto(relativePath, this.fileCache);
const resolvedPath = await this.storage.resolveLogicalPath(relativePath, false, false);
if (resolvedPath !== null) await this.scanDirectoryInto(resolvedPath, this.fileCache);
}
private async scanDirectoryInto(relativePath: string, files: Map<string, NodeFile>): Promise<void> {
const fullPath = this.resolvePath(relativePath);
try {
const directoryStat = await this.storage.stat(relativePath);
const directoryStat = this.storage.isUnicodePathNormalizationEnabled
? await this.storage.statExact(relativePath)
: await this.storage.stat(relativePath, false);
if (directoryStat?.type !== "folder") throw new Error(`Directory does not exist: ${fullPath}`);
const entries = await this.storage.list(relativePath);
const entries = await this.storage.listExact(relativePath);
for (const entryPath of entries.files) {
const stat = await this.storage.stat(entryPath);
const stat = await this.storage.statExact(entryPath);
if (stat?.type !== "file") continue;
const file: NodeFile = {
path: entryPath as FilePath,
@@ -169,6 +209,7 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
await this.scanDirectoryInto(entryPath, files);
}
} catch (error) {
if (error instanceof AmbiguousNormalisedPathError) throw error;
// Directory doesn't exist or is not readable
this.reportDiagnostic(`Error scanning directory ${fullPath}:`, error);
}
src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts
+387 -1
View File
@@ -1,5 +1,12 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { fsPromises as fs, os, path } from "@vrtmrz/livesync-commonlib/node";
import { fsPromises as fs, NodeStorageAdapter, os, path } from "@vrtmrz/livesync-commonlib/node";
import { DEFAULT_SETTINGS, type FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
import {
ServiceRebuilder,
type ServiceRebuilderDependencies,
} from "@vrtmrz/livesync-commonlib/compat/serviceModules/Rebuilder";
import { PathServiceCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectablePathService";
import { ServiceContext } from "@vrtmrz/livesync-commonlib/compat/services/base/ServiceBase";
import { NodeFileSystemAdapter } from "./NodeFileSystemAdapter";
describe("NodeFileSystemAdapter file enumeration", () => {
@@ -116,3 +123,382 @@ describe("NodeFileSystemAdapter file enumeration", () => {
await expect(adapter.getFiles()).resolves.toEqual([]);
});
});
describe("NodeFileSystemAdapter Unicode path resolution", () => {
const tempDirs: string[] = [];
async function createVault() {
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-cli-unicode-"));
tempDirs.push(directory);
return directory;
}
function createNormalisingAdapter(directory: string, caseInsensitive = false) {
return new NodeFileSystemAdapter(
directory,
undefined,
() => 1,
() => caseInsensitive
);
}
afterEach(async () => {
await Promise.all(tempDirs.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true })));
});
it.each([0, 1])("keeps an NFD-only file in the storage listing in identity mode %s", async (version) => {
const directory = await createVault();
const originalPath = "café.md".normalize("NFD");
await fs.writeFile(path.join(directory, originalPath), "original");
const report = vi.fn();
const adapter = new NodeFileSystemAdapter(directory, report, () => version);
expect((await adapter.getFiles()).map((file) => file.path)).toEqual([originalPath]);
expect(report).not.toHaveBeenCalled();
await expect(fs.readFile(path.join(directory, originalPath), "utf8")).resolves.toBe("original");
expect(await fs.readdir(directory)).toEqual([originalPath]);
});
it.each([false, true])(
"enumerates nested mixed names once per directory with case-insensitive matching %s",
async (caseInsensitive) => {
const directory = await createVault();
const parent = "café".normalize("NFD");
const child = `${parent}/Étage`;
await fs.mkdir(path.join(directory, child), { recursive: true });
const names = Array.from(
{ length: 12 },
(_, index) =>
`${child}/${index}-résumé.md`
.normalize(index % 2 ? "NFC" : "NFD")
.split("/")
.pop()!
);
for (const name of names) await fs.writeFile(path.join(directory, child, name), "content");
const adapter = createNormalisingAdapter(directory, caseInsensitive);
const list = vi.spyOn(NodeStorageAdapter.prototype, "list");
try {
const files = await adapter.getFiles();
expect(files.map((file) => file.path).sort()).toEqual(names.map((name) => `${child}/${name}`).sort());
expect(list).toHaveBeenCalledTimes(3);
} finally {
list.mockRestore();
}
}
);
it("uses the existing mixed parent during directory preparation and preserves the new leaf spelling", async () => {
const directory = await createVault();
const parent = "café".normalize("NFD");
const child = `${parent}/Étage`;
await fs.mkdir(path.join(directory, child), { recursive: true });
const adapter = createNormalisingAdapter(directory);
await adapter.storage.mkdir("café");
await adapter.storage.mkdir("café/Étage");
const newName = "résumé.md".normalize("NFD");
const file = await adapter.vault.create(`café/Étage/${newName}`, "created");
expect(file.path).toBe(`${child}/${newName}`);
expect(await fs.readdir(directory)).toEqual([parent]);
expect(await fs.readdir(path.join(directory, parent))).toEqual(["Étage"]);
expect(await fs.readdir(path.join(directory, child))).toEqual([newName]);
await expect(adapter.storage.stat(`café/Étage/${newName.normalize("NFC")}`)).resolves.toMatchObject({
size: 7,
});
});
it("rejects an NFC-equivalent file and directory before scanning or writing a child", async () => {
const directory = await createVault();
const composed = "café";
const decomposed = composed.normalize("NFD");
await fs.writeFile(path.join(directory, composed), "file");
await fs.mkdir(path.join(directory, decomposed));
const adapter = createNormalisingAdapter(directory);
await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i);
await expect(adapter.storage.write(`${composed}/new.md`, "new")).rejects.toThrow(/ambiguous/i);
await expect(fs.readFile(path.join(directory, composed), "utf8")).resolves.toBe("file");
expect(await fs.readdir(path.join(directory, decomposed))).toEqual([]);
});
it.each([false, true])(
"enumerates each directory once for a flat NFC scan with case-insensitive matching %s",
async (caseInsensitive) => {
const directory = await createVault();
for (let index = 0; index < 12; index++) {
await fs.writeFile(path.join(directory, `${index}.md`), "content");
}
const adapter = createNormalisingAdapter(directory, caseInsensitive);
const list = vi.spyOn(NodeStorageAdapter.prototype, "list");
try {
expect(await adapter.getFiles()).toHaveLength(12);
expect(list).toHaveBeenCalledTimes(1);
} finally {
list.mockRestore();
}
}
);
it("reads, updates, and deletes an NFD file through its NFC path", async () => {
const directory = await createVault();
const nfcFolder = "café";
const nfdFolder = nfcFolder.normalize("NFD");
const nfcName = "Ångström.md";
const nfdName = nfcName.normalize("NFD");
await fs.mkdir(path.join(directory, nfdFolder));
await fs.writeFile(path.join(directory, nfdFolder, nfdName), "before", "utf8");
const adapter = createNormalisingAdapter(directory);
const file = await adapter.getAbstractFileByPath(`${nfcFolder}/${nfcName}`);
expect(file?.path).toBe(`${nfdFolder}/${nfdName}`);
if (!file) throw new Error("Expected the NFC path to resolve to the NFD file");
await expect(adapter.vault.read(file)).resolves.toBe("before");
await adapter.vault.modify(file, "after");
await expect(fs.readFile(path.join(directory, nfdFolder, nfdName), "utf8")).resolves.toBe("after");
expect(await fs.readdir(directory)).toEqual([nfdFolder]);
expect(await fs.readdir(path.join(directory, nfdFolder))).toEqual([nfdName]);
await adapter.vault.delete(file);
await expect(fs.readdir(path.join(directory, nfdFolder))).resolves.toEqual([]);
});
it("creates a new file inside an existing NFD directory using that directory spelling", async () => {
const directory = await createVault();
const nfdFolder = "café".normalize("NFD");
await fs.mkdir(path.join(directory, nfdFolder));
const adapter = createNormalisingAdapter(directory);
const file = await adapter.vault.create("café/new.md", "content");
expect(file.path).toBe(`${nfdFolder}/new.md`);
await expect(fs.readFile(path.join(directory, nfdFolder, "new.md"), "utf8")).resolves.toBe("content");
expect(await fs.readdir(directory)).toEqual([nfdFolder]);
});
it("renames a file by case inside an NFD directory while preserving that directory spelling", async () => {
const directory = await createVault();
const nfcFolder = "Caf\u00e9";
const nfdFolder = nfcFolder.normalize("NFD");
await fs.mkdir(path.join(directory, nfdFolder));
await fs.writeFile(path.join(directory, nfdFolder, "Note.md"), "content", "utf8");
const adapter = createNormalisingAdapter(directory, true);
const file = await adapter.getAbstractFileByPathInsensitive(`${nfcFolder}/Note.md`);
if (!file) throw new Error("Expected the NFC path to resolve to the NFD file");
const renamedFile = await adapter.renameFile(file, `${nfcFolder}/note.md`);
expect(renamedFile.path).toBe(`${nfdFolder}/note.md`);
expect(await fs.readdir(directory)).toEqual([nfdFolder]);
await expect(fs.readdir(path.join(directory, nfdFolder))).resolves.toEqual(["note.md"]);
await expect(fs.readFile(path.join(directory, nfdFolder, "note.md"), "utf8")).resolves.toBe("content");
});
it("rejects an NFC/NFD file collision for lookup and vault mutations", async () => {
const directory = await createVault();
const nfcName = "café.md";
const nfdName = nfcName.normalize("NFD");
await fs.writeFile(path.join(directory, nfcName), "NFC", "utf8");
await fs.writeFile(path.join(directory, nfdName), "NFD", "utf8");
const adapter = createNormalisingAdapter(directory);
const exactFile = {
path: nfcName as FilePath,
stat: { ctime: 1, mtime: 2, size: 3, type: "file" as const },
};
await expect(adapter.getAbstractFileByPath(nfcName)).rejects.toThrow(/ambiguous/i);
await expect(adapter.vault.modify(exactFile, "changed")).rejects.toThrow(/ambiguous/i);
await expect(adapter.vault.delete(exactFile)).rejects.toThrow(/ambiguous/i);
await expect(fs.readFile(path.join(directory, nfcName), "utf8")).resolves.toBe("NFC");
await expect(fs.readFile(path.join(directory, nfdName), "utf8")).resolves.toBe("NFD");
expect((await fs.readdir(directory)).sort()).toEqual([nfcName, nfdName].sort());
});
it("rejects an NFC/NFD directory collision before creating a child", async () => {
const directory = await createVault();
const nfcFolder = "café";
const nfdFolder = nfcFolder.normalize("NFD");
await fs.mkdir(path.join(directory, nfcFolder));
await fs.mkdir(path.join(directory, nfdFolder));
await fs.writeFile(path.join(directory, nfcFolder, "inside.md"), "NFC", "utf8");
await fs.writeFile(path.join(directory, nfdFolder, "inside.md"), "NFD", "utf8");
const adapter = createNormalisingAdapter(directory);
await expect(adapter.getAbstractFileByPath(`${nfcFolder}/inside.md`)).rejects.toThrow(/ambiguous/i);
await expect(adapter.storage.write(`${nfcFolder}/created.md`, "created")).rejects.toThrow(/ambiguous/i);
await expect(fs.readdir(path.join(directory, nfcFolder))).resolves.toEqual(["inside.md"]);
await expect(fs.readdir(path.join(directory, nfdFolder))).resolves.toEqual(["inside.md"]);
await expect(fs.readFile(path.join(directory, nfcFolder, "inside.md"), "utf8")).resolves.toBe("NFC");
await expect(fs.readFile(path.join(directory, nfdFolder, "inside.md"), "utf8")).resolves.toBe("NFD");
});
it("rejects enumeration when NFC-equivalent directories contain different files", async () => {
const directory = await createVault();
const nfcFolder = "café";
const nfdFolder = nfcFolder.normalize("NFD");
await fs.mkdir(path.join(directory, nfcFolder));
await fs.mkdir(path.join(directory, nfdFolder));
await fs.writeFile(path.join(directory, nfcFolder, "a.md"), "NFC", "utf8");
await fs.writeFile(path.join(directory, nfdFolder, "b.md"), "NFD", "utf8");
const adapter = createNormalisingAdapter(directory);
await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i);
await expect(adapter.scanDirectory()).rejects.toThrow(/ambiguous/i);
await expect(fs.readFile(path.join(directory, nfcFolder, "a.md"), "utf8")).resolves.toBe("NFC");
await expect(fs.readFile(path.join(directory, nfdFolder, "b.md"), "utf8")).resolves.toBe("NFD");
});
it("rechecks the directory after a unique file is cached", async () => {
const directory = await createVault();
const nfcName = "café.md";
const nfdName = nfcName.normalize("NFD");
await fs.writeFile(path.join(directory, nfdName), "NFD", "utf8");
const adapter = createNormalisingAdapter(directory);
await expect(adapter.getAbstractFileByPath(nfcName)).resolves.toMatchObject({ path: nfdName });
await fs.writeFile(path.join(directory, nfcName), "NFC", "utf8");
await expect(adapter.getAbstractFileByPath(nfcName)).rejects.toThrow(/ambiguous/i);
await expect(fs.readFile(path.join(directory, nfcName), "utf8")).resolves.toBe("NFC");
await expect(fs.readFile(path.join(directory, nfdName), "utf8")).resolves.toBe("NFD");
});
it.each([
["Foo", "foo"],
["Café", "café".normalize("NFD")],
])("rejects case-insensitive parent aliases %s and %s containing different files", async (first, second) => {
const directory = await createVault();
for (const [parent, name] of [
[first, "one.md"],
[second, "two.md"],
]) {
await fs.mkdir(path.join(directory, parent));
await fs.writeFile(path.join(directory, parent, name), name);
}
const adapter = createNormalisingAdapter(directory, true);
await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i);
await expect(adapter.scanDirectory()).rejects.toThrow(/ambiguous/i);
await expect(fs.readFile(path.join(directory, first, "one.md"), "utf8")).resolves.toBe("one.md");
await expect(fs.readFile(path.join(directory, second, "two.md"), "utf8")).resolves.toBe("two.md");
});
it.each([
{ version: 0, caseInsensitive: true },
{ version: 1, caseInsensitive: false },
])(
"compatibility: retains distinct case paths in mode $version with case-insensitive matching $caseInsensitive",
async ({ version, caseInsensitive }) => {
const directory = await createVault();
const names = ["Café/one.md", "café/two.md".normalize("NFD")];
for (const name of names) {
await fs.mkdir(path.dirname(path.join(directory, name)));
await fs.writeFile(path.join(directory, name), name);
}
const adapter = new NodeFileSystemAdapter(
directory,
undefined,
() => version,
() => caseInsensitive
);
expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual([...names].sort());
for (const name of names) await expect(adapter.storage.read(name)).resolves.toBe(name);
}
);
it("refuses reconstruction before starting remote activity when parent aliases are ambiguous", async () => {
const directory = await createVault();
for (const name of ["Café/one.md", "café/two.md".normalize("NFD")]) {
await fs.mkdir(path.dirname(path.join(directory, name)));
await fs.writeFile(path.join(directory, name), name);
}
const adapter = createNormalisingAdapter(directory, true);
const settings = {
...DEFAULT_SETTINGS,
pathNormalizationVersion: 1 as const,
handleFilenameCaseSensitive: false,
};
const setting = { currentSettings: () => settings, suspendAllSync: { addHandler: vi.fn() } };
const runBoundedRemoteActivity = vi.fn(async () => undefined);
const rebuilder = new ServiceRebuilder({
setting,
storageAccess: { getFileNames: async () => (await adapter.getFiles()).map((file) => file.path) },
database: {
onDatabaseReset: { addHandler: vi.fn() },
isDatabaseReady: () => true,
localDatabase: { findAllDocs: async function* () {} },
},
vault: { isTargetFile: async () => true },
path: new PathServiceCompat(new ServiceContext(), { settingService: setting as never }),
replicator: { runBoundedRemoteActivity },
UI: { showMarkdownDialog: vi.fn(async () => undefined) },
} as unknown as ServiceRebuilderDependencies);
await expect(rebuilder.$rebuildEverything()).rejects.toThrow(/ambiguous/i);
expect(runBoundedRemoteActivity).not.toHaveBeenCalled();
});
it("uses fresh listings after external additions, renames, deletions, and collisions", async () => {
const directory = await createVault();
const nfdName = "café.md".normalize("NFD");
await fs.writeFile(path.join(directory, nfdName), "original");
const adapter = createNormalisingAdapter(directory);
expect((await adapter.getFiles()).map((file) => file.path)).toEqual([nfdName]);
const renamed = "résumé.md".normalize("NFD");
await fs.rename(path.join(directory, nfdName), path.join(directory, renamed));
await fs.writeFile(path.join(directory, "new.md"), "new");
expect((await adapter.getFiles()).map((file) => file.path).sort()).toEqual(["new.md", renamed]);
await fs.rm(path.join(directory, "new.md"));
expect((await adapter.getFiles()).map((file) => file.path)).toEqual([renamed]);
await fs.writeFile(path.join(directory, renamed.normalize("NFC")), "collision");
await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i);
});
it("retains the storage boundary for exact metadata reads and directory scans", async () => {
const directory = await createVault();
const outside = await createVault();
await fs.writeFile(path.join(outside, "note.md"), "outside");
await fs.symlink(outside, path.join(directory, "linked"), "dir");
const adapter = createNormalisingAdapter(directory);
await expect(adapter.getFiles()).resolves.toEqual([]);
await expect(adapter.storage.statExact("linked/note.md")).rejects.toThrow(/symbolic links/i);
await expect(adapter.storage.write("linked/note.md", "changed")).rejects.toThrow(/symbolic links/i);
await expect(fs.readFile(path.join(outside, "note.md"), "utf8")).resolves.toBe("outside");
});
it("detects NFC and case collisions when lookup is case-insensitive", async () => {
const directory = await createVault();
const upperName = "Café.md";
const lowerNfdName = "café.md".normalize("NFD");
await fs.writeFile(path.join(directory, upperName), "upper", "utf8");
await fs.writeFile(path.join(directory, lowerNfdName), "lower", "utf8");
const adapter = createNormalisingAdapter(directory, true);
await expect(adapter.getAbstractFileByPathInsensitive("CAFÉ.md")).rejects.toThrow(/ambiguous/i);
await expect(adapter.getFiles()).rejects.toThrow(/ambiguous/i);
await expect(adapter.scanDirectory()).rejects.toThrow(/ambiguous/i);
await expect(fs.readFile(path.join(directory, upperName), "utf8")).resolves.toBe("upper");
await expect(fs.readFile(path.join(directory, lowerNfdName), "utf8")).resolves.toBe("lower");
});
it("retains exact-path behaviour when path normalisation is disabled", async () => {
const directory = await createVault();
const nfcName = "café.md";
const nfdName = nfcName.normalize("NFD");
await fs.writeFile(path.join(directory, nfdName), "NFD", "utf8");
const adapter = new NodeFileSystemAdapter(directory);
await expect(adapter.getAbstractFileByPath(nfcName)).resolves.toBeNull();
await adapter.storage.write(nfcName, "NFC");
await expect(fs.readFile(path.join(directory, nfcName), "utf8")).resolves.toBe("NFC");
await expect(fs.readFile(path.join(directory, nfdName), "utf8")).resolves.toBe("NFD");
});
});
src/apps/cli/adapters/NodeVaultAdapter.ts
+11 -2
View File
@@ -2,6 +2,7 @@ import type { FilePath, UXDataWriteOptions } from "@vrtmrz/livesync-commonlib/co
import type { IVaultAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
import type { NodeFile, NodeFolder } from "./NodeTypes";
import { NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node";
import { UnicodeResolvingNodeStorageAdapter } from "./UnicodeResolvingNodeStorageAdapter";
/**
* Vault adapter implementation for Node.js
@@ -73,8 +74,16 @@ export class NodeVaultAdapter implements IVaultAdapter<NodeFile> {
}
private async toNodeFile(path: string): Promise<NodeFile> {
const stat = await this.storage.stat(path);
const resolvedPath =
this.storage instanceof UnicodeResolvingNodeStorageAdapter
? await this.storage.resolveLogicalPath(path, undefined, false)
: path;
if (resolvedPath === null) throw new Error(`Could not read created file metadata: ${path}`);
const stat =
this.storage instanceof UnicodeResolvingNodeStorageAdapter
? await this.storage.statExact(resolvedPath)
: await this.storage.stat(resolvedPath);
if (stat?.type !== "file") throw new Error(`Could not read created file metadata: ${path}`);
return { path: path as FilePath, stat };
return { path: resolvedPath as FilePath, stat };
}
}
src/apps/cli/adapters/UnicodeResolvingNodeStorageAdapter.ts
+205
View File
@@ -0,0 +1,205 @@
import { NodeStorageAdapter, validateStoragePath } from "@vrtmrz/livesync-commonlib/node";
export class AmbiguousNormalisedPathError extends Error {
constructor(path: string) {
super(`Ambiguous path after Unicode normalisation: ${path}`);
this.name = "AmbiguousNormalisedPathError";
}
}
/** Resolves NFC-equivalent logical paths against current Node storage listings. */
export class UnicodeResolvingNodeStorageAdapter extends NodeStorageAdapter {
constructor(
rootPath: string,
private readonly getPathNormalizationVersion: () => number = () => 0,
private readonly isCaseInsensitive: () => boolean = () => false
) {
super(rootPath);
}
get isUnicodePathNormalizationEnabled(): boolean {
return this.getPathNormalizationVersion() === 1;
}
/** Read metadata for an entry already resolved or obtained from a storage listing. */
async statExact(path: string) {
return await super.trystat(path);
}
/** Inspect a physical directory once before descending into its enumerated entries. */
async listExact(path: string) {
const entries = await super.list(path);
if (this.isUnicodePathNormalizationEnabled) {
const caseInsensitive = this.isCaseInsensitive();
const names = new Map<string, string>();
for (const entry of [...entries.files, ...entries.folders]) {
const key = this.comparisonKey(entry, caseInsensitive);
if (names.has(key)) throw new AmbiguousNormalisedPathError(entry);
names.set(key, entry);
}
}
return entries;
}
async resolveLogicalPath(
path: string,
caseInsensitive = this.isCaseInsensitive(),
allowMissing = true
): Promise<string | null> {
if (!this.isUnicodePathNormalizationEnabled) return path;
validateStoragePath(path, true);
if (path === "") return "";
const segments = path.split("/");
let resolvedParent = "";
for (let index = 0; index < segments.length; index++) {
const segment = segments[index];
const entries = await super.list(resolvedParent);
const candidates = [...entries.files, ...entries.folders];
const requestedKey = this.comparisonKey(segment, caseInsensitive);
const matches = candidates.filter((entryPath) => {
const entryName = entryPath.slice(resolvedParent === "" ? 0 : resolvedParent.length + 1);
return this.comparisonKey(entryName, caseInsensitive) === requestedKey;
});
if (matches.length > 1) throw new AmbiguousNormalisedPathError(path);
if (matches.length === 1) {
resolvedParent = matches[0];
continue;
}
if (!allowMissing) return null;
const remainingPath = segments.slice(index).join("/");
return resolvedParent === "" ? remainingPath : `${resolvedParent}/${remainingPath}`;
}
return resolvedParent;
}
override async exists(path: string, caseInsensitive = this.isCaseInsensitive()): Promise<boolean> {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false);
if (resolvedPath === null) return false;
return await super.exists(resolvedPath);
}
override async trystat(path: string, caseInsensitive = this.isCaseInsensitive()) {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false);
if (resolvedPath === null) return null;
return await super.trystat(resolvedPath);
}
override async stat(path: string, caseInsensitive = this.isCaseInsensitive()) {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false);
if (resolvedPath === null) return null;
return await super.trystat(resolvedPath);
}
override async mkdir(path: string, caseInsensitive = this.isCaseInsensitive()): Promise<void> {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, true);
if (resolvedPath === null) throw new Error(`Could not resolve path: ${path}`);
await super.mkdir(resolvedPath);
}
override async remove(path: string, caseInsensitive = this.isCaseInsensitive()): Promise<void> {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false);
await super.remove(resolvedPath ?? path);
}
override async read(path: string, caseInsensitive = this.isCaseInsensitive()): Promise<string> {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false);
return await super.read(resolvedPath ?? path);
}
override async readBinary(path: string, caseInsensitive = this.isCaseInsensitive()): Promise<ArrayBuffer> {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, false);
return await super.readBinary(resolvedPath ?? path);
}
override async write(
path: string,
data: string,
options?: Parameters<NodeStorageAdapter["write"]>[2],
caseInsensitive = this.isCaseInsensitive()
): Promise<void> {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, true);
if (resolvedPath === null) throw new Error(`Could not resolve path: ${path}`);
await super.write(resolvedPath, data, options);
}
override async writeBinary(
path: string,
data: ArrayBuffer,
options?: Parameters<NodeStorageAdapter["writeBinary"]>[2],
caseInsensitive = this.isCaseInsensitive()
): Promise<void> {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, true);
if (resolvedPath === null) throw new Error(`Could not resolve path: ${path}`);
await super.writeBinary(resolvedPath, data, options);
}
override async append(
path: string,
data: string,
options?: Parameters<NodeStorageAdapter["append"]>[2],
caseInsensitive = this.isCaseInsensitive()
): Promise<void> {
const resolvedPath = await this.resolveLogicalPath(path, caseInsensitive, true);
if (resolvedPath === null) throw new Error(`Could not resolve path: ${path}`);
await super.append(resolvedPath, data, options);
}
override async rename(
sourcePath: string,
targetPath: string,
caseInsensitive = this.isCaseInsensitive()
): Promise<void> {
const resolvedSource = await this.resolveLogicalPath(sourcePath, caseInsensitive, false);
if (resolvedSource === null) return await super.rename(sourcePath, targetPath);
const resolvedTarget = await this.resolveLogicalPath(targetPath, caseInsensitive, true);
if (resolvedTarget === null) throw new Error(`Could not resolve path: ${targetPath}`);
// Preserve explicit case-only or Unicode-spelling renames when the target resolves to the source.
const sourceKey = this.comparisonKey(resolvedSource, caseInsensitive);
const targetKey = this.comparisonKey(resolvedTarget, caseInsensitive);
const target =
sourceKey === targetKey && resolvedTarget === resolvedSource && targetPath !== resolvedSource
? await this.resolveRenameTargetSpelling(targetPath, caseInsensitive)
: resolvedTarget;
await super.rename(resolvedSource, target);
}
override async list(basePath: string, caseInsensitive = this.isCaseInsensitive()) {
const resolvedPath = await this.resolveLogicalPath(basePath, caseInsensitive, false);
if (resolvedPath === null) return { files: [], folders: [] };
return await super.list(resolvedPath);
}
private async resolveRenameTargetSpelling(path: string, caseInsensitive: boolean): Promise<string> {
validateStoragePath(path, false);
const segments = path.split("/");
const leaf = segments.pop();
if (leaf === undefined) throw new Error(`Could not resolve path: ${path}`);
const parentPath = segments.join("/");
const resolvedParent = await this.resolveLogicalPath(parentPath, caseInsensitive, true);
if (resolvedParent === null) throw new Error(`Could not resolve path: ${path}`);
const entries = await super.list(resolvedParent);
const requestedKey = this.comparisonKey(leaf, caseInsensitive);
const matches = [...entries.files, ...entries.folders].filter((entryPath) => {
const entryName = entryPath.slice(resolvedParent === "" ? 0 : resolvedParent.length + 1);
return this.comparisonKey(entryName, caseInsensitive) === requestedKey;
});
if (matches.length > 1) throw new AmbiguousNormalisedPathError(path);
return resolvedParent === "" ? leaf : `${resolvedParent}/${leaf}`;
}
private comparisonKey(segment: string, caseInsensitive: boolean): string {
const normalisedSegment = segment.normalize("NFC");
return caseInsensitive ? normalisedSegment.toLowerCase() : normalisedSegment;
}
}
src/apps/cli/commands/runCommand.ts
+43 -2
View File
@@ -6,6 +6,7 @@ import {
type ObsidianLiveSyncSettings,
type EntryDoc,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { isRemediationModeActive } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
import {
activateRemoteConfiguration,
@@ -13,7 +14,7 @@ import {
} from "@vrtmrz/livesync-commonlib/remote-configurations";
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import type { CLICommandContext, CLIOptions } from "./types";
import { toArrayBuffer, toDatabaseRelativePath } from "./utils";
import { getForceRequirementError, toArrayBuffer, toDatabaseRelativePath } from "./utils";
import { collectPeers, openP2PHost, parseTimeoutSeconds, syncWithPeer } from "./p2p";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
import { fsPromises as fs, getNodeFileCreationTime, path } from "@vrtmrz/livesync-commonlib/node";
@@ -37,8 +38,25 @@ function redactConnectionString(uri: string): string {
}
export async function runCommand(options: CLIOptions, context: CLICommandContext): Promise<boolean> {
const { databasePath, core, replicationScheduling, settingsPath } = context;
const { core } = context;
const { standardIo } = core.services.context;
const forceRequirementError = getForceRequirementError(options.command, options.force);
if (forceRequirementError) {
writeStderrLine(standardIo, `[Error] ${forceRequirementError}`);
return false;
}
if (
options.command === "fetch" &&
isRemediationModeActive(core.services.setting.currentSettings())
) {
writeStderrLine(
standardIo,
`[Error] The fetch command cannot run while remediation mode is active. Clear the modification-time limit before fetching.`
);
return false;
}
const { databasePath, replicationScheduling, settingsPath } = context;
const vaultPath = context.vaultPath || databasePath;
await core.services.control.activated;
@@ -519,6 +537,29 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
return true;
}
if (options.command === "rebuild") {
writeStderrLine(standardIo, "[Command] rebuild");
try {
await core.serviceModules.rebuilder.$rebuildEverything();
return true;
} catch (error) {
writeStderrLine(standardIo, "[Error] Rebuild failed:", error);
return false;
}
}
if (options.command === "fetch") {
writeStderrLine(standardIo, "[Command] fetch");
try {
// Fetch replaces the local database from remote; do not persist legacy local files first.
await core.serviceModules.rebuilder.$fetchLocal(false, true);
return true;
} catch (error) {
writeStderrLine(standardIo, "[Error] Fetch failed:", error);
return false;
}
}
if (options.command === "remote-add") {
if (options.commandArgs.length < 2) {
throw new Error("remote-add requires two arguments: <name> <connstr>");
src/apps/cli/commands/runCommand.unit.spec.ts
+71
View File
@@ -118,6 +118,10 @@ function createCoreMock() {
fetch: vi.fn(async () => undefined),
storeContent: vi.fn(async () => true),
},
rebuilder: {
$rebuildEverything: vi.fn(async () => undefined),
$fetchLocal: vi.fn(async () => undefined),
},
},
} as any;
}
@@ -268,6 +272,73 @@ describe("runCommand abnormal cases", () => {
vi.restoreAllMocks();
});
it.each(["rebuild", "fetch"] as const)("requires --force before activating for %s", async (command) => {
const core = createCoreMock();
const activated = vi.fn(() => Promise.resolve());
Object.defineProperty(core.services.control, "activated", { get: activated });
await expect(runCommand(makeOptions(command, []), { ...context, core })).resolves.toBe(false);
expect(activated).not.toHaveBeenCalled();
expect(core.serviceModules.rebuilder.$rebuildEverything).not.toHaveBeenCalled();
expect(core.serviceModules.rebuilder.$fetchLocal).not.toHaveBeenCalled();
expect(core.services.context.standardIo.writeStderr).toHaveBeenCalledWith(
expect.stringContaining("requires --force")
);
});
it("dispatches a forced rebuild to the existing Rebuilder service", async () => {
const core = createCoreMock();
const options = makeOptions("rebuild", []);
options.force = true;
await expect(runCommand(options, { ...context, core })).resolves.toBe(true);
expect(core.serviceModules.rebuilder.$rebuildEverything).toHaveBeenCalledOnce();
expect(core.serviceModules.rebuilder.$fetchLocal).not.toHaveBeenCalled();
});
it("dispatches a forced fetch without saving legacy Vault files first", async () => {
const core = createCoreMock();
const options = makeOptions("fetch", []);
options.force = true;
await expect(runCommand(options, { ...context, core })).resolves.toBe(true);
expect(core.serviceModules.rebuilder.$fetchLocal).toHaveBeenCalledOnce();
expect(core.serviceModules.rebuilder.$fetchLocal).toHaveBeenCalledWith(false, true);
expect(core.serviceModules.rebuilder.$rebuildEverything).not.toHaveBeenCalled();
});
it("rejects fetch in remediation mode even when forced", async () => {
const core = createCoreMock();
core.services.setting.currentSettings().maxMTimeForReflectEvents = 1;
const options = makeOptions("fetch", []);
options.force = true;
await expect(runCommand(options, { ...context, core })).resolves.toBe(false);
expect(core.serviceModules.rebuilder.$fetchLocal).not.toHaveBeenCalled();
expect(core.services.context.standardIo.writeStderr).toHaveBeenCalledWith(
expect.stringContaining("remediation mode")
);
});
it("surfaces an internal rebuild refusal as a failed command", async () => {
const core = createCoreMock();
core.serviceModules.rebuilder.$rebuildEverything.mockRejectedValue(
new Error("Rebuilding is not available while remediation mode is active.")
);
const options = makeOptions("rebuild", []);
options.force = true;
await expect(runCommand(options, { ...context, core })).resolves.toBe(false);
expect(core.services.context.standardIo.writeStderr).toHaveBeenCalledWith(
expect.stringContaining("Rebuilding is not available while remediation mode is active")
);
});
it("retains visible progress for the interactive sync command", async () => {
const core = createCoreMock();
src/apps/cli/commands/types.ts
+4
View File
@@ -22,6 +22,8 @@ export type CLICommand =
| "rm"
| "resolve"
| "mirror"
| "rebuild"
| "fetch"
| "remote-add"
| "remote-rm"
| "remote-ls"
@@ -100,6 +102,8 @@ export const VALID_COMMANDS = new Set([
"rm",
"resolve",
"mirror",
"rebuild",
"fetch",
"remote-add",
"remote-rm",
"remote-ls",
src/apps/cli/commands/utils.ts
+12
View File
@@ -1,4 +1,16 @@
import { path } from "@vrtmrz/livesync-commonlib/node";
import type { CLICommand } from "./types";
export function getForceRequirementError(command: CLICommand, force = false): string | undefined {
if (force) return undefined;
if (command === "rebuild") {
return "Command 'rebuild' requires --force because it replaces the local and remote databases.";
}
if (command === "fetch") {
return "Command 'fetch' requires --force because it replaces the local database with remote state.";
}
return undefined;
}
export function toArrayBuffer(data: Buffer): ArrayBuffer {
return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
src/apps/cli/commands/utils.unit.spec.ts
+15 -1
View File
@@ -1,6 +1,20 @@
import { path } from "@vrtmrz/livesync-commonlib/node";
import { describe, expect, it } from "vitest";
import { toDatabaseRelativePath } from "./utils";
import { getForceRequirementError, toDatabaseRelativePath } from "./utils";
describe("getForceRequirementError", () => {
it.each([
["rebuild", "Command 'rebuild' requires --force because it replaces the local and remote databases."],
["fetch", "Command 'fetch' requires --force because it replaces the local database with remote state."],
] as const)("requires force for %s", (command, expected) => {
expect(getForceRequirementError(command, false)).toBe(expected);
expect(getForceRequirementError(command, true)).toBeUndefined();
});
it("does not require force for ordinary commands", () => {
expect(getForceRequirementError("sync", false)).toBeUndefined();
});
});
describe("toDatabaseRelativePath", () => {
const databasePath = path.resolve("/tmp/livesync-vault");
src/apps/cli/main.bootstrap.unit.spec.ts
+79 -2
View File
@@ -3,6 +3,7 @@ import { appendFile, mkdtemp, mkdir, readFile, rm, stat, writeFile } from "node:
import { tmpdir } from "node:os";
import { join } from "node:path";
import * as chokidar from "chokidar";
import * as offlineScanner from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService";
import {
AutoAccepting,
@@ -20,6 +21,7 @@ import { cloneSettings } from "./settingsPersistence";
import type { CLICommand } from "./commands/types";
vi.mock("chokidar", { spy: true });
vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", { spy: true });
function createStandardIoMock() {
return {
@@ -41,6 +43,7 @@ describe("CLI database preparation", () => {
beforeEach(async () => {
vi.mocked(chokidar.watch).mockClear();
vi.mocked(offlineScanner.useOfflineScanner).mockClear();
directory = await mkdtemp(join(tmpdir(), "livesync-cli-bootstrap-"));
vaultPath = join(directory, "vault");
settingsPath = join(directory, "settings.json");
@@ -69,12 +72,43 @@ describe("CLI database preparation", () => {
await rm(directory, { recursive: true, force: true });
});
async function start(command: CLICommand, runner: CliCommandRunner, exitCode = 1) {
process.argv = ["node", "livesync-cli", directory, "--vault", vaultPath, "--settings", settingsPath, command];
async function start(command: CLICommand, runner: CliCommandRunner, exitCode = 1, force = false) {
process.argv = [
"node",
"livesync-cli",
directory,
...(force ? ["--force"] : []),
"--vault",
vaultPath,
"--settings",
settingsPath,
command,
];
// Daemon probes return false so the real core unloads without keeping a daemon alive.
await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`);
}
it("loads legacy device values and persists configuration through the service-owned store", async () => {
const localStoragePath = join(directory, ".livesync", "runtime", "local-storage.json");
await mkdir(join(directory, ".livesync", "runtime"), { recursive: true });
await writeFile(localStoragePath, JSON.stringify({ ["__proto__"]: "legacy value" }));
let legacyValue: string | null | undefined;
const runner = vi.fn<CliCommandRunner>(async (_options, { core }) => {
legacyValue = await core.services.setting.deviceValues.getValue("__proto__");
expect(core.services.config).toBe(core.services.setting.config);
await core.services.config.setValue("bootstrap-probe", "config value");
return false;
});
await start("ls", runner);
expect(legacyValue).toBe("legacy value");
const stored = JSON.parse(await readFile(localStoragePath, "utf-8")) as Record<string, string>;
expect(Object.prototype.hasOwnProperty.call(stored, "__proto__")).toBe(true);
expect(stored["__proto__"]).toBe("legacy value");
expect(Object.values(stored)).toContain("config value");
});
it.each([REMOTE_P2P, REMOTE_COUCHDB])(
"restores configured P2P automation before the host connects (%s)",
async (remoteType) => {
@@ -255,6 +289,49 @@ describe("CLI database preparation", () => {
expect(watch).not.toHaveBeenCalled();
});
it.each(["rebuild", "fetch"] as const)("does not scan or mirror legacy Vault files before %s", async (command) => {
const storedPaths: string[] = [];
const runner = vi.fn<CliCommandRunner>(async (_options, { core }) => {
for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) {
storedPaths.push(doc.path);
}
return false;
});
await start(command, runner, 1, true);
expect(runner).toHaveBeenCalledOnce();
expect(storedPaths).toEqual([]);
expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content");
expect(vi.mocked(chokidar.watch)).not.toHaveBeenCalled();
});
it("registers the Rebuild scanner after readiness and makes its Vault scan available", async () => {
const enumerate = vi.spyOn(ServiceFileAccessCLI.prototype, "getFiles");
const useScanner = vi.mocked(offlineScanner.useOfflineScanner);
let scansBeforeCommand = -1;
let scannerRegistrationsBeforeCommand = -1;
let storedPaths: string[] = [];
const runner = vi.fn<CliCommandRunner>(async (_options, { core }) => {
scansBeforeCommand = enumerate.mock.calls.length;
scannerRegistrationsBeforeCommand = useScanner.mock.calls.length;
await core.services.vault.scanVault(true, true);
for await (const doc of core.services.database.localDatabase.findAllNormalDocs()) {
storedPaths.push(doc.path);
}
return false;
});
await start("rebuild", runner, 1, true);
expect(useScanner).toHaveBeenCalledOnce();
expect(scannerRegistrationsBeforeCommand).toBe(1);
expect(scansBeforeCommand).toBe(0);
expect(enumerate).toHaveBeenCalledOnce();
expect(storedPaths).toEqual(["notes/local.md"]);
});
it.each([
{ command: "daemon" as const, commandRuns: true },
{ command: "mirror" as const, commandRuns: false },
src/apps/cli/main.ts
+29 -6
View File
@@ -1,6 +1,5 @@
import { useP2PSettingsPreparation } from "@/serviceFeatures/useP2PSettingsPreparation";
import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub";
import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage";
import { LiveSyncBaseCore, type StartupDatabaseOptions } from "@/LiveSyncBaseCore";
import { initialiseServiceModulesCLI } from "./serviceModules/CLIServiceModules";
import {
@@ -21,6 +20,7 @@ import {
import { runCommand } from "./commands/runCommand";
import { isCLICommand } from "./commands/types";
import type { CLICommand, CLICommandContext, CLIOptions } from "./commands/types";
import { getForceRequirementError } from "./commands/utils";
import { getPathFromUXFileInfo } from "@vrtmrz/livesync-commonlib/compat/common/typeUtils";
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import { IgnoreRules } from "./serviceModules/IgnoreRules";
@@ -47,6 +47,7 @@ const SETTINGS_FILE = ".livesync/settings.json";
interface CLIVaultSyncMode {
readonly watchFiles: boolean;
readonly reflectReplicationResults: boolean;
readonly scanOnStartup?: boolean;
readonly startupDatabaseOptions: StartupDatabaseOptions;
}
@@ -62,9 +63,20 @@ const VAULT_SYNC_MODES: Readonly<Partial<Record<CLICommand, CLIVaultSyncMode>>>
reflectReplicationResults: false,
startupDatabaseOptions: { ignoreSuspending: true, continueOnFileFailure: false },
},
rebuild: {
watchFiles: false,
reflectReplicationResults: false,
scanOnStartup: false,
startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: false },
},
fetch: {
watchFiles: false,
reflectReplicationResults: true,
scanOnStartup: false,
startupDatabaseOptions: { ignoreSuspending: false, continueOnFileFailure: false },
},
};
ensureGlobalNodeLocalStorage();
defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG;
/** Injectable command boundary used by CLI integration probes. */
@@ -90,6 +102,8 @@ Commands:
p2p-sync <peer> <timeout>
Sync with the specified peer-id or peer-name
p2p-host Start P2P host mode and wait until interrupted
rebuild Rebuild local and remote databases from the selected Vault (requires --force)
fetch Replace the local database with remote state and reflect it to the Vault (requires --force)
push <src> <dst> Push local file <src> into local database path <dst>
pull <src> <dst> Pull file <src> from local database into local file <dst>
pull-rev <src> <dst> <rev> Pull file <src> at specific revision <rev> into local file <dst>
@@ -122,8 +136,9 @@ Commands:
Show remote database status
Options:
--vault <path>, -V <path> (daemon/mirror) Path to the vault directory containing .md files
--vault <path>, -V <path> (daemon/mirror/rebuild/fetch) Path to the vault directory containing .md files
(defaults to database-path; allows separate PouchDB and vault dirs)
--force, -f Required for rebuild and fetch because they replace databases
--interval <N>, -i <N> (daemon only) Poll CouchDB every N seconds instead of using the _changes feed
--write-settings Write setting changes after a successful command
--compat-remote-admin-exit-zero
@@ -319,6 +334,12 @@ export async function main(
commandRunner: CliCommandRunner = runCommand
) {
const options = parseArgs(standardIo);
const forceRequirementError = getForceRequirementError(options.command, options.force);
if (forceRequirementError) {
writeStderrLine(standardIo, `[Error] ${forceRequirementError}`);
process.exitCode = 1;
return;
}
const vaultSyncMode = VAULT_SYNC_MODES[options.command];
if (options.interval && options.command !== "daemon") {
writeStderrLine(
@@ -377,8 +398,6 @@ export async function main(
const settingsPath = options.settingsPath
? path.resolve(options.settingsPath)
: path.join(databasePath, SETTINGS_FILE);
configureNodeLocalStorage(path.join(databasePath, ".livesync", "runtime", "local-storage.json"));
// Resolve vault path: mirror positional argument takes priority,
// then --vault flag, otherwise fall back to databasePath.
const vaultPath =
@@ -521,7 +540,7 @@ export async function main(
() => [], // No add-ons
(core, coreFeatureViews) => {
replicationScheduling = coreFeatureViews.replicationScheduling;
if (vaultSyncMode) {
if (vaultSyncMode && vaultSyncMode.scanOnStartup !== false) {
useOfflineScanner(core);
}
// Register P2P replicator feature.
@@ -641,6 +660,10 @@ export async function main(
if (!readyResult) {
throw new Error("Failed to initialise LiveSync.");
}
if (vaultSyncMode?.scanOnStartup === false) {
// Keep the startup reset clean, then enable scans for the explicit Rebuild or Fetch operation.
useOfflineScanner(core);
}
const settingsBeforeCommand = cloneSettings(core.services.setting.currentSettings());
const transientSettingKeys = changedSettingKeys(settingsBeforeSuspend, settingsAfterSuspend);
for (const key of CLI_RUNTIME_ONLY_SETTING_KEYS) {
src/apps/cli/main.unit.spec.ts
+25 -1
View File
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { parseArgs as parseCliArgs } from "./main";
import { main as runCliMain, parseArgs as parseCliArgs } from "./main";
function createStandardIoMock() {
return {
@@ -19,6 +19,7 @@ function mockProcessExit() {
describe("CLI parseArgs", () => {
const originalArgv = process.argv.slice();
const originalExitCode = process.exitCode;
let standardIo: ReturnType<typeof createStandardIoMock>;
beforeEach(() => {
@@ -29,6 +30,7 @@ describe("CLI parseArgs", () => {
afterEach(() => {
process.argv = originalArgv.slice();
process.exitCode = originalExitCode;
vi.restoreAllMocks();
});
@@ -99,6 +101,28 @@ describe("CLI parseArgs", () => {
expect(parsed.commandArgs).toEqual([]);
});
it.each(["rebuild", "fetch"] as const)("parses %s and its force flag", (command) => {
process.argv = ["node", "livesync-cli", "./databasePath", "--force", command];
const parsed = parseArgs();
expect(parsed.command).toBe(command);
expect(parsed.force).toBe(true);
expect(parsed.commandArgs).toEqual([]);
});
it.each(["rebuild", "fetch"] as const)("rejects %s without force before database access", async (command) => {
process.argv = ["node", "livesync-cli", "./missing-database", command];
process.exitCode = undefined;
const runner = vi.fn(async () => true);
await expect(runCliMain(standardIo, runner as any)).resolves.toBeUndefined();
expect(runner).not.toHaveBeenCalled();
expect(process.exitCode).toBe(1);
expect(standardIo.writeStderr).toHaveBeenCalledWith(expect.stringContaining("requires --force"));
expect(standardIo.writeStderr).not.toHaveBeenCalledWith(expect.stringContaining("Database directory"));
});
it("parses remote-add command", () => {
process.argv = [
"node",
src/apps/cli/package.json
+4 -4
View File
@@ -1,7 +1,7 @@
{
"name": "self-hosted-livesync-cli",
"private": true,
"version": "1.0.37-cli",
"version": "1.0.42-cli",
"main": "dist/index.cjs",
"type": "module",
"scripts": {
@@ -12,7 +12,7 @@
"buildRun": "npm run build && npm run cli --",
"build:docker": "docker build -f Dockerfile -t livesync-cli ../../..",
"check": "tsc -p tsconfig.json",
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts",
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts",
"test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh",
"test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh",
"test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh",
@@ -52,7 +52,7 @@
},
"devDependencies": {
"typescript": "6.0.3",
"vite": "^8.0.16",
"vitest": "^4.1.11"
"vite": "^8.1.6",
"vitest": "^4.1.12"
}
}
src/apps/cli/serviceModules/CLIServiceModules.ts
+1
View File
@@ -99,6 +99,7 @@ export function initialiseServiceModulesCLI(
// Rebuilder (platform-independent)
const rebuilder = new ServiceRebuilder({
path: services.path,
events: services.context.events,
API: services.API,
database: services.database,
src/apps/cli/serviceModules/FileAccessCLI.ts
+22 -7
View File
@@ -1,6 +1,10 @@
import { FileAccessBase, type FileAccessBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/serviceModules/FileAccessBase";
import {
FileAccessBase,
type FileAccessBaseDependencies,
} from "@vrtmrz/livesync-commonlib/compat/serviceModules/FileAccessBase";
import { NodeFileSystemAdapter } from "@/apps/cli/adapters/NodeFileSystemAdapter";
import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
import { getPathNormalizationVersion } from "@vrtmrz/livesync-commonlib/settings";
/**
* CLI-specific implementation of FileAccessBase
@@ -8,12 +12,23 @@ import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
*/
export class FileAccessCLI extends FileAccessBase<NodeFileSystemAdapter> {
constructor(basePath: string, dependencies: FileAccessBaseDependencies) {
const adapter = new NodeFileSystemAdapter(basePath, (message, detail) => {
dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE);
if (detail !== undefined) {
dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE);
}
});
const pathNormalizationVersion = () =>
getPathNormalizationVersion(dependencies.settingService.currentSettings());
const isCaseInsensitive = () => {
const settings = dependencies.settingService.currentSettings();
return !settings.handleFilenameCaseSensitive || dependencies.vaultService.isStorageInsensitive();
};
const adapter = new NodeFileSystemAdapter(
basePath,
(message, detail) => {
dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE);
if (detail !== undefined) {
dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE);
}
},
pathNormalizationVersion,
isCaseInsensitive
);
super(adapter, dependencies);
}
src/apps/cli/services/NodeLocalStorage.ts
+65 -94
View File
@@ -1,115 +1,86 @@
import { createDeletableValueStore, sanitiseValueStoreError, ValueStoreError } from "@vrtmrz/value-store";
import type { ValueDeleter, ValueStore } from "@vrtmrz/value-store";
import { fs as nodeFs, path as nodePath } from "@vrtmrz/livesync-commonlib/node";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
type LocalStorageShape = {
getItem(key: string): string | null;
setItem(key: string, value: string): void;
removeItem(key: string): void;
clear(): void;
};
type NodeLocalStorage = ValueStore & ValueDeleter;
class PersistentNodeLocalStorage {
private storagePath: string | undefined;
private localStore: Record<string, string> = {};
class JsonFileValueBackend {
private values: Map<string, string>;
configure(storagePath: string) {
if (this.storagePath === storagePath) {
return;
}
this.storagePath = storagePath;
this.loadFromFile();
constructor(private readonly storagePath: string) {
this.values = this.loadValues();
}
private loadFromFile() {
if (!this.storagePath) {
this.localStore = {};
get(key: string): string | null {
return this.values.get(key) ?? null;
}
set(key: string, value: string): void {
const nextValues = new Map(this.values);
nextValues.set(key, value);
this.writeValues(nextValues);
this.values = nextValues;
}
delete(key: string): void {
if (!this.values.has(key)) {
return;
}
const nextValues = new Map(this.values);
nextValues.delete(key);
this.writeValues(nextValues);
this.values = nextValues;
}
private loadValues(): Map<string, string> {
let contents: string;
try {
const loaded = JSON.parse(nodeFs.readFileSync(this.storagePath, "utf-8")) as Record<string, string>;
this.localStore = { ...loaded };
contents = nodeFs.readFileSync(this.storagePath, "utf-8");
} catch (error) {
if (isMissingFileError(error)) {
return new Map();
}
throw sanitiseValueStoreError(error);
}
let document: unknown;
try {
document = JSON.parse(contents);
} catch {
this.localStore = {};
throw new ValueStoreError("invalid-data");
}
if (document === null || typeof document !== "object" || Array.isArray(document)) {
throw new ValueStoreError("invalid-data");
}
const values = new Map<string, string>();
for (const [key, value] of Object.entries(document)) {
if (typeof value !== "string") {
throw new ValueStoreError("invalid-data");
}
values.set(key, value);
}
return values;
}
private flushToFile() {
if (!this.storagePath) {
return;
}
private writeValues(values: ReadonlyMap<string, string>): void {
nodeFs.mkdirSync(nodePath.dirname(this.storagePath), { recursive: true });
nodeFs.writeFileSync(this.storagePath, JSON.stringify(this.localStore, null, 2), "utf-8");
}
getItem(key: string): string | null {
return this.localStore[key] ?? null;
}
setItem(key: string, value: string) {
this.localStore[key] = value;
this.flushToFile();
}
removeItem(key: string) {
if (!(key in this.localStore)) {
return;
}
delete this.localStore[key];
this.flushToFile();
}
clear() {
this.localStore = {};
this.flushToFile();
nodeFs.writeFileSync(this.storagePath, JSON.stringify(Object.fromEntries(values), null, 2), "utf-8");
}
}
const persistentNodeLocalStorage = new PersistentNodeLocalStorage();
function createNodeLocalStorageShim(): LocalStorageShape {
return {
getItem(key: string) {
return persistentNodeLocalStorage.getItem(key);
},
setItem(key: string, value: string) {
persistentNodeLocalStorage.setItem(key, value);
},
removeItem(key: string) {
persistentNodeLocalStorage.removeItem(key);
},
clear() {
persistentNodeLocalStorage.clear();
},
};
}
export function ensureGlobalNodeLocalStorage() {
const g = compatGlobal as unknown as Record<string, unknown>;
if (
!("localStorage" in g) ||
typeof (g.localStorage as Record<string, unknown> | undefined)?.getItem !== "function"
) {
g.localStorage = createNodeLocalStorageShim();
function isMissingFileError(error: unknown): boolean {
if (typeof error !== "object" || error === null) {
return false;
}
try {
const code = Object.getOwnPropertyDescriptor(error, "code");
return code !== undefined && Object.prototype.hasOwnProperty.call(code, "value") && code.value === "ENOENT";
} catch {
return false;
}
}
export function configureNodeLocalStorage(storagePath: string) {
persistentNodeLocalStorage.configure(storagePath);
ensureGlobalNodeLocalStorage();
}
export function getNodeLocalStorageItem(key: string): string {
return persistentNodeLocalStorage.getItem(key) ?? "";
}
export function setNodeLocalStorageItem(key: string, value: string) {
persistentNodeLocalStorage.setItem(key, value);
}
export function deleteNodeLocalStorageItem(key: string) {
persistentNodeLocalStorage.removeItem(key);
}
export function clearNodeLocalStorage() {
persistentNodeLocalStorage.clear();
export function createNodeLocalStorage(storagePath: string): NodeLocalStorage {
return createDeletableValueStore(new JsonFileValueBackend(storagePath));
}
src/apps/cli/services/NodeLocalStorage.unit.spec.ts
+98 -29
View File
@@ -1,58 +1,127 @@
import { fs, os, path } from "@vrtmrz/livesync-commonlib/node";
import { ValueStoreError } from "@vrtmrz/value-store";
import { afterEach, describe, expect, it } from "vitest";
import {
clearNodeLocalStorage,
configureNodeLocalStorage,
ensureGlobalNodeLocalStorage,
getNodeLocalStorageItem,
setNodeLocalStorageItem,
} from "./NodeLocalStorage";
import { createNodeLocalStorage } from "./NodeLocalStorage";
describe("NodeLocalStorage", () => {
const tempDirs: string[] = [];
afterEach(() => {
clearNodeLocalStorage();
for (const tempDir of tempDirs.splice(0)) {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
it("persists values to the configured file", () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-"));
tempDirs.push(tempDir);
const storagePath = path.join(tempDir, "runtime", "local-storage.json");
configureNodeLocalStorage(storagePath);
setNodeLocalStorageItem("checkpoint", "42");
const saved = JSON.parse(fs.readFileSync(storagePath, "utf-8")) as Record<string, string>;
expect(saved.checkpoint).toBe("42");
});
it("reloads persisted values when configured again", () => {
it("loads and preserves legacy flat string data, including empty values and exact keys", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-"));
tempDirs.push(tempDir);
const storagePath = path.join(tempDir, "runtime", "local-storage.json");
const specialKey = "__proto__";
const specialValue = "line one\n雪";
fs.mkdirSync(path.dirname(storagePath), { recursive: true });
fs.writeFileSync(storagePath, JSON.stringify({ persisted: "value" }, null, 2), "utf-8");
fs.writeFileSync(storagePath, JSON.stringify({ [specialKey]: specialValue, empty: "" }), "utf-8");
configureNodeLocalStorage(storagePath);
const store = createNodeLocalStorage(storagePath);
expect(getNodeLocalStorageItem("persisted")).toBe("value");
await expect(store.getValue(specialKey)).resolves.toBe(specialValue);
await expect(store.getValue("empty", "default")).resolves.toBe("");
const saved = JSON.parse(fs.readFileSync(storagePath, "utf-8")) as Record<string, string>;
expect(saved[specialKey]).toBe(specialValue);
expect(saved.empty).toBe("");
});
it("installs a global localStorage shim backed by the same store", () => {
it("persists strings and objects, reloads them, and deletes keys", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-"));
tempDirs.push(tempDir);
const storagePath = path.join(tempDir, "runtime", "local-storage.json");
const store = createNodeLocalStorage(storagePath);
const object = { enabled: false, entries: ["value", null, 0] };
configureNodeLocalStorage(storagePath);
ensureGlobalNodeLocalStorage();
await store.setValue("empty", "");
await store.setValue("text", "stored value");
await store.setObject("object", object);
globalThis.localStorage.setItem("shared", "state");
const reloaded = createNodeLocalStorage(storagePath);
await expect(reloaded.getValue("empty", "default")).resolves.toBe("");
await expect(reloaded.getValue("text")).resolves.toBe("stored value");
await expect(reloaded.getObject("object")).resolves.toEqual(object);
expect(getNodeLocalStorageItem("shared")).toBe("state");
await reloaded.delete("text");
await expect(reloaded.getValue("text")).resolves.toBeNull();
const saved = JSON.parse(fs.readFileSync(storagePath, "utf-8")) as Record<string, string>;
expect(saved.text).toBeUndefined();
expect(saved.object).toBe(JSON.stringify(object));
});
it("treats a missing file as empty and creates its parent on the first write", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-"));
tempDirs.push(tempDir);
const storagePath = path.join(tempDir, "runtime", "local-storage.json");
const store = createNodeLocalStorage(storagePath);
await expect(store.getValue("absent")).resolves.toBeNull();
expect(fs.existsSync(storagePath)).toBe(false);
await store.setValue("created", "value");
expect(fs.existsSync(storagePath)).toBe(true);
});
it("keeps separate store instances isolated by their file paths", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-"));
tempDirs.push(tempDir);
const first = createNodeLocalStorage(path.join(tempDir, "first", "local-storage.json"));
const second = createNodeLocalStorage(path.join(tempDir, "second", "local-storage.json"));
await first.setValue("shared-key", "first");
await second.setValue("shared-key", "second");
await expect(first.getValue("shared-key")).resolves.toBe("first");
await expect(second.getValue("shared-key")).resolves.toBe("second");
});
it("surfaces malformed and unreadable files as sanitised value-store errors", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-"));
tempDirs.push(tempDir);
const malformedPath = path.join(tempDir, "malformed.json");
fs.writeFileSync(malformedPath, "{ malformed", "utf-8");
let malformedError: unknown;
try {
createNodeLocalStorage(malformedPath);
} catch (error) {
malformedError = error;
}
expect(malformedError).toBeInstanceOf(ValueStoreError);
expect(malformedError).toMatchObject({ code: "invalid-data" });
expect(fs.readFileSync(malformedPath, "utf-8")).toBe("{ malformed");
const unreadablePath = path.join(tempDir, "unreadable.json");
fs.mkdirSync(unreadablePath);
expect(() => createNodeLocalStorage(unreadablePath)).toThrowError(ValueStoreError);
});
it("does not change its visible values when a write fails", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-"));
tempDirs.push(tempDir);
const storagePath = path.join(tempDir, "runtime", "local-storage.json");
const store = createNodeLocalStorage(storagePath);
await store.setValue("retained", "before failure");
fs.rmSync(storagePath);
fs.mkdirSync(storagePath);
await expect(store.setValue("new", "write fails")).rejects.toBeInstanceOf(ValueStoreError);
await expect(store.getValue("retained")).resolves.toBe("before failure");
await expect(store.getValue("new")).resolves.toBeNull();
});
it("does not install or modify a global localStorage value", () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-local-storage-"));
tempDirs.push(tempDir);
const before = Object.getOwnPropertyDescriptor(globalThis, "localStorage");
createNodeLocalStorage(path.join(tempDir, "runtime", "local-storage.json"));
expect(Object.getOwnPropertyDescriptor(globalThis, "localStorage")).toEqual(before);
});
});
src/apps/cli/services/NodeServiceHub.ts
+8 -11
View File
@@ -1,6 +1,5 @@
import type { AppLifecycleServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService";
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { ConfigServiceBrowserCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/browser/ConfigServiceBrowserCompat";
import type {
ComponentHasResult,
SvelteDialogManager,
@@ -11,6 +10,7 @@ import { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat
import { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService";
import { InjectableDatabaseEventService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableDatabaseEventService";
import { InjectableFileProcessingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableFileProcessingService";
import { InjectableSettingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableSettingService";
import { PathServiceCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectablePathService";
import { InjectableRemoteService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableRemoteService";
import { InjectableReplicationService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableReplicationService";
@@ -21,7 +21,7 @@ import { InjectableVaultServiceCompat } from "@vrtmrz/livesync-commonlib/compat/
import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService";
import { HeadlessAPIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/headless/HeadlessAPIService";
import { NodeKeyValueDBService } from "./NodeKeyValueDBService";
import { NodeSettingService } from "./NodeSettingService";
import { createNodeLocalStorage } from "./NodeLocalStorage";
import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService";
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { path as nodePath } from "@vrtmrz/livesync-commonlib/node";
@@ -96,11 +96,11 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
const conflict = new InjectableConflictService(context);
const fileProcessing = new InjectableFileProcessingService(context);
const setting = new NodeSettingService(
context,
{ APIService: API, onDisplayLanguageChanged: setLang },
localStoragePath
);
const setting = new InjectableSettingService<T>(context, {
APIService: API,
onDisplayLanguageChanged: setLang,
deviceValues: createNodeLocalStorage(localStoragePath),
});
const appLifecycle = new NodeAppLifecycleService<T>(context, {
settingService: setting,
@@ -132,10 +132,7 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
setting,
});
const config = new ConfigServiceBrowserCompat<T>(context, {
settingService: setting,
APIService: API,
});
const config = setting.config;
const replicator = new InjectableReplicatorService(context, {
settingService: setting,
src/apps/cli/services/NodeSettingService.ts
-42
View File
@@ -1,42 +0,0 @@
import { EVENT_SETTING_SAVED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents";
import { EVENT_REQUEST_RELOAD_SETTING_TAB } from "@/common/events";
import { handlers } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils";
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { SettingService, type SettingServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/SettingService";
import {
configureNodeLocalStorage,
deleteNodeLocalStorageItem,
getNodeLocalStorageItem,
setNodeLocalStorageItem,
} from "./NodeLocalStorage";
export class NodeSettingService<T extends ServiceContext> extends SettingService<T> {
constructor(context: T, dependencies: SettingServiceDependencies, storagePath: string) {
super(context, dependencies);
configureNodeLocalStorage(storagePath);
this.onSettingSaved.addHandler((settings) => {
this.context.events.emitEvent(EVENT_SETTING_SAVED, settings);
return Promise.resolve(true);
});
this.onSettingLoaded.addHandler((settings) => {
this.context.events.emitEvent(EVENT_REQUEST_RELOAD_SETTING_TAB);
return Promise.resolve(true);
});
}
protected setItem(key: string, value: string) {
setNodeLocalStorageItem(key, value);
}
protected getItem(key: string): string {
return getNodeLocalStorageItem(key);
}
protected deleteItem(key: string): void {
deleteNodeLocalStorageItem(key);
}
public saveData = handlers<{ saveData: (data: ObsidianLiveSyncSettings) => Promise<void> }>().binder("saveData");
public loadData = handlers<{ loadData: () => Promise<ObsidianLiveSyncSettings | undefined> }>().binder("loadData");
}
src/apps/cli/testdeno/test_dev_deno.md
+3 -3
View File
@@ -146,7 +146,7 @@ These flows do not require Bash on the host. The S3 matrix tasks, environment va
### `test-mirror.ts`
- Verifies six core mirror scenarios.
- Verifies seven core mirror scenarios, including an unrecognised revision origin.
- Does not require an external remote.
### `test-sync-two-local-databases.ts`
@@ -247,11 +247,12 @@ deno task test:daemon
deno task test:push-pull
deno task test:sync-locked-remote
# CouchDB-based tests
# CouchDB and Object Storage tests
deno task test:sync-two-local
deno task test:decoupled-vault
deno task test:remote-commands
deno task test:e2e-couchdb
deno task test:e2e-matrix
# Explicit host-direct P2P diagnostics for cross-platform investigations.
# These are not part of the default suite or release evidence.
@@ -260,7 +261,6 @@ deno task test:p2p-peers
deno task test:p2p-sync
deno task test:p2p-three-nodes
deno task test:p2p-upload-download
deno task test:e2e-matrix
```
---
src/apps/webapp/WebAppRuntime.ts
+12 -2
View File
@@ -21,6 +21,8 @@ import { useCheckRemoteSize } from "@vrtmrz/livesync-commonlib/compat/serviceFea
import { useRemoteConfiguration } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/remoteConfig";
import { useP2PReplicatorFeature } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/useP2PReplicatorFeature";
import type { UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/UseP2PReplicatorResult";
import type { ValueStore, ValueDeleter } from "@vrtmrz/value-store";
import { createWebStorageValueStore } from "@vrtmrz/value-store/web-storage";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
import {
createLiveSyncBrowserServiceHub,
@@ -56,6 +58,7 @@ const DEFAULT_SETTINGS: Partial<ObsidianLiveSyncSettings> = {
export type WebAppRuntimeStatusKind = "info" | "warning" | "error" | "success";
export interface WebAppRuntimeOptions {
deviceValues?: ValueStore & ValueDeleter;
reportStatus?: (kind: WebAppRuntimeStatusKind, message: string) => void;
scheduleReload?: (delayMilliseconds: number) => void;
}
@@ -71,7 +74,10 @@ export class WebAppRuntime {
private paneHost: P2PReplicatorPaneHost | null = null;
private restartScheduled = false;
constructor(rootHandle: FileSystemDirectoryHandle, options: WebAppRuntimeOptions = {}) {
constructor(
rootHandle: FileSystemDirectoryHandle,
private readonly options: WebAppRuntimeOptions = {}
) {
this.rootHandle = rootHandle;
this.reportStatus = options.reportStatus ?? (() => {});
this.scheduleReload =
@@ -89,6 +95,8 @@ export class WebAppRuntime {
private createServiceOptions(): LiveSyncBrowserServiceHubOptions<ServiceContext> {
return {
deviceValues:
this.options.deviceValues ?? createWebStorageValueStore(compatGlobal.localStorage, { prefix: "" }),
getSystemVaultName: () => this.rootHandle.name || "livesync-webapp",
settings: {
save: async (data) => {
@@ -219,7 +227,9 @@ export class WebAppRuntime {
useCheckRemoteSize(core);
useRemoteConfiguration(core);
this.p2p = useP2PReplicatorFeature(core, undefined, undefined, {
prepareP2PSettings: useP2PSettingsPreparation(core.services.API.webCompatFetch.bind(core.services.API)),
prepareP2PSettings: useP2PSettingsPreparation(
core.services.API.webCompatFetch.bind(core.services.API)
),
});
this.paneHost = {
services: core.services,
src/apps/webapp/package.json
+2 -2
View File
@@ -1,7 +1,7 @@
{
"name": "livesync-webapp",
"private": true,
"version": "1.0.37-webapp",
"version": "1.0.42-webapp",
"type": "module",
"description": "Browser-based Self-hosted LiveSync using FileSystem API",
"scripts": {
@@ -21,6 +21,6 @@
"@sveltejs/vite-plugin-svelte": "^7.1.2",
"svelte": "5.56.3",
"typescript": "6.0.3",
"vite": "^8.0.16"
"vite": "^8.1.6"
}
}
src/apps/webapp/serviceModules/FSAPIServiceModules.ts
+1
View File
@@ -90,6 +90,7 @@ export function initialiseServiceModulesFSAPI(
// Rebuilder (platform-independent)
const rebuilder = new ServiceRebuilder({
path: services.path,
events: services.context.events,
API: services.API,
database: services.database,
src/apps/webpeer/README.md
+2
View File
@@ -49,6 +49,8 @@ After the first device connects, **Try another device without resetting** keeps
Serve the production build over HTTPS or from `localhost`. Downloading `check.html` alone is not supported because the page uses built module assets and origin-scoped browser storage. The generated credentials are temporary and must not replace the settings of a production Vault.
The generated Setup URI is Time-bound, and the page shows when its current fixed UTC window ends. After that time, the page hides the URI and QR code and disables the additional-device action. Start a fresh check to configure another device. A monitor for a device which already imported the settings remains available.
The detailed scope and result semantics are recorded in the [browser-assisted P2P connection-check ADR](../../../docs/adr/2026_07_p2p_connection_check.md).
## Storage and lifecycle
src/apps/webpeer/package.json
+2 -2
View File
@@ -1,7 +1,7 @@
{
"name": "webpeer",
"private": true,
"version": "1.0.37-webpeer",
"version": "1.0.42-webpeer",
"type": "module",
"scripts": {
"dev": "vite",
@@ -24,6 +24,6 @@
"svelte": "5.56.3",
"svelte-check": "^4.6.0",
"typescript": "6.0.3",
"vite": "^8.0.16"
"vite": "^8.1.6"
}
}
src/apps/webpeer/src/WebPeerRuntime.ts
+7 -3
View File
@@ -7,6 +7,8 @@ import { P2PLogCollector } from "@vrtmrz/livesync-commonlib/compat/replication/t
import { useP2PReplicatorFeature } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/useP2PReplicatorFeature";
import { ServiceContext, type LiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context";
import type { P2PServiceViews } from "@vrtmrz/livesync-commonlib/p2p";
import type { ValueStore, ValueDeleter } from "@vrtmrz/value-store";
import { createWebStorageValueStore } from "@vrtmrz/value-store/web-storage";
import { unique } from "octagonal-wheels/collection";
import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase";
@@ -20,6 +22,7 @@ import { translateLiveSyncMessage } from "@/common/translation";
import { WEBPEER_STORE_NAME, createWebPeerPersistence } from "./WebPeerPersistence";
export interface WebPeerRuntimeOptions {
deviceValues?: ValueStore & ValueDeleter;
context?: ServiceContext;
store?: SimpleStore<unknown>;
deviceName?: string;
@@ -62,6 +65,7 @@ export class WebPeerRuntime {
this.context = options.context ?? new ServiceContext({ translate: translateLiveSyncMessage });
this.services = createLiveSyncBrowserServiceHub<ServiceContext>({
context: this.context,
deviceValues: options.deviceValues ?? createWebStorageValueStore(compatGlobal.localStorage, { prefix: "" }),
getSystemVaultName: () => options.systemVaultName ?? WEBPEER_STORE_NAME,
settings: persistence.settings,
restart: {
@@ -103,7 +107,7 @@ export class WebPeerRuntime {
await this.services.setting.loadSettings();
const deviceName = this.options.deviceName?.trim();
if (deviceName) {
this.services.config.setSmallConfig(SETTING_KEY_P2P_DEVICE_NAME, deviceName);
await this.services.config.setValue(SETTING_KEY_P2P_DEVICE_NAME, deviceName);
}
const opened = await this.services.database.openDatabase({
replicator: this.services.replicator,
@@ -192,7 +196,7 @@ export class WebPeerRuntime {
);
}
getDeviceName(): string {
return this.services.config.getSmallConfig(SETTING_KEY_P2P_DEVICE_NAME) || this.services.vault.getVaultName();
async getDeviceName(): Promise<string> {
return (await this.services.config.getValue(SETTING_KEY_P2P_DEVICE_NAME)) || this.services.vault.getVaultName();
}
}
src/common/events.ts
+2
View File
@@ -19,6 +19,7 @@ export const EVENT_REQUEST_RUN_DOCTOR = "request-run-doctor";
export const EVENT_REQUEST_RUN_FIX_INCOMPLETE = "request-run-fix-incomplete";
export const EVENT_ANALYSE_DB_USAGE = "analyse-db-usage";
export const EVENT_INSPECT_DB_INTEGRITY = "inspect-db-integrity";
export const EVENT_REQUEST_PERFORM_GC_V3 = "request-perform-gc-v3";
// export const EVENT_REQUEST_CHECK_REMOTE_SIZE = "request-check-remote-size";
// export const EVENT_FILE_CHANGED = "file-changed";
@@ -37,6 +38,7 @@ declare global {
[EVENT_REQUEST_RUN_DOCTOR]: string;
[EVENT_REQUEST_RUN_FIX_INCOMPLETE]: undefined;
[EVENT_ANALYSE_DB_USAGE]: undefined;
[EVENT_INSPECT_DB_INTEGRITY]: undefined;
[EVENT_REQUEST_PERFORM_GC_V3]: undefined;
}
}
src/common/messages/LiveSyncProvisionalMessages.ts
+147
View File
@@ -7,6 +7,111 @@
* remove it from this map in the same change.
*/
export const liveSyncProvisionalEnglishMessages = {
"Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files.":
"Hidden File Sync is disabled on this device. Enable it in Sync Settings before using Automatic or initialising hidden files.",
"Hidden File Sync initialisation was not started.": "Hidden File Sync initialisation was not started.",
"(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns.":
"(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns.",
"(RegExp) Skip changes to matching normal and hidden files, locally and remotely.":
"(RegExp) Skip changes to matching normal and hidden files, locally and remotely.",
"Unknown platform": "Unknown platform",
"This removes only this device's record from the remote.":
"This removes only this device's record from the remote.",
"This remote is unlocked. A device that returns can register again.":
"This remote is unlocked. A device that returns can register again.",
"This remote is locked. Use the existing recovery process for devices that need to join again.":
"This remote is locked. Use the existing recovery process for devices that need to join again.",
"This remote is locked. If the device returns, use the existing recovery process to join it again.":
"This remote is locked. If the device returns, use the existing recovery process to join it again.",
"This remote does not support conditional updates, so the device record could not be removed.":
"This remote does not support conditional updates, so the device record could not be removed.",
"This remote does not currently have a record of joined devices.":
"This remote does not currently have a record of joined devices.",
"This list shows device information recorded by the remote. It does not show whether a device is online now.":
"This list shows device information recorded by the remote. It does not show whether a device is online now.",
"This device has been locked out. Use the existing recovery process to join again.":
"This device has been locked out. Use the existing recovery process to join again.",
"This device": "This device",
"This remote has been cleaned. Use the existing recovery process to join devices again as needed.":
"This remote has been cleaned. Use the existing recovery process to join devices again as needed.",
"The device record was forgotten.": "The device record was forgotten.",
"The device record was already absent.": "The device record was already absent.",
"The device list changed while you were trying to forget this device. Refresh the list before trying again.":
"The device list changed while you were trying to forget this device. Refresh the list before trying again.",
"The device information could not be loaded. Close this dialogue and open Joined devices again if the remote configuration has changed.":
"The device information could not be loaded. Close this dialogue and open Joined devices again if the remote configuration has changed.",
"Review device records for the active CouchDB or Object Storage connection.":
"Review device records for the active CouchDB or Object Storage connection.",
Refresh: "Refresh",
"Plug-in version": "Plug-in version",
Platform: "Platform",
"Older Self-hosted LiveSync versions detected: ${versions}.":
"Older Self-hosted LiveSync versions detected: ${versions}.",
"Newer Self-hosted LiveSync versions detected: ${versions}.":
"Newer Self-hosted LiveSync versions detected: ${versions}.",
"Reviewing and aligning Self-hosted LiveSync versions across devices is recommended.":
"Reviewing and aligning Self-hosted LiveSync versions across devices is recommended.",
Open: "Open",
"Not recorded": "Not recorded",
"Not accepted": "Not accepted",
"No joined devices are recorded on this remote.": "No joined devices are recorded on this remote.",
"Locked out": "Locked out",
"Loading joined devices…": "Loading joined devices…",
"Last recorded": "Last recorded",
"Joined devices could not be updated. Refresh the list and try again. If the remote configuration has changed, close this dialogue and open Joined devices again.":
"Joined devices could not be updated. Refresh the list and try again. If the remote configuration has changed, close this dialogue and open Joined devices again.",
"Joined devices are available after database startup with an active CouchDB or Object Storage connection.":
"Joined devices are available after database startup with an active CouchDB or Object Storage connection.",
"Joined devices": "Joined devices",
"Review devices": "Review devices",
"Synchronisation between different plug-in versions is generally safe unless the connection is explicitly blocked. However, it may be less efficient or use more storage, and P2P synchronisation may be unavailable.":
"Synchronisation between different plug-in versions is generally safe unless the connection is explicitly blocked. However, it may be less efficient or use more storage, and P2P synchronisation may be unavailable.",
"Forgetting…": "Forgetting…",
"Forgetting the record does not delete Vault files or revoke credentials.":
"Forgetting the record does not delete Vault files or revoke credentials.",
"Forget this device record?": "Forget this device record?",
"Forget device": "Forget device",
Close: "Close",
Cancel: "Cancel",
"App version": "App version",
Accepted: "Accepted",
"Preparing follow-up report…": "Preparing follow-up report…",
"Cancelling local search. Checked ${COUNT} entries.": "Cancelling local search. Checked ${COUNT} entries.",
"Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.":
"Search current Vault paths and current local Metadata only. Older history and remote-only files are not searched.",
"The lookup does not read file contents or contact a remote database. Follow-up reports include current conflicts and merge bases.":
"The lookup does not read file contents or contact a remote database. Follow-up reports include current conflicts and merge bases.",
"Local file diagnostic report": "Local file diagnostic report",
"Checked ${COUNT} local entries.": "Checked ${COUNT} local entries.",
"Enter a valid report path pseudonym.": "Enter a valid report path pseudonym.",
"Search cancelled. Matches found so far are shown.": "Search cancelled. Matches found so far are shown.",
"No matches were found in current Vault paths or current local Metadata. Older history and remote-only files are not searched.":
"No matches were found in current Vault paths or current local Metadata. Older history and remote-only files are not searched.",
"Found ${COUNT} local matches.": "Found ${COUNT} local matches.",
"The local search could not finish.": "The local search could not finish.",
"The file could not be opened.": "The file could not be opened.",
"The follow-up report could not be prepared.": "The follow-up report could not be prepared.",
"Report path pseudonym": "Report path pseudonym",
Source: "Source",
Metadata: "Metadata",
Vault: "Vault",
"Metadata document ID": "Metadata document ID",
"No current Metadata document": "No current Metadata document",
"Current Vault file": "Current Vault file",
Present: "Present",
Missing: "Missing",
"Open file": "Open file",
"Prepare follow-up report": "Prepare follow-up report",
"This report uses pseudonyms. Review its coverage before sharing.":
"This report uses pseudonyms. Review its coverage before sharing.",
"Back to matches": "Back to matches",
"Reading revisions": "Reading revisions",
"Calculating database usage": "Calculating database usage",
"Discovering revisions": "Discovering revisions",
"Find a file from a report pseudonym": "Find a file from a report pseudonym",
"Find matching paths in this Vault and local Metadata, then prepare a follow-up report using the same pseudonyms.":
"Find matching paths in this Vault and local Metadata, then prepare a follow-up report using the same pseudonyms.",
"Find file": "Find file",
"⚠️ Initialise or overwrite the remote": "⚠️ Initialise or overwrite the remote",
"🔗 Join this device": "🔗 Join this device",
"⚙️ Apply settings only (advanced)": "⚙️ Apply settings only (advanced)",
@@ -251,6 +356,48 @@ export const liveSyncProvisionalEnglishMessages = {
"An ID source is required to enable this option.": "An ID source is required to enable this option.",
"The ID source or recovery code is invalid. Check it and try again.":
"The ID source or recovery code is invalid. Check it and try again.",
"Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.":
"Read normal-file Metadata and current conflicts locally and in CouchDB, then compare content for suspect groups. Choose original names or pseudonyms before starting.",
Inspect: "Inspect",
"Inspect database integrity": "Inspect database integrity",
"Inspection completed. Review the findings and coverage before sharing.":
"Inspection completed. Review the findings and coverage before sharing.",
"Inspection partially completed. Some information could not be checked; review the reported reasons.":
"Inspection partially completed. Some information could not be checked; review the reported reasons.",
"Inspection failed. The report records the available evidence and reasons.":
"Inspection failed. The report records the available evidence and reasons.",
"Inspection cancelled. The report contains only the evidence collected so far.":
"Inspection cancelled. The report contains only the evidence collected so far.",
"Report copied.": "Report copied.",
"Copy failed. Select the report text and copy it manually.":
"Copy failed. Select the report text and copy it manually.",
"Download requested. You can also copy the report if downloads are unavailable on this device.":
"Download requested. You can also copy the report if downloads are unavailable on this device.",
"Pseudonyms include a public salt, so guessed paths can be checked. Original names include private filenames. Reports omit file contents and credentials.":
"Pseudonyms include a public salt, so guessed paths can be checked. Original names include private filenames. Reports omit file contents and credentials.",
"Read available local revisions and report their Chunk references. Settings are left unchanged.":
"Read available local revisions and report their Chunk references. Settings are left unchanged.",
"Inspect normal-file Metadata, current conflicts, and suspect content against the Vault. Remote inspection supports CouchDB. Older history is excluded; findings do not authorise deletion.":
"Inspect normal-file Metadata, current conflicts, and suspect content against the Vault. Remote inspection supports CouchDB. Older history is excluded; findings do not authorise deletion.",
"Include remote database": "Include remote database",
"Names in this report": "Names in this report",
"Pseudonyms (recommended)": "Pseudonyms (recommended)",
"Original names": "Original names",
"Cancelling inspection…": "Cancelling inspection…",
"Items inspected": "Items inspected",
"Cancel inspection": "Cancel inspection",
"This report uses temporary pseudonyms. Counts and relationships remain visible.":
"This report uses temporary pseudonyms. Counts and relationships remain visible.",
"This report contains original paths and identifiers. Review them before sharing.":
"This report contains original paths and identifiers. Review them before sharing.",
"Partial Usage results are available as JSON so their coverage and reasons stay with the rows.":
"Partial Usage results are available as JSON so their coverage and reasons stay with the rows.",
"Report format": "Report format",
"Diagnostic report": "Diagnostic report",
"Copy report": "Copy report",
"Save report": "Save report",
"The inspection could not finish. Close this dialogue and try again after database startup.":
"The inspection could not finish. Close this dialogue and try again after database startup.",
} as const;
export type LiveSyncProvisionalMessageKey = keyof typeof liveSyncProvisionalEnglishMessages;
src/common/messages/combinedMessages.prod.ts
+13
View File
@@ -1297,6 +1297,10 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
zh: "CouchDB 连接调优",
"zh-tw": "CouchDB 連線調校",
},
"CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1.":
{
def: "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1.",
},
"CouchDB validates the database name when you connect. The name must not be empty.": {
def: "CouchDB validates the database name when you connect. The name must not be empty.",
es: "CouchDB valida el nombre de la base de datos al conectar. El nombre no puede estar vacío.",
@@ -3871,6 +3875,9 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
zh: "稍后",
"zh-tw": "稍後",
},
"Legacy path identity": {
def: "Legacy path identity",
},
"Limit: {datetime} ({timestamp})": {
def: "Limit: {datetime} ({timestamp})",
es: "Límite: {datetime} ({timestamp})",
@@ -7937,6 +7944,9 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
zh: "路径",
"zh-tw": "路徑",
},
"Path identity mode": {
def: "Path identity mode",
},
"Path Obfuscation": {
def: "Path Obfuscation",
es: "Ofuscación de rutas",
@@ -13774,6 +13784,9 @@ export const allMessages: Readonly<Record<string, Readonly<Record<string, string
zh: "输入服务器信息",
"zh-tw": "選擇同步遠端",
},
"Unicode NFC path identity": {
def: "Unicode NFC path identity",
},
"Unique name between all synchronized devices. To edit this setting, please disable customization sync once.": {
def: "Unique name between all synchronized devices. To edit this setting, please disable customization sync once.",
es: "Nombre único entre dispositivos sincronizados. Para editarlo, desactive sincronización de personalización",
src/common/messagesJson/en.json
+4
View File
@@ -151,6 +151,7 @@
"Copy Report to clipboard": "Copy Report to clipboard",
"CouchDB Configuration": "CouchDB Configuration",
"CouchDB Connection Tweak": "CouchDB Connection Tweak",
"CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1.": "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1.",
"CouchDB validates the database name when you connect. The name must not be empty.": "CouchDB validates the database name when you connect. The name must not be empty.",
"Create or connect to database and continue": "Create or connect to database and continue",
"Create P2P remote": "Create P2P remote",
@@ -446,6 +447,7 @@
"lang-zh-tw": "繁體中文",
"Last tweak modified timestamp": "Last tweak modified timestamp",
"Later": "Later",
"Legacy path identity": "Legacy path identity",
"Limit: {datetime} ({timestamp})": "Limit: {datetime} ({timestamp})",
"live": "live",
"LiveSync could not handle multiple vaults which have same name without different prefix, This should be automatically configured.": "LiveSync could not handle multiple vaults which have same name without different prefix, This should be automatically configured.",
@@ -841,6 +843,7 @@
"Paste a connection string": "Paste a connection string",
"Paste the Setup URI generated from one of your active devices.": "Paste the Setup URI generated from one of your active devices.",
"Path": "Path",
"Path identity mode": "Path identity mode",
"Path Obfuscation": "Path Obfuscation",
"Patterns to match files for overwriting instead of merging": "Patterns to match files for overwriting instead of merging",
"Patterns to match files for syncing": "Patterns to match files for syncing",
@@ -1523,6 +1526,7 @@
"Ui.SetupWizard.SetupRemote.ProceedCouchDb": "Continue to CouchDB setup",
"Ui.SetupWizard.SetupRemote.ProceedP2P": "Continue to P2P setup",
"Ui.SetupWizard.SetupRemote.Title": "Choose a synchronisation remote",
"Unicode NFC path identity": "Unicode NFC path identity",
"Unique name between all synchronized devices. To edit this setting, please disable customization sync once.": "Unique name between all synchronized devices. To edit this setting, please disable customization sync once.",
"Unless you are certain, selecting this options is bit dangerous. It assumes that the server configuration is compatible with this device. If this is not the case, data loss may occur. Please ensure you know what you are doing.": "Unless you are certain, selecting this options is bit dangerous. It assumes that the server configuration is compatible with this device. If this is not the case, data loss may occur. Please ensure you know what you are doing.",
"Updating list...": "Updating list...",
src/common/messagesYAML/en.yaml
+5
View File
@@ -169,6 +169,8 @@ Copy: Copy
Copy Report to clipboard: Copy Report to clipboard
CouchDB Configuration: CouchDB Configuration
CouchDB Connection Tweak: CouchDB Connection Tweak
"CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1.":
"CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1."
CouchDB validates the database name when you connect. The name must not be empty.:
CouchDB validates the database name when you connect. The name must not be
empty.
@@ -556,6 +558,7 @@ lang-ru: Русский
lang-zh: 简体中文
lang-zh-tw: 繁體中文
Later: Later
Legacy path identity: Legacy path identity
"Limit: {datetime} ({timestamp})": "Limit: {datetime} ({timestamp})"
live: live
LiveSync could not handle multiple vaults which have same name without different prefix, This should be automatically configured.:
@@ -1251,6 +1254,7 @@ password: password
Password: Password
Paste a connection string: Paste a connection string
Path: Path
Path identity mode: Path identity mode
Path Obfuscation: Path Obfuscation
Patterns to match files for overwriting instead of merging: Patterns to match files for overwriting instead of merging
Patterns to match files for syncing: Patterns to match files for syncing
@@ -2042,6 +2046,7 @@ TweakMismatchResolve:
TweakResolving: Configuration Mismatch Detected
UseRemoteConfig: Use Remote Configuration
AutoAcceptCompatible: Auto-Accept Available
Unicode NFC path identity: Unicode NFC path identity
Unique name between all synchronized devices. To edit this setting, please disable customization sync once.:
Unique name between all synchronized devices. To edit this setting, please
disable customization sync once.
src/common/replicatorConfigurationIdentity.ts
+10 -8
View File
@@ -1,5 +1,6 @@
import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { usesEncryptedInternalMetadata } from "@vrtmrz/livesync-commonlib/replication";
import { getPathNormalizationVersion } from "@vrtmrz/livesync-commonlib/settings";
type EndpointProjection = readonly [kind: "url" | "invalid-url", value: string];
@@ -44,17 +45,18 @@ function projectHeaders(value: string): readonly (readonly [name: string, value:
function projectRemoteSecurity(settings: RemoteDBSettings) {
return [
getPathNormalizationVersion(settings),
settings.idDerivationVersion,
settings.idDerivationKey,
settings.encrypt
? ([
"encrypted",
settings.passphrase,
settings.useDynamicIterationCount,
settings.E2EEAlgorithm,
settings.permitEmptyPassphrase,
] as const)
: (["plain"] as const),
? ([
"encrypted",
settings.passphrase,
settings.useDynamicIterationCount,
settings.E2EEAlgorithm,
settings.permitEmptyPassphrase,
] as const)
: (["plain"] as const),
] as const;
}
src/common/replicatorConfigurationIdentity.unit.spec.ts
+24
View File
@@ -7,6 +7,30 @@ import {
} from "./replicatorConfigurationIdentity";
describe("active Replicator configuration identity", () => {
it("replaces a connection when the path identity mode changes", () => {
const previous = configuredSettings({ pathNormalizationVersion: 0 });
const next = { ...previous, pathNormalizationVersion: 1 as const };
expect(getCouchDBReplicatorConfigurationIdentity(next)).not.toBe(
getCouchDBReplicatorConfigurationIdentity(previous)
);
expect(getObjectStorageReplicatorConfigurationIdentity(next)).not.toBe(
getObjectStorageReplicatorConfigurationIdentity(previous)
);
});
it("treats a missing path identity mode as version 0", () => {
const missingVersion = configuredSettings();
Reflect.deleteProperty(missingVersion, "pathNormalizationVersion");
const explicitVersion0 = { ...missingVersion, pathNormalizationVersion: 0 as const };
expect(getCouchDBReplicatorConfigurationIdentity(explicitVersion0)).toBe(
getCouchDBReplicatorConfigurationIdentity(missingVersion)
);
expect(getObjectStorageReplicatorConfigurationIdentity(explicitVersion0)).toBe(
getObjectStorageReplicatorConfigurationIdentity(missingVersion)
);
});
function configuredSettings(overrides: Partial<ObsidianLiveSyncSettings> = {}): ObsidianLiveSyncSettings {
return Object.assign(createNewVaultSettings(), {
activeConfigurationId: "profile-a",
src/common/replicatorProviders.ts
+13
View File
@@ -28,7 +28,10 @@ import {
getObjectStorageReplicatorConfigurationIdentity,
} from "./replicatorConfigurationIdentity";
import {
createCouchDBJoinedDevicesResourceFactory,
createObjectStorageJoinedDevicesResourceFactory,
createCouchDBConnectionProbeFactory,
createCouchDBInitialFetchFastForwardResourceFactory,
createCouchDBPreferredTweakProbeFactory,
createCouchDBSecuritySeedResourceFactory,
createCouchDBSynchronisationInformationResourceFactory,
@@ -115,6 +118,9 @@ export function createCentralReplicatorProviderDefinitions(
configurationIdentity: getCouchDBReplicatorConfigurationIdentity,
create: () => Promise.resolve(new LiveSyncCouchDBReplicator(host)),
remoteResources: {
[REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability(
createCouchDBJoinedDevicesResourceFactory(host)
),
[REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability(createCouchDBConnectionProbeFactory(host)),
[REMOTE_RESOURCE_KINDS.PREFERRED_TWEAK]: supportedCapability(
createCouchDBPreferredTweakProbeFactory(host)
@@ -125,6 +131,9 @@ export function createCentralReplicatorProviderDefinitions(
[REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION]: supportedCapability(
createCouchDBSynchronisationInformationResourceFactory(host)
),
[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]: supportedCapability(
createCouchDBInitialFetchFastForwardResourceFactory(host)
),
},
centralRemoteAdministration: COUCHDB_CENTRAL_REMOTE_ADMINISTRATION_CAPABILITY,
userInitiatedOneShot: supportedCapability(couchDBUserInitiatedOneShot),
@@ -141,6 +150,9 @@ export function createCentralReplicatorProviderDefinitions(
configurationIdentity: getObjectStorageReplicatorConfigurationIdentity,
create: () => Promise.resolve(new LiveSyncJournalReplicator(host)),
remoteResources: {
[REMOTE_RESOURCE_KINDS.JOINED_DEVICES]: supportedCapability(
createObjectStorageJoinedDevicesResourceFactory(host)
),
[REMOTE_RESOURCE_KINDS.CONNECTION]: supportedCapability(
createObjectStorageConnectionProbeFactory(host)
),
@@ -151,6 +163,7 @@ export function createCentralReplicatorProviderDefinitions(
createObjectStorageSecuritySeedResourceFactory(host)
),
[REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION]: CAPABILITY_NOT_APPLICABLE,
[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]: CAPABILITY_NOT_APPLICABLE,
},
centralRemoteAdministration: OBJECT_STORAGE_CENTRAL_REMOTE_ADMINISTRATION_CAPABILITY,
userInitiatedOneShot: supportedCapability(objectStorageUserInitiatedOneShot),
src/common/replicatorProviders.unit.spec.ts
+21 -3
View File
@@ -44,7 +44,16 @@ describe("central Replicator provider definitions", () => {
it("keeps the retained remote-resource catalogue bounded", () => {
expect
.soft(Object.values(REMOTE_RESOURCE_KINDS).sort())
.toEqual(["connection", "preferred-tweak", "security-seed", "synchronisation-information"].sort());
.toEqual(
[
"connection",
"initial-fetch-fast-forward",
"joined-devices",
"preferred-tweak",
"security-seed",
"synchronisation-information",
].sort()
);
});
it("lets Journal prepare its own fresh Security Seed while CouchDB uses central preparation", () => {
@@ -106,18 +115,27 @@ describe("central Replicator provider definitions", () => {
const couchAdministration = definitions.get(REMOTE_COUCHDB)?.centralRemoteAdministration;
const objectAdministration = definitions.get(REMOTE_MINIO)?.centralRemoteAdministration;
expect(Object.keys(couchResources ?? {}).sort()).toEqual(Object.values(REMOTE_RESOURCE_KINDS).sort());
expect(Object.keys(objectResources ?? {}).sort()).toEqual(Object.values(REMOTE_RESOURCE_KINDS).sort());
const resources = Object.values(REMOTE_RESOURCE_KINDS).sort();
expect(Object.keys(couchResources ?? {}).sort()).toEqual(resources);
expect(Object.keys(objectResources ?? {}).sort()).toEqual(resources);
expect(couchResources?.[REMOTE_RESOURCE_KINDS.JOINED_DEVICES]?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED);
expect(objectResources?.[REMOTE_RESOURCE_KINDS.JOINED_DEVICES]?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED);
expect(couchResources?.[REMOTE_RESOURCE_KINDS.CONNECTION].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED);
expect(couchResources?.[REMOTE_RESOURCE_KINDS.PREFERRED_TWEAK].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED);
expect(couchResources?.[REMOTE_RESOURCE_KINDS.SECURITY_SEED].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED);
expect(couchResources?.[REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION].kind).toBe(
CAPABILITY_SUPPORT_KINDS.SUPPORTED
);
expect(couchResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]?.kind).toBe(
CAPABILITY_SUPPORT_KINDS.SUPPORTED
);
expect(objectResources?.[REMOTE_RESOURCE_KINDS.SECURITY_SEED].kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED);
expect(objectResources?.[REMOTE_RESOURCE_KINDS.SYNCHRONISATION_INFORMATION].kind).toBe(
CAPABILITY_SUPPORT_KINDS.NOT_APPLICABLE
);
expect(objectResources?.[REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]?.kind).toBe(
CAPABILITY_SUPPORT_KINDS.NOT_APPLICABLE
);
expect(couchAdministration?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED);
expect(objectAdministration?.kind).toBe(CAPABILITY_SUPPORT_KINDS.SUPPORTED);
expect("activeRemoteReads" in definitions.get(REMOTE_COUCHDB)!).toBe(false);
src/common/replicatorResources.unit.spec.ts
+99
View File
@@ -5,6 +5,7 @@ import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings";
const mocks = vi.hoisted(() => ({
logger: vi.fn(),
initialFetchFastForwardResource: undefined as unknown,
couchDB: [] as Array<{
host: unknown;
isMobile: ReturnType<typeof vi.fn>;
@@ -12,6 +13,9 @@ const mocks = vi.hoisted(() => ({
getRemoteStatus: ReturnType<typeof vi.fn>;
getRemotePreferredTweakValues: ReturnType<typeof vi.fn>;
getReplicationPBKDF2Salt: ReturnType<typeof vi.fn>;
getJoinedDevices: ReturnType<typeof vi.fn>;
forgetJoinedDevice: ReturnType<typeof vi.fn>;
createInitialFetchFastForwardResource: ReturnType<typeof vi.fn>;
closeReplication: ReturnType<typeof vi.fn>;
}>,
objectStorage: [] as Array<{
@@ -20,6 +24,8 @@ const mocks = vi.hoisted(() => ({
getRemoteStatus: ReturnType<typeof vi.fn>;
getRemotePreferredTweakValues: ReturnType<typeof vi.fn>;
getReplicationPBKDF2Salt: ReturnType<typeof vi.fn>;
getJoinedDevices: ReturnType<typeof vi.fn>;
forgetJoinedDevice: ReturnType<typeof vi.fn>;
closeReplication: ReturnType<typeof vi.fn>;
}>,
checkSyncInfo: vi.fn(async () => true),
@@ -42,6 +48,9 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicato
getRemoteStatus = vi.fn();
getRemotePreferredTweakValues = vi.fn();
getReplicationPBKDF2Salt = vi.fn();
getJoinedDevices = vi.fn();
forgetJoinedDevice = vi.fn();
createInitialFetchFastForwardResource = vi.fn(() => mocks.initialFetchFastForwardResource);
closeReplication = vi.fn();
constructor(host: unknown) {
@@ -58,6 +67,8 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalRe
getRemoteStatus = vi.fn();
getRemotePreferredTweakValues = vi.fn();
getReplicationPBKDF2Salt = vi.fn();
getJoinedDevices = vi.fn();
forgetJoinedDevice = vi.fn();
closeReplication = vi.fn();
constructor(host: unknown) {
@@ -69,10 +80,13 @@ vi.mock("@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalRe
import {
createCouchDBConnectionProbeFactory,
createCouchDBJoinedDevicesResourceFactory,
createCouchDBInitialFetchFastForwardResourceFactory,
createCouchDBPreferredTweakProbeFactory,
createCouchDBSecuritySeedResourceFactory,
createCouchDBSynchronisationInformationResourceFactory,
createObjectStorageConnectionProbeFactory,
createObjectStorageJoinedDevicesResourceFactory,
createObjectStoragePreferredTweakProbeFactory,
createObjectStorageSecuritySeedResourceFactory,
} from "./replicatorResources";
@@ -92,6 +106,7 @@ describe("replicator probe factories", () => {
beforeEach(() => {
mocks.couchDB.length = 0;
mocks.objectStorage.length = 0;
mocks.initialFetchFastForwardResource = undefined;
mocks.checkSyncInfo.mockReset().mockResolvedValue(true);
mocks.logger.mockClear();
});
@@ -226,6 +241,65 @@ describe("replicator probe factories", () => {
expect(preferredReplicator.getRemotePreferredTweakValues).toHaveBeenCalledWith(snapshot);
});
it.each([
{
kind: "CouchDB",
remoteType: REMOTE_COUCHDB,
createFactory: createCouchDBJoinedDevicesResourceFactory,
instances: () => mocks.couchDB,
},
{
kind: "Object Storage",
remoteType: REMOTE_MINIO,
createFactory: createObjectStorageJoinedDevicesResourceFactory,
instances: () => mocks.objectStorage,
},
])(
"keeps $kind joined-device operations on an owned settings snapshot",
async ({ remoteType, createFactory, instances }) => {
const host = { name: "host" };
const source = createSettings({ remoteType });
const snapshot = { ...source };
const factory = createFactory(host as never);
const resource = await factory(source);
const nextResource = await factory(source);
const [replicator, nextReplicator] = instances();
expect(instances()).toHaveLength(2);
expect(replicator.host).toBe(host);
expect(replicator.getJoinedDevices).not.toHaveBeenCalled();
expect(replicator.forgetJoinedDevice).not.toHaveBeenCalled();
const devices = {
milestonePresent: true,
locked: false,
cleaned: false,
devices: [{ id: "other", accepted: true }],
};
replicator.getJoinedDevices.mockResolvedValue(devices);
replicator.forgetJoinedDevice.mockResolvedValue("conflict");
source.couchDB_DBNAME = "changed-vault";
source.bucket = "changed-bucket";
await expect(resource.read()).resolves.toBe(devices);
await expect(resource.forget("other")).resolves.toBe("conflict");
expect(replicator.getJoinedDevices).toHaveBeenCalledWith(snapshot);
expect(replicator.getJoinedDevices.mock.calls[0][0]).not.toBe(source);
expect(replicator.forgetJoinedDevice).toHaveBeenCalledWith(snapshot, "other");
const failure = new Error("remote unavailable");
replicator.getJoinedDevices.mockRejectedValue(failure);
await expect(resource.read()).rejects.toBe(failure);
replicator.forgetJoinedDevice.mockRejectedValue(failure);
await expect(resource.forget("other")).rejects.toBe(failure);
const disposal = resource.dispose();
expect(resource.dispose()).toBe(disposal);
await disposal;
expect(replicator.closeReplication).toHaveBeenCalledOnce();
expect(nextReplicator.closeReplication).not.toHaveBeenCalled();
await nextResource.dispose();
expect(nextReplicator.closeReplication).toHaveBeenCalledOnce();
}
);
it("shares one successful asynchronous disposal promise for every probe kind", async () => {
const couchProbe = await createCouchDBPreferredTweakProbeFactory({} as never)(createSettings());
const objectProbe = await createObjectStoragePreferredTweakProbeFactory({} as never)(
@@ -370,4 +444,29 @@ describe("replicator probe factories", () => {
await resource.dispose();
expect(replicator.closeReplication).toHaveBeenCalledOnce();
});
it("creates an Initial Fetch Fast Forward resource from a settings snapshot and leaves ownership to the resource", async () => {
const host = { name: "host" };
const settings = createSettings();
const snapshot = { ...settings };
const prepare = vi.fn(async () => undefined);
const dispose = vi.fn(async () => undefined);
const resource = { prepare, dispose };
const factory = createCouchDBInitialFetchFastForwardResourceFactory(host as never);
mocks.initialFetchFastForwardResource = resource;
const result = await factory(settings);
const replicator = mocks.couchDB[0];
settings.couchDB_URI = "https://changed.example.test";
expect(replicator.createInitialFetchFastForwardResource).toHaveBeenCalledWith(snapshot);
expect(replicator.createInitialFetchFastForwardResource.mock.calls[0][0]).not.toBe(settings);
expect(replicator.host).toBe(host);
expect(result).toBe(resource);
await expect(result.prepare()).resolves.toBeUndefined();
await result.dispose();
expect(prepare).toHaveBeenCalledOnce();
expect(dispose).toHaveBeenCalledOnce();
});
});
src/common/replicatorResources/index.ts
+6
View File
@@ -14,3 +14,9 @@ export {
type SecuritySeedResourceHost,
} from "./securitySeed";
export { createCouchDBSynchronisationInformationResourceFactory } from "./synchronisationInformation";
export {
createCouchDBJoinedDevicesResourceFactory,
createObjectStorageJoinedDevicesResourceFactory,
} from "./joinedDevices";
export { createCouchDBInitialFetchFastForwardResourceFactory } from "./initialFetchFastForward";
src/common/replicatorResources/initialFetchFastForward.ts
+17
View File
@@ -0,0 +1,17 @@
import type { InitialFetchFastForwardResourceFactory } from "@vrtmrz/livesync-commonlib/replication";
import {
LiveSyncCouchDBReplicator,
type LiveSyncCouchDBReplicatorEnv,
} from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator";
import { snapshotRemoteSettings } from "./shared";
/** Create a private CouchDB Replicator for a fresh Fast Fetch checkpoint handoff. */
export function createCouchDBInitialFetchFastForwardResourceFactory(
host: LiveSyncCouchDBReplicatorEnv
): InitialFetchFastForwardResourceFactory {
return (setting) => {
const snapshot = snapshotRemoteSettings(setting);
const replicator = new LiveSyncCouchDBReplicator(host);
return Promise.resolve(replicator.createInitialFetchFastForwardResource(snapshot));
};
}
src/common/replicatorResources/joinedDevices.ts
+40
View File
@@ -0,0 +1,40 @@
import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type {
JoinedDevicesResource,
JoinedDevicesResourceFactory,
JoinedDevicesSnapshot,
ForgetJoinedDeviceResult,
} from "@vrtmrz/livesync-commonlib/replication";
import {
LiveSyncCouchDBReplicator,
type LiveSyncCouchDBReplicatorEnv,
} from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator";
import { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator";
import { createReplicatorDisposer, snapshotRemoteSettings, type ResourceReplicator } from "./shared";
interface JoinedDevicesReplicator extends ResourceReplicator {
getJoinedDevices(setting: RemoteDBSettings): Promise<JoinedDevicesSnapshot>;
forgetJoinedDevice(setting: RemoteDBSettings, deviceId: string): Promise<ForgetJoinedDeviceResult>;
}
function createResource(replicator: JoinedDevicesReplicator, snapshot: RemoteDBSettings): JoinedDevicesResource {
return {
read: () => replicator.getJoinedDevices(snapshot),
forget: (deviceId) => replicator.forgetJoinedDevice(snapshot, deviceId),
dispose: createReplicatorDisposer(replicator),
};
}
export function createCouchDBJoinedDevicesResourceFactory(
host: LiveSyncCouchDBReplicatorEnv
): JoinedDevicesResourceFactory {
return (setting) =>
Promise.resolve(createResource(new LiveSyncCouchDBReplicator(host), snapshotRemoteSettings(setting)));
}
export function createObjectStorageJoinedDevicesResourceFactory(
host: LiveSyncCouchDBReplicatorEnv
): JoinedDevicesResourceFactory {
return (setting) =>
Promise.resolve(createResource(new LiveSyncJournalReplicator(host), snapshotRemoteSettings(setting)));
}
src/common/turnSettingsPrivacy.unit.spec.ts
+2 -10
View File
@@ -1,3 +1,4 @@
import { MemoryValueStore } from "@vrtmrz/value-store";
import { describe, expect, it } from "vitest";
import {
DEFAULT_SETTINGS,
@@ -18,17 +19,7 @@ import {
} from "./turnSettingsPrivacy";
class MemorySettingService extends SettingService {
readonly items = new Map<string, string>();
saved?: ObsidianLiveSyncSettings;
protected setItem(key: string, value: string) {
this.items.set(key, value);
}
protected getItem(key: string) {
return this.items.get(key) ?? "";
}
protected deleteItem(key: string) {
this.items.delete(key);
}
protected saveData(settings: ObsidianLiveSyncSettings) {
this.saved = structuredClone(settings);
return Promise.resolve();
@@ -70,6 +61,7 @@ describe("managed TURN settings privacy", () => {
const originalURI = ConnectionStringParser.serialize({ type: "p2p", settings: current });
current.remoteConfigurations.managed.uri = originalURI;
const service = new MemorySettingService(new ServiceContext(), {
deviceValues: new MemoryValueStore(),
APIService: {
getSystemVaultName: () => "test-vault",
getAppID: () => "test-app",
src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts
+298
View File
@@ -0,0 +1,298 @@
import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("@/deps.ts", () => ({
addIcon: vi.fn(),
diff_match_patch: class DiffMatchPatch {},
normalizePath: (path: string) => path,
parseYaml: vi.fn(),
Platform: {},
}));
vi.mock("./PluginDialogModal.ts", () => ({ PluginDialogModal: class PluginDialogModal {} }));
vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} }));
vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({
ConflictResolveModal: class ConflictResolveModal {},
}));
vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" }));
vi.mock("@/common/utils.ts", () => ({
cancelTask: vi.fn(),
EVEN: Symbol("even"),
isCustomisationSyncMetadata: vi.fn(),
isPluginMetadata: vi.fn(),
scheduleTask: vi.fn(),
}));
vi.mock("@/common/PeriodicProcessor.ts", () => ({ PeriodicProcessor: class PeriodicProcessor {} }));
vi.mock("@/common/events.ts", () => ({
EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync",
eventHub: { onEvent: vi.fn() },
}));
vi.mock("@/common/translation", () => ({ $msg: (message: string) => message }));
vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ getObsidianCommunityPluginManager: vi.fn() }));
import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { createOptionalFileSyncDatabase, optionalFileEntry } from "../../../test/fixtures/optionalFileSyncDatabase";
import { ConfigSync, PluginDataExDisplayV2, pluginManifests } from "./CmdConfigSync";
const filenames = ["main.js", "manifest.json", "styles.css"];
const content: Record<string, string> = {
"main.js": "module.exports = class ExamplePlugin {};",
"manifest.json": JSON.stringify({ id: "example", name: "Example", version: "1.0.0" }),
"styles.css": ".example { color: red; }",
};
const fixtureOwners: Awaited<ReturnType<typeof createOptionalFileSyncDatabase>>[] = [];
function deferred() {
let resolve!: () => void;
const promise = new Promise<void>((done) => {
resolve = done;
});
return { promise, resolve };
}
async function catalogue() {
const fixture = await createOptionalFileSyncDatabase();
fixtureOwners.push(fixture);
fixture.settings.syncOnlyRegEx = "\\.(?:md|png|pdf)$" as typeof fixture.settings.syncOnlyRegEx;
const paths = filenames.map((name) => `ix:device/PLUGIN_MAIN/example%${name}` as FilePathWithPrefix);
for (const [index, path] of paths.entries()) {
const wireContent = `fixture-header\u200b\n\u200c${btoa(content[filenames[index]])}`;
await expect(fixture.localDatabase.putDBEntry(optionalFileEntry(path, wireContent))).resolves.toMatchObject({
ok: true,
});
}
const gates = new Map(paths.map((path) => [path, deferred()]));
const readEntry = fixture.localDatabase.getDBEntry.bind(fixture.localDatabase);
vi.spyOn(fixture.localDatabase, "getDBEntry").mockImplementation(async (path, ...args) => {
await gates.get(path)?.promise;
return readEntry(path, ...args);
});
const written = new Map<string, string>();
const configSync = Object.create(ConfigSync.prototype) as ConfigSync;
Object.assign(configSync, {
core: {
localDatabase: fixture.localDatabase,
settings: fixture.settings,
services: fixture.services,
storageAccess: {
ensureDir: vi.fn(async () => true),
readHiddenFileText: vi.fn(async (path: string) => written.get(path) ?? ""),
writeHiddenFileAuto: vi.fn(async (path: string, data: string) => {
written.set(path, data);
}),
},
},
pluginList: [],
pluginListGeneration: 0,
loadedManifest_mTime: new Map(),
updatingV2Count: 0,
pluginScanProcessor: { clearQueue: vi.fn(), enqueue: vi.fn() },
_log: vi.fn(),
storeCustomisationFileV2: vi.fn(async () => true),
});
const updates: Promise<void>[] = [];
const update = configSync.updatePluginListV2.bind(configSync);
vi.spyOn(configSync, "updatePluginListV2").mockImplementation((...args) => {
const promise = update(...args);
updates.push(promise);
return promise;
});
const release = (order = filenames) => {
for (const name of order) gates.get(`ix:device/PLUGIN_MAIN/example%${name}` as FilePathWithPrefix)!.resolve();
};
return { ...fixture, configSync, updates, paths, release, written };
}
afterEach(async () => {
vi.restoreAllMocks();
pluginManifests.clear();
for (const fixture of fixtureOwners.splice(0)) await fixture.dispose();
});
describe("issue #1271: rebuilding a multi-file Customisation Sync item", () => {
it.each([
{ order: filenames },
{ order: [...filenames].reverse() },
{ order: ["manifest.json", "main.js", "styles.css"] },
])("retains all per-file documents when reads are released as $order", async ({ order }) => {
const fixture = await catalogue();
await fixture.configSync.reloadPluginList(false);
fixture.release(order);
await Promise.all(fixture.updates);
const documents = await fixture.database.allDocs({ keys: fixture.paths });
expect(documents.rows).toHaveLength(3);
expect(fixture.configSync.pluginList).toHaveLength(1);
expect(fixture.configSync.pluginList[0].files.map((file) => file.filename).sort()).toEqual(
filenames.map((name) => `plugins/example/${name}`).sort()
);
});
it("applies all three files after the first reload", async () => {
const fixture = await catalogue();
await fixture.configSync.reloadPluginList(false);
fixture.release();
await Promise.all(fixture.updates);
await expect(
fixture.configSync.applyDataV2(fixture.configSync.pluginList[0] as PluginDataExDisplayV2)
).resolves.toBe(true);
expect([...fixture.written.keys()].sort()).toEqual(
filenames.map((name) => `.obsidian/plugins/example/${name}`).sort()
);
for (const name of filenames)
expect(fixture.written.get(`.obsidian/plugins/example/${name}`)).toBe(content[name]);
});
it("fills an existing row during a second enumeration", async () => {
const fixture = await catalogue();
await fixture.configSync.reloadPluginList(false);
fixture.release();
await Promise.all(fixture.updates);
await fixture.configSync.updatePluginList(false);
await Promise.all(fixture.updates);
expect(fixture.configSync.pluginList[0].files.map((file) => file.filename).sort()).toEqual(
filenames.map((name) => `plugins/example/${name}`).sort()
);
});
it("retains all files when item updates run sequentially", async () => {
const fixture = await catalogue();
fixture.release();
for (const path of fixture.paths) await fixture.configSync.updatePluginListV2(false, path);
expect(fixture.configSync.pluginList[0].files.map((file) => file.filename).sort()).toEqual(
filenames.map((name) => `plugins/example/${name}`).sort()
);
});
it("updates a different item while another item's reads are pending", async () => {
const fixture = await catalogue();
const otherPath = "ix:device/PLUGIN_MAIN/other%main.js" as FilePathWithPrefix;
await fixture.localDatabase.putDBEntry(
optionalFileEntry(otherPath, `fixture-header\u200b\n\u200c${btoa(content["main.js"])}`)
);
const pending = fixture.paths.map((path) => fixture.configSync.updatePluginListV2(false, path));
try {
await fixture.configSync.updatePluginListV2(false, otherPath);
expect(fixture.configSync.pluginList.map((entry) => entry.name)).toEqual(["other"]);
expect(fixture.configSync.updatingV2Count).toBe(3);
} finally {
fixture.release();
await Promise.all(pending);
}
expect(fixture.configSync.updatingV2Count).toBe(0);
});
it("continues updating an item after an earlier read fails", async () => {
const fixture = await catalogue();
fixture.release();
vi.spyOn(fixture.configSync, "createPluginDataExFileV2").mockRejectedValueOnce(new Error("Read failed"));
const failed = fixture.configSync.updatePluginListV2(false, fixture.paths[0]);
const next = fixture.configSync.updatePluginListV2(false, fixture.paths[1]);
await expect(failed).rejects.toThrow("Read failed");
await next;
expect(fixture.configSync.pluginList[0].files.map((file) => file.filename)).toEqual([
"plugins/example/manifest.json",
]);
expect(fixture.configSync.updatingV2Count).toBe(0);
});
it.each(["read", "file update"])("reload discards cached files while an earlier %s is pending", async (phase) => {
const fixture = await catalogue();
fixture.release();
for (const path of fixture.paths) await fixture.configSync.updatePluginListV2(false, path);
const earlierRead = deferred();
const freshRead = deferred();
const entered = deferred();
const createFile = fixture.configSync.createPluginDataExFileV2.bind(fixture.configSync);
let reads = 0;
vi.spyOn(fixture.configSync, "createPluginDataExFileV2").mockImplementation(async (path) => {
if (reads++ === 0) {
if (phase === "read") {
entered.resolve();
await earlierRead.promise;
}
} else {
await freshRead.promise;
}
return createFile(path);
});
if (phase === "file update") {
const entry = fixture.configSync.pluginList[0] as PluginDataExDisplayV2;
const setFile = entry.setFile.bind(entry);
vi.spyOn(entry, "setFile").mockImplementationOnce(async (file) => {
entered.resolve();
await earlierRead.promise;
await setFile(file);
});
}
const pending = fixture.configSync.updatePluginListV2(false, fixture.paths[0]);
try {
await entered.promise;
for (const path of fixture.paths.slice(1)) await fixture.database.remove(await fixture.database.get(path));
await fixture.configSync.reloadPluginList(false);
earlierRead.resolve();
await pending;
} finally {
earlierRead.resolve();
freshRead.resolve();
await Promise.all(fixture.updates);
}
expect(fixture.configSync.pluginList[0].files.map((file) => file.filename)).toEqual([
"plugins/example/main.js",
]);
});
it("clearing the disabled feature prevents pending updates from repopulating the list", async () => {
const fixture = await catalogue();
const pending = fixture.paths.map((path) => fixture.configSync.updatePluginListV2(false, path));
fixture.settings.usePluginSync = false;
await fixture.configSync.updatePluginList(false);
fixture.release();
await Promise.all(pending);
expect(fixture.configSync.pluginList).toEqual([]);
expect(fixture.configSync.updatingV2Count).toBe(0);
});
it("a superseded enumeration cannot reintroduce a deleted item", async () => {
const fixture = await catalogue();
const obsoletePath = "ix:device/PLUGIN_MAIN/obsolete%main.js" as FilePathWithPrefix;
await fixture.localDatabase.putDBEntry(
optionalFileEntry(obsoletePath, `fixture-header\u200b\n\u200c${btoa(content["main.js"])}`)
);
const enumerate = fixture.localDatabase.findEntries.bind(fixture.localDatabase);
type EnumeratedEntry = ReturnType<typeof enumerate> extends AsyncGenerator<infer Entry> ? Entry : never;
const obsoleteDocs: EnumeratedEntry[] = [];
for await (const doc of enumerate(obsoletePath, `${obsoletePath}\u{10ffff}`, { include_docs: true })) {
obsoleteDocs.push(doc);
}
const entered = deferred();
const release = deferred();
let scans = 0;
vi.spyOn(fixture.localDatabase, "findEntries").mockImplementation((...args) => {
if (scans++ !== 0) return enumerate(...args);
return (async function* () {
entered.resolve();
await release.promise;
yield* obsoleteDocs;
})();
});
const pending = fixture.configSync.updatePluginList(false);
try {
await entered.promise;
await fixture.database.remove(await fixture.database.get(obsoletePath));
await fixture.configSync.reloadPluginList(false);
} finally {
release.resolve();
fixture.release();
await pending;
await Promise.all(fixture.updates);
}
expect(fixture.configSync.pluginList.map((entry) => entry.name)).toEqual(["example"]);
expect(fixture.configSync.pluginList[0].files).toHaveLength(3);
expect(fixture.configSync.updatingV2Count).toBe(0);
});
});
src/features/ConfigSync/CmdConfigSync.completion.unit.spec.ts
+220
View File
@@ -0,0 +1,220 @@
import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("@/deps.ts", () => ({
addIcon: vi.fn(),
diff_match_patch: class DiffMatchPatch {},
normalizePath: vi.fn((path: string) => path),
parseYaml: vi.fn(),
Platform: {},
}));
vi.mock("./PluginDialogModal.ts", () => ({
PluginDialogModal: class PluginDialogModal {},
}));
vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({
JsonResolveModal: class JsonResolveModal {},
}));
vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({
ConflictResolveModal: class ConflictResolveModal {},
}));
// The readiness helpers come from the real LiveSyncCommands, so only its log view dependency is replaced.
vi.mock("@/modules/features/ModuleLog.ts", () => ({
MARK_DONE: "",
}));
vi.mock("@/common/types.ts", () => ({
ICXHeader: "ix:",
PERIODIC_PLUGIN_SWEEP: 60,
}));
vi.mock("@/common/utils.ts", () => ({
cancelTask: vi.fn(),
EVEN: Symbol("even"),
isCustomisationSyncMetadata: vi.fn(),
isPluginMetadata: vi.fn(),
scheduleTask: vi.fn(),
}));
vi.mock("@/common/PeriodicProcessor.ts", () => ({
PeriodicProcessor: class PeriodicProcessor {},
}));
vi.mock("@/common/events.ts", () => ({
EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync",
eventHub: {
onEvent: vi.fn(),
},
}));
vi.mock("@/common/translation", () => ({
$msg: vi.fn((message: string) => message),
}));
vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({
getObsidianCommunityPluginManager: vi.fn(),
}));
import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { getDocData } from "@vrtmrz/livesync-commonlib/compat/common/utils";
import { ConfigSync } from "./CmdConfigSync";
import { createOptionalFileSyncDatabase } from "../../../test/fixtures/optionalFileSyncDatabase";
const path = ".obsidian/snippets/test.css" as FilePath;
const fixtures: Awaited<ReturnType<typeof createOptionalFileSyncDatabase>>[] = [];
async function createConfigSync(useV2 = false) {
const fixture = await createOptionalFileSyncDatabase();
fixtures.push(fixture);
fixture.settings.usePluginSyncV2 = useV2;
const files = new Map<string, { content: string; mtime: number }>([
[path, { content: "body { color: red; }", mtime: 2000 }],
]);
const storageAccess = {
statHidden: vi.fn(async (file: string) => {
const entry = files.get(file);
return entry
? { type: "file" as const, ctime: 1000, mtime: entry.mtime, size: entry.content.length }
: null;
}),
readHiddenFileBinary: vi.fn(async (file: string) => {
const entry = files.get(file);
if (!entry) throw new Error(`Missing fixture: ${file}`);
return new TextEncoder().encode(entry.content).buffer;
}),
};
const services = {
...fixture.services,
appLifecycle: { isReady: () => false, isSuspended: () => true },
database: { ...fixture.services.database, isDatabaseReady: vi.fn(() => true) },
path: {
...fixture.services.path,
isMarkedAsSameChanges: vi.fn(),
markChangesAreSame: vi.fn(),
},
};
const config = Object.create(ConfigSync.prototype) as ConfigSync;
Object.assign(config, {
core: { settings: fixture.settings, localDatabase: fixture.localDatabase, storageAccess, services },
_log: vi.fn(),
updatePluginList: vi.fn(async () => undefined),
updatePluginListV2: vi.fn(async () => undefined),
recentProcessedInternalFiles: [],
});
const databasePath = useV2 ? config.filenameWithUnifiedKey(path) : config.filenameToUnifiedKey(path);
return { fixture, config, files, services, storageAccess, databasePath };
}
afterEach(async () => {
vi.restoreAllMocks();
for (const fixture of fixtures.splice(0)) await fixture.dispose();
});
describe("Customisation Sync optional save completion", () => {
const flush = { whenFlushRequested: Promise.resolve() };
it("waits for the database write while the application is suspended or still starting", async () => {
const { fixture, config } = await createConfigSync();
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const put = fixture.localDatabase.putDBEntry.bind(fixture.localDatabase);
const spy = vi.spyOn(fixture.localDatabase, "putDBEntry").mockImplementation(async (...args) => {
await gate;
return await put(...args);
});
let completed = false;
const result = config._processOptionalFileEventWithResult(path, flush).then((outcome) => {
completed = true;
return outcome;
});
await vi.waitFor(() => expect(spy).toHaveBeenCalledOnce());
expect(completed).toBe(false);
release();
await expect(result).resolves.toEqual({ kind: "completed" });
});
it.each([false, true])("confirms identical contents and stores a same-time edit (V2: %s)", async (useV2) => {
const { fixture, config, databasePath, files } = await createConfigSync(useV2);
await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" });
const first = await fixture.database.get(databasePath);
await config._processOptionalFileEventWithResult(path, flush);
expect((await fixture.database.get(databasePath))._rev).toBe(first._rev);
files.get(path)!.content = "body { color: tan; }";
await config._processOptionalFileEventWithResult(path, flush);
expect((await fixture.database.get(databasePath))._rev).not.toBe(first._rev);
});
it.each([
{ useV2: false, throws: false },
{ useV2: true, throws: false },
{ useV2: false, throws: true },
{ useV2: true, throws: true },
])("rejects a failed write and permits the unchanged notification to retry: %j", async ({ useV2, throws }) => {
const { fixture, config, databasePath } = await createConfigSync(useV2);
const failure = new Error("temporary write failure");
const put = vi.spyOn(fixture.localDatabase, "putDBEntry");
if (throws) put.mockRejectedValueOnce(failure);
else put.mockResolvedValueOnce(false);
const failed = config._processOptionalFileEventWithResult(path, flush);
if (throws) await expect(failed).rejects.toBe(failure);
else await expect(failed).rejects.toThrow("Could not save customisation");
await expect(fixture.localDatabase.getDBEntry(databasePath)).resolves.toBe(false);
await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" });
expect(put).toHaveBeenCalledTimes(2);
expect(await fixture.localDatabase.getDBEntry(databasePath)).not.toBe(false);
});
it.each([false, true])("confirms deletion and treats a repeated deletion as complete (V2: %s)", async (useV2) => {
const { fixture, config, databasePath, files } = await createConfigSync(useV2);
await config._processOptionalFileEventWithResult(path, flush);
files.delete(path);
const put = vi.spyOn(fixture.localDatabase, "putRaw");
put.mockResolvedValueOnce({ ok: false, id: "failed", rev: "" });
await expect(config._processOptionalFileEventWithResult(path, flush)).rejects.toThrow(
"Could not delete customisation"
);
await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" });
const deleted = await fixture.database.get(databasePath);
expect("deleted" in deleted && deleted.deleted).toBe(true);
await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({ kind: "completed" });
expect((await fixture.database.get(databasePath))._rev).toBe(deleted._rev);
});
it("distinguishes a different owner and unmet prerequisites", async () => {
const { config, services, storageAccess } = await createConfigSync();
await expect(config._processOptionalFileEventWithResult("notes.md" as FilePath, flush)).resolves.toEqual({
kind: "not-applicable",
});
services.database.isDatabaseReady.mockReturnValue(false);
await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({
kind: "veto",
value: "database-not-ready",
});
services.database.isDatabaseReady.mockReturnValue(true);
vi.spyOn(services.setting, "getDeviceAndVaultName").mockReturnValue("");
await expect(config._processOptionalFileEventWithResult(path, flush)).resolves.toEqual({
kind: "veto",
value: "device-name-not-configured",
});
expect(storageAccess.readHiddenFileBinary).not.toHaveBeenCalled();
});
it("preserves every V2 file when neighbouring plug-in files arrive together", async () => {
const { fixture, config, files } = await createConfigSync(true);
const main = ".obsidian/plugins/example/main.js" as FilePath;
const style = ".obsidian/plugins/example/styles.css" as FilePath;
files.set(main, { content: "module.exports = {};", mtime: 3000 });
files.set(style, { content: "body {}", mtime: 3000 });
await Promise.all([
config._processOptionalFileEventWithResult(main, flush),
config._processOptionalFileEventWithResult(style, flush),
]);
expect(await fixture.localDatabase.getDBEntry(config.filenameWithUnifiedKey(main))).not.toBe(false);
expect(await fixture.localDatabase.getDBEntry(config.filenameWithUnifiedKey(style))).not.toBe(false);
});
it("does not acknowledge a database deletion which was not accepted", async () => {
const { fixture, config, databasePath } = await createConfigSync();
await config.storeCustomizationFiles(path);
const original = await fixture.localDatabase.getDBEntry(databasePath);
expect(original).not.toBe(false);
vi.spyOn(fixture.localDatabase, "putRaw").mockResolvedValueOnce({ ok: false, id: "failed", rev: "" });
await expect(config.deleteConfigOnDatabase(databasePath)).resolves.toBe(false);
const unchanged = await fixture.localDatabase.getDBEntry(databasePath);
expect(unchanged && getDocData(unchanged.data)).toBe(original && getDocData(original.data));
});
});
src/features/ConfigSync/CmdConfigSync.ts
+124 -34
View File
@@ -69,6 +69,10 @@ import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError"
import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts";
import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlugins.ts";
import type { HandlerOutcome } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils";
import type { OptionalFileEventOptions } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
import { PendingConfigSaves } from "./PendingConfigSaves";
const d = "\u200b";
const d2 = "\n";
const UPDATED_CONFIGURATION_NOTICE_KEY = "config-sync:updated-configuration";
@@ -412,6 +416,7 @@ export class ConfigSync extends LiveSyncCommands {
periodicPluginSweepProcessor = new PeriodicProcessor(this.core, async () => await this.scanAllConfigFiles(false));
pluginList: IPluginDataExDisplay[] = [];
private pluginListGeneration = 0;
showPluginSyncModal() {
if (!this.isThisModuleEnabled()) {
return;
@@ -532,6 +537,7 @@ export class ConfigSync extends LiveSyncCommands {
return Promise.resolve(true);
}
async reloadPluginList(showMessage: boolean) {
this.pluginListGeneration++;
this.pluginList = [];
this.loadedManifest_mTime.clear();
pluginList.set(this.pluginList);
@@ -761,39 +767,46 @@ export class ConfigSync extends LiveSyncCommands {
updatingV2Count = 0;
async updatePluginListV2(showMessage: boolean, unifiedFilenameWithKey: FilePathWithPrefix): Promise<void> {
const generation = this.pluginListGeneration;
const isCurrent = () => generation === this.pluginListGeneration && this.isThisModuleEnabled();
try {
this.updatingV2Count++;
pluginV2Progress.set(this.updatingV2Count);
// const unifiedFilenameWithKey = this.filenameWithUnifiedKey(updatedDocumentPath);
const { pathV1 } = this.parseUnifiedPath(unifiedFilenameWithKey);
const oldEntry = this.pluginList.find((e) => e.documentPath == pathV1);
let entry: PluginDataExDisplayV2 | undefined = undefined;
await serialized(`config:catalogue:${pathV1}`, async () => {
if (!isCurrent()) return;
const oldEntry = this.pluginList.find((e) => e.documentPath == pathV1);
let entry: PluginDataExDisplayV2 | undefined = undefined;
if (!oldEntry || !(oldEntry instanceof PluginDataExDisplayV2)) {
const newEntry = this.createPluginDataFromV2(unifiedFilenameWithKey);
if (newEntry) {
entry = newEntry;
if (!oldEntry || !(oldEntry instanceof PluginDataExDisplayV2)) {
const newEntry = this.createPluginDataFromV2(unifiedFilenameWithKey);
if (newEntry) {
entry = newEntry;
}
} else if (oldEntry instanceof PluginDataExDisplayV2) {
entry = oldEntry;
}
} else if (oldEntry instanceof PluginDataExDisplayV2) {
entry = oldEntry;
}
if (!entry) return;
const file = await this.createPluginDataExFileV2(unifiedFilenameWithKey);
if (file) {
await entry.setFile(file);
} else {
entry.deleteFile(unifiedFilenameWithKey);
if (entry.files.length == 0) {
this.pluginList = this.pluginList.filter((e) => e.documentPath != pathV1);
if (!entry) return;
const file = await this.createPluginDataExFileV2(unifiedFilenameWithKey);
if (!isCurrent()) return;
if (file) {
await entry.setFile(file);
} else {
entry.deleteFile(unifiedFilenameWithKey);
if (entry.files.length == 0) {
this.pluginList = this.pluginList.filter((e) => e.documentPath != pathV1);
}
}
}
const newList = this.pluginList.filter((e) => e.documentPath != entry.documentPath);
newList.push(entry);
this.pluginList = newList;
if (!isCurrent()) return;
const newList = this.pluginList.filter((e) => e.documentPath != entry.documentPath);
newList.push(entry);
this.pluginList = newList;
scheduleTask("updatePluginListV2", 100, () => {
pluginList.set(this.pluginList);
scheduleTask("updatePluginListV2", 100, () => {
pluginList.set(this.pluginList);
});
});
} finally {
this.updatingV2Count--;
@@ -871,11 +884,13 @@ export class ConfigSync extends LiveSyncCommands {
async updatePluginList(showMessage: boolean, updatedDocumentPath?: FilePathWithPrefix): Promise<void> {
if (!this.isThisModuleEnabled()) {
this.pluginListGeneration++;
this.pluginScanProcessor.clearQueue();
this.pluginList = [];
pluginList.set(this.pluginList);
return;
}
const generation = this.pluginListGeneration;
try {
this.updatingV2Count++;
pluginV2Progress.set(this.updatingV2Count);
@@ -888,6 +903,7 @@ export class ConfigSync extends LiveSyncCommands {
})
: this.localDatabase.findEntries(ICXHeader + "", `${ICXHeader}\u{10ffff}`, { include_docs: true });
for await (const v of plugins) {
if (generation !== this.pluginListGeneration || !this.isThisModuleEnabled()) return;
if (v.deleted || v._deleted) continue;
if (v.path.indexOf("%") !== -1) {
fireAndForget(() => this.updatePluginListV2(showMessage, v.path));
@@ -1206,7 +1222,7 @@ export class ConfigSync extends LiveSyncCommands {
}
recentProcessedInternalFiles = [] as string[];
async makeEntryFromFile(path: FilePath): Promise<false | PluginDataExFile> {
async makeEntryFromFile(path: FilePath, requireCompletion = false): Promise<false | PluginDataExFile> {
const stat = await this.core.storageAccess.statHidden(path);
let version: string | undefined;
let displayName: string | undefined;
@@ -1240,6 +1256,7 @@ export class ConfigSync extends LiveSyncCommands {
} catch (ex) {
this._log(`The file ${path} could not be encoded`);
this._log(ex, LOG_LEVEL_VERBOSE);
if (requireCompletion) throw ex;
return false;
}
const mtime = stat.mtime;
@@ -1254,6 +1271,15 @@ export class ConfigSync extends LiveSyncCommands {
}
async storeCustomisationFileV2(path: FilePath, term: string, force = false) {
return await this.storeCustomisationFileV2Core(path, term, force, false);
}
private async storeCustomisationFileV2Core(
path: FilePath,
term: string,
force: boolean,
requireCompletion: boolean
) {
const vf = this.filenameWithUnifiedKey(path, term);
return await serialized(`plugin-${vf}`, async () => {
const prefixedFileName = vf;
@@ -1261,7 +1287,8 @@ export class ConfigSync extends LiveSyncCommands {
const id = await this.path2id(prefixedFileName);
const stat = await this.core.storageAccess.statHidden(path);
if (!stat) {
return false;
if (!requireCompletion) return false;
return await this.deleteConfigOnDatabaseCore(vf, force, true);
}
const mtime = stat.mtime;
const content = await this.core.storageAccess.readHiddenFileBinary(path);
@@ -1285,7 +1312,10 @@ export class ConfigSync extends LiveSyncCommands {
eden: {},
};
} else {
if (this.services.path.isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN) {
if (
!requireCompletion &&
this.services.path.isMarkedAsSameChanges(prefixedFileName, [old.mtime, mtime + 1]) == EVEN
) {
this._log(
`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Already checked the same)`,
LOG_LEVEL_DEBUG
@@ -1300,7 +1330,7 @@ export class ConfigSync extends LiveSyncCommands {
const dataStart = dataSrc.indexOf(DUMMY_END);
const oldContent = dataSrc.substring(dataStart + DUMMY_END.length);
const oldContentArray = base64ToArrayBuffer(oldContent);
if (await isDocContentSame(oldContentArray, content)) {
if (!old.deleted && (await isDocContentSame(oldContentArray, content))) {
this._log(
`STORAGE --> DB:${prefixedFileName}: (config) Skipped (the same content)`,
LOG_LEVEL_VERBOSE
@@ -1320,24 +1350,36 @@ export class ConfigSync extends LiveSyncCommands {
};
}
const ret = await this.localDatabase.putDBEntry(saveData);
if (requireCompletion && (!ret || !ret.ok))
throw new LiveSyncError(`Could not save customisation: ${path}`);
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`);
fireAndForget(() => this.updatePluginListV2(false, this.filenameWithUnifiedKey(path)));
return ret;
} catch (ex) {
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
if (requireCompletion) throw ex;
return false;
}
});
}
async storeCustomizationFiles(path: FilePath, termOverRide?: string) {
return await this.storeCustomizationFilesCore(path, termOverRide, false);
}
private async storeCustomizationFilesCore(
path: FilePath,
termOverRide: string | undefined,
requireCompletion: boolean
) {
const term = termOverRide || this.services.setting.getDeviceAndVaultName();
if (term == "") {
this._log($msg("We have to configure the device name"), LOG_LEVEL_NOTICE);
if (requireCompletion) throw new LiveSyncError("The device name is not configured");
return;
}
if (this.useV2) {
return await this.storeCustomisationFileV2(path, term);
return await this.storeCustomisationFileV2Core(path, term, false, requireCompletion);
}
const vf = this.filenameToUnifiedKey(path, term);
// console.warn(`Storing ${path} to ${bareVF} :--> ${keyedVF}`);
@@ -1378,7 +1420,7 @@ export class ConfigSync extends LiveSyncCommands {
fileTargets = ["manifest.json", "theme.css"].map((e) => `${parentPath}/${e}` as FilePath);
}
for (const target of fileTargets) {
const data = await this.makeEntryFromFile(target);
const data = await this.makeEntryFromFile(target, requireCompletion);
if (data == false) {
this._log(`Config: skipped (Possibly is not exist): ${target} `, LOG_LEVEL_VERBOSE);
continue;
@@ -1398,9 +1440,9 @@ export class ConfigSync extends LiveSyncCommands {
// this._log(`Configuration saving: ${prefixedFileName}`);
if (dt.files.length == 0) {
this._log(`Nothing left: deleting.. ${path}`);
await this.deleteConfigOnDatabase(prefixedFileName);
const deleted = await this.deleteConfigOnDatabaseCore(prefixedFileName, false, requireCompletion);
await this.updatePluginList(false, prefixedFileName);
return;
return requireCompletion ? deleted : undefined;
}
const content = createTextBlob(serialize(dt));
@@ -1422,7 +1464,7 @@ export class ConfigSync extends LiveSyncCommands {
eden: {},
};
} else {
if (old.mtime == mtime) {
if (!requireCompletion && old.mtime == mtime) {
// this._log(`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same time)`, LOG_LEVEL_VERBOSE);
return true;
}
@@ -1443,7 +1485,7 @@ export class ConfigSync extends LiveSyncCommands {
}
});
const isSame = (await Promise.all(diffs)).every((e) => e == true);
if (isSame) {
if (isSame && !old.deleted) {
this._log(
`STORAGE --> DB:${prefixedFileName}: (config) Skipped (Same content)`,
LOG_LEVEL_VERBOSE
@@ -1464,16 +1506,51 @@ export class ConfigSync extends LiveSyncCommands {
};
}
const ret = await this.localDatabase.putDBEntry(saveData);
if (requireCompletion && (!ret || !ret.ok))
throw new LiveSyncError(`Could not save customisation: ${path}`);
await this.updatePluginList(false, saveData.path);
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Done`);
return ret;
} catch (ex) {
this._log(`STORAGE --> DB:${prefixedFileName}: (config) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
if (requireCompletion) throw ex;
return false;
}
});
}
private pendingOptionalSaves?: PendingConfigSaves;
async _processOptionalFileEventWithResult(
path: FilePath,
options?: OptionalFileEventOptions
): Promise<HandlerOutcome<void, string>> {
if (!this.isThisModuleEnabled() || !this.isTargetPath(path)) return { kind: "not-applicable" };
const configDir = normalizePath(this.configDir);
const handledElsewhere = Object.values(this.settings.pluginSyncExtendedSetting)
.filter((entry) => entry.mode !== MODE_SELECTIVE && entry.mode !== MODE_SHINY)
.flatMap((entry) => entry.files)
.some((file) => `${configDir}/${file}`.toLowerCase().startsWith(path.toLowerCase()));
if (handledElsewhere) return { kind: "not-applicable" };
if (!this._isDatabaseReady()) return { kind: "veto", value: "database-not-ready" };
const term = this.services.setting.getDeviceAndVaultName();
if (!term) return { kind: "veto", value: "device-name-not-configured" };
const stat = await this.core.storageAccess.statHidden(path);
if (stat && stat.type !== "file") return { kind: "not-applicable" };
// V1 saves a complete file group; V2 persists each file separately.
const key = this.useV2 ? this.filenameWithUnifiedKey(path, term) : this.filenameToUnifiedKey(path, term);
this.pendingOptionalSaves ??= new PendingConfigSaves();
await this.pendingOptionalSaves.enqueue(
key,
async () => {
await this.storeCustomizationFilesCore(path, term, true);
},
options?.whenFlushRequested
);
return { kind: "completed" };
}
async _anyProcessOptionalFileEvent(path: FilePath): Promise<boolean> {
return await this.watchVaultRawEventsAsync(path);
}
@@ -1615,6 +1692,14 @@ export class ConfigSync extends LiveSyncCommands {
}
async deleteConfigOnDatabase(prefixedFileName: FilePathWithPrefix, forceWrite = false) {
return await this.deleteConfigOnDatabaseCore(prefixedFileName, forceWrite, false);
}
private async deleteConfigOnDatabaseCore(
prefixedFileName: FilePathWithPrefix,
forceWrite: boolean,
requireCompletion: boolean
) {
// const id = await this.path2id(prefixedFileName);
const mtime = new Date().getTime();
return await serialized("file-x-" + prefixedFileName, async () => {
@@ -1640,13 +1725,15 @@ export class ConfigSync extends LiveSyncCommands {
type: "newnote",
};
}
await this.localDatabase.putRaw(saveData);
const result = await this.localDatabase.putRaw(saveData);
if (!result.ok) throw new LiveSyncError(`Could not delete customisation: ${prefixedFileName}`);
await this.updatePluginList(false, prefixedFileName);
this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Done`);
return true;
} catch (ex) {
this._log(`STORAGE -x> DB:${prefixedFileName}: (config) Failed`);
this._log(ex, LOG_LEVEL_VERBOSE);
if (requireCompletion) throw ex;
return false;
}
});
@@ -1756,6 +1843,9 @@ export class ConfigSync extends LiveSyncCommands {
}
override onBindFunction(core: LiveSyncCore, services: InjectableServiceHub): void {
services.fileProcessing.processOptionalFileEvent.addHandler(this._anyProcessOptionalFileEvent.bind(this));
services.fileProcessing.processOptionalFileEventWithResult?.addHandler(
this._processOptionalFileEventWithResult.bind(this)
);
services.conflict.getOptionalConflictCheckMethod.addHandler(this._anyGetOptionalConflictCheckMethod.bind(this));
services.replication.processVirtualDocument.addHandler(this._anyModuleParsedReplicationResultItem.bind(this));
services.setting.onRealiseSetting.addHandler(this._everyRealizeSettingSyncMode.bind(this));
src/features/ConfigSync/PendingConfigSaves.ts
+49
View File
@@ -0,0 +1,49 @@
import { compatGlobal, type CompatTimeoutHandle } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
type PendingSave = {
save: () => Promise<void>;
completion: Promise<void>;
resolve: () => void;
reject: (error: unknown) => void;
timer?: CompatTimeoutHandle;
started: boolean;
};
/** Owns the quiet-period delay and completion of coalesced customisation saves. */
export class PendingConfigSaves {
private readonly pending = new Map<string, PendingSave>();
enqueue(key: string, save: () => Promise<void>, whenFlushRequested?: Promise<void>): Promise<void> {
let group = this.pending.get(key);
if (!group) {
let resolve!: () => void;
let reject!: (error: unknown) => void;
const completion = new Promise<void>((onResolve, onReject) => {
resolve = onResolve;
reject = onReject;
});
group = { save, completion, resolve, reject, started: false };
this.pending.set(key, group);
} else {
group.save = save;
if (group.timer !== undefined) compatGlobal.clearTimeout(group.timer);
}
const captured = group;
group.timer = compatGlobal.setTimeout(() => this.run(key, captured), 100);
// Capture this group: a late flush of an earlier event must not release
// a newer group which happens to use the same customisation key.
void whenFlushRequested?.then(
() => this.run(key, captured),
() => {}
);
return group.completion;
}
private run(key: string, group: PendingSave): void {
if (group.started) return;
group.started = true;
if (group.timer !== undefined) compatGlobal.clearTimeout(group.timer);
this.pending.delete(key);
void Promise.resolve().then(group.save).then(group.resolve, group.reject);
}
}
Loaded 100 of 247 files, more files were not shown because too many files have changed in this diff. Show more