Compare commits

..

10 Commits

Author SHA1 Message Date
vorotamoroz 7439b75cd0 feat: distinguish remote operation and request activity 2026-07-16 07:20:55 +00:00
vorotamoroz 9ffde2dd3e fix: align chunk reads with remote delivery
Consume commonlib's delivery lifecycle, classify finite replication entry points, and document why the five-minute inactivity fuse is only a leak safety valve. Record the remote-activity counter correction and preserve continuous replication's unbounded live channel.
2026-07-16 01:44:58 +00:00
vorotamoroz f54d162ef9 Merge pull request #1024 from vrtmrz/0_25_82
Releasing 0.25.82
2026-07-16 00:07:44 +09:00
vorotamoroz 9d70ac8064 Synchronise workspace lock dependencies 2026-07-15 14:13:43 +00:00
vorotamoroz f44dc76087 Polish 0.25.82 release notes 2026-07-15 14:03:15 +00:00
github-actions[bot] 962589a40f Releasing 0.25.82 2026-07-15 14:00:29 +00:00
vorotamoroz d8bbab6bed Fix Deno setup for release preparation 2026-07-15 22:55:19 +09:00
vorotamoroz b475ea64e2 ci: publish CLI images for AMD64 and ARM64 2026-07-15 22:12:07 +09:00
vorotamoroz 908acc4a84 test: add CLI-to-Obsidian compatibility E2E 2026-07-15 22:12:07 +09:00
vorotamoroz 31e9186930 Protect bounded remote activity across app lifecycle 2026-07-15 20:53:27 +09:00
37 changed files with 1830 additions and 118 deletions
+9 -3
View File
@@ -57,7 +57,7 @@ jobs:
MAJOR_MINOR=$(echo "${VERSION}" | cut -d. -f1,2) MAJOR_MINOR=$(echo "${VERSION}" | cut -d. -f1,2)
SHORT_SHA=$(git rev-parse --short HEAD) SHORT_SHA=$(git rev-parse --short HEAD)
IMAGE="ghcr.io/${{ github.repository_owner }}/livesync-cli" IMAGE="ghcr.io/${{ github.repository_owner }}/livesync-cli"
# Build tag list based on the event and git ref # Build tag list based on the event and git ref
TAGS="" TAGS=""
if [[ "${{ github.ref }}" == refs/tags/* ]]; then if [[ "${{ github.ref }}" == refs/tags/* ]]; then
@@ -70,7 +70,7 @@ jobs:
# Other branches / manual run fallback # Other branches / manual run fallback
TAGS="${IMAGE}:${VERSION}-dev-sha-${SHORT_SHA}-cli" TAGS="${IMAGE}:${VERSION}-dev-sha-${SHORT_SHA}-cli"
fi fi
# Determine if the image should be pushed # Determine if the image should be pushed
PUSH="true" PUSH="true"
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
@@ -78,7 +78,7 @@ jobs:
PUSH="false" PUSH="false"
fi fi
fi fi
echo "tags=${TAGS}" >> $GITHUB_OUTPUT echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "push=${PUSH}" >> $GITHUB_OUTPUT echo "push=${PUSH}" >> $GITHUB_OUTPUT
@@ -89,6 +89,11 @@ jobs:
username: ${{ github.actor }} username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: arm64
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3 uses: docker/setup-buildx-action@v3
@@ -126,5 +131,6 @@ jobs:
file: src/apps/cli/Dockerfile file: src/apps/cli/Dockerfile
push: ${{ steps.meta.outputs.push }} push: ${{ steps.meta.outputs.push }}
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
platforms: linux/amd64,linux/arm64
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
+5
View File
@@ -45,6 +45,11 @@ jobs:
node-version: "24.x" node-version: "24.x"
cache: npm cache: npm
- name: Use Deno
uses: denoland/setup-deno@v2
with:
deno-version: v2.x
- name: Install dependencies - name: Install dependencies
run: npm ci run: npm ci
+2 -1
View File
@@ -64,7 +64,8 @@ This plug-in may be particularly useful for researchers, engineers, and develope
Synchronisation status is shown in the status bar with the following icons. Synchronisation status is shown in the status bar with the following icons.
- Activity Indicator - Activity Indicator
- 📲 Remote activity - 📲 A finite remote operation is in progress
- 🌐N Approximate remote requests currently in progress
- Status - Status
- ⏹️ Stopped - ⏹️ Stopped
- 💤 LiveSync enabled. Waiting for changes - 💤 LiveSync enabled. Waiting for changes
+22 -10
View File
@@ -31,13 +31,15 @@ The boundary has the following contract:
- allow overlapping tasks, so transitions may be `0 → 1 → 2 → 1 → 0`; and - allow overlapping tasks, so transitions may be `0 → 1 → 2 → 1 → 0`; and
- count logical operations, not physical connections, sockets, HTTP requests, queued work, or retry delays outside the bounded task. - count logical operations, not physical connections, sockets, HTTP requests, queued work, or retry delays outside the bounded task.
`ReplicatorService` also owns a narrower `finiteReplicationActivityCount`. Callers enter it through the typed `runFiniteReplicationActivity` method; diagnostic labels do not determine behaviour. The narrower count describes operations which may still place replicated documents in the local database; it excludes rebuilds, chunk-fetch claims, and other bounded work which cannot satisfy an arbitrary missing-chunk read. It is a delivery-lifecycle capability, not a second Wake Lock policy or a connection counter.
The Obsidian host injects the screen wake-lock manager from the `octagonal-wheels` package in the Fancy Kit monorepo as the activity runner on both mobile and desktop. Unsupported or rejected wake-lock requests remain best effort and do not prevent the remote task from running. The manager is disposed when the plug-in unloads. The Obsidian host injects the screen wake-lock manager from the `octagonal-wheels` package in the Fancy Kit monorepo as the activity runner on both mobile and desktop. Unsupported or rejected wake-lock requests remain best effort and do not prevent the remote task from running. The manager is disposed when the plug-in unloads.
Finite replication enters the boundary only after readiness checks have succeeded and leaves it after `openReplication(..., continuous: false, ...)` settles. Failure handling runs afterwards so a mismatch or recovery dialogue does not retain the activity. This includes the direct start-up synchronisation path as well as manual, event-driven, and periodic calls through `ReplicationService`. Continuous replication does not enter the boundary. Finite replication enters both counts only after readiness checks have succeeded and leaves them after `openReplication(..., continuous: false, ...)` settles. A successful completion has reached the latest sequence in that operation's scope and is therefore an authoritative quiescence boundary for chunk retrieval. A failed operation does not prove latest state, but can no longer deliver documents from that attempt. Failure handling runs afterwards so a mismatch or recovery dialogue does not retain the activity. This includes the direct start-up synchronisation path as well as manual, event-driven, and periodic calls through `ReplicationService`. The unbounded continuous channel does not enter either boundary, but its finite initial pull-only catch-up does; the one-shot parameter fallback chain remains inside that boundary.
Manual P2P commands which bypass `ReplicationService` enter the same boundary. Direct P2P pull and push entry points also create finite transfer activities, covering the Obsidian panes, CLI, and Webapp. Automatic synchronisation on peer discovery, a pull requested by a remote peer, and a watched pull following a peer progress notification enter the same boundary. A normal P2P peer-selection dialogue represents one finite session: it remains inside the boundary while waiting for a peer and while the person may perform repeated synchronisations, then settles when the dialogue closes and any in-flight synchronisation has finished. Closing without synchronising returns a failed result and releases the boundary. The 'Start Sync & Close' action completes its synchronisation before closing. This deliberately protects peer discovery and selection, because display sleep can interrupt discovery or connection establishment and require the person to start detection again. It may therefore retain a Wake Lock longer than the network transfer alone. A transfer performed inside that session temporarily adds a nested activity; the count remains a logical-operation count rather than a connection total. Manual P2P commands which bypass `ReplicationService` enter the broad boundary. Direct P2P pull and push entry points are therefore both protected as finite remote work, covering the Obsidian panes, CLI, and Webapp. A pull or bidirectional synchronisation also enters the narrower finite-replication boundary because it can place documents in the local database. A push-only request remains broad-only: it cannot satisfy a local missing-chunk read and must not present itself as a delivery source. Automatic synchronisation on peer discovery, a pull requested by a remote peer, and a watched pull following a peer progress notification enter both boundaries because each can deliver local documents. A normal P2P peer-selection dialogue represents one broad finite session: it remains inside the boundary while waiting for a peer and while the person may perform repeated synchronisations, then settles when the dialogue closes and any in-flight synchronisation has finished. Closing without synchronising returns a failed result and releases the boundary. The 'Start Sync & Close' action completes its synchronisation before closing. This deliberately protects peer discovery and selection, because display sleep can interrupt discovery or connection establishment and require the person to start detection again. It may therefore retain a Wake Lock longer than the network transfer alone. A transfer performed inside that session temporarily adds a nested activity; the count remains a logical-operation count rather than a connection total.
`ChunkFetcher` enters the same boundary only around the actual `fetchRemoteChunks` request. Queue waiting, interval throttling, and local chunk validation or persistence remain outside the remote activity scope. `ChunkFetcher` enters the broad boundary synchronously when it accepts newly missing chunk identifiers, but it does not increment the finite-replication count. A typed per-identifier claim keeps the broad boundary active through queue waiting, interval throttling, `fetchRemoteChunks`, validation, local persistence, and terminal event delivery. Duplicate requests share the existing claim. Explicit absence, failure, cancellation, or a conservative five-minute period without fetcher-observable progress settles the affected claim. The five-minute value is only a last-resort leak fuse: it prevents a never-settling integration from retaining the per-identifier claim and waiter indefinitely and, once the activity runner has entered the claim task, lets the associated Wake Lock, lifecycle deferral, and indicator finish. It is not an arrival estimate, proof of remote absence, or a transport deadline. This scope is defined in detail by the chunk-arrival-quiescence ADR.
Rebuild operations use the same boundary at their destructive or remote phase: Rebuild operations use the same boundary at their destructive or remote phase:
@@ -56,11 +58,17 @@ Fetch rebuilds temporarily suspend file watching. Their visibility event still r
If the desktop background setting applies, its existing continuous or periodic policy remains authoritative. When a Desktop LiveSync window becomes visible during bounded activity, the normal continuous-channel teardown and resume sequence is also deferred until that activity ends, so recovery does not abort the finite operation. If the desktop background setting applies, its existing continuous or periodic policy remains authoritative. When a Desktop LiveSync window becomes visible during bounded activity, the normal continuous-channel teardown and resume sequence is also deferred until that activity ends, so recovery does not abort the finite operation.
The status-bar remote-work indicator is shown when either the existing HTTP request balance is non-zero or the bounded remote activity count is non-zero. This preserves coverage from the existing counters while adding finite replication, peer waiting, and remote chunk fetching. The combined icon reports broader remote work, not an exact physical connection state; connection-level telemetry remains a separate concern. The status bar separates the two meanings. `📲` is shown while the bounded remote activity count is non-zero. It therefore reports a finite logical operation, including periods such as P2P peer selection or chunk-fetch queueing when no request is currently crossing the network. An adjacent `🌐N` reports the approximate number of tracked physical-request units currently in progress. The icon values and the physical indicator's 150 ms minimum display time are named constants so their presentation can be revised without changing the activity contract.
Each physical HTTP attempt owns one balanced counter pair. The request counter is incremented immediately before invoking the selected fetch implementation, and the response counter is incremented in `finally`, whether the attempt returns or rejects. A web-fetch failure followed by the native fallback is two physical attempts and therefore contributes two balanced pairs. Callers must not add another pair around `performFetch`, because duplicated or missing increments leave the status indicator permanently active.
Object Storage contributes one approximate unit for each AWS SDK command issued by its adapter, including upload, download, listing, deletion, availability, and usage requests. A download remains active until its response body has been consumed. This boundary is above the request-handler choice, so it covers both the standard SDK handler and Obsidian's internal request API without double counting. SDK-internal retries remain within one reported command. The displayed value is therefore intentionally approximate and is not an exact count of sockets, HTTP exchanges, or bytes transferred.
P2P does not yet contribute to the physical-request count because it does not have a request unit comparable with CouchDB HTTP attempts or Object Storage SDK commands. Its finite operations remain visible through `📲`. A future P2P transfer metric should be added only when it has a stable meaning, rather than being inferred from the broad logical-operation count.
## Ownership ## Ownership
`ReplicatorService` is the shared ownership point because both `ReplicationService` and `ChunkFetcher` already depend on it. Placing the activity state in `ReplicationService` would make chunk fetching depend in the opposite direction and risk a service dependency cycle. Adding another Service Hub service would introduce a wider capability surface without a distinct lifecycle owner. `ReplicatorService` is the shared ownership point because both `ReplicationService` and `ChunkFetcher` already depend on it. It owns the broad activity count and classifies the semantic subset which represents finite replication. Placing either activity state in `ReplicationService` would make chunk fetching depend in the opposite direction and risk a service dependency cycle. Adding another Service Hub service would introduce a wider capability surface without a distinct lifecycle owner.
The platform activity runner remains injected. Common library and headless consumers can omit it while retaining the same bounded activity count and operation semantics. The platform activity runner remains injected. Common library and headless consumers can omit it while retaining the same bounded activity count and operation semantics.
@@ -68,7 +76,7 @@ The platform activity runner remains injected. Common library and headless consu
- Do not count continuous replication as a bounded activity. - Do not count continuous replication as a bounded activity.
- Do not reinterpret the count as an exact number of network connections or HTTP requests. - Do not reinterpret the count as an exact number of network connections or HTTP requests.
- Do not use this logical-operation count as a replacement for future connection-level telemetry. - Do not use either diagnostic count for replication completion, throttling, protocol correctness, or power-policy decisions.
- Do not claim or implement privileged mobile background execution. - Do not claim or implement privileged mobile background execution.
- Do not guarantee protection against operating-system suspension, closing a laptop lid, forced termination, network loss, or a user-initiated sleep action. - Do not guarantee protection against operating-system suspension, closing a laptop lid, forced termination, network loss, or a user-initiated sleep action.
- Do not add a lifecycle timeout which would abort an unusually slow rebuild. A genuinely stalled operation may postpone LiveSync's visibility suspension until it settles, but the platform may still suspend or terminate background work. - Do not add a lifecycle timeout which would abort an unusually slow rebuild. A genuinely stalled operation may postpone LiveSync's visibility suspension until it settles, but the platform may still suspend or terminate background work.
@@ -83,13 +91,15 @@ Unit tests cover:
- count cleanup after rejection; - count cleanup after rejection;
- entry into the boundary only after replication readiness succeeds; - entry into the boundary only after replication readiness succeeds;
- replication failure handling occurring after the finite activity ends; - replication failure handling occurring after the finite activity ends;
- start-up one-shot replication entering the boundary while continuous start-up does not; - start-up one-shot replication and continuous start-up's finite pull-only catch-up entering the boundary, while the unbounded live channel does not;
- start-up readiness failure avoiding the boundary; - start-up readiness failure avoiding the boundary;
- direct P2P commands entering the boundary; - direct P2P commands entering the boundary;
- direct P2P pull and push entry points entering the boundary; - direct P2P pull and push entry points entering the broad boundary, while only pull and bidirectional operations enter the finite-delivery boundary;
- automatic synchronisation on peer discovery, remote pull requests, and watched peer progress entering the boundary; - automatic synchronisation on peer discovery, remote pull requests, and watched peer progress entering the boundary;
- P2P peer-selection sessions settling on close, including cancellation, repeated synchronisation, and a close during in-flight work; - P2P peer-selection sessions settling on close, including cancellation, repeated synchronisation, and a close during in-flight work;
- remote chunk fetching through the shared boundary; - remote chunk fetching remaining inside the shared boundary from synchronous queue acceptance through local persistence and terminal notification;
- missing-chunk waiters rechecking local storage when observed per-identifier claims and finite replication have settled;
- the finite-replication count excluding other bounded work;
- standard, fast, remote, and combined rebuild activity boundaries; - standard, fast, remote, and combined rebuild activity boundaries;
- Rebuilder-owned confirmation and completion dialogues remaining outside rebuild activity; - Rebuilder-owned confirmation and completion dialogues remaining outside rebuild activity;
- fallback from fast fetch avoiding a nested activity boundary; - fallback from fast fetch avoiding a nested activity boundary;
@@ -100,6 +110,8 @@ Unit tests cover:
- hidden-to-visible transitions before the final activity avoiding an unmatched resume; and - hidden-to-visible transitions before the final activity avoiding an unmatched resume; and
- continuous-channel recovery being deferred when a desktop window becomes visible during bounded activity. - continuous-channel recovery being deferred when a desktop window becomes visible during bounded activity.
Additional tests cover balanced physical-request counters after success and rejection, each Object Storage command boundary, response-body consumption for downloads, the split `📲` and `🌐N` status labels, and a deterministic real-Obsidian CouchDB request held while the physical indicator is observed. The Object Storage integration and real-Obsidian MinIO workflows verify that actual AWS SDK operations advance and rebalance the shared counters.
The exact Fancy Kit screen wake-lock behaviour is covered by its package and Harness tests. A real Obsidian smoke test remains appropriate when changing the platform adapter or lifecycle integration, but is not required for changes confined to the already-tested injected activity-runner contract. The exact Fancy Kit screen wake-lock behaviour is covered by its package and Harness tests. A real Obsidian smoke test remains appropriate when changing the platform adapter or lifecycle integration, but is not required for changes confined to the already-tested injected activity-runner contract.
## Consequences ## Consequences
@@ -107,4 +119,4 @@ The exact Fancy Kit screen wake-lock behaviour is covered by its package and Har
- One finite activity definition drives Wake Lock, lifecycle protection, and status UI without coupling common library code to Obsidian or browser globals. - One finite activity definition drives Wake Lock, lifecycle protection, and status UI without coupling common library code to Obsidian or browser globals.
- Callers can observe accurate logical activity even in CLI and Webapp hosts which do not inject a Wake Lock implementation. - Callers can observe accurate logical activity even in CLI and Webapp hosts which do not inject a Wake Lock implementation.
- Rebuild operations now retain Wake Lock and lifecycle protection across their longest interruption-sensitive phases without retaining them for Rebuilder-owned pre-operation or completion dialogues. Post-reset P2P discovery and selection remain protected as an intentional part of completing the rebuild. - Rebuild operations now retain Wake Lock and lifecycle protection across their longest interruption-sensitive phases without retaining them for Rebuilder-owned pre-operation or completion dialogues. Post-reset P2P discovery and selection remain protected as an intentional part of completing the rebuild.
- Future remote-work indicators should distinguish logical activity from connection-level telemetry when that distinction matters to users. - Users can now distinguish the lifetime of a finite remote operation from approximate request activity within it.
@@ -0,0 +1,144 @@
# Architectural Decision Record: Chunk Arrival Quiescence
## Status
Accepted
## Context
A file metadata document refers to one or more content-addressed chunk documents. LiveSync normally writes the chunks before writing the metadata, but those writes are separate database operations and are not replicated as one atomic unit. A local reader can consequently observe metadata before every referenced chunk is locally readable. This is expected when CouchDB on-demand chunk fetching is enabled, and can also occur around conflict resolution, replication event processing, and historical data.
Before this decision, a missing-chunk waiter used a fixed 5-second or 30-second timeout. Its budget included queueing, throttling, network transfer, validation, local persistence, and event delivery. A healthy operation could therefore time out immediately before it delivered the requested chunk. Conversely, when no operation was capable of delivering the chunk, the timer merely delayed the same unavailable result.
Finite replication provides a stronger boundary than elapsed wall-clock time. In the absence of an error, a finite replication does not complete until it has reached the latest sequence in its scope. Once it completes, that replication cannot deliver another chunk. An on-demand fetch is a separate finite delivery path and needs its own per-identifier boundary.
## Decision
Wait only for a delivery lifecycle which is observable when the local miss is handled. Do not guess how long an unobserved producer might take.
Two lifecycle signals are relevant:
- `finiteReplicationActivityCount` records finite replication operations which can still place database documents in the local database; and
- a `ChunkDeliveryCoordinator` claim records the complete lifetime of an accepted on-demand request for each missing chunk identifier.
The finite replication count is distinct from the broader `boundedRemoteActivityCount`. Rebuilds and other bounded remote work may need Wake Lock and lifecycle protection without being able to satisfy an arbitrary missing-chunk read. A P2P push-only operation is broad for the same reason: it sends documents away but cannot place one in the local database. P2P pull and bidirectional operations are finite-delivery sources. `ReplicatorService` updates the narrower count only through the typed `runFiniteReplicationActivity` boundary; the diagnostic activity label is not used as a behavioural discriminator.
The waiting layer follows these rules:
1. Register the waiter before requesting on-demand delivery.
2. Dispatch `missingChunks` synchronously when direct fetch is permitted. `ChunkFetcher` must claim accepted identifiers before dispatch returns, closing the scheduling gap without a timer.
3. If a matching claim or finite replication is active, wait for that observable producer.
4. A valid chunk arrival resolves the waiter immediately.
5. An explicit remote-missing result resolves it as missing immediately.
6. Once every observed producer completes, read the requested identifier from the local database once more, bypassing the cache.
7. Return the rechecked chunk, or return unavailable. Do not add a fixed grace period after the producer has stopped.
8. If no producer is observable after synchronous dispatch, return unavailable immediately. There is no operation for a duration to represent.
If new relevant activity starts while the final database recheck is pending, that result is stale. The waiter remains active until the newer producer completes and a current recheck finishes.
A successful finite replication completion is the authoritative latest boundary. A failed operation does not prove that the remote lacks the chunk, but it is still terminal for that attempt: it can no longer deliver data. The same local recheck preserves any documents received before the failure, after which normal replication error and retry handling remains responsible for recovery. Missing-chunk code must not misreport that case as an explicit remote absence.
### On-demand fetch boundary
`ChunkFetcher` claims newly requested identifiers synchronously while handling the `missingChunks` event. The claim remains active through:
- queueing and concurrency scheduling;
- configured interval throttling;
- entry into the injected bounded-activity runner;
- `fetchRemoteChunks`;
- response validation;
- local database persistence; and
- fetched or missing event delivery.
The claim settles on every terminal path, including explicit absence, no active replicator, rejection, invalid results, destruction, and cancellation. Its completion Promise is the task passed to the bounded `chunk-fetch` activity, keeping Wake Lock, application lifecycle deferral, the remote-work indicator, and missing-chunk delivery aligned.
### Five-minute leak fuse
An accepted on-demand claim has a separate five-minute inactivity fuse. This is a last-resort leak safety valve, not a chunk-arrival budget or a remote-request timeout.
Its purpose is to prevent a faulty integration, a never-settling Promise, or a stalled transport from retaining logical ownership indefinitely. When it fires, the coordinator releases the per-identifier claim and its waiter. If the bounded activity callback has been entered, resolving the claim also allows the associated Wake Lock, application-lifecycle deferral, and remote-work indicator to be released. `ChunkFetcher` refreshes the fuse only at observable progress points, such as entering the activity boundary, beginning and completing throttling or transfer, and completing persistence.
Five minutes is deliberately a conservative operational limit, not a value derived from a network protocol, a benchmark, or evidence that a missing chunk will arrive within that period. Firing the fuse neither proves remote absence nor makes the underlying request safe to abort. The current `fetchRemoteChunks` contract has no `AbortSignal`, so a physical request may still complete after its logical claim has been released. A future cancellable transport contract should add transport-specific deadlines and explicit cancellation without changing the lifecycle-based wait rule.
### Continuous replication
The unbounded live channel is not a quiescence gate because it has no natural end. Its initial pull-only catch-up is finite, however, and must enter `runFiniteReplicationActivity`. This includes the one-shot parameter fallback chain: every retry remains within the catch-up boundary until it succeeds or stops. If continuous replication later restarts with adjusted parameters, the new initial catch-up enters a new finite boundary.
Once the live channel has begun, a chunk delivered through it still resolves an existing waiter immediately, but the channel itself does not keep a new waiter open. CouchDB on-demand fetching supplies its own per-identifier claim. If a future defect demonstrates a delivery race inside a live batch, that batch lifecycle should be exposed explicitly rather than approximated with another elapsed delay.
## Ownership
`ReplicatorService` owns both the broad bounded-operation count and the narrower finite-replication count. It is the common lifecycle owner for CouchDB, sequential, and P2P replicators.
`LayeredChunkManager` owns one `ChunkDeliveryCoordinator` and supplies it to its arrival layer and `ChunkFetcher`. `ArrivalWaitLayer` owns waiter resolution and the final local database recheck. It depends on the narrow coordinator capability rather than on `ReplicatorService` itself.
`ChunkFetcher` owns per-identifier claims because it knows when each identifier enters its queue and reaches a terminal result.
## Compatibility
- Preserve immediate reads when `waitForDelivery` is false or the deprecated call-site `timeout` is zero or negative.
- Treat a positive deprecated `timeout` only as source-compatible opt-in to lifecycle waiting. Its numeric value no longer represents an arrival duration.
- Preserve `preventRemoteRequest`: no on-demand request is dispatched, although an already-active finite replication may satisfy the waiter.
- Preserve Promise sharing for concurrent reads of the same chunk identifier.
- Preserve immediate explicit remote-missing results.
- Do not change which remote types support direct on-demand fetching.
## Historical Evidence and Scope
This decision addresses the lifecycle-race class rather than treating every Load failed report as a timeout:
- [Issue #166](https://github.com/vrtmrz/obsidian-livesync/issues/166) contained logs where chunk collection failed shortly before related chunk writes appeared. It is evidence for the timing class, although that issue's hidden-file start-up path was repaired separately and is not claimed as a direct regression test here.
- The 2021 timing fixes in [commit `39e2eab0`](https://github.com/vrtmrz/obsidian-livesync/commit/39e2eab0238d9c37e3653cdec884cbeed543fc23) and the extended leaf timeout in [commit `9facb577`](https://github.com/vrtmrz/obsidian-livesync/commit/9facb577601d8aceff7df547cd2a6f9357fdaa29) show that elapsed timeout values have historically been used to absorb the same ordering uncertainty. They do not provide a protocol basis for retaining 5-second or 30-second delays.
- Replication pacing introduced by [commit `8d66c372`](https://github.com/vrtmrz/obsidian-livesync/commit/8d66c372e15c43a2de84a223c6385077b7724eec) and commonlib [commit `051b50c`](https://github.com/vrtmrz/livesync-commonlib/commit/051b50ca38ec4c05a11e8216ac259b4488b825f0) is a direct precedent for preventing replication progress from outrunning chunk collection. The present design expresses that dependency as an explicit lifecycle and completion recheck.
- [Issue #505](https://github.com/vrtmrz/obsidian-livesync/issues/505) was traced to chunks which were genuinely absent after the former bulk-send option broke the chunks-before-metadata guarantee. Waiting cannot recreate missing data, so this decision does not claim to fix it.
- [Issue #771](https://github.com/vrtmrz/obsidian-livesync/issues/771) and [Issue #986](https://github.com/vrtmrz/obsidian-livesync/issues/986) contain ambiguous or version-dependent `Load failed` reports. They remain unclaimed until the original writer and database state can be reproduced.
The detailed setting and replicator matrix is recorded in [Chunk Retrieval and Waiting](../design_docs/chunk_retrieval_and_waiting.md).
## Alternatives Rejected
### Increase the fixed timeout constants
Any fixed total duration can still expire immediately before a queued or progressing operation reports its result. Larger values also make genuine failures slower without defining what the system is waiting for.
### Start another grace period after finite completion
A successful finite replication has already reached its latest sequence, and the completion recheck observes documents persisted without a waiter event. Waiting an additional 5 or 30 seconds has no identified producer to wait for and merely retains the historical approximation.
### Observe all bounded remote activity
The broad count includes operations which cannot provide the requested chunk. Using it as the delivery gate lets unrelated work delay a read and makes its completion semantically meaningless. A separate finite-replication count avoids that leak.
### Keep a fallback timer for unobserved delivery
An unobserved producer has no defined start, progress, or completion semantics. A timer would therefore be a guess rather than a safety property. Relevant delivery paths must claim their work synchronously or expose a finite replication boundary; otherwise the read returns unavailable.
### Remove every timer
The arrival wait has no elapsed timer, but an implementation fault can leave a delivery claim unresolved forever. The five-minute inactivity fuse bounds that leaked logical state without being used as a successful delivery condition.
## Verification
Unit tests use deterministic clocks and deferred Promises to cover:
- a finite replication which lasts well beyond the former arrival values;
- successful finite completion causing a cache-bypassing local database recheck;
- immediate unavailability when no producer is observable;
- per-identifier claims covering queueing, throttling, remote fetch, validation, persistence, and event delivery;
- explicit missing, no-replicator, rejection, invalid response, cancellation, runner rejection, and teardown paths;
- overlapping claims and finite replications;
- a runner which never enters the task and a request which never settles;
- the five-minute fuse being refreshed by observable progress;
- continuous replication's finite initial catch-up, including its parameter fallback path; and
- the setting and replicator decision matrix.
Integration-style unit tests exercise `LayeredChunkManager`, `ChunkFetcher`, a memory-backed PouchDB database, and a deferred fake replicator together. A real Obsidian test is not required because the change remains behind the existing database, service, and event boundaries and does not alter platform UI or an adapter contract.
## Consequences
- A healthy finite replication or on-demand request no longer loses a race against an unrelated wall-clock estimate.
- Successful finite replication completion provides a precise latest boundary for missing-chunk reads.
- The local recheck closes event-delivery and cache timing gaps without extending the wait after completion.
- Reads no longer pause for 5 or 30 seconds when no observable operation can deliver the chunk.
- Relevant producers must expose a lifecycle and must continue to prove cleanup on every exceptional path.
- The five-minute fuse bounds leaked logical activity, but it neither establishes remote absence nor cancels a physical request.
@@ -0,0 +1,118 @@
# Chunk Retrieval and Waiting
## Purpose
This document records how LiveSync retrieves chunks after file metadata has been found, which operation provides each terminal condition, and what the remaining time value means. It is an implementation specification for developers; it is not a user configuration guide.
The architectural decision and historical rationale are in [Chunk Arrival Quiescence](../adr/2026_07_chunk_arrival_quiescence.md).
## Invariants and Sources of Apparent Reordering
A normal local save creates and persists the chunks before it writes the metadata document which refers to them. LiveSync must preserve this invariant: publishing metadata first can expose a reference which no client can satisfy.
This ordering is not an atomic transaction across documents. A reader may still see metadata before a referenced chunk for these reasons:
- CouchDB replication transfers individual documents and does not expose the chunk and metadata writes as one atomic unit.
- With `readChunksOnline` enabled, CouchDB pull replication deliberately excludes chunk documents. Seeing metadata first is then the intended design, and the chunk is fetched by identifier.
- A winning metadata conflict revision may refer to chunks created by another revision or client which have not yet reached the local database.
- Replication persists documents before every downstream change callback and file-reflection task has necessarily observed them.
- Historical versions and removed transfer modes may have produced data which did not preserve the normal ordering invariant.
Waiting may resolve temporary visibility and processing gaps only when a known operation can still deliver the chunk. It cannot repair a chunk which is absent from every available source.
## Retrieval Capabilities
Direct on-demand fetch is currently available only for CouchDB when `useOnlyLocalChunk` is false. This capability deliberately does not depend on `readChunksOnline`:
- when `readChunksOnline` is true, direct fetch is the normal way to obtain a chunk omitted from pull replication; and
- when `readChunksOnline` is false, direct fetch remains a recovery path if a normally replicated chunk is locally absent.
MinIO's sequential replicator and P2P do not implement direct `fetchRemoteChunks` delivery through this path. Their chunks must arrive through a finite replication operation. A P2P pull or bidirectional synchronisation is such a producer. A push-only P2P request remains broad remote activity for Wake Lock and lifecycle reporting, but it is deliberately excluded from `finiteReplicationActivityCount` because it cannot deliver a local document.
`waitForReady` is a call-site policy, not a persisted user setting. `true` permits waiting for an already-observable producer. `false` normally requests an immediate local result, except that CouchDB on-demand delivery still waits for the claim which synchronous dispatch creates.
## Policy Matrix
The matrix selects whether lifecycle waiting is permitted and whether the waiter may dispatch a direct request. It does not assign elapsed arrival budgets.
| Remote | `waitForReady` | `useOnlyLocalChunk` | Direct fetch | Wait for observed producer | Intended behaviour |
| ------- | -------------: | ------------------: | -----------: | -------------------------: | --------------------------------------------------------------------------------------- |
| CouchDB | `false` | `false` | Yes | Yes | Dispatch on-demand fetch and finish at its per-identifier claim boundary. |
| CouchDB | `true` | `false` | Yes | Yes | Accept an active finite replication or dispatch direct fetch. |
| CouchDB | `false` | `true` | No | No | Return immediately after the local miss. |
| CouchDB | `true` | `true` | No | Yes | Wait for an already-active finite replication; otherwise return unavailable. |
| MinIO | `false` | Either | No | No | Return immediately after the local miss. |
| MinIO | `true` | Either | No | Yes | Wait for an already-active finite sequential replication; otherwise return unavailable. |
| P2P | `false` | Either | No | No | Return immediately after the local miss. |
| P2P | `true` | Either | No | Yes | Wait for an already-active finite P2P replication; otherwise return unavailable. |
For CouchDB, `readChunksOnline` changes what normal replication includes, not the direct-fetch capability or this matrix:
| `readChunksOnline` | CouchDB pull contains chunks | Role of direct fetch |
| -----------------: | ---------------------------: | ------------------------------------------------------------------------ |
| `true` | No | Primary chunk delivery after metadata arrives. |
| `false` | Yes | Recovery fallback for a chunk which is unexpectedly unavailable locally. |
`concurrencyOfReadChunksOnline` and `minimumIntervalOfReadChunksOnline` affect only the scheduling of CouchDB on-demand requests. They do not change whether a request may be dispatched or which lifecycle a reader observes. Accepted identifiers remain claimed while they wait for a concurrency slot and while the configured interval is applied. A minimum interval of five minutes or more is an exceptional value: the inactivity fuse may release the logical claim before that deliberate pause completes. This safety precedence does not abort the delayed physical request.
## Wait State Machine
1. Read the cache and local database.
2. If every requested chunk is present, return it without entering a wait.
3. Register one shared waiter per missing identifier.
4. If policy permits direct fetch, emit `missingChunks`. `ChunkFetcher` synchronously creates the per-identifier claim before the event dispatch returns.
5. Observe both the matching claim and `finiteReplicationActivityCount`.
6. Resolve immediately if a valid chunk or explicit remote-missing event arrives.
7. If an observed producer remains active, do not charge elapsed time against an arrival budget.
8. When all observed producers end, bypass the cache and read the identifiers from the local database once.
9. Return the rechecked chunk, or return unavailable. Do not add another fixed grace after the authoritative boundary.
10. If no producer is observable after synchronous dispatch, return unavailable immediately.
If new relevant activity starts while the final database recheck is pending, that result becomes stale. The waiter remains active until the newer producer completes and a current recheck finishes.
## Meaning of Finite Replication Completion
Finite replication enters the typed `runFiniteReplicationActivity` boundary and is represented by the narrower `finiteReplicationActivityCount`. The optional `replication` label remains diagnostic and does not control this behaviour.
For a successful finite operation, completion means that its replicator has reached the latest sequence in the operation's scope and processed its replication change callbacks. No more database documents can arrive from that operation. This is the primary semantic cutoff.
If the operation fails, it has not proved remote absence or latest state. It has nevertheless stopped being a producer. The waiting layer rechecks documents which may have arrived before the failure and then returns unavailable; the replication error and retry workflow owns further recovery.
Overlapping finite replications keep the count above zero until the final operation settles. The local recheck therefore occurs only after every observed finite producer is quiescent.
The continuous live channel is intentionally excluded because it has no completion boundary and would otherwise make a chunk read unbounded. The pull-only catch-up run before opening that channel is finite and enters the same typed boundary. Its one-shot batch-size fallback remains inside that boundary. A live-channel fallback starts another continuous attempt and therefore another bounded initial catch-up.
## Meaning of an On-demand Claim
An accepted identifier remains claimed from synchronous queue acceptance through throttling, physical fetch, validation, local persistence, and terminal event delivery. The claim is identifier-scoped because a global remote-work count cannot say whether unrelated work can provide this chunk.
The claim finishes when the fetcher has recorded an outcome for the identifier. A transport error, missing active replicator, or invalid result releases the claim without emitting an explicit remote-missing result unless the remote actually supplied that information.
## Meaning of the Five-minute Value
The five-minute value is an inactivity leak fuse for an accepted on-demand claim. It is the only elapsed duration in this state machine, and it is not a normal terminal condition.
The fuse bounds retention if a faulty activity runner never enters its task, a Promise never settles, or a transport stops making observable progress. It prevents the per-identifier claim and waiter from remaining live forever. Once the bounded activity callback has entered, releasing the claim also allows Wake Lock, application-lifecycle deferral, and the remote-work indicator associated with that callback to finish. Observable progress rearms the fuse.
Five minutes is a conservative operational ceiling rather than a measured chunk-arrival expectation. It must not be used to infer that the remote lacks a chunk, and it does not abort the physical request. `fetchRemoteChunks` does not yet accept an `AbortSignal`, so the request may complete after the logical state has been released. Transport cancellation and transport-specific deadlines are separate future work.
The old 5-second and 30-second constants remain exported for source compatibility only. A positive deprecated `ChunkReadOptions.timeout` opts into lifecycle waiting, but its numeric value is ignored. Zero or a negative value still requests an immediate result. New code uses `waitForDelivery` explicitly.
## Test Obligations
Changes to this behaviour must keep automated coverage for:
- chunks-before-metadata save ordering;
- every row in the retrieval policy matrix;
- a finite replication which remains active well beyond the former 5-second and 30-second values;
- successful completion with the chunk already persisted but no arrival event delivered;
- immediate unavailability when no producer is observable;
- overlapping finite operations and overlapping per-identifier claims;
- activity restarting while a local recheck is pending;
- direct fetch queueing, throttling, persistence, and terminal notification;
- explicit remote absence versus transport or replicator failure;
- runner rejection, cancellation, teardown, and an operation which never enters its task;
- leak-fuse refresh at observable progress points; and
- continuous replication's finite initial catch-up and parameter fallback.
The service, database, and event boundaries are testable with memory-backed PouchDB and injected activity sources. A real Obsidian test is required only when a change crosses into the platform adapter, application lifecycle, or visible UI rather than for this retrieval state machine alone.
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"id": "obsidian-livesync", "id": "obsidian-livesync",
"name": "Self-hosted LiveSync", "name": "Self-hosted LiveSync",
"version": "0.25.81", "version": "0.25.82",
"minAppVersion": "1.7.2", "minAppVersion": "1.7.2",
"description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"author": "vorotamoroz", "author": "vorotamoroz",
+8 -8
View File
@@ -1,12 +1,12 @@
{ {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.25.81", "version": "0.25.82",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.25.81", "version": "0.25.82",
"license": "MIT", "license": "MIT",
"workspaces": [ "workspaces": [
"src/apps/cli", "src/apps/cli",
@@ -16226,11 +16226,11 @@
}, },
"src/apps/cli": { "src/apps/cli": {
"name": "self-hosted-livesync-cli", "name": "self-hosted-livesync-cli",
"version": "0.25.81-cli", "version": "0.25.82-cli",
"dependencies": { "dependencies": {
"chokidar": "^4.0.0", "chokidar": "^4.0.0",
"minimatch": "^10.2.5", "minimatch": "^10.2.5",
"octagonal-wheels": "^0.1.50", "octagonal-wheels": "^0.1.51",
"pouchdb-adapter-http": "^9.0.0", "pouchdb-adapter-http": "^9.0.0",
"pouchdb-adapter-leveldb": "^9.0.0", "pouchdb-adapter-leveldb": "^9.0.0",
"pouchdb-core": "^9.0.0", "pouchdb-core": "^9.0.0",
@@ -16252,9 +16252,9 @@
}, },
"src/apps/webapp": { "src/apps/webapp": {
"name": "livesync-webapp", "name": "livesync-webapp",
"version": "0.25.81-webapp", "version": "0.25.82-webapp",
"dependencies": { "dependencies": {
"octagonal-wheels": "^0.1.50" "octagonal-wheels": "^0.1.51"
}, },
"devDependencies": { "devDependencies": {
"@playwright/test": "^1.58.2", "@playwright/test": "^1.58.2",
@@ -16267,9 +16267,9 @@
} }
}, },
"src/apps/webpeer": { "src/apps/webpeer": {
"version": "0.25.81-webpeer", "version": "0.25.82-webpeer",
"dependencies": { "dependencies": {
"octagonal-wheels": "^0.1.50" "octagonal-wheels": "^0.1.51"
}, },
"devDependencies": { "devDependencies": {
"@sveltejs/vite-plugin-svelte": "^7.1.2", "@sveltejs/vite-plugin-svelte": "^7.1.2",
+2 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "obsidian-livesync", "name": "obsidian-livesync",
"version": "0.25.81", "version": "0.25.82",
"description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.",
"main": "main.js", "main": "main.js",
"type": "module", "type": "module",
@@ -45,6 +45,7 @@
"test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts", "test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts",
"test:e2e:obsidian:vault-reflection": "tsx test/e2e-obsidian/scripts/vault-reflection.ts", "test:e2e:obsidian:vault-reflection": "tsx test/e2e-obsidian/scripts/vault-reflection.ts",
"test:e2e:obsidian:couchdb-upload": "tsx test/e2e-obsidian/scripts/couchdb-upload.ts", "test:e2e:obsidian:couchdb-upload": "tsx test/e2e-obsidian/scripts/couchdb-upload.ts",
"test:e2e:obsidian:cli-to-obsidian-sync": "tsx test/e2e-obsidian/scripts/cli-to-obsidian-sync.ts",
"test:e2e:obsidian:minio-upload": "tsx test/e2e-obsidian/scripts/minio-upload.ts", "test:e2e:obsidian:minio-upload": "tsx test/e2e-obsidian/scripts/minio-upload.ts",
"test:e2e:obsidian:startup-scan": "tsx test/e2e-obsidian/scripts/startup-scan.ts", "test:e2e:obsidian:startup-scan": "tsx test/e2e-obsidian/scripts/startup-scan.ts",
"test:e2e:obsidian:two-vault-sync": "tsx test/e2e-obsidian/scripts/two-vault-sync.ts", "test:e2e:obsidian:two-vault-sync": "tsx test/e2e-obsidian/scripts/two-vault-sync.ts",
+2 -2
View File
@@ -1,7 +1,7 @@
{ {
"name": "self-hosted-livesync-cli", "name": "self-hosted-livesync-cli",
"private": true, "private": true,
"version": "0.25.81-cli", "version": "0.25.82-cli",
"main": "dist/index.cjs", "main": "dist/index.cjs",
"type": "module", "type": "module",
"scripts": { "scripts": {
@@ -41,7 +41,7 @@
"dependencies": { "dependencies": {
"chokidar": "^4.0.0", "chokidar": "^4.0.0",
"minimatch": "^10.2.5", "minimatch": "^10.2.5",
"octagonal-wheels": "^0.1.50", "octagonal-wheels": "^0.1.51",
"pouchdb-adapter-http": "^9.0.0", "pouchdb-adapter-http": "^9.0.0",
"pouchdb-adapter-leveldb": "^9.0.0", "pouchdb-adapter-leveldb": "^9.0.0",
"pouchdb-core": "^9.0.0", "pouchdb-core": "^9.0.0",
+2 -2
View File
@@ -1,7 +1,7 @@
{ {
"name": "livesync-webapp", "name": "livesync-webapp",
"private": true, "private": true,
"version": "0.25.81-webapp", "version": "0.25.82-webapp",
"type": "module", "type": "module",
"description": "Browser-based Self-hosted LiveSync using FileSystem API", "description": "Browser-based Self-hosted LiveSync using FileSystem API",
"scripts": { "scripts": {
@@ -12,7 +12,7 @@
"preview": "vite preview" "preview": "vite preview"
}, },
"dependencies": { "dependencies": {
"octagonal-wheels": "^0.1.50" "octagonal-wheels": "^0.1.51"
}, },
"devDependencies": { "devDependencies": {
"@playwright/test": "^1.58.2", "@playwright/test": "^1.58.2",
+2 -2
View File
@@ -1,7 +1,7 @@
{ {
"name": "webpeer", "name": "webpeer",
"private": true, "private": true,
"version": "0.25.81-webpeer", "version": "0.25.82-webpeer",
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "vite", "dev": "vite",
@@ -12,7 +12,7 @@
"check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json" "check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json"
}, },
"dependencies": { "dependencies": {
"octagonal-wheels": "^0.1.50" "octagonal-wheels": "^0.1.51"
}, },
"devDependencies": { "devDependencies": {
"eslint-plugin-svelte": "^3.19.0", "eslint-plugin-svelte": "^3.19.0",
+1 -1
Submodule src/lib updated: a58965f9cd...09bfafbd6c
+5 -1
View File
@@ -150,7 +150,11 @@ Even if you choose to clean up, you will see this option again if you exit Obsid
await purgeUnreferencedChunks(this.localDatabase.localDatabase, false); await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
this.localDatabase.clearCaches(); this.localDatabase.clearCaches();
// Perform the synchronisation once. // Perform the synchronisation once.
if (await this.core.replicator.openReplication(this.settings, false, showMessage, true)) { const replicated = await this.services.replicator.runFiniteReplicationActivity(
() => this.core.replicator.openReplication(this.settings, false, showMessage, true),
{ label: "replication" }
);
if (replicated) {
await balanceChunkPurgedDBs(this.localDatabase.localDatabase, remoteDB.db); await balanceChunkPurgedDBs(this.localDatabase.localDatabase, remoteDB.db);
await purgeUnreferencedChunks(this.localDatabase.localDatabase, false); await purgeUnreferencedChunks(this.localDatabase.localDatabase, false);
this.localDatabase.clearCaches(); this.localDatabase.clearCaches();
@@ -69,6 +69,7 @@ describe("ModuleReplicator legacy cleanup", () => {
activityFinished(); activityFinished();
} }
}); });
const runFiniteReplicationActivity = vi.fn(async (task: () => unknown) => await task());
const openReplication = vi.fn(async () => true); const openReplication = vi.fn(async () => true);
const activeReplicator = Object.assign(new LiveSyncCouchDBReplicator({} as any), { const activeReplicator = Object.assign(new LiveSyncCouchDBReplicator({} as any), {
connectRemoteCouchDBWithSetting: vi.fn(async () => ({ db: {} })), connectRemoteCouchDBWithSetting: vi.fn(async () => ({ db: {} })),
@@ -90,6 +91,7 @@ describe("ModuleReplicator legacy cleanup", () => {
replicator: { replicator: {
getActiveReplicator: vi.fn(() => activeReplicator), getActiveReplicator: vi.fn(() => activeReplicator),
runBoundedRemoteActivity, runBoundedRemoteActivity,
runFiniteReplicationActivity,
}, },
}; };
const localDatabase = { const localDatabase = {
@@ -111,6 +113,9 @@ describe("ModuleReplicator legacy cleanup", () => {
expect(runBoundedRemoteActivity).toHaveBeenCalledWith(expect.any(Function), { expect(runBoundedRemoteActivity).toHaveBeenCalledWith(expect.any(Function), {
label: "database-cleanup", label: "database-cleanup",
}); });
expect(runFiniteReplicationActivity).toHaveBeenCalledWith(expect.any(Function), {
label: "replication",
});
expect(openReplication).toHaveBeenCalledOnce(); expect(openReplication).toHaveBeenCalledOnce();
expect(openReplication.mock.invocationCallOrder[0]).toBeLessThan(activityFinished.mock.invocationCallOrder[0]); expect(openReplication.mock.invocationCallOrder[0]).toBeLessThan(activityFinished.mock.invocationCallOrder[0]);
expect(chunkMocks.balanceChunkPurgedDBs).toHaveBeenCalledOnce(); expect(chunkMocks.balanceChunkPurgedDBs).toHaveBeenCalledOnce();
+1 -1
View File
@@ -33,7 +33,7 @@ export class ModuleReplicatorCouchDB extends AbstractModule {
if (continuous) { if (continuous) {
void openReplication(); void openReplication();
} else { } else {
await this.services.replicator.runBoundedRemoteActivity(openReplication, { await this.services.replicator.runFiniteReplicationActivity(openReplication, {
label: "replication", label: "replication",
}); });
} }
@@ -3,7 +3,7 @@ import { ModuleReplicatorCouchDB } from "./ModuleReplicatorCouchDB.ts";
function createModule(settings: { liveSync: boolean; syncOnStart: boolean }, isReplicationReady = true) { function createModule(settings: { liveSync: boolean; syncOnStart: boolean }, isReplicationReady = true) {
const openReplication = vi.fn(async () => true); const openReplication = vi.fn(async () => true);
const runBoundedRemoteActivity = vi.fn(async (task: () => unknown) => await task()); const runFiniteReplicationActivity = vi.fn(async (task: () => unknown) => await task());
const services = { const services = {
API: { API: {
addLog: vi.fn(), addLog: vi.fn(),
@@ -20,7 +20,7 @@ function createModule(settings: { liveSync: boolean; syncOnStart: boolean }, isR
isReplicationReady: vi.fn(async () => isReplicationReady), isReplicationReady: vi.fn(async () => isReplicationReady),
}, },
replicator: { replicator: {
runBoundedRemoteActivity, runFiniteReplicationActivity,
}, },
setting: { setting: {
saveSettingData: vi.fn(async () => undefined), saveSettingData: vi.fn(async () => undefined),
@@ -38,13 +38,13 @@ function createModule(settings: { liveSync: boolean; syncOnStart: boolean }, isR
return { return {
module: new ModuleReplicatorCouchDB(core), module: new ModuleReplicatorCouchDB(core),
openReplication, openReplication,
runBoundedRemoteActivity, runFiniteReplicationActivity,
}; };
} }
describe("ModuleReplicatorCouchDB resume replication activity", () => { describe("ModuleReplicatorCouchDB resume replication activity", () => {
it("runs start-up one-shot replication through bounded remote activity", async () => { it("exposes start-up one-shot replication as finite replication activity", async () => {
const { module, openReplication, runBoundedRemoteActivity } = createModule({ const { module, openReplication, runFiniteReplicationActivity } = createModule({
liveSync: false, liveSync: false,
syncOnStart: true, syncOnStart: true,
}); });
@@ -52,14 +52,14 @@ describe("ModuleReplicatorCouchDB resume replication activity", () => {
await module._everyAfterResumeProcess(); await module._everyAfterResumeProcess();
await vi.waitFor(() => expect(openReplication).toHaveBeenCalledOnce()); await vi.waitFor(() => expect(openReplication).toHaveBeenCalledOnce());
expect(runBoundedRemoteActivity).toHaveBeenCalledWith(expect.any(Function), { expect(runFiniteReplicationActivity).toHaveBeenCalledWith(expect.any(Function), {
label: "replication", label: "replication",
}); });
expect(openReplication).toHaveBeenCalledWith(expect.any(Object), false, false, false); expect(openReplication).toHaveBeenCalledWith(expect.any(Object), false, false, false);
}); });
it("does not treat continuous replication as bounded activity", async () => { it("does not wrap the unbounded continuous channel in another finite activity", async () => {
const { module, openReplication, runBoundedRemoteActivity } = createModule({ const { module, openReplication, runFiniteReplicationActivity } = createModule({
liveSync: true, liveSync: true,
syncOnStart: false, syncOnStart: false,
}); });
@@ -67,12 +67,12 @@ describe("ModuleReplicatorCouchDB resume replication activity", () => {
await module._everyAfterResumeProcess(); await module._everyAfterResumeProcess();
await vi.waitFor(() => expect(openReplication).toHaveBeenCalledOnce()); await vi.waitFor(() => expect(openReplication).toHaveBeenCalledOnce());
expect(runBoundedRemoteActivity).not.toHaveBeenCalled(); expect(runFiniteReplicationActivity).not.toHaveBeenCalled();
expect(openReplication).toHaveBeenCalledWith(expect.any(Object), true, false, false); expect(openReplication).toHaveBeenCalledWith(expect.any(Object), true, false, false);
}); });
it("does not start a one-shot activity when start-up readiness fails", async () => { it("does not start a one-shot activity when start-up readiness fails", async () => {
const { module, openReplication, runBoundedRemoteActivity } = createModule( const { module, openReplication, runFiniteReplicationActivity } = createModule(
{ {
liveSync: false, liveSync: false,
syncOnStart: true, syncOnStart: true,
@@ -83,7 +83,7 @@ describe("ModuleReplicatorCouchDB resume replication activity", () => {
await module._everyAfterResumeProcess(); await module._everyAfterResumeProcess();
await new Promise((resolve) => setTimeout(resolve, 0)); await new Promise((resolve) => setTimeout(resolve, 0));
expect(runBoundedRemoteActivity).not.toHaveBeenCalled(); expect(runFiniteReplicationActivity).not.toHaveBeenCalled();
expect(openReplication).not.toHaveBeenCalled(); expect(openReplication).not.toHaveBeenCalled();
}); });
}); });
+41 -41
View File
@@ -31,7 +31,12 @@ import { LogPaneView, VIEW_TYPE_LOG } from "./Log/LogPaneView.ts";
import { serialized } from "octagonal-wheels/concurrency/lock"; import { serialized } from "octagonal-wheels/concurrency/lock";
import { $msg } from "@lib/common/i18n.ts"; import { $msg } from "@lib/common/i18n.ts";
import { P2PLogCollector } from "@lib/replication/trystero/P2PLogCollector.ts"; import { P2PLogCollector } from "@lib/replication/trystero/P2PLogCollector.ts";
import { hasRemoteActivity } from "./RemoteActivityStatus.ts"; import {
REMOTE_REQUEST_ACTIVITY_MINIMUM_VISIBLE_MS,
formatRemoteActivityStatusLabel,
getTrackedRequestCount,
} from "./RemoteActivityStatus.ts";
import { createMinimumVisibleActivityCount, createPaddedCounterLabel } from "./StatusBarDisplay.ts";
import type { LiveSyncCore } from "@/main.ts"; import type { LiveSyncCore } from "@/main.ts";
import { LiveSyncError } from "@lib/common/LSError.ts"; import { LiveSyncError } from "@lib/common/LSError.ts";
import { isValidPath } from "@/common/utils.ts"; import { isValidPath } from "@/common/utils.ts";
@@ -118,48 +123,43 @@ export class ModuleLog extends AbstractObsidianModule {
p2pLogCollector = new P2PLogCollector(); p2pLogCollector = new P2PLogCollector();
observeForLogs() { observeForLogs() {
const padSpaces = `\u{2007}`.repeat(10); const registerDisplay = <T extends { dispose(): void }>(display: T): T => {
// const emptyMark = `\u{2003}`; this.plugin.register(() => display.dispose());
function padLeftSpComputed(numI: ReactiveValue<number>, mark: string) { return display;
const formatted = reactiveSource(""); };
let timer: number | undefined = undefined; const labelReplication = registerDisplay(
let maxLen = 1; createPaddedCounterLabel(this.services.replication.replicationResultCount, `📥`)
numI.onChanged((numX) => { );
const num = numX.value; const labelDBCount = registerDisplay(
const numLen = `${Math.abs(num)}`.length + 1; createPaddedCounterLabel(this.services.replication.databaseQueueCount, `📄`)
maxLen = maxLen < numLen ? numLen : maxLen; );
if (timer) compatGlobal.clearTimeout(timer); const labelStorageCount = registerDisplay(
if (num == 0) { createPaddedCounterLabel(this.services.replication.storageApplyingCount, `💾`)
timer = compatGlobal.setTimeout(() => { );
formatted.value = ""; const labelChunkCount = registerDisplay(createPaddedCounterLabel(collectingChunks, `🧩`));
maxLen = 1; const labelPluginScanCount = registerDisplay(createPaddedCounterLabel(pluginScanningCount, `🔌`));
}, 3000); const labelConflictProcessCount = registerDisplay(
} createPaddedCounterLabel(this.services.conflict.conflictProcessQueueCount, `🔩`)
formatted.value = ` ${mark}${`${padSpaces}${num}`.slice(-maxLen)}`; );
});
return computed(() => formatted.value);
}
const labelReplication = padLeftSpComputed(this.services.replication.replicationResultCount, `📥`);
const labelDBCount = padLeftSpComputed(this.services.replication.databaseQueueCount, `📄`);
const labelStorageCount = padLeftSpComputed(this.services.replication.storageApplyingCount, `💾`);
const labelChunkCount = padLeftSpComputed(collectingChunks, `🧩`);
const labelPluginScanCount = padLeftSpComputed(pluginScanningCount, `🔌`);
const labelConflictProcessCount = padLeftSpComputed(this.services.conflict.conflictProcessQueueCount, `🔩`);
const hiddenFilesCount = reactive(() => hiddenFilesEventCount.value - hiddenFilesProcessingCount.value); const hiddenFilesCount = reactive(() => hiddenFilesEventCount.value - hiddenFilesProcessingCount.value);
const labelHiddenFilesCount = padLeftSpComputed(hiddenFilesCount, `⚙️`); const labelHiddenFilesCount = registerDisplay(createPaddedCounterLabel(hiddenFilesCount, `⚙️`));
const queueCountLabelX = reactive(() => { const queueCountLabelX = reactive(() => {
return `${labelReplication()}${labelDBCount()}${labelStorageCount()}${labelChunkCount()}${labelPluginScanCount()}${labelHiddenFilesCount()}${labelConflictProcessCount()}`; return `${labelReplication.value}${labelDBCount.value}${labelStorageCount.value}${labelChunkCount.value}${labelPluginScanCount.value}${labelHiddenFilesCount.value}${labelConflictProcessCount.value}`;
}); });
const queueCountLabel = () => queueCountLabelX.value; const queueCountLabel = () => queueCountLabelX.value;
const trackedRequestCount = reactive(() => {
return getTrackedRequestCount(this.services.API.requestCount.value, this.services.API.responseCount.value);
});
const displayedTrackedRequestCount = registerDisplay(
createMinimumVisibleActivityCount(trackedRequestCount, REMOTE_REQUEST_ACTIVITY_MINIMUM_VISIBLE_MS)
);
const requestingStatLabel = computed(() => { const requestingStatLabel = computed(() => {
return hasRemoteActivity( return formatRemoteActivityStatusLabel({
this.services.API.requestCount.value, remoteOperationCount: Math.max(0, this.services.replicator.boundedRemoteActivityCount.value),
this.services.API.responseCount.value, trackedRequestCount: displayedTrackedRequestCount.value,
this.services.replicator.boundedRemoteActivityCount.value });
)
? "📲 "
: "";
}); });
const replicationStatLabel = computed(() => { const replicationStatLabel = computed(() => {
@@ -215,11 +215,11 @@ export class ModuleLog extends AbstractObsidianModule {
} }
return { w, sent, pushLast, arrived, pullLast }; return { w, sent, pushLast, arrived, pullLast };
}); });
const labelProc = padLeftSpComputed(this.services.fileProcessing.processing, ``); const labelProc = registerDisplay(createPaddedCounterLabel(this.services.fileProcessing.processing, ``));
const labelPend = padLeftSpComputed(this.services.fileProcessing.totalQueued, `🛫`); const labelPend = registerDisplay(createPaddedCounterLabel(this.services.fileProcessing.totalQueued, `🛫`));
const labelInBatchDelay = padLeftSpComputed(this.services.fileProcessing.batched, `📬`); const labelInBatchDelay = registerDisplay(createPaddedCounterLabel(this.services.fileProcessing.batched, `📬`));
const waitingLabel = computed(() => { const waitingLabel = computed(() => {
return `${labelProc()}${labelPend()}${labelInBatchDelay()}`; return `${labelProc.value}${labelPend.value}${labelInBatchDelay.value}`;
}); });
const statusLineLabel = computed(() => { const statusLineLabel = computed(() => {
const { w, sent, pushLast, arrived, pullLast } = replicationStatLabel(); const { w, sent, pushLast, arrived, pullLast } = replicationStatLabel();
@@ -1,13 +0,0 @@
import { describe, expect, it } from "vitest";
import { hasRemoteActivity } from "./RemoteActivityStatus.ts";
describe("hasRemoteActivity", () => {
it("preserves the existing HTTP request balance signal", () => {
expect(hasRemoteActivity(2, 1, 0)).toBe(true);
expect(hasRemoteActivity(2, 2, 0)).toBe(false);
});
it("reports bounded remote activity without an HTTP request imbalance", () => {
expect(hasRemoteActivity(2, 2, 1)).toBe(true);
});
});
+26 -3
View File
@@ -1,4 +1,27 @@
/** Returns whether the status UI should report HTTP traffic or a finite remote operation in progress. */ /** Status icon for a finite remote operation whose lifetime is known. */
export function hasRemoteActivity(requestCount: number, responseCount: number, boundedRemoteActivityCount: number) { export const REMOTE_OPERATION_ACTIVITY_ICON = "📲";
return requestCount - responseCount !== 0 || boundedRemoteActivityCount !== 0;
/** Status icon for approximate physical remote-request activity. */
export const REMOTE_REQUEST_ACTIVITY_ICON = "🌐";
/** Avoids hiding very short remote requests before the status bar can render them. */
export const REMOTE_REQUEST_ACTIVITY_MINIMUM_VISIBLE_MS = 150;
export type RemoteActivityStatus = {
remoteOperationCount: number;
trackedRequestCount: number;
};
/** Returns the non-negative difference between tracked request starts and completions. */
export function getTrackedRequestCount(requestCount: number, responseCount: number): number {
return Math.max(0, requestCount - responseCount);
}
/** Formats the compact prefix shown before the replication status. */
export function formatRemoteActivityStatusLabel(status: RemoteActivityStatus): string {
const labels = [
status.remoteOperationCount > 0 ? REMOTE_OPERATION_ACTIVITY_ICON : "",
status.trackedRequestCount > 0 ? `${REMOTE_REQUEST_ACTIVITY_ICON}${status.trackedRequestCount}` : "",
].filter((label) => label !== "");
return labels.length > 0 ? `${labels.join(" ")} ` : "";
} }
@@ -0,0 +1,35 @@
import { describe, expect, it } from "vitest";
import {
REMOTE_OPERATION_ACTIVITY_ICON,
REMOTE_REQUEST_ACTIVITY_ICON,
formatRemoteActivityStatusLabel,
getTrackedRequestCount,
} from "./RemoteActivityStatus.ts";
describe("getTrackedRequestCount", () => {
it("reports the non-negative difference between starts and completions", () => {
expect(getTrackedRequestCount(3, 2)).toBe(1);
expect(getTrackedRequestCount(2, 2)).toBe(0);
expect(getTrackedRequestCount(2, 3)).toBe(0);
});
});
describe("formatRemoteActivityStatusLabel", () => {
it("separates a finite remote operation from tracked physical requests", () => {
expect(formatRemoteActivityStatusLabel({ remoteOperationCount: 1, trackedRequestCount: 0 })).toBe(
`${REMOTE_OPERATION_ACTIVITY_ICON} `
);
expect(formatRemoteActivityStatusLabel({ remoteOperationCount: 0, trackedRequestCount: 1 })).toBe(
`${REMOTE_REQUEST_ACTIVITY_ICON}1 `
);
expect(formatRemoteActivityStatusLabel({ remoteOperationCount: 1, trackedRequestCount: 2 })).toBe(
`${REMOTE_OPERATION_ACTIVITY_ICON} ${REMOTE_REQUEST_ACTIVITY_ICON}2 `
);
});
it("omits inactive and invalid negative activity counts", () => {
expect(formatRemoteActivityStatusLabel({ remoteOperationCount: 0, trackedRequestCount: 0 })).toBe("");
expect(formatRemoteActivityStatusLabel({ remoteOperationCount: -1, trackedRequestCount: -1 })).toBe("");
});
});
+133
View File
@@ -0,0 +1,133 @@
import { reactiveSource, type ReactiveSource, type ReactiveValue } from "octagonal-wheels/dataobject/reactive";
import { compatGlobal } from "@lib/common/coreEnvFunctions.ts";
const STATUS_COUNTER_PADDING = "\u2007".repeat(10);
export const STATUS_COUNTER_INACTIVE_LINGER_MS = 3_000;
export type DisposableReactiveValue<T> = ReactiveValue<T> & {
dispose(): void;
};
function asDisposableReactiveValue<T>(value: ReactiveSource<T>, dispose: () => void): DisposableReactiveValue<T> {
return {
get value() {
return value.value;
},
onChanged(handler) {
value.onChanged(handler);
},
offChanged(handler) {
value.offChanged(handler);
},
dispose,
};
}
/**
* Mirrors an activity count while keeping each visible period on screen for a
* minimum total lifetime. The delay applies only when the source becomes zero.
*/
export function createMinimumVisibleActivityCount(
source: ReactiveValue<number>,
minimumVisibleMs: number
): DisposableReactiveValue<number> {
const minimumLifetime = Math.max(0, minimumVisibleMs);
const displayed = reactiveSource(Math.max(0, source.value));
let visibleSince = displayed.value > 0 ? Date.now() : undefined;
let hideTimer: number | undefined;
let disposed = false;
const cancelHide = () => {
if (hideTimer === undefined) return;
compatGlobal.clearTimeout(hideTimer);
hideTimer = undefined;
};
const hideIfIdle = () => {
hideTimer = undefined;
if (disposed || Math.max(0, source.value) > 0) return;
displayed.value = 0;
visibleSince = undefined;
};
const update = () => {
if (disposed) return;
const nextCount = Math.max(0, source.value);
cancelHide();
if (nextCount > 0) {
if (displayed.value === 0) {
visibleSince = Date.now();
}
displayed.value = nextCount;
return;
}
if (displayed.value === 0) {
visibleSince = undefined;
return;
}
const elapsed = Date.now() - (visibleSince ?? Date.now());
const remaining = Math.max(0, minimumLifetime - elapsed);
if (remaining === 0) {
hideIfIdle();
} else {
hideTimer = compatGlobal.setTimeout(hideIfIdle, remaining);
}
};
source.onChanged(update);
return asDisposableReactiveValue(displayed, () => {
if (disposed) return;
disposed = true;
cancelHide();
source.offChanged(update);
});
}
/**
* Formats a counter with a stable width and briefly retains its zero value so
* that the completion of queued work remains visible.
*/
export function createPaddedCounterLabel(
source: ReactiveValue<number>,
mark: string,
inactiveLingerMs = STATUS_COUNTER_INACTIVE_LINGER_MS
): DisposableReactiveValue<string> {
const linger = Math.max(0, inactiveLingerMs);
const formatted = reactiveSource("");
let maximumLength = 1;
let clearTimer: number | undefined;
let disposed = false;
const cancelClear = () => {
if (clearTimer === undefined) return;
compatGlobal.clearTimeout(clearTimer);
clearTimer = undefined;
};
const format = (count: number) => {
const requiredLength = `${Math.abs(count)}`.length + 1;
maximumLength = Math.max(maximumLength, requiredLength);
return ` ${mark}${`${STATUS_COUNTER_PADDING}${count}`.slice(-maximumLength)}`;
};
const update = () => {
if (disposed) return;
cancelClear();
const count = source.value;
formatted.value = format(count);
if (count !== 0) return;
clearTimer = compatGlobal.setTimeout(() => {
clearTimer = undefined;
if (disposed) return;
formatted.value = "";
maximumLength = 1;
}, linger);
};
source.onChanged(update);
return asDisposableReactiveValue(formatted, () => {
if (disposed) return;
disposed = true;
cancelClear();
source.offChanged(update);
});
}
@@ -0,0 +1,139 @@
import { reactive, reactiveSource } from "octagonal-wheels/dataobject/reactive";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
STATUS_COUNTER_INACTIVE_LINGER_MS,
createMinimumVisibleActivityCount,
createPaddedCounterLabel,
} from "./StatusBarDisplay.ts";
describe("createMinimumVisibleActivityCount", () => {
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-07-16T00:00:00Z"));
});
afterEach(() => {
vi.useRealTimers();
});
it("keeps a short activity visible for the configured minimum lifetime", () => {
const source = reactiveSource(0);
const display = createMinimumVisibleActivityCount(source, 150);
const rendered = reactive(() => `active:${display.value}`);
expect(rendered.value).toBe("active:0");
source.value = 1;
expect(rendered.value).toBe("active:1");
vi.advanceTimersByTime(50);
source.value = 0;
expect(display.value).toBe(1);
vi.advanceTimersByTime(99);
expect(display.value).toBe(1);
vi.advanceTimersByTime(1);
expect(display.value).toBe(0);
expect(rendered.value).toBe("active:0");
display.dispose();
});
it("updates overlapping activity and starts a new minimum lifetime after becoming idle", () => {
const source = reactiveSource(0);
const display = createMinimumVisibleActivityCount(source, 150);
source.value = 1;
vi.advanceTimersByTime(25);
source.value = 2;
expect(display.value).toBe(2);
source.value = 0;
vi.advanceTimersByTime(50);
source.value = 1;
expect(display.value).toBe(1);
vi.advanceTimersByTime(75);
source.value = 0;
expect(display.value).toBe(0);
source.value = 3;
source.value = 0;
expect(display.value).toBe(3);
vi.advanceTimersByTime(150);
expect(display.value).toBe(0);
display.dispose();
});
it("cancels pending work and stops observing its source when disposed", () => {
const source = reactiveSource(0);
const display = createMinimumVisibleActivityCount(source, 150);
source.value = 1;
source.value = 0;
display.dispose();
vi.advanceTimersByTime(150);
source.value = 2;
expect(display.value).toBe(1);
});
});
describe("createPaddedCounterLabel", () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it("keeps the widest counter label until its inactive linger period ends", () => {
const source = reactiveSource(0);
const display = createPaddedCounterLabel(source, "📥");
expect(display.value).toBe("");
source.value = 9;
expect(display.value).toBe(" 📥\u20079");
source.value = 123;
expect(display.value).toBe(" 📥\u2007123");
source.value = 0;
expect(display.value).toBe(" 📥\u2007\u2007\u20070");
vi.advanceTimersByTime(STATUS_COUNTER_INACTIVE_LINGER_MS - 1);
expect(display.value).toBe(" 📥\u2007\u2007\u20070");
vi.advanceTimersByTime(1);
expect(display.value).toBe("");
source.value = 7;
expect(display.value).toBe(" 📥\u20077");
display.dispose();
});
it("cancels the pending clear when counter activity resumes", () => {
const source = reactiveSource(0);
const display = createPaddedCounterLabel(source, "📄");
source.value = 1;
source.value = 0;
vi.advanceTimersByTime(1_000);
source.value = 2;
vi.advanceTimersByTime(STATUS_COUNTER_INACTIVE_LINGER_MS);
expect(display.value).toBe(" 📄\u20072");
display.dispose();
});
it("cancels its inactive timer and source subscription when disposed", () => {
const source = reactiveSource(0);
const display = createPaddedCounterLabel(source, "📄");
source.value = 4;
source.value = 0;
display.dispose();
vi.advanceTimersByTime(STATUS_COUNTER_INACTIVE_LINGER_MS);
source.value = 5;
expect(display.value).toBe(" 📄\u20070");
});
});
+1 -1
View File
@@ -87,7 +87,7 @@ export function useP2PReplicatorUI(
const activeReplicator = replicator.replicator; const activeReplicator = replicator.replicator;
if (!activeReplicator) return; if (!activeReplicator) return;
const settings = host.services.setting.currentSettings(); const settings = host.services.setting.currentSettings();
void host.services.replicator.runBoundedRemoteActivity( void host.services.replicator.runFiniteReplicationActivity(
() => activeReplicator.openReplication(settings, false, true, false), () => activeReplicator.openReplication(settings, false, true, false),
{ label: "replication" } { label: "replication" }
); );
@@ -12,11 +12,11 @@ vi.mock("@/features/P2PSync/P2PReplicator/P2PServerStatusPaneView", () => ({
import { useP2PReplicatorUI } from "./useP2PReplicatorUI"; import { useP2PReplicatorUI } from "./useP2PReplicatorUI";
describe("useP2PReplicatorUI commands", () => { describe("useP2PReplicatorUI commands", () => {
it("runs a direct modal P2P replication command through the shared activity boundary", async () => { it("exposes a direct modal P2P replication command as finite replication activity", async () => {
const commands: Array<{ id: string; checkCallback?: (isChecking: boolean) => unknown }> = []; const commands: Array<{ id: string; checkCallback?: (isChecking: boolean) => unknown }> = [];
let initialise: (() => Promise<unknown>) | undefined; let initialise: (() => Promise<unknown>) | undefined;
const openReplication = vi.fn(async () => true); const openReplication = vi.fn(async () => true);
const runBoundedRemoteActivity = vi.fn(async (task: () => unknown) => await task()); const runFiniteReplicationActivity = vi.fn(async (task: () => unknown) => await task());
const host = { const host = {
services: { services: {
API: { API: {
@@ -35,7 +35,7 @@ describe("useP2PReplicatorUI commands", () => {
onLayoutReady: { addHandler: vi.fn() }, onLayoutReady: { addHandler: vi.fn() },
}, },
setting: { currentSettings: vi.fn(() => ({ remoteType: "COUCHDB" })) }, setting: { currentSettings: vi.fn(() => ({ remoteType: "COUCHDB" })) },
replicator: { runBoundedRemoteActivity }, replicator: { runFiniteReplicationActivity },
}, },
} as any; } as any;
const p2p = { const p2p = {
@@ -51,7 +51,7 @@ describe("useP2PReplicatorUI commands", () => {
commands.find((command) => command.id === "replicate-now-by-p2p")?.checkCallback?.(false); commands.find((command) => command.id === "replicate-now-by-p2p")?.checkCallback?.(false);
await vi.waitFor(() => expect(openReplication).toHaveBeenCalledOnce()); await vi.waitFor(() => expect(openReplication).toHaveBeenCalledOnce());
expect(runBoundedRemoteActivity).toHaveBeenCalledWith(expect.any(Function), { expect(runFiniteReplicationActivity).toHaveBeenCalledWith(expect.any(Function), {
label: "replication", label: "replication",
}); });
}); });
+28 -2
View File
@@ -51,6 +51,7 @@ npm run test:e2e:obsidian:cli-help -- vaults verbose
npm run test:e2e:obsidian:smoke npm run test:e2e:obsidian:smoke
npm run test:e2e:obsidian:vault-reflection npm run test:e2e:obsidian:vault-reflection
npm run test:e2e:obsidian:couchdb-upload npm run test:e2e:obsidian:couchdb-upload
npm run test:e2e:obsidian:cli-to-obsidian-sync
npm run test:e2e:obsidian:minio-upload npm run test:e2e:obsidian:minio-upload
npm run test:e2e:obsidian:startup-scan npm run test:e2e:obsidian:startup-scan
npm run test:e2e:obsidian:two-vault-sync npm run test:e2e:obsidian:two-vault-sync
@@ -61,11 +62,32 @@ npm run test:e2e:obsidian:local-suite
npm run test:e2e:obsidian:local-suite:services npm run test:e2e:obsidian:local-suite:services
``` ```
`test:e2e:obsidian:local-suite` runs `npm run build`, discovery, smoke, vault reflection, CouchDB upload, Object Storage upload, startup scan, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB and MinIO fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run. `test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, vault reflection, CouchDB upload, CLI-to-Obsidian synchronisation, Object Storage upload, startup scan, two-vault synchronisation, Hidden File Sync, Customisation Sync, and setting Markdown export in sequence. Start the local CouchDB and MinIO fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run.
`test:e2e:obsidian:couchdb-upload` reuses the CouchDB variables from `.test.env` or the process environment. It expects a reachable CouchDB service, creates a unique database, configures Self-hosted LiveSync through `obsidian-cli eval`, creates a note in real Obsidian, commits the note into the local database, runs one-shot synchronisation, and verifies that the remote database contains both the metadata document and its chunk documents. `test:e2e:obsidian:couchdb-upload` reuses the CouchDB variables from `.test.env` or the process environment. It expects a reachable CouchDB service, creates a unique database, configures Self-hosted LiveSync through `obsidian-cli eval`, creates a note in real Obsidian, commits the note into the local database, runs one-shot synchronisation, and verifies that the remote database contains both the metadata document and its chunk documents.
`test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service, configures Self-hosted LiveSync for Object Storage through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix. The same workflow checks the two remote-activity status boundaries. It first holds a real CouchDB request at the selected fetch implementation and confirms that `🌐N` is visible while `📲` is absent. It then holds the real one-shot replication immediately before its replicator call, confirms that `📲` is visible while no physical request is active, releases it, and requires the finite and bounded activity counts to return to zero, the request and response counts to balance, and both indicators to disappear. Finally, it creates a remote-only chunk, holds the real on-demand fetch immediately before its remote call, makes the same logical active and idle assertions, and verifies that the fetched chunk is written into the local database. These gates make the active states deterministic without replacing the remote request or operation.
If this status workflow fails while Obsidian is running, it writes a full-page screenshot and a JSON snapshot of the status text and counters under `/tmp/obsidian-livesync-e2e`. Set `E2E_OBSIDIAN_DIAGNOSTICS_DIR` to use another directory.
`test:e2e:obsidian:cli-to-obsidian-sync` is the cross-runtime compatibility check for the official LiveSync CLI and the real Obsidian plug-in. Build the plug-in first, and build the local CLI too when no external CLI command is selected. The script uses E2EE, Path Obfuscation, and the current preferred chunk settings to create and synchronise a note through the CLI, starts real Obsidian with an isolated Vault and profile, synchronises the same CouchDB database, and verifies that the plug-in materialises identical note content. This covers the boundary that CLI-only and plug-in-only round trips do not exercise.
By default, the compatibility check runs `node src/apps/cli/dist/index.cjs`. Set `LIVESYNC_CLI_COMMAND` to test another CLI build or distribution. The value may be a quoted command line or a JSON array of executable and prefix arguments; the scenario arguments are appended without going through a shell.
For example, to test an executable on `PATH`:
```bash
LIVESYNC_CLI_COMMAND='livesync-cli' npm run test:e2e:obsidian:cli-to-obsidian-sync
```
On Linux, a multi-architecture published Docker image can run against the local CouchDB fixture by sharing the temporary directory, using host networking, preserving the host user's file ownership, and overriding the image entrypoint so that the runner can supply its explicit database path. Images published before ARM64 support remain AMD64-only and require configured Docker emulation on an ARM host.
```bash
LIVESYNC_CLI_COMMAND="docker run --rm --network host --user $(id -u):$(id -g) --volume /tmp:/tmp --entrypoint node ghcr.io/vrtmrz/livesync-cli:edge /app/dist/index.cjs" \
npm run test:e2e:obsidian:cli-to-obsidian-sync
```
`test:e2e:obsidian:minio-upload` reuses the Object Storage variables from `.test.env` or the process environment. It expects a reachable S3-compatible service, configures Self-hosted LiveSync for Object Storage through `obsidian-cli eval`, creates a note in real Obsidian, runs one-shot Journal Sync, and verifies through the AWS SDK that objects were written under a unique bucket prefix. Adapter tests separately observe an in-progress SDK command, while this real-runtime workflow verifies the resulting request counters advance and rebalance.
`test:e2e:obsidian:startup-scan` configures a temporary CouchDB database, stops Obsidian, writes a note directly into the vault, restarts Obsidian, and verifies from CouchDB that the boot-time scan picked up the offline file. `test:e2e:obsidian:startup-scan` configures a temporary CouchDB database, stops Obsidian, writes a note directly into the vault, restarts Obsidian, and verifies from CouchDB that the boot-time scan picked up the offline file.
@@ -104,10 +126,14 @@ Useful environment variables:
- `E2E_OBSIDIAN_READY_TIMEOUT_MS`: plug-in readiness timeout in milliseconds. - `E2E_OBSIDIAN_READY_TIMEOUT_MS`: plug-in readiness timeout in milliseconds.
- `E2E_OBSIDIAN_CLI_READY_TIMEOUT_MS`: timeout for waiting until the vault-side Obsidian CLI exposes the plug-in catalogue. - `E2E_OBSIDIAN_CLI_READY_TIMEOUT_MS`: timeout for waiting until the vault-side Obsidian CLI exposes the plug-in catalogue.
- `E2E_OBSIDIAN_CLI_TIMEOUT_MS`: timeout for each `obsidian-cli` invocation. - `E2E_OBSIDIAN_CLI_TIMEOUT_MS`: timeout for each `obsidian-cli` invocation.
- `E2E_LIVESYNC_CLI_TIMEOUT_MS`: timeout for each official LiveSync CLI invocation in the CLI-to-Obsidian compatibility check; default is 60 seconds.
- `LIVESYNC_CLI_COMMAND`: optional LiveSync CLI executable and prefix arguments used by the CLI-to-Obsidian compatibility check. The default is the locally built CLI.
- `E2E_OBSIDIAN_FILE_TIMEOUT_MS`: timeout for waiting until a note created through Obsidian's vault API is reflected to disk. - `E2E_OBSIDIAN_FILE_TIMEOUT_MS`: timeout for waiting until a note created through Obsidian's vault API is reflected to disk.
- `E2E_OBSIDIAN_CORE_READY_TIMEOUT_MS`: timeout for waiting until Self-hosted LiveSync reports that its core lifecycle and local database are ready. - `E2E_OBSIDIAN_CORE_READY_TIMEOUT_MS`: timeout for waiting until Self-hosted LiveSync reports that its core lifecycle and local database are ready.
- `E2E_OBSIDIAN_LOCAL_DB_TIMEOUT_MS`: timeout for waiting until a file appears in Self-hosted LiveSync's local database. - `E2E_OBSIDIAN_LOCAL_DB_TIMEOUT_MS`: timeout for waiting until a file appears in Self-hosted LiveSync's local database.
- `E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS`: timeout for waiting until CouchDB contains uploaded E2E documents. - `E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS`: timeout for waiting until CouchDB contains uploaded E2E documents.
- `E2E_OBSIDIAN_REMOTE_ACTIVITY_TIMEOUT_MS`: timeout for an observed remote activity to enter or leave its status boundary; default is 30 seconds.
- `E2E_OBSIDIAN_DIAGNOSTICS_DIR`: directory for screenshots and status snapshots captured after a remote-activity failure; default is `/tmp/obsidian-livesync-e2e`.
- `E2E_OBSIDIAN_OBJECT_STORAGE_TIMEOUT_MS`: timeout for waiting until Object Storage contains uploaded E2E objects. - `E2E_OBSIDIAN_OBJECT_STORAGE_TIMEOUT_MS`: timeout for waiting until Object Storage contains uploaded E2E objects.
- `E2E_OBSIDIAN_KEEP_COUCHDB=true`: keep the temporary CouchDB database for inspection. - `E2E_OBSIDIAN_KEEP_COUCHDB=true`: keep the temporary CouchDB database for inspection.
- `E2E_OBSIDIAN_KEEP_OBJECT_STORAGE=true`: keep the temporary Object Storage prefix for inspection. - `E2E_OBSIDIAN_KEEP_OBJECT_STORAGE=true`: keep the temporary Object Storage prefix for inspection.
+20
View File
@@ -126,6 +126,26 @@ export async function createCouchDbDatabase(config: CouchDbConfig, dbName: strin
} }
} }
export async function putCouchDbDocument(
config: CouchDbConfig,
dbName: string,
document: CouchDbDocument
): Promise<void> {
const response = await fetch(databaseUrl(config, dbName, `/${encodeURIComponent(document._id)}`), {
method: "PUT",
headers: {
authorization: authHeader(config),
"content-type": "application/json",
},
body: JSON.stringify(document),
});
if (!response.ok) {
throw new Error(
`Failed to write CouchDB document ${document._id}. HTTP ${response.status}: ${await response.text()}`
);
}
}
export async function deleteCouchDbDatabase(config: CouchDbConfig, dbName: string): Promise<void> { export async function deleteCouchDbDatabase(config: CouchDbConfig, dbName: string): Promise<void> {
const response = await fetch(databaseUrl(config, dbName), { const response = await fetch(databaseUrl(config, dbName), {
method: "DELETE", method: "DELETE",
+237
View File
@@ -0,0 +1,237 @@
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { Page } from "playwright";
import { withObsidianPage } from "./ui.ts";
import {
REMOTE_OPERATION_ACTIVITY_ICON,
REMOTE_REQUEST_ACTIVITY_ICON,
} from "../../../src/modules/features/RemoteActivityStatus.ts";
export const REMOTE_ACTIVITY_E2E_STATE_KEY = "__livesyncE2ERemoteActivity";
export const REMOTE_ACTIVITY_GATE_KIND = {
chunkFetch: "chunk-fetch",
oneShot: "one-shot",
trackedRequest: "tracked-request",
} as const;
export const REMOTE_ACTIVITY_EXPECTED_STATE = {
chunkFetchActive: "chunk-fetch-active",
finiteReplicationActive: "finite-replication-active",
idle: "idle",
trackedRequestActive: "tracked-request-active",
} as const;
export type RemoteActivityGateKind = (typeof REMOTE_ACTIVITY_GATE_KIND)[keyof typeof REMOTE_ACTIVITY_GATE_KIND];
export type RemoteActivitySnapshot = {
boundedRemoteActivityCount: number;
finiteReplicationActivityCount: number;
gateDone?: boolean;
gateEntered?: boolean;
gateError?: string;
gateKind?: RemoteActivityGateKind;
requestCount: number;
remoteOperationIndicatorVisible: boolean;
remoteRequestIndicatorVisible: boolean;
responseCount: number;
statusBarFound: boolean;
statusBarText: string;
};
export type ExpectedRemoteActivityState =
(typeof REMOTE_ACTIVITY_EXPECTED_STATE)[keyof typeof REMOTE_ACTIVITY_EXPECTED_STATE];
type RuntimeCounter = { value?: number };
type RuntimeCore = {
services?: {
API?: {
requestCount?: RuntimeCounter;
responseCount?: RuntimeCounter;
};
replicator?: {
boundedRemoteActivityCount?: RuntimeCounter;
finiteReplicationActivityCount?: RuntimeCounter;
};
};
};
type RuntimeGate = {
done?: boolean;
entered?: boolean;
error?: string;
kind?: RemoteActivityGateKind;
};
type RendererGlobals = typeof globalThis & {
app?: {
plugins?: {
plugins?: Record<string, { core?: RuntimeCore }>;
};
};
};
async function readRemoteActivitySnapshotFromPage(page: Page): Promise<RemoteActivitySnapshot> {
return await page.evaluate(
({ operationIcon, pluginId, requestIcon, stateKey }) => {
const globals = globalThis as RendererGlobals;
const core = globals.app?.plugins?.plugins?.[pluginId]?.core;
if (!core) throw new Error(`Obsidian plug-in is not loaded: ${pluginId}`);
const gate = (globalThis as unknown as Record<string, RuntimeGate | undefined>)[stateKey];
const statusBars = Array.from(document.querySelectorAll<HTMLElement>(".syncstatusbar"));
return {
boundedRemoteActivityCount: Number(core.services?.replicator?.boundedRemoteActivityCount?.value ?? -1),
finiteReplicationActivityCount: Number(
core.services?.replicator?.finiteReplicationActivityCount?.value ?? -1
),
gateDone: gate?.done,
gateEntered: gate?.entered,
gateError: gate?.error,
gateKind: gate?.kind,
requestCount: Number(core.services?.API?.requestCount?.value ?? -1),
remoteOperationIndicatorVisible: statusBars.some((element) =>
(element.textContent ?? "").includes(operationIcon)
),
remoteRequestIndicatorVisible: statusBars.some((element) =>
(element.textContent ?? "").includes(requestIcon)
),
responseCount: Number(core.services?.API?.responseCount?.value ?? -1),
statusBarFound: statusBars.length > 0,
statusBarText: statusBars.map((element) => element.textContent ?? "").join("\n"),
} satisfies RemoteActivitySnapshot;
},
{
operationIcon: REMOTE_OPERATION_ACTIVITY_ICON,
pluginId: "obsidian-livesync",
requestIcon: REMOTE_REQUEST_ACTIVITY_ICON,
stateKey: REMOTE_ACTIVITY_E2E_STATE_KEY,
}
);
}
export async function readRemoteActivitySnapshot(port: number): Promise<RemoteActivitySnapshot> {
return await withObsidianPage(port, async (page) => await readRemoteActivitySnapshotFromPage(page));
}
function formatWaitFailure(
expected: ExpectedRemoteActivityState,
snapshot: RemoteActivitySnapshot | undefined,
error: unknown
): Error {
return new Error(
[
`Timed out waiting for remote activity state: ${expected}`,
snapshot ? `Last snapshot: ${JSON.stringify(snapshot)}` : undefined,
error instanceof Error ? error.message : String(error),
]
.filter((line): line is string => line !== undefined)
.join("\n")
);
}
export async function waitForRemoteActivityState(
port: number,
expected: ExpectedRemoteActivityState,
timeoutMs = Number(process.env.E2E_OBSIDIAN_REMOTE_ACTIVITY_TIMEOUT_MS ?? 30000)
): Promise<RemoteActivitySnapshot> {
try {
return await withObsidianPage(port, async (page) => {
await page.waitForFunction(
({ expectedState, expectedStates, gateKinds, operationIcon, pluginId, requestIcon, stateKey }) => {
const globals = globalThis as RendererGlobals;
const core = globals.app?.plugins?.plugins?.[pluginId]?.core;
if (!core) return false;
const gate = (globalThis as unknown as Record<string, RuntimeGate | undefined>)[stateKey];
const statusBarText = Array.from(document.querySelectorAll<HTMLElement>(".syncstatusbar"))
.map((element) => element.textContent ?? "")
.join("\n");
const bounded = Number(core.services?.replicator?.boundedRemoteActivityCount?.value ?? -1);
const finite = Number(core.services?.replicator?.finiteReplicationActivityCount?.value ?? -1);
const requests = Number(core.services?.API?.requestCount?.value ?? -1);
const responses = Number(core.services?.API?.responseCount?.value ?? -1);
const operationIconVisible = statusBarText.includes(operationIcon);
const requestIconVisible = statusBarText.includes(requestIcon);
const trackedRequests = Math.max(0, requests - responses);
if (expectedState === expectedStates.finiteReplicationActive) {
return (
gate?.kind === gateKinds.oneShot &&
gate.entered === true &&
bounded > 0 &&
finite > 0 &&
operationIconVisible &&
!requestIconVisible &&
trackedRequests === 0
);
}
if (expectedState === expectedStates.chunkFetchActive) {
return (
gate?.kind === gateKinds.chunkFetch &&
gate.entered === true &&
bounded > 0 &&
finite === 0 &&
operationIconVisible &&
!requestIconVisible &&
trackedRequests === 0
);
}
if (expectedState === expectedStates.trackedRequestActive) {
return (
gate?.kind === gateKinds.trackedRequest &&
gate.entered === true &&
bounded === 0 &&
finite === 0 &&
trackedRequests > 0 &&
!operationIconVisible &&
statusBarText.includes(`${requestIcon}${trackedRequests}`)
);
}
return (
bounded === 0 &&
finite === 0 &&
requests === responses &&
!operationIconVisible &&
!requestIconVisible
);
},
{
expectedState: expected,
expectedStates: REMOTE_ACTIVITY_EXPECTED_STATE,
gateKinds: REMOTE_ACTIVITY_GATE_KIND,
operationIcon: REMOTE_OPERATION_ACTIVITY_ICON,
pluginId: "obsidian-livesync",
requestIcon: REMOTE_REQUEST_ACTIVITY_ICON,
stateKey: REMOTE_ACTIVITY_E2E_STATE_KEY,
},
{ timeout: timeoutMs }
);
return await readRemoteActivitySnapshotFromPage(page);
});
} catch (error) {
const snapshot = await readRemoteActivitySnapshot(port).catch(() => undefined);
throw formatWaitFailure(expected, snapshot, error);
}
}
export type RemoteActivityDiagnostics = {
screenshotPath: string;
snapshot: RemoteActivitySnapshot;
snapshotPath: string;
};
export async function captureRemoteActivityDiagnostics(
port: number,
label: string
): Promise<RemoteActivityDiagnostics> {
const outputDirectory = process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e";
await mkdir(outputDirectory, { recursive: true });
const safeLabel = label.replace(/[^a-z0-9_-]+/gi, "-").replace(/^-+|-+$/g, "") || "remote-activity";
const prefix = `${safeLabel}-${new Date().toISOString().replace(/[:.]/g, "-")}`;
const screenshotPath = join(outputDirectory, `${prefix}.png`);
const snapshotPath = join(outputDirectory, `${prefix}.json`);
const snapshot = await withObsidianPage(port, async (page) => {
const current = await readRemoteActivitySnapshotFromPage(page);
await page.screenshot({ path: screenshotPath, fullPage: true });
return current;
});
await writeFile(snapshotPath, `${JSON.stringify(snapshot, undefined, 2)}\n`, "utf8");
return { screenshotPath, snapshot, snapshotPath };
}
@@ -0,0 +1,236 @@
import { evalObsidianJson } from "./cli.ts";
import {
REMOTE_ACTIVITY_E2E_STATE_KEY,
REMOTE_ACTIVITY_GATE_KIND,
type RemoteActivityGateKind,
} from "./remoteActivity.ts";
export type HeldRemoteActivityResult = {
done: boolean;
entered: boolean;
error?: string;
kind: RemoteActivityGateKind;
requestedIds?: string[];
result?: boolean;
resultCount?: number;
};
const stateKeySource = JSON.stringify(REMOTE_ACTIVITY_E2E_STATE_KEY);
export async function startHeldOneShotReplication(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
await evalObsidianJson<{ started: boolean }>(
cliBinary,
[
"(async()=>{",
`const stateKey=${stateKeySource};`,
"const host=globalThis;",
"if(host[stateKey]) throw new Error('A remote activity E2E gate is already installed.');",
"const core=app.plugins.plugins['obsidian-livesync'].core;",
"const replicator=core.services.replicator.getActiveReplicator();",
"if(!replicator) throw new Error('No active replicator is available.');",
"const original=replicator.openReplication;",
"let releaseGate;",
"const gate=new Promise((resolve)=>{releaseGate=resolve;});",
`const state={kind:${JSON.stringify(REMOTE_ACTIVITY_GATE_KIND.oneShot)},entered:false,done:false,released:false,error:undefined,result:undefined,promise:undefined,release:undefined,restore:undefined};`,
"state.release=()=>{if(!state.released){state.released=true;releaseGate();}};",
"state.restore=()=>{replicator.openReplication=original;};",
"host[stateKey]=state;",
"replicator.openReplication=async function(...args){",
"state.entered=true;",
"await gate;",
"return await original.apply(this,args);",
"};",
"state.promise=(async()=>{",
"try{",
"if(!(await core.services.fileProcessing.commitPendingFileEvents())) throw new Error('Pending file events could not be committed.');",
"state.result=!!(await core.services.replication.replicate(true));",
"}catch(error){",
"state.error=error instanceof Error?error.message:String(error);",
"}finally{",
"state.restore();",
"state.done=true;",
"}",
"})();",
"return JSON.stringify({started:true});",
"})()",
].join(""),
env
);
}
export async function startHeldChunkFetch(cliBinary: string, env: NodeJS.ProcessEnv, chunkId: string): Promise<void> {
await evalObsidianJson<{ started: boolean }>(
cliBinary,
[
"(async()=>{",
`const stateKey=${stateKeySource};`,
`const chunkId=${JSON.stringify(chunkId)};`,
"const host=globalThis;",
"if(host[stateKey]) throw new Error('A remote activity E2E gate is already installed.');",
"const core=app.plugins.plugins['obsidian-livesync'].core;",
"const replicator=core.services.replicator.getActiveReplicator();",
"if(!replicator) throw new Error('No active replicator is available.');",
"const localDb=core.localDatabase.localDatabase;",
"const existing=await localDb.get(chunkId).catch(()=>undefined);",
"if(existing&&!existing._deleted) throw new Error(`The remote-only chunk already exists locally: ${chunkId}`);",
"const original=replicator.fetchRemoteChunks;",
"let releaseGate;",
"let resolveDone;",
"const gate=new Promise((resolve)=>{releaseGate=resolve;});",
"const donePromise=new Promise((resolve)=>{resolveDone=resolve;});",
`const state={kind:${JSON.stringify(REMOTE_ACTIVITY_GATE_KIND.chunkFetch)},entered:false,done:false,released:false,error:undefined,resultCount:undefined,requestedIds:undefined,promise:donePromise,release:undefined,restore:undefined};`,
"state.release=()=>{if(!state.released){state.released=true;releaseGate();}};",
"state.restore=()=>{replicator.fetchRemoteChunks=original;};",
"host[stateKey]=state;",
"replicator.fetchRemoteChunks=async function(...args){",
"state.entered=true;",
"state.requestedIds=Array.isArray(args[0])?[...args[0]]:[];",
"await gate;",
"try{",
"const result=await original.apply(this,args);",
"state.resultCount=Array.isArray(result)?result.length:0;",
"return result;",
"}catch(error){",
"state.error=error instanceof Error?error.message:String(error);",
"throw error;",
"}finally{",
"state.restore();",
"state.done=true;",
"resolveDone();",
"}",
"};",
"core.localDatabase.managers.chunkFetcher.onEvent([chunkId]);",
"return JSON.stringify({started:true});",
"})()",
].join(""),
env
);
}
export async function startHeldTrackedRequest(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
await evalObsidianJson<{ started: boolean }>(
cliBinary,
[
"(async()=>{",
`const stateKey=${stateKeySource};`,
"const host=globalThis;",
"if(host[stateKey]) throw new Error('A remote activity E2E gate is already installed.');",
"const core=app.plugins.plugins['obsidian-livesync'].core;",
"const remote=core.services.remote;",
"const api=core.services.API;",
"const settings=core.services.setting.currentSettings();",
"const original=api.webCompatFetch;",
"let releaseGate;",
"const gate=new Promise((resolve)=>{releaseGate=resolve;});",
`const state={kind:${JSON.stringify(REMOTE_ACTIVITY_GATE_KIND.trackedRequest)},entered:false,done:false,released:false,error:undefined,result:undefined,promise:undefined,release:undefined,restore:undefined};`,
"state.release=()=>{if(!state.released){state.released=true;releaseGate();}};",
"state.restore=()=>{api.webCompatFetch=original;};",
"host[stateKey]=state;",
"api.webCompatFetch=async function(...args){",
"state.entered=true;",
"await gate;",
"return await original.apply(this,args);",
"};",
"state.promise=(async()=>{",
"try{",
"const base=String(settings.couchDB_URI).replace(/\\/$/,'');",
"const database=encodeURIComponent(settings.couchDB_DBNAME);",
"const credentials=btoa(`${settings.couchDB_USER}:${settings.couchDB_PASSWORD}`);",
"const response=await remote.performFetch(`${base}/${database}/_all_docs?limit=0`,{headers:{Authorization:`Basic ${credentials}`}});",
"state.result=response.ok;",
"}catch(error){",
"state.error=error instanceof Error?error.message:String(error);",
"}finally{",
"state.restore();",
"state.done=true;",
"}",
"})();",
"return JSON.stringify({started:true});",
"})()",
].join(""),
env
);
}
export async function finishHeldRemoteActivity(
cliBinary: string,
env: NodeJS.ProcessEnv
): Promise<HeldRemoteActivityResult> {
return await evalObsidianJson<HeldRemoteActivityResult>(
cliBinary,
[
"(async()=>{",
`const stateKey=${stateKeySource};`,
"const state=globalThis[stateKey];",
"if(!state) throw new Error('No remote activity E2E gate is installed.');",
"state.release();",
"await state.promise;",
"return JSON.stringify({kind:state.kind,entered:state.entered,done:state.done,error:state.error,result:state.result,resultCount:state.resultCount,requestedIds:state.requestedIds});",
"})()",
].join(""),
env
);
}
export async function waitForRestoredChunk(
cliBinary: string,
env: NodeJS.ProcessEnv,
chunkId: string,
timeoutMs = Number(process.env.E2E_OBSIDIAN_REMOTE_ACTIVITY_TIMEOUT_MS ?? 30000)
): Promise<{ id: string; type: string }> {
return await evalObsidianJson<{ id: string; type: string }>(
cliBinary,
[
"(async()=>{",
`const chunkId=${JSON.stringify(chunkId)};`,
`const deadline=Date.now()+${JSON.stringify(timeoutMs)};`,
"const core=app.plugins.plugins['obsidian-livesync'].core;",
"const localDb=core.localDatabase.localDatabase;",
"const sleep=(ms)=>new Promise((resolve)=>setTimeout(resolve,ms));",
"while(Date.now()<deadline){",
"const chunk=await localDb.get(chunkId).catch(()=>undefined);",
"if(chunk&&!chunk._deleted&&chunk.type==='leaf'&&typeof chunk.data==='string') return JSON.stringify({id:chunk._id,type:chunk.type});",
"await sleep(100);",
"}",
"throw new Error(`Timed out waiting for the fetched chunk to return: ${chunkId}`);",
"})()",
].join(""),
env
);
}
export async function clearHeldRemoteActivity(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
await evalObsidianJson<{ cleared: boolean }>(
cliBinary,
[
"(async()=>{",
`const stateKey=${stateKeySource};`,
"const state=globalThis[stateKey];",
"if(!state) return JSON.stringify({cleared:false});",
"if(!state.done) throw new Error('The remote activity E2E gate is still running.');",
"delete globalThis[stateKey];",
"return JSON.stringify({cleared:true});",
"})()",
].join(""),
env
);
}
export async function cleanUpHeldRemoteActivity(cliBinary: string, env: NodeJS.ProcessEnv): Promise<void> {
await evalObsidianJson<{ cleared: boolean }>(
cliBinary,
[
"(async()=>{",
`const stateKey=${stateKeySource};`,
"const state=globalThis[stateKey];",
"if(!state) return JSON.stringify({cleared:false});",
"state.release?.();",
"await Promise.race([Promise.resolve(state.promise),new Promise((resolve)=>setTimeout(resolve,5000))]);",
"state.restore?.();",
"delete globalThis[stateKey];",
"return JSON.stringify({cleared:true});",
"})()",
].join(""),
env
);
}
@@ -0,0 +1,356 @@
import { spawn } from "node:child_process";
import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import {
assertCouchDbReachable,
createCouchDbDatabase,
deleteCouchDbDatabase,
loadCouchDbConfig,
makeUniqueDatabaseName,
waitForCouchDbDocs,
} from "../runner/couchdb.ts";
import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
import {
assertEqual,
configureCouchDb,
prepareRemote,
pushLocalChanges,
waitForLiveSyncCoreReady,
} from "../runner/liveSyncWorkflow.ts";
import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
import { createTemporaryVault } from "../runner/vault.ts";
process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000";
process.env.E2E_OBSIDIAN_COUCHDB_TIMEOUT_MS ??= "30000";
process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ??= "30000";
const liveSyncCli = resolve("src/apps/cli/dist/index.cjs");
const notePath = "E2E/cli-to-obsidian.md";
const noteContent = [
"# CLI to real Obsidian",
"",
"This note was created by the Self-hosted LiveSync CLI.",
"The real Obsidian plug-in must retrieve the same content from CouchDB.",
"0123456789 abcdefghijklmnopqrstuvwxyz 0123456789 abcdefghijklmnopqrstuvwxyz",
"",
].join("\n");
const e2eePassphrase = "real-obsidian-cli-compatibility-e2e";
type LiveSyncCliCommand = {
executable: string;
prefixArgs: string[];
};
type CliResult = {
stdout: string;
stderr: string;
};
type CliFileInfo = {
id: string;
children: string[];
};
function parseCommandLine(value: string): string[] {
const trimmed = value.trim();
if (trimmed.startsWith("[")) {
const parsed = JSON.parse(trimmed) as unknown;
if (!Array.isArray(parsed) || parsed.length === 0 || parsed.some((part) => typeof part !== "string")) {
throw new Error("LIVESYNC_CLI_COMMAND JSON form must be a non-empty array of strings.");
}
return parsed;
}
const parts: string[] = [];
let current = "";
let quote: "'" | '"' | undefined;
let tokenStarted = false;
for (let index = 0; index < trimmed.length; index++) {
const character = trimmed[index];
if (quote) {
if (character === quote) {
quote = undefined;
continue;
}
if (character === "\\" && quote === '"' && ['"', "\\"].includes(trimmed[index + 1] ?? "")) {
current += trimmed[++index];
continue;
}
current += character;
continue;
}
if (character === "'" || character === '"') {
quote = character;
tokenStarted = true;
continue;
}
if (character === "\\" && ["'", '"', "\\", " ", "\t"].includes(trimmed[index + 1] ?? "")) {
current += trimmed[++index];
tokenStarted = true;
continue;
}
if (/\s/u.test(character)) {
if (tokenStarted) {
parts.push(current);
current = "";
tokenStarted = false;
}
continue;
}
current += character;
tokenStarted = true;
}
if (quote) {
throw new Error("LIVESYNC_CLI_COMMAND contains an unterminated quoted value.");
}
if (tokenStarted) {
parts.push(current);
}
if (parts.length === 0) {
throw new Error("LIVESYNC_CLI_COMMAND must not be empty.");
}
return parts;
}
function resolveLiveSyncCliCommand(): LiveSyncCliCommand {
const override = process.env.LIVESYNC_CLI_COMMAND;
if (override !== undefined) {
const [executable, ...prefixArgs] = parseCommandLine(override);
return { executable, prefixArgs };
}
return { executable: process.execPath, prefixArgs: [liveSyncCli] };
}
async function runLiveSyncCli(command: LiveSyncCliCommand, args: string[]): Promise<CliResult> {
return await new Promise((resolvePromise, reject) => {
const timeoutMs = Number(process.env.E2E_LIVESYNC_CLI_TIMEOUT_MS ?? 60000);
const child = spawn(command.executable, [...command.prefixArgs, ...args], {
cwd: process.cwd(),
env: process.env,
stdio: ["ignore", "pipe", "pipe"],
});
let stdout = "";
let stderr = "";
child.stdout.on("data", (chunk: Buffer) => {
stdout += chunk.toString("utf-8");
});
child.stderr.on("data", (chunk: Buffer) => {
stderr += chunk.toString("utf-8");
});
let timedOut = false;
const timeout = setTimeout(() => {
timedOut = true;
child.kill("SIGTERM");
}, timeoutMs);
child.on("error", (error) => {
clearTimeout(timeout);
reject(error);
});
child.on("exit", (code, signal) => {
clearTimeout(timeout);
const result = {
stdout,
stderr,
};
if (timedOut) {
reject(
new Error(
`LiveSync CLI timed out after ${timeoutMs} ms\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`
)
);
return;
}
if (code === 0) {
resolvePromise(result);
return;
}
reject(
new Error(
`LiveSync CLI failed with ${signal ? `signal ${signal}` : `exit code ${String(code)}`}\n` +
`stdout:\n${result.stdout}\nstderr:\n${result.stderr}`
)
);
});
});
}
async function configureLiveSyncCli(
command: LiveSyncCliCommand,
settingsPath: string,
couchDb: Awaited<ReturnType<typeof loadCouchDbConfig>>,
dbName: string
): Promise<void> {
await runLiveSyncCli(command, ["init-settings", "--force", settingsPath]);
const settings = JSON.parse(await readFile(settingsPath, "utf-8")) as Record<string, unknown>;
Object.assign(settings, {
couchDB_URI: couchDb.uri,
couchDB_USER: couchDb.username,
couchDB_PASSWORD: couchDb.password,
couchDB_DBNAME: dbName,
remoteType: "",
liveSync: false,
syncOnStart: false,
syncOnSave: false,
usePluginSync: false,
usePluginSyncV2: true,
useEden: false,
customChunkSize: 60,
sendChunksBulk: false,
sendChunksBulkMaxSize: 1,
chunkSplitterVersion: "v3-rabin-karp",
readChunksOnline: true,
disableCheckingConfigMismatch: false,
enableCompression: false,
hashAlg: "xxhash64",
handleFilenameCaseSensitive: false,
doNotUseFixedRevisionForChunks: true,
E2EEAlgorithm: "v2",
encrypt: true,
passphrase: e2eePassphrase,
usePathObfuscation: true,
doctorProcessedVersion: "0.25.27",
isConfigured: true,
});
await writeFile(settingsPath, `${JSON.stringify(settings, null, 2)}\n`, "utf-8");
}
async function writeCliNote(
command: LiveSyncCliCommand,
databasePath: string,
settingsPath: string,
sourcePath: string
): Promise<CliFileInfo> {
await mkdir(dirname(sourcePath), { recursive: true });
await writeFile(sourcePath, noteContent, "utf-8");
await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "push", sourcePath, notePath]);
const info = await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "info", notePath]);
const fileInfo = JSON.parse(info.stdout) as CliFileInfo;
if (!fileInfo.id || !Array.isArray(fileInfo.children) || fileInfo.children.length === 0) {
throw new Error(`LiveSync CLI did not create complete metadata for ${notePath}: ${info.stdout}`);
}
await runLiveSyncCli(command, [databasePath, "--settings", settingsPath, "sync"]);
return fileInfo;
}
async function waitForVaultContent(
vaultPath: string,
path: string,
timeoutMs = Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS)
): Promise<string> {
const fullPath = join(vaultPath, path);
const deadline = Date.now() + timeoutMs;
let lastContent = "";
while (Date.now() < deadline) {
try {
lastContent = await readFile(fullPath, "utf-8");
if (lastContent === noteContent) {
return lastContent;
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
throw error;
}
}
await new Promise((resolvePromise) => setTimeout(resolvePromise, 250));
}
throw new Error(`Timed out waiting for CLI-created note at ${fullPath}. Last content:\n${lastContent}`);
}
async function main(): Promise<void> {
const liveSyncCliCommand = resolveLiveSyncCliCommand();
if (process.env.LIVESYNC_CLI_COMMAND === undefined) {
await access(liveSyncCli).catch(() => {
throw new Error(
`Built LiveSync CLI was not found at ${liveSyncCli}. Run 'npm run build -w self-hosted-livesync-cli' first, or set LIVESYNC_CLI_COMMAND.`
);
});
}
const binary = requireObsidianBinary();
const obsidianCli = discoverObsidianCli();
if (!obsidianCli.binary) {
throw new Error(`Could not find obsidian-cli. Checked paths: ${obsidianCli.checked.join(", ")}`);
}
const couchDb = await loadCouchDbConfig();
const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "cli-to-obsidian");
const cliState = await mkdtemp(join(tmpdir(), "livesync-cli-to-obsidian-e2e-"));
const cliDatabasePath = join(cliState, "database");
const cliSettingsPath = join(cliState, "settings.json");
const cliSourcePath = join(cliState, "source", "cli-to-obsidian.md");
const vault = await createTemporaryVault();
let session: ObsidianLiveSyncSession | undefined;
try {
await assertCouchDbReachable(couchDb);
await createCouchDbDatabase(couchDb, dbName);
await mkdir(cliDatabasePath, { recursive: true });
await configureLiveSyncCli(liveSyncCliCommand, cliSettingsPath, couchDb, dbName);
if (process.env.LIVESYNC_CLI_COMMAND === undefined) {
console.log(`Using locally built LiveSync CLI: ${liveSyncCli}`);
} else {
console.log(
`Using LiveSync CLI command override: ${JSON.stringify(liveSyncCliCommand.executable)} ` +
`with ${liveSyncCliCommand.prefixArgs.length} prefix argument(s)`
);
}
console.log(`Using Obsidian executable: ${binary}`);
console.log(`Temporary Obsidian vault: ${vault.path}`);
console.log(`Temporary CouchDB database: ${dbName}`);
const cliFileInfo = await writeCliNote(liveSyncCliCommand, cliDatabasePath, cliSettingsPath, cliSourcePath);
await waitForCouchDbDocs(couchDb, dbName, (docs) => {
const ids = new Set(docs.map((doc) => doc._id));
return ids.has(cliFileInfo.id) && cliFileInfo.children.every((childId) => ids.has(childId));
});
session = await startObsidianLiveSyncSession({
binary,
cliBinary: obsidianCli.binary,
vault,
startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000),
});
await waitForLiveSyncCoreReady(obsidianCli.binary, session.cliEnv);
await configureCouchDb(
obsidianCli.binary,
session.cliEnv,
{
uri: couchDb.uri,
username: couchDb.username,
password: couchDb.password,
dbName,
},
{
encrypt: true,
passphrase: e2eePassphrase,
usePathObfuscation: true,
E2EEAlgorithm: "v2",
}
);
await waitForLiveSyncCoreReady(obsidianCli.binary, session.cliEnv);
await prepareRemote(obsidianCli.binary, session.cliEnv);
await pushLocalChanges(obsidianCli.binary, session.cliEnv);
const received = await waitForVaultContent(vault.path, notePath);
assertEqual(received, noteContent, "The real Obsidian plug-in did not materialise the CLI-created note.");
console.log("CLI-created encrypted note was retrieved by the real Obsidian plug-in with identical content.");
} finally {
if (session) {
await session.app.stop();
}
await vault.dispose();
await rm(cliState, { recursive: true, force: true });
if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") {
await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => {
console.warn(error instanceof Error ? error.message : error);
});
}
}
}
main().catch((error: unknown) => {
console.error(error instanceof Error ? error.stack : error);
process.exit(1);
});
+128 -2
View File
@@ -5,6 +5,7 @@ import {
deleteCouchDbDatabase, deleteCouchDbDatabase,
loadCouchDbConfig, loadCouchDbConfig,
makeUniqueDatabaseName, makeUniqueDatabaseName,
putCouchDbDocument,
waitForCouchDbDocs, waitForCouchDbDocs,
} from "../runner/couchdb.ts"; } from "../runner/couchdb.ts";
import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts";
@@ -12,10 +13,23 @@ import {
assertEqual, assertEqual,
configureCouchDb, configureCouchDb,
prepareRemote, prepareRemote,
pushLocalChanges,
waitForLiveSyncCoreReady, waitForLiveSyncCoreReady,
type LocalDatabaseEntry, type LocalDatabaseEntry,
} from "../runner/liveSyncWorkflow.ts"; } from "../runner/liveSyncWorkflow.ts";
import {
REMOTE_ACTIVITY_EXPECTED_STATE,
captureRemoteActivityDiagnostics,
waitForRemoteActivityState,
} from "../runner/remoteActivity.ts";
import {
cleanUpHeldRemoteActivity,
clearHeldRemoteActivity,
finishHeldRemoteActivity,
startHeldChunkFetch,
startHeldOneShotReplication,
startHeldTrackedRequest,
waitForRestoredChunk,
} from "../runner/remoteActivityWorkflow.ts";
import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
import { createTemporaryVault } from "../runner/vault.ts"; import { createTemporaryVault } from "../runner/vault.ts";
@@ -72,6 +86,7 @@ async function main(): Promise<void> {
const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "obsidian-upload"); const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "obsidian-upload");
const vault = await createTemporaryVault(); const vault = await createTemporaryVault();
let session: ObsidianLiveSyncSession | undefined; let session: ObsidianLiveSyncSession | undefined;
let activityStage = "session-startup";
try { try {
await assertCouchDbReachable(couchDb); await assertCouchDbReachable(couchDb);
@@ -104,8 +119,50 @@ async function main(): Promise<void> {
assertEqual(configured.syncOnSave, false, "Sync on save should remain disabled during this workflow."); assertEqual(configured.syncOnSave, false, "Sync on save should remain disabled during this workflow.");
await prepareRemote(cli.binary, session.cliEnv); await prepareRemote(cli.binary, session.cliEnv);
activityStage = "initial-idle";
const initialIdle = await waitForRemoteActivityState(
session.remoteDebuggingPort,
REMOTE_ACTIVITY_EXPECTED_STATE.idle
);
const localEntry = await createNoteAndWaitForLocalDb(cli.binary, session.cliEnv); const localEntry = await createNoteAndWaitForLocalDb(cli.binary, session.cliEnv);
await pushLocalChanges(cli.binary, session.cliEnv);
activityStage = REMOTE_ACTIVITY_EXPECTED_STATE.trackedRequestActive;
await startHeldTrackedRequest(cli.binary, session.cliEnv);
const trackedRequestActive = await waitForRemoteActivityState(
session.remoteDebuggingPort,
REMOTE_ACTIVITY_EXPECTED_STATE.trackedRequestActive
);
const trackedRequestResult = await finishHeldRemoteActivity(cli.binary, session.cliEnv);
assertEqual(trackedRequestResult.error, undefined, "The observed CouchDB request failed.");
assertEqual(trackedRequestResult.result, true, "The observed CouchDB request did not report success.");
activityStage = "tracked-request-idle";
const trackedRequestIdle = await waitForRemoteActivityState(
session.remoteDebuggingPort,
REMOTE_ACTIVITY_EXPECTED_STATE.idle
);
if (trackedRequestIdle.requestCount <= initialIdle.requestCount) {
throw new Error("The held CouchDB request did not advance the tracked remote-request count.");
}
await clearHeldRemoteActivity(cli.binary, session.cliEnv);
activityStage = "one-shot-active";
await startHeldOneShotReplication(cli.binary, session.cliEnv);
const oneShotActive = await waitForRemoteActivityState(
session.remoteDebuggingPort,
REMOTE_ACTIVITY_EXPECTED_STATE.finiteReplicationActive
);
const oneShotResult = await finishHeldRemoteActivity(cli.binary, session.cliEnv);
assertEqual(oneShotResult.error, undefined, "One-shot replication failed while its activity was observed.");
assertEqual(oneShotResult.result, true, "One-shot replication did not report success.");
activityStage = "one-shot-idle";
const oneShotIdle = await waitForRemoteActivityState(
session.remoteDebuggingPort,
REMOTE_ACTIVITY_EXPECTED_STATE.idle
);
if (oneShotIdle.requestCount <= trackedRequestIdle.requestCount) {
throw new Error("One-shot replication did not make an observed remote request.");
}
await clearHeldRemoteActivity(cli.binary, session.cliEnv);
const remoteDocs = await waitForCouchDbDocs(couchDb, dbName, (docs) => { const remoteDocs = await waitForCouchDbDocs(couchDb, dbName, (docs) => {
const ids = new Set(docs.map((doc) => doc._id)); const ids = new Set(docs.map((doc) => doc._id));
@@ -118,11 +175,80 @@ async function main(): Promise<void> {
"Remote metadata path did not match the local database entry." "Remote metadata path did not match the local database entry."
); );
const sourceChunkId = localEntry.children[0];
if (!sourceChunkId) throw new Error("The uploaded note did not produce a chunk for the fetch workflow.");
const sourceChunk = remoteDocs.find((document) => document._id === sourceChunkId);
if (!sourceChunk || sourceChunk.type !== "leaf") {
throw new Error(`The uploaded source chunk was not found in CouchDB: ${sourceChunkId}`);
}
const { _rev: _sourceRevision, ...remoteOnlyChunk } = sourceChunk;
const chunkId = `h:e2e-remote-activity-${Date.now().toString(36)}`;
await putCouchDbDocument(couchDb, dbName, { ...remoteOnlyChunk, _id: chunkId });
activityStage = REMOTE_ACTIVITY_EXPECTED_STATE.chunkFetchActive;
await startHeldChunkFetch(cli.binary, session.cliEnv, chunkId);
const chunkFetchActive = await waitForRemoteActivityState(
session.remoteDebuggingPort,
REMOTE_ACTIVITY_EXPECTED_STATE.chunkFetchActive
);
const chunkFetchResult = await finishHeldRemoteActivity(cli.binary, session.cliEnv);
assertEqual(
chunkFetchResult.error,
undefined,
"On-demand chunk fetching failed while its activity was observed."
);
if (!chunkFetchResult.requestedIds?.includes(chunkId)) {
throw new Error(`The on-demand chunk request did not include the selected chunk: ${chunkId}`);
}
if ((chunkFetchResult.resultCount ?? 0) < 1) {
throw new Error(`The remote did not return the selected chunk: ${chunkId}`);
}
const restoredChunk = await waitForRestoredChunk(cli.binary, session.cliEnv, chunkId);
assertEqual(restoredChunk.id, chunkId, "The restored chunk ID did not match the requested chunk.");
activityStage = "chunk-fetch-idle";
const chunkFetchIdle = await waitForRemoteActivityState(
session.remoteDebuggingPort,
REMOTE_ACTIVITY_EXPECTED_STATE.idle
);
if (chunkFetchIdle.requestCount <= oneShotIdle.requestCount) {
throw new Error("On-demand chunk fetching did not make an observed remote request.");
}
await clearHeldRemoteActivity(cli.binary, session.cliEnv);
console.log( console.log(
`Uploaded metadata ${localEntry.id} and ${localEntry.children.length} chunk(s) to CouchDB database ${dbName}` `Uploaded metadata ${localEntry.id} and ${localEntry.children.length} chunk(s) to CouchDB database ${dbName}`
); );
console.log(
[
`Tracked request: ${trackedRequestActive.statusBarText.trim()} -> idle`,
`One-shot activity: ${oneShotActive.statusBarText.trim()} -> idle`,
`Chunk-fetch activity: ${chunkFetchActive.statusBarText.trim()} -> idle`,
`Balanced remote requests: ${chunkFetchIdle.requestCount}/${chunkFetchIdle.responseCount}`,
].join("\n")
);
} catch (error) {
if (session) {
const diagnostics = await captureRemoteActivityDiagnostics(
session.remoteDebuggingPort,
`couchdb-upload-${activityStage}`
).catch((diagnosticError: unknown) => {
console.warn(
`Could not capture remote activity diagnostics: ${
diagnosticError instanceof Error ? diagnosticError.message : String(diagnosticError)
}`
);
return undefined;
});
if (diagnostics) {
console.error(`Remote activity screenshot: ${diagnostics.screenshotPath}`);
console.error(`Remote activity snapshot: ${diagnostics.snapshotPath}`);
}
}
throw error;
} finally { } finally {
if (session) { if (session) {
await cleanUpHeldRemoteActivity(cli.binary, session.cliEnv).catch((error: unknown) => {
console.warn(error instanceof Error ? error.message : error);
});
await session.app.stop(); await session.app.stop();
} }
await vault.dispose(); await vault.dispose();
+7
View File
@@ -8,10 +8,17 @@ type Step = {
const testSteps: Step[] = [ const testSteps: Step[] = [
{ name: "build", args: ["run", "build"] }, { name: "build", args: ["run", "build"] },
...(process.env.LIVESYNC_CLI_COMMAND === undefined
? [{ name: "CLI build", args: ["run", "build", "-w", "self-hosted-livesync-cli"] }]
: []),
{ name: "discover", args: ["run", "test:e2e:obsidian:discover"] }, { name: "discover", args: ["run", "test:e2e:obsidian:discover"] },
{ name: "smoke", args: ["run", "test:e2e:obsidian:smoke"] }, { name: "smoke", args: ["run", "test:e2e:obsidian:smoke"] },
{ name: "vault reflection", args: ["run", "test:e2e:obsidian:vault-reflection"] }, { name: "vault reflection", args: ["run", "test:e2e:obsidian:vault-reflection"] },
{ name: "CouchDB upload", args: ["run", "test:e2e:obsidian:couchdb-upload"] }, { name: "CouchDB upload", args: ["run", "test:e2e:obsidian:couchdb-upload"] },
{
name: "CLI to real Obsidian synchronisation",
args: ["run", "test:e2e:obsidian:cli-to-obsidian-sync"],
},
{ name: "Object Storage upload", args: ["run", "test:e2e:obsidian:minio-upload"] }, { name: "Object Storage upload", args: ["run", "test:e2e:obsidian:minio-upload"] },
{ name: "startup scan", args: ["run", "test:e2e:obsidian:startup-scan"] }, { name: "startup scan", args: ["run", "test:e2e:obsidian:startup-scan"] },
{ name: "two-vault synchronisation", args: ["run", "test:e2e:obsidian:two-vault-sync"] }, { name: "two-vault synchronisation", args: ["run", "test:e2e:obsidian:two-vault-sync"] },
+18 -1
View File
@@ -17,6 +17,7 @@ import {
} from "../runner/objectStorage.ts"; } from "../runner/objectStorage.ts";
import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts";
import { createTemporaryVault } from "../runner/vault.ts"; import { createTemporaryVault } from "../runner/vault.ts";
import { REMOTE_ACTIVITY_EXPECTED_STATE, waitForRemoteActivityState } from "../runner/remoteActivity.ts";
process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000"; process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "30000";
@@ -115,13 +116,29 @@ async function main(): Promise<void> {
assertEqual(configured.liveSync, false, "LiveSync should remain disabled during this one-shot workflow."); assertEqual(configured.liveSync, false, "LiveSync should remain disabled during this one-shot workflow.");
await prepareRemote(cli.binary, session.cliEnv); await prepareRemote(cli.binary, session.cliEnv);
const activityBeforeUpload = await waitForRemoteActivityState(
session.remoteDebuggingPort,
REMOTE_ACTIVITY_EXPECTED_STATE.idle
);
const localEntry = await createNoteAndWaitForLocalDb(cli.binary, session.cliEnv); const localEntry = await createNoteAndWaitForLocalDb(cli.binary, session.cliEnv);
await pushLocalChanges(cli.binary, session.cliEnv); await pushLocalChanges(cli.binary, session.cliEnv);
const activityAfterUpload = await waitForRemoteActivityState(
session.remoteDebuggingPort,
REMOTE_ACTIVITY_EXPECTED_STATE.idle
);
if (activityAfterUpload.requestCount <= activityBeforeUpload.requestCount) {
throw new Error("Object Storage synchronisation did not advance the tracked remote-request count.");
}
assertEqual(
activityAfterUpload.responseCount,
activityAfterUpload.requestCount,
"Object Storage remote-request counters did not rebalance after synchronisation."
);
const keys = await waitForObjectStorageObjects(bucketPrefix); const keys = await waitForObjectStorageObjects(bucketPrefix);
console.log( console.log(
`Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s))` `Uploaded ${localEntry.path} through Journal Sync to ${objectStorage.bucket}/${bucketPrefix} (${keys.length} object(s)); tracked requests: ${activityAfterUpload.requestCount - activityBeforeUpload.requestCount}`
); );
} finally { } finally {
if (session) { if (session) {
+14
View File
@@ -105,6 +105,13 @@ for (const dir of workspaceDirs) {
console.log(` Dependency '${dep}': ${oldVer} -> ${newVer}`); console.log(` Dependency '${dep}': ${oldVer} -> ${newVer}`);
pkg.dependencies[dep] = newVer; pkg.dependencies[dep] = newVer;
} }
if (lockWorkspace) {
lockWorkspace.dependencies ??= {};
if (lockWorkspace.dependencies[dep] !== newVer) {
lockWorkspace.dependencies[dep] = newVer;
packageLockChanged = true;
}
}
} }
} }
} }
@@ -119,6 +126,13 @@ for (const dir of workspaceDirs) {
console.log(` DevDependency '${dep}': ${oldVer} -> ${newVer}`); console.log(` DevDependency '${dep}': ${oldVer} -> ${newVer}`);
pkg.devDependencies[dep] = newVer; pkg.devDependencies[dep] = newVer;
} }
if (lockWorkspace) {
lockWorkspace.devDependencies ??= {};
if (lockWorkspace.devDependencies[dep] !== newVer) {
lockWorkspace.devDependencies[dep] = newVer;
packageLockChanged = true;
}
}
} }
} }
} }
+24 -1
View File
@@ -7,7 +7,22 @@ The head note of 0.25 is now in [updates_old.md](https://github.com/vrtmrz/obsid
### Fixed ### Fixed
- Refresh the remote Security Seed before each replication, preventing a client that remained open during a remote database rebuild from uploading data encrypted with the previous seed (#1018). - Fixed the 📲 remote-activity indicator remaining visible after CouchDB requests had completed.
- Fixed missing chunks being reported unavailable while an in-progress on-demand fetch or finite replication could still deliver them. Reads now follow the actual delivery lifecycle and recheck the local database when it finishes.
### Improved
- Split the status-bar remote activity display into `📲` for a finite remote operation and `🌐N` for the approximate number of tracked CouchDB or Object Storage requests currently in progress.
## 0.25.82
15th July, 2026
Recently, I created a repository called Fancy Kit and have been trying to build some proper infrastructure around it. Does any of this look like cumbersome bureaucracy? No need to worry: you can carry on as usual. Codex is simply tidying up my usual rambling prose, terrible pull requests, and disjointed remarks.
### Fixed
- Refreshed the remote Security Seed before each replication, preventing a client that remained open during a remote database rebuild from uploading data encrypted with the previous seed (#1018).
- The P2P **Start Sync & Close** action now waits for synchronisation to settle before closing the dialogue, avoiding premature release of screen-awake protection while work remains in flight. - The P2P **Start Sync & Close** action now waits for synchronisation to settle before closing the dialogue, avoiding premature release of screen-awake protection while work remains in flight.
### Improved ### Improved
@@ -15,6 +30,14 @@ The head note of 0.25 is now in [updates_old.md](https://github.com/vrtmrz/obsid
- On supported mobile and desktop devices, one-shot replication, P2P peer discovery and selection, database rebuild and fetch operations, and remote chunk fetching now keep the screen awake for the duration of these finite remote operations. LiveSync also postpones its normal visibility suspension until the operation finishes, although platform restrictions may still pause or terminate background work. - On supported mobile and desktop devices, one-shot replication, P2P peer discovery and selection, database rebuild and fetch operations, and remote chunk fetching now keep the screen awake for the duration of these finite remote operations. LiveSync also postpones its normal visibility suspension until the operation finishes, although platform restrictions may still pause or terminate background work.
- The 📲 status-bar indicator now covers finite remote work as well as HTTP requests. It reports broad remote activity, such as P2P peer selection and remote chunk fetching, rather than an exact physical connection count. - The 📲 status-bar indicator now covers finite remote work as well as HTTP requests. It reports broad remote activity, such as P2P peer selection and remote chunk fetching, rather than an exact physical connection count.
### Improved (CLI)
- CLI container images are now published for both AMD64 and ARM64 platforms.
### Testing
- Added a local real-Obsidian compatibility test that writes an encrypted, path-obfuscated note through the LiveSync CLI and verifies that the plug-in materialises the same content.
## 0.25.81 ## 0.25.81
14th July, 2026 14th July, 2026
+41 -5
View File
@@ -113,6 +113,15 @@ describe("release workflow", () => {
expect(workflow).toMatch(/git add[^\n]*_types/); expect(workflow).toMatch(/git add[^\n]*_types/);
}); });
it("installs Deno before post-processing fallback type definitions", () => {
const workflow = readFileSync(prepareReleaseWorkflow, "utf8");
const setupDeno = workflow.indexOf("denoland/setup-deno@v2");
const buildTypes = workflow.indexOf("npm run build:lib:types");
expect(setupDeno).toBeGreaterThan(-1);
expect(setupDeno).toBeLessThan(buildTypes);
});
it("keeps the release PR in draft until BRAT validation", () => { it("keeps the release PR in draft until BRAT validation", () => {
const workflow = readFileSync(prepareReleaseWorkflow, "utf8"); const workflow = readFileSync(prepareReleaseWorkflow, "utf8");
@@ -158,9 +167,18 @@ describe("workspace version update", () => {
it("keeps workspace package and lockfile versions together", () => { it("keeps workspace package and lockfile versions together", () => {
const directory = makeTemporaryDirectory(); const directory = makeTemporaryDirectory();
const workspaces = ["src/apps/cli", "src/apps/webpeer", "src/apps/webapp"]; const workspaces = ["src/apps/cli", "src/apps/webpeer", "src/apps/webapp"];
writeJson(directory, "package.json", { version: "0.25.81", workspaces }); writeJson(directory, "package.json", {
version: "0.25.81",
workspaces,
dependencies: { "octagonal-wheels": "^0.1.51" },
devDependencies: { typescript: "^5.9.3" },
});
for (const workspace of ["cli", "webpeer", "webapp"]) { for (const workspace of ["cli", "webpeer", "webapp"]) {
writeJson(directory, `src/apps/${workspace}/package.json`, { version: `0.25.80-${workspace}` }); writeJson(directory, `src/apps/${workspace}/package.json`, {
version: `0.25.80-${workspace}`,
dependencies: { "octagonal-wheels": "^0.1.50" },
devDependencies: { typescript: "^5.8.0" },
});
} }
writeJson(directory, "package-lock.json", { writeJson(directory, "package-lock.json", {
name: "obsidian-livesync", name: "obsidian-livesync",
@@ -168,9 +186,21 @@ describe("workspace version update", () => {
lockfileVersion: 3, lockfileVersion: 3,
packages: { packages: {
"": { version: "0.25.80", workspaces }, "": { version: "0.25.80", workspaces },
"src/apps/cli": { version: "0.25.80-cli" }, "src/apps/cli": {
"src/apps/webpeer": { version: "0.25.80-webpeer" }, version: "0.25.80-cli",
"src/apps/webapp": { version: "0.25.80-webapp" }, dependencies: { "octagonal-wheels": "^0.1.50" },
devDependencies: { typescript: "^5.8.0" },
},
"src/apps/webpeer": {
version: "0.25.80-webpeer",
dependencies: { "octagonal-wheels": "^0.1.50" },
devDependencies: { typescript: "^5.8.0" },
},
"src/apps/webapp": {
version: "0.25.80-webapp",
dependencies: { "octagonal-wheels": "^0.1.50" },
devDependencies: { typescript: "^5.8.0" },
},
}, },
}); });
@@ -180,6 +210,8 @@ describe("workspace version update", () => {
for (const workspace of ["cli", "webpeer", "webapp"]) { for (const workspace of ["cli", "webpeer", "webapp"]) {
const packageJson = JSON.parse(readFileSync(join(directory, `src/apps/${workspace}/package.json`), "utf8")); const packageJson = JSON.parse(readFileSync(join(directory, `src/apps/${workspace}/package.json`), "utf8"));
expect(packageJson.version).toBe(`0.25.81-${workspace}`); expect(packageJson.version).toBe(`0.25.81-${workspace}`);
expect(packageJson.dependencies["octagonal-wheels"]).toBe("^0.1.51");
expect(packageJson.devDependencies.typescript).toBe("^5.9.3");
} }
const packageLock = JSON.parse(readFileSync(join(directory, "package-lock.json"), "utf8")); const packageLock = JSON.parse(readFileSync(join(directory, "package-lock.json"), "utf8"));
expect(packageLock.version).toBe("0.25.81"); expect(packageLock.version).toBe("0.25.81");
@@ -187,5 +219,9 @@ describe("workspace version update", () => {
expect(packageLock.packages["src/apps/cli"].version).toBe("0.25.81-cli"); expect(packageLock.packages["src/apps/cli"].version).toBe("0.25.81-cli");
expect(packageLock.packages["src/apps/webpeer"].version).toBe("0.25.81-webpeer"); expect(packageLock.packages["src/apps/webpeer"].version).toBe("0.25.81-webpeer");
expect(packageLock.packages["src/apps/webapp"].version).toBe("0.25.81-webapp"); expect(packageLock.packages["src/apps/webapp"].version).toBe("0.25.81-webapp");
for (const workspace of workspaces) {
expect(packageLock.packages[workspace].dependencies["octagonal-wheels"]).toBe("^0.1.51");
expect(packageLock.packages[workspace].devDependencies.typescript).toBe("^5.9.3");
}
}); });
}); });
+2 -1
View File
@@ -3,5 +3,6 @@
"0.25.60": "1.7.2", "0.25.60": "1.7.2",
"1.0.1": "0.9.12", "1.0.1": "0.9.12",
"1.0.0": "0.9.7", "1.0.0": "0.9.7",
"0.25.81": "1.7.2" "0.25.81": "1.7.2",
"0.25.82": "1.7.2"
} }