diff --git a/.dockerignore b/.dockerignore index a3ee67b0..d6d47438 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,7 +19,7 @@ production/ coverage/ _testdata/ test/bench-network/bench-results/ -src/apps/cli/testdeno/bench-results/ +test/e2e-cli/bench-results/ # Local environment / secrets .env diff --git a/.github/workflows/cli-deno-tests.yml b/.github/workflows/cli-node-tests.yml similarity index 65% rename from .github/workflows/cli-deno-tests.yml rename to .github/workflows/cli-node-tests.yml index 66f13e77..56f47f86 100644 --- a/.github/workflows/cli-deno-tests.yml +++ b/.github/workflows/cli-node-tests.yml @@ -1,4 +1,4 @@ -name: cli-deno-tests +name: cli-node-tests on: push: @@ -6,9 +6,11 @@ on: - main - beta paths: - - '.github/workflows/cli-deno-tests.yml' + - '.github/workflows/cli-node-tests.yml' - 'src/apps/cli/**' - '!src/apps/cli/**/*.md' + - 'test/e2e-cli/**' + - '!test/e2e-cli/**/*.md' - 'src/LiveSyncBaseCore.ts' - 'src/common/**' - 'src/serviceFeatures/**' @@ -24,9 +26,11 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review] paths: - - '.github/workflows/cli-deno-tests.yml' + - '.github/workflows/cli-node-tests.yml' - 'src/apps/cli/**' - '!src/apps/cli/**/*.md' + - 'test/e2e-cli/**' + - '!test/e2e-cli/**/*.md' - 'src/LiveSyncBaseCore.ts' - 'src/common/**' - 'src/serviceFeatures/**' @@ -42,7 +46,7 @@ on: workflow_dispatch: inputs: test_task: - description: 'Deno test task to run' + description: 'Node.js test task to run' type: choice options: - test:ci @@ -67,7 +71,7 @@ concurrency: jobs: prepare: - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && (github.event_name != 'workflow_dispatch' || inputs.test_task != 'test:local') }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -89,10 +93,10 @@ jobs: case "$SELECTED_TASK" in test:ci) - TASK_MATRIX=$(jq -c . src/apps/cli/testdeno/ci-tasks.json) + TASK_MATRIX=$(jq -c . test/e2e-cli/ci-tasks.json) ;; test:local) - TASK_MATRIX='["test:setup-put-cat","test:mirror","test:daemon"]' + TASK_MATRIX='["test:local"]' ;; test:e2e-matrix) TASK_MATRIX='["test:e2e-matrix:couchdb-enc0","test:e2e-matrix:couchdb-enc1","test:e2e-matrix:minio-enc0","test:e2e-matrix:minio-enc1"]' @@ -108,7 +112,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v5 with: - node-version: '24.x' + node-version: '26.x' cache: 'npm' - name: Install dependencies @@ -127,12 +131,12 @@ jobs: retention-days: 1 test: - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && (github.event_name != 'workflow_dispatch' || inputs.test_task != 'test:local') }} needs: prepare runs-on: ubuntu-latest timeout-minutes: 60 env: - DENO_DIR: ~/.cache/deno + NODE_OPTIONS: --max-old-space-size=3072 strategy: fail-fast: false matrix: @@ -144,23 +148,10 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v5 with: - node-version: '24.x' + node-version: '26.x' cache: 'npm' cache-dependency-path: package-lock.json - - name: Setup Deno - uses: denoland/setup-deno@v2 - with: - deno-version: v2.x - - - name: Cache Deno dependencies - uses: actions/cache@v5 - with: - path: ~/.cache/deno - key: ${{ runner.os }}-deno-${{ hashFiles('src/apps/cli/testdeno/deno.lock', 'src/apps/cli/testdeno/deno.json') }} - restore-keys: | - ${{ runner.os }}-deno- - - name: Install dependencies run: npm ci @@ -184,8 +175,8 @@ jobs: bucketName=livesync-test-bucket-ci EOF - - name: Run Deno tests - working-directory: src/apps/cli/testdeno + - name: Run Node.js tests + working-directory: test/e2e-cli env: LIVESYNC_DOCKER_MODE: native LIVESYNC_CLI_RETRY: 3 @@ -194,11 +185,61 @@ jobs: LIVESYNC_USE_COTURN: ${{ inputs.use_coturn == true && '1' || '0' }} run: | TASK="${{ matrix.task }}" - echo "[INFO] Running Deno task: $TASK" - deno task "$TASK" + echo "[INFO] Running Node.js task: $TASK" + node run-task.ts "$TASK" - name: Stop leftover containers if: always() run: | docker stop couchdb-test minio-test relay-test coturn-test >/dev/null 2>&1 || true docker rm couchdb-test minio-test relay-test coturn-test >/dev/null 2>&1 || true + + local-smoke: + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && (github.event_name != 'workflow_dispatch' || inputs.test_task == 'test:local') }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + env: + NODE_OPTIONS: --max-old-space-size=3072 + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: '26.x' + cache: 'npm' + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build CLI + if: ${{ matrix.os != 'ubuntu-latest' }} + env: + LIVESYNC_CLI_TEST_SUPPORT: '1' + run: npm run build --workspace self-hosted-livesync-cli + + - name: Run local smoke tests + if: ${{ matrix.os != 'ubuntu-latest' }} + working-directory: src/apps/cli + run: node ../../../test/e2e-cli/run-task.ts test:local + + - name: Run local smoke tests with coverage + if: ${{ matrix.os == 'ubuntu-latest' }} + run: npm run test:e2e:coverage --workspace self-hosted-livesync-cli + + - name: Upload local smoke coverage reports + if: ${{ always() && matrix.os == 'ubuntu-latest' }} + uses: actions/upload-artifact@v6 + with: + name: cli-e2e-local-coverage + path: | + coverage/cli-e2e/** + !coverage/cli-e2e/raw/** + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/cli-p2p-tests.yml b/.github/workflows/cli-p2p-tests.yml index 478afad4..5db10f35 100644 --- a/.github/workflows/cli-p2p-tests.yml +++ b/.github/workflows/cli-p2p-tests.yml @@ -9,6 +9,8 @@ on: - '.github/workflows/cli-p2p-tests.yml' - 'src/apps/cli/**' - '!src/apps/cli/**/*.md' + - 'test/e2e-cli/**' + - '!test/e2e-cli/**/*.md' - 'src/LiveSyncBaseCore.ts' - 'src/common/**' - 'src/serviceFeatures/**' @@ -29,6 +31,8 @@ on: - '.github/workflows/cli-p2p-tests.yml' - 'src/apps/cli/**' - '!src/apps/cli/**/*.md' + - 'test/e2e-cli/**' + - '!test/e2e-cli/**/*.md' - 'src/LiveSyncBaseCore.ts' - 'src/common/**' - 'src/serviceFeatures/**' diff --git a/.github/workflows/unit-ci.yml b/.github/workflows/unit-ci.yml index 80147fd4..7afc8320 100644 --- a/.github/workflows/unit-ci.yml +++ b/.github/workflows/unit-ci.yml @@ -39,7 +39,7 @@ on: - '.github/workflows/prepare-release.yml' - '.github/workflows/finalise-release.yml' - '.github/workflows/release.yml' - - '.github/workflows/cli-deno-tests.yml' + - '.github/workflows/cli-node-tests.yml' - '.github/workflows/cli-p2p-tests.yml' - '.github/workflows/cli-docker-validation.yml' - '.github/workflows/cli-docker.yml' @@ -78,7 +78,7 @@ on: - '.github/workflows/prepare-release.yml' - '.github/workflows/finalise-release.yml' - '.github/workflows/release.yml' - - '.github/workflows/cli-deno-tests.yml' + - '.github/workflows/cli-node-tests.yml' - '.github/workflows/cli-p2p-tests.yml' - '.github/workflows/cli-docker-validation.yml' - '.github/workflows/cli-docker.yml' diff --git a/AGENTS.md b/AGENTS.md index 93172ba0..a84f2832 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ Always adhere to the following stylistic and spelling rules: - Treat `@vrtmrz/livesync-commonlib` as an external, authoritative package. Make Commonlib changes in its repository, validate the packed artefact and downstream LiveSync consumer, and update the exact dependency here. Do not recreate a `src/lib` source mirror or generated `_types` fallback. 4. **Application Directories**: - The directory [src/apps](src/apps) contains independent application modules: - - `cli`: A Command Line Interface application. Tests specifically for the CLI (both unit and End-to-End tests) are located and executed within [src/apps/cli](src/apps/cli) using its local `package.json` scripts. + - `cli`: A Command Line Interface application. CLI unit tests are next to their source under [src/apps/cli](src/apps/cli), while CLI End-to-End tests are in [test/e2e-cli](test/e2e-cli) and run through the local [CLI package scripts](src/apps/cli/package.json). - `webapp`: A Web-based application. - `webpeer`: A Web-based peer utility. diff --git a/devs.md b/devs.md index 5fdbbb2b..d936734c 100644 --- a/devs.md +++ b/devs.md @@ -49,6 +49,12 @@ npm run test:e2e:browser-apps:interop # Run WebApp → WebPeer → CLI in Com npm run test:e2e:obsidian:local-suite # Run the real Obsidian local suite ``` +### Production minification + +Production builds remove unused declarations and shorten local identifiers while retaining class and function names used by module discovery and diagnostic stacks. `terser.config.mjs` also preserves property names, top-level bindings, and identifiers visible to direct `eval` calls. Keep these boundaries when changing compression settings; `test/build/terserConfig.unit.spec.ts` checks the generated code against host properties, persisted keys, reflective names, and exception stacks. + +Compare bundle bytes and gzip bytes from the same source, dependency artefacts, and build inputs. Validate import-time registration and the resulting plug-in in the affected real-runtime workflows. The iOS syntax check is one part of compatibility verification; it does not measure performance or replace physical-device testing. + ### Tips Select the narrowest unit, integration, CLI E2E, or real Obsidian E2E command that owns the behaviour being changed. The obsolete mocked browser Harness has been retired. @@ -78,7 +84,7 @@ See the [test procedures](test/README.md) for clean-installation checks, local v Regression tests remain in the suite owned by the implementation under test. Plug-in tests may be co-located with their source, while independent application tests remain under `test/apps/` or `test/browser-apps/` so that they stay outside the Community Review source boundary. Prefix a case or group with `compatibility:` when it protects a persisted input or state which current releases still accept, and with `retirement guard:` when it prevents a removed setting, control, or notification from returning. Remove or replace a compatibility case only when the corresponding input is no longer accepted or an equivalent maintained case preserves the contract. Remove a retirement guard only when another current contract makes the old behaviour unreachable. Do not preserve a disconnected historical test as an executable specification when no maintained runner invokes it; Git history is the reference for retired test infrastructure. -- **CLI E2E** (`src/apps/cli/testdeno/`): Host-independent consumer workflows. The canonical Compose P2P suite covers ordinary two-peer synchronisation, replacement of the current Replicator followed by transfer with the same peer, and explicit relay disconnection followed by paused and resumed reconnection. Its lifecycle entry point is included only in the Docker test build and does not add a public CLI command. Run `npm run test:e2e:cli` for the ordinary suite or `npm run test:e2e:cli:p2p` for P2P validation. +- **CLI E2E** (`test/e2e-cli/`): Node.js 26 native TypeScript and `node:test` subprocess workflows. Host-independent consumer workflows. The canonical Compose P2P suite covers ordinary two-peer synchronisation, replacement of the current Replicator followed by transfer with the same peer, and explicit relay disconnection followed by paused and resumed reconnection. Its lifecycle entry point is included only in the Docker test build and does not add a public CLI command. Run `npm run test:e2e:cli` for the ordinary suite or `npm run test:e2e:cli:p2p` for P2P validation. - **Self-hosted setup tools** (`utils/couchdb/`, `utils/setup/`, and `utils/flyio/`): Deno contract tests consume the exact locked Commonlib registry package, verify current CouchDB, Object Storage, and random-room P2P Setup URI defaults and remote profiles, and keep CouchDB administration separate from package-owned LiveSync database-version negotiation. `unit-ci` also provisions a real temporary CouchDB database and verifies its version document against the installed Commonlib package. Run `npm run test:setup-tools` for the local contract gate. - **Real Obsidian E2E** (`test/e2e-obsidian/`): Local-first scripts that launch real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in. Use these for boot-up sequence, vault reflection, RedFlag flows, Fast Setup (Simple Fetch), settings dialogues, restart-sensitive workflows, Object Storage regressions, and other behaviour that depends on Obsidian itself. After source changes, run the build-owning wrapper, for example `npm run test:e2e:obsidian:focused -- two-vault-sync`, with the required fixture already running. Direct scenario scripts reuse the existing bundle and are suitable for reruns only while that build remains unchanged. Use `npm run test:e2e:obsidian:local-suite:services` to run the broader local suite with CouchDB and RustFS fixtures managed by the wrapper. @@ -146,6 +152,8 @@ The application is composed from Services, ServiceModules, serviceFeatures, add- The normal composition order is the Service Hub, Replicator provider registration, ServiceModules, serviceFeatures, add-ons, and finally legacy Module binding. A serviceFeature may therefore consume an already constructed ServiceModule. Preferring a serviceFeature for new composition is a dependency-boundary rule, not an initialisation-order rule. +Declare the feature host type explicitly when a narrow dependency contract is intended. `NecessaryServices` selects service keys from `THub` and module keys from `ServiceModules`; it retains `context` only when the selected source provides it, and defaults to `ServiceHub`. `createServiceFeature()` returns the supplied function with its complete signature and result type, but it does not infer dependency keys from the callback body. Unannotated legacy callbacks retain the full default `ServiceHub` contract for compatibility. Put feature-only collaborators, such as a borrowed `LogFunction`, in the feature's own dependency type. + Mutable state is permitted in a serviceFeature. State alone is not a reason to create a class, a ServiceModule, or retain an AbstractModule. Prefer one private context, with module-level functions which receive that context, when identity and polymorphism are not part of the contract. Separate the state, transitions, and invariants from the surrounding function which registers lifecycle handlers and connects downstream effects. Give the stateful boundary narrow collaborators rather than `LiveSyncBaseCore`. Use a class when stable object identity, replaceable implementations, or an explicit external-resource lifecycle such as serialised ownership, `dispose()`, or `abort()` is part of the contract. Use a ServiceModule when that operational capability or resource lifecycle must also be shared explicitly by several consumers. Do not introduce a class merely to group dependencies or make private functions callable. @@ -158,7 +166,7 @@ Commonlib's `targetFilter.ts` and `prepareDatabaseForUse.ts` demonstrate the int Use interaction-based, London School unit tests for the composition boundary. Verify collaborator calls, ordering, failure short-circuiting, and handler registration, then test the focused state owner for its transitions and invariants. If a test needs a broad core fixture, a large class mock, deep mock chains, or unrelated Services, treat that friction as a design-review signal and consider a private context with narrower functions before adding more test machinery. -See [Service feature and legacy Module boundaries](docs/design_docs/service_feature_and_legacy_module_boundaries.md) for the selection criteria, current examples, reasons to avoid new `AbstractModule` subclasses, incremental migration guidance, and test shapes. Commonlib's [service feature composition guide](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/service-feature-composition.md) defines the shared host-neutral boundary. +See [Service feature and legacy Module boundaries](docs/design_docs/service_feature_and_legacy_module_boundaries.md) for the selection criteria, current examples, reasons to avoid new `AbstractModule` subclasses, incremental migration guidance, and test shapes. Commonlib's [service feature composition guide](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/service-feature-composition.md) defines the shared host-neutral boundary. Fancy Kit's [shared runtime composition guide](https://github.com/vrtmrz/fancy-kit/blob/main/docs/runtime-composition.md) maps content application, cleanup, progress, and host adapters to their owning APIs, with their publication status and consumer responsibilities. Legacy Modules remain grouped by directory: @@ -176,6 +184,18 @@ Declare shared UI requests with `declareUICommand` from `@vrtmr The registry belongs to one UIService instance. Reuse the same declaration object at each call site, and keep active UI cleanup in the feature and dialogue owner. `UIService` disposes the registry when the host's context emits the plug-in-unloaded event. +Each browser service hub creates a local `BrowserUiScope` with a `document` and +`signal`, exposed as readonly `hub.browserUi`. Pass overrides through +the service-hub option `ui: { document, signal }`; when `document` is omitted, +the scope snapshots legacy `_activeDocument`. The scope is bound to +its root `ServiceContext` unload event and optional parent `AbortSignal`, and is +shared by browser confirmation and the Svelte dialogue manager. A dialogue +resolves with its result or `undefined` on ordinary close; cancellation rejects +with the abort reason, including `Error('Plug-in unloaded')` on root-context +unload. A `Menu` can receive `hub.browserUi`; closing, replacing, or aborting it +unmounts its Svelte component, and callbacks from a closed session cannot close +the replacement. + ### Key Architectural Components - **LiveSyncLocalDB** (`@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB`): Local PouchDB database wrapper @@ -190,6 +210,8 @@ steps required to add a built-in provider. Commonlib owns one stable `LiveSyncP2PService`, its `P2PRoomSessionOwner`, and the replaceable Trystero room session. Host commands, event handlers, and views consume the focused transport, connection-probe admission, directory, peer-admission, transfer, change-relay, configuration, and diagnostic views returned by the service feature. They must not retain the deprecated compatibility Replicator as an ordinary service locator, close Trystero-owned raw peers, or install another Trystero transport generation at the application root. The exact implemented ownership and shutdown boundaries are recorded in Commonlib's [P2P transport lifecycle](https://github.com/vrtmrz/livesync-commonlib/blob/main/docs/p2p-transport-lifecycle.md) design document. +The Obsidian `useP2PReplicatorUI` feature declares only `API`, `appLifecycle`, `setting`, and `replicator`, and consumes the public `P2PServiceViews` type instead of the broader `UseP2PReplicatorResult`. Its separate `LiveSyncCore` argument and global `eventHub` remain the legacy pane bridge; this type narrowing leaves the existing presentation and lifecycle wiring intact. + The [TURN connection settings design](docs/design_docs/renewable_turn_credentials.md) describes how the host prepares temporary ICE credentials in a connection-only settings copy. It covers room reuse and expiry, replication continuation, profile persistence and sharing, and report redaction. ### Conflict Merge Policy @@ -279,6 +301,8 @@ The scenario uses only fixed in-memory inputs. It does not read or write the Vau - Dev mode creates `ls-debug/` folder in `.obsidian/` for debug outputs (e.g., missing translations) - This causes pretty significant performance overhead. +`ModuleLog` owns an `OwnedResource` for its `API.addLog.setHandler` registration and keeps service log input in a per-instance buffer. The host wraps the complete asynchronous `control.onUnload()` in `withOwnedResource`. On successful shutdown, the handler and its global and per-instance observers remain registered through the final database close and final unload log. If `control.onUnload()` rejects, `withOwnedResource` releases them when the operation settles. Do not release it from an `onBeforeUnload` or module `onUnload` handler; both run before the host's complete unload finishes. The import-time global `Logger` bridge and display and report histories remain shared, while service log calls stay with their owning instance. + #### Diagnostic and notice ownership - A Commonlib or service operation should normally record detailed diagnostics at `LOG_LEVEL_VERBOSE` and return a typed result which lets its caller distinguish complete, partial, and failed outcomes. Do not make callers infer an outcome by parsing log text. diff --git a/docs/README.md b/docs/README.md index 2938d7c1..a93d8fe8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -38,6 +38,7 @@ Use these references for the area being changed, together with its relevant ADR. | Encryption and sharing | [Metadata encryption](design_docs/internal_metadata_encryption.md) and [time-bound Setup URIs](design_docs/time_bound_setup_uri.md) | | P2P connection preparation | [TURN connection settings](design_docs/renewable_turn_credentials.md) | | Journal storage internals | [Journal Replicator 2nd Edition](design_docs_of_journalsync_2nd.md) | +| CLI database selection and interrupted reconstruction | [CLI database continuity](design_docs/cli_database_continuity.md) | The installed `@vrtmrz/livesync-commonlib` package is authoritative for shared runtime contracts. Use the exact dependency selected by [package.json](../package.json) and its lockfile when checking behaviour. The [Commonlib documentation](https://github.com/vrtmrz/livesync-commonlib/tree/main/docs) is maintained in that repository; this index does not replace its package contracts. diff --git a/docs/design_docs/cli_database_continuity.md b/docs/design_docs/cli_database_continuity.md new file mode 100644 index 00000000..3c1d0b66 --- /dev/null +++ b/docs/design_docs/cli_database_continuity.md @@ -0,0 +1,101 @@ +--- +date: 2026-11-01 +commonlib-version: "0.2.2" +self-hosted-livesync-version: "1.0.43" +status: accepted +--- + +# CLI database continuity + +Publication note: the correction for #1304 is implemented and locally validated, +but has not been released. The reviewed baseline is Self-hosted LiveSync `0e458e5e` with +Commonlib `0.2.2`. The [CLI guide](../../src/apps/cli/README.md) describes the +command interface; the [test procedure](../../test/README.md#cli-fetch-database-continuity) +describes the Object Storage reproduction. + +## Required behaviour + +A successful `fetch` or `rebuild` must leave subsequent CLI commands using the +reconstructed local database. The Commonlib Rebuilder selects the host app ID +as the database suffix. The CLI saves that selection on success even without +`--write-settings`; the option continues to govern other lasting settings +changes. Temporary suspension values remain runtime state. + +Before reconstruction can mutate a database, the CLI records the operation, +database directory, Vault path, and settings path. An interrupted operation +blocks ordinary mutation and synchronisation. Retrying the same reconstruction +with the same paths and `--force` is an explicit recovery action. It can repeat +the destructive operation, so users must preserve unique edits and establish +the authoritative copy before retrying. + +Completion records the settings update before applying it. A later invocation +can finish a pending settings commit; if the settings file has independently +changed, it must stop for review rather than overwrite those changes. The +pending record is removed only after completion and settings persistence. + +## Ownership of persistent state + +The selected local database name identifies the dataset. Its runtime state is +further scoped by the canonical physical Vault path. This includes Journal +checkpoints, scanner state, replication-result queues, file-reflection +provenance, and storage-event snapshots. Paths identify local ownership; they +are not sent to the remote as a new document identity. + +A local database reset invalidates every Vault scope associated with that +database, because each scope refers to the same discarded revision history. +Other local database suffixes retain their state. PouchDB database names and +Commonlib's reconstruction contract remain unchanged. + +KV writes replace the file atomically. Invalid persisted data must fail +initialisation instead of becoming an empty checkpoint history. Closing a KV +handle must not overwrite the file from an old in-memory snapshot. + +## Legacy state + +The former shared `keyvalue-db.json` and Vault-local `.livesync-snapshot.json` +do not record their complete owner. A single surviving database directory does +not prove which physical Vault produced those records. Automatically importing +them can replay events against the wrong database; dropping them and immediately +scanning can misinterpret file absence before remote history is recovered. + +Preserve unowned legacy state. When it contains work, stop ordinary mutating +commands and require explicit reconstruction from a verified source. Inspection +remains available. This deliberately requires more user action than automatic +migration, and must not silently delete the old records. + +Check the selected scope's completion flag before reading legacy files. For an +unreviewed scope, inspect both legacy files after settings and scoped KV loading, +but before readiness starts scanning or event replay. Distinguish non-empty +unowned state, invalid JSON or shape, and read failures. Report affected paths +once per invocation on stderr without logging file contents. Read commands may +continue inspection; ordinary writes stop, and explicit reconstruction retains +the files. Once reconstruction has succeeded for that scope, legacy parse or +read failures must not block it again. The active scoped KV still fails +initialisation on corruption. + +## Evidence and limits + +The original Object Storage scenario uses encryption, path obfuscation, and a +second-generation file revision. Without settings writeback, A reopens an empty +database while sharing the fetched database's Journal history, creates an +independent root, and receives B's same-content resolution tombstone without +the original surviving revision. The original assertion fails because A's +physical file disappears. The explicit `--write-settings` control preserves +the file and the original revision. The remote and B retain the content in this +reproduction; it does not demonstrate irreversible loss. + +Validation of the correction must check bytes and revision ancestry after +restart, interrupted reconstruction admission, pending settings completion, +legacy-state preservation, and reset isolation. A successful exit or an empty +queue alone does not establish preservation. + +Local validation passed the encrypted Object Storage CLI regression and the +explicit-settings control on Node.js 26. Unit tests cover failed-operation +admission, saved running/completed markers, settings-edit preservation, scoped +stores, reset isolation, corrupt KV files, and legacy-state refusal. The marker +tests inject interruption states; they do not emulate a machine power failure. + +The shared device-value JSON file retains its existing concurrency limitations. +This change does not establish concurrent CLI writers to one database directory +as a supported workflow. Native Obsidian and other transport behaviour require +their own runtime evidence. diff --git a/docs/design_docs/service_feature_and_legacy_module_boundaries.md b/docs/design_docs/service_feature_and_legacy_module_boundaries.md index 65ec41ec..8dd3539d 100644 --- a/docs/design_docs/service_feature_and_legacy_module_boundaries.md +++ b/docs/design_docs/service_feature_and_legacy_module_boundaries.md @@ -1,7 +1,7 @@ --- -date: 2026-09-04 -commonlib-version: "0.1.21" -self-hosted-livesync-version: "1.0.24" +date: 2026-10-25 +commonlib-version: "0.2.0" +self-hosted-livesync-version: "1.0.40" status: accepted --- @@ -104,14 +104,22 @@ For new behaviour, keep Obsidian-specific presentation or registration in an Obs ### A small serviceFeature: language initialisation -`src/serviceFeatures/onLayoutReady/enablei18n.ts` declares only `setting`, `API`, and `appLifecycle`: +`src/serviceFeatures/onLayoutReady/enablei18n.ts` declares only `setting`, `API`, and `appLifecycle`, with an explicit dependency type: ```typescript -export const enableI18nFeature = createServiceFeature(async ({ services: { setting, API, appLifecycle } }) => { - // Apply the language, persist a change, and register unload clean-up. -}); +type EnableI18nHost = NecessaryServices<"setting" | "API" | "appLifecycle", never>; + +export const enableI18nFeature = createServiceFeature( + async (host: EnableI18nHost): Promise => { + const { setting, API, appLifecycle } = host.services; + // Apply the language, persist a change, and register unload clean-up. + return true; + } +); ``` +The host annotation keeps the dependency contract visible because `createServiceFeature()` does not infer dependency keys from the callback body. This feature retains its `Promise` result. + The local `ObsidianLanguageAppliedNotice` class is still appropriate. It owns one replaceable Obsidian `Notice` and has an explicit `clear()` lifetime operation. The class is not used as a service locator, and the serviceFeature owns its construction and host binding. ### Operation and composition: database preparation @@ -130,6 +138,22 @@ The operation does not need an application Module identity. ### Ordered start-up composition and registration-only features +Compatibility review uses distinct controller and composer dependencies: + +```typescript +type CompatibilityReviewDependencies = NecessaryServicesInterfaces< + "setting" | "config" | "vault" | "control", + never +>; + +type CompatibilityReviewHost = NecessaryServices< + "setting" | "config" | "vault" | "control" | "appLifecycle" | "API", + never +>; +``` + +The controller receives the four interface-only Services it uses. The composer adds `appLifecycle` and `API` for ordered lifecycle handlers and command registration; the priority-30 compatibility review remains after flag-file recovery. + Configured Vault admission and the checks which follow database preparation are composed by `src/serviceFeatures/startupLifecycle/`. The directory keeps onboarding admission, compromised-chunk inspection, incomplete-document repair, Config Doctor, and the obsolete bulk-send setting migration as separate operations. One feature composer owns their order and receives the compatibility-review wait operation explicitly; an individual operation does not call the composer. The layout-ready admission handler uses priority 1. This preserves ordinary priority-0 host integration before admission, while keeping an unconfigured Vault outside the flag-file recovery handlers at priorities 5, 10, and 20, and the compatibility review at priority 30. Admission belongs to one plug-in process: an initially unconfigured process remains inert until setup restarts it, and declining the requested restart does not trigger an in-process reconfiguration. Changing an admitted process back to unconfigured retires its Config Doctor and incomplete-document repair request handlers. The handlers also recheck the current configured state and database readiness when invoked, so a pending restart cannot expose partially initialised or retired state. The first-initialise handler rechecks admission before retaining the established order after the file watcher has been started: database readiness, compromised chunks, incomplete documents, compatibility review, Config Doctor, and the bulk-send setting migration. @@ -144,7 +168,19 @@ The state remains private to the composed feature. It does not become a `LiveSyn ### Implemented composition: conflict resolution -Conflict checking and resolution are composed for every host by `useConflictResolutionFeature`. The feature owns its `QueueProcessor` privately and registers the conflict Service handlers directly. Its operations receive explicit collaborators for settings, active-file state, database and storage access, replication, logging, and host events. No consumer locates a conflict Module or retains the queue. +Conflict checking and resolution are composed for every host by `useConflictResolutionFeature`. The feature owns its `QueueProcessor` privately and registers the conflict Service handlers directly. Its host type selects the required service and module contracts and adds the borrowed logger explicitly: + +```typescript +type ConflictResolutionHost = NecessaryServices< + "appLifecycle" | "conflict" | "database" | "replication" | "setting" | "vault", + "databaseFileAccess" | "fileHandler" | "storageAccess", + InjectableServiceHub +> & { + readonly log: LogFunction; +}; +``` + +The operations receive explicit collaborators for settings, active-file state, database and storage access, replication, a borrowed `LogFunction`, and host events. `LiveSyncBaseCore` supplies `log` using `createInstanceLogFunction("SF:ConflictResolution", this.services.API)`. Because feature composition precedes legacy Module binding, the host supplies the existing lazy service logger and does not read `logResource.value` before binding. `ModuleLog` and the host retain resource ownership. No consumer locates a conflict Module or retains the queue. The scheduling queue remains one state owner. It publishes `conflictProcessQueueCount`, coalesces pending checks for the same path, and makes `ensureAllProcessed()` wait for conflict resolution to finish. Repeated resolver invocations for one path retain only the newest waiting request and close an active comparison for that path before waiting for the per-file resolver, while comparisons for other paths remain open. Resolution remains host-neutral and communicates dialogue cancellation through `services.context.events`, so CLI, WebApp, and Obsidian compositions use their own selected event channel. diff --git a/docs/design_docs/unicode_path_identity.md b/docs/design_docs/unicode_path_identity.md index c5808f92..b153420e 100644 --- a/docs/design_docs/unicode_path_identity.md +++ b/docs/design_docs/unicode_path_identity.md @@ -155,10 +155,11 @@ preflight failure. If a required setup decision cannot be made in a headless CLI invocation, the command must fail with a clear remediation message rather than report success after cancellation. -`--write-settings` retains the Rebuilder's selected local database name and -lasting settings changes for subsequent invocations. Without this option, the -existing CLI policy leaves the settings file unchanged, so a later command can -reopen the previously selected database. +The unreleased CLI correction for #1304 always retains the Rebuilder's selected +local database suffix after successful `fetch` or `rebuild`. `--write-settings` +still controls other lasting settings changes. Earlier versions required that +option to retain the database selection; omitting it could reopen the previous +database on the next invocation. The examples retain the option for compatibility. ## State which follows the selected identity diff --git a/docs/recovery.md b/docs/recovery.md index b73cf9fd..8340a17e 100644 --- a/docs/recovery.md +++ b/docs/recovery.md @@ -18,6 +18,37 @@ Use the least destructive operation which matches the evidence: Do not switch transport, enable P2P, or run Garbage Collection as a substitute for diagnosing a stopped CouchDB or Object Storage setup. +## Recover files truncated during Object Storage reception + +This procedure addresses [issue #1303](https://github.com/vrtmrz/obsidian-livesync/issues/1303): an older Journal receiver can truncate a Chunk at a raw unit separator (`U+001F`) within its data. This can damage a received text file, including a plug-in's `main.js`, while the original batch remains intact in the bucket. + +The receiver correction is available in Commonlib 0.2.3 and is included in Self-hosted LiveSync 1.0.44. Use a build containing that correction on every participating client before recovery; versions 1.0.42 and 1.0.43 still contain the defect. Updating alone does not repair Chunks already stored locally. Resetting only the Journal received history is also insufficient because ordinary reception skips existing Chunk IDs. + +### Preserve the intact source + +1. Stop editing and synchronisation on every device. If necessary, use the [emergency suspension flag](#suspend-before-diagnosis). +2. Back up every Vault, including hidden files such as `.obsidian`, and the complete remote bucket before resetting anything. +3. Verify the actual contents of the proposed recovery source. A file's existence, recorded size, or successful synchronisation status does not prove that its content is intact. Preserve unique unsynchronised edits and any needed content which exists only in database history separately. +4. Update all participating clients to a build containing the correction, keeping the other devices stopped while recovering one source and then one receiving device at a time. + +### Rebuild from a complete, intact Vault + +When one Vault contains all the intended current files, a deliberate Rebuild avoids relying on potentially damaged local databases. Use [Overwrite Server Data with This Device's Files](#overwrite-server-data-with-this-devices-files) on that authoritative device, then prepare an empty recovery Vault on each other device and [Reset Synchronisation on This Device](#reset-synchronisation-on-this-device) from the rebuilt remote. Keep the previous Vaults as backups until verification is complete. + +Rebuild reconstructs the local and remote databases from the current Vault files. It does not retain revisions available only in the previous local database, remote history, or another device. Keep the backups and preserve any required historical content before proceeding. An intact copy of one affected file does not establish that the whole Vault is a complete recovery source. + +### Reuse a verified intact remote + +If the selected remote file contents are verified intact, the remote need not be overwritten. Prepare a new empty recovery Vault with the same remote configuration and use [Reset Synchronisation on This Device](#reset-synchronisation-on-this-device). Keep the previous Vault, including its hidden files and unsynchronised work, as a separate backup. This reconstructs the local database and its receive checkpoints, allowing the corrected receiver to read the original batches again. + +Fetch does not unconditionally overwrite existing Vault files. A differing local file can remain and be preserved as a separate conflict branch alongside the intact remote content. An empty recovery Vault avoids retaining damaged local files during this process. In the CLI, use the empty Vault with `fetch --force --write-settings`; `--force` confirms database replacement, not forced file overwriting. + +### Restore optional files and verify the result + +Rebuild and Fetch disable Hidden File Sync and Customisation Sync. Complete ordinary note recovery first, then initialise the optional feature deliberately. For Hidden File Sync, follow its [initial source selection](tips/hidden-file-sync.md#choose-the-initial-source): after a full Rebuild, use `Overwrite` on the device with the intact hidden files, then `Fetch` on additional devices. When retaining an intact remote hidden-file state, use `Fetch` on the recovering device. Restore the intended Customisation Sync selections separately; do not enable both features for the same files. + +Compare the recovered files with the preserved originals, including the affected hidden files, then restart Obsidian and synchronise again. Confirm that the contents remain intact and that any affected plug-in loads before resuming ordinary work on the remaining devices. Keep the backups until this verification is complete. + ## Migrate to Unicode NFC path identity Use this coordinated migration when CLI and Obsidian clients share a CouchDB database and composed and decomposed Unicode spellings should identify the same path. Version `0` remains the default and preserves legacy identity. Version `1` is supported with CouchDB only; it is unsupported with Object Storage and P2P. diff --git a/docs/releases/1.0.md b/docs/releases/1.0.md index 88bd667f..6edeb99c 100644 --- a/docs/releases/1.0.md +++ b/docs/releases/1.0.md @@ -2,6 +2,90 @@ This document contains earlier published releases from the 1.0 line of the [current Self-hosted LiveSync release history](../../updates.md). Beta and release-candidate builds published before 1.0.0 are recorded in the [1.0 preview history](1.0-previews.md). Earlier release lines continue in the [0.25 history](0.25.md) and the [legacy history](legacy.md). +## 1.0.38 + +17th October, 2026 + +### Synchronisation + +#### Added + +- We can opt into NFC path identity for CouchDB synchronisation between the CLI and Obsidian. This prevents decomposed or mixed Unicode file names from becoming inaccessible through a receiving device's normalised path. (#1259) + - Existing Vaults retain their current identity rules. Migration requires updated clients, a Rebuild from a complete authoritative Vault, and a Fetch on the other devices. + - Reconstruction stops when the supported host file listing or local Metadata exposes ambiguous files or parent folders under NFC and the configured case handling. The CLI retains existing physical file names. P2P and Object Storage do not support the new mode. + +#### Fixed + +- We can synchronise file deletions and renames when **Do not keep metadata of deleted files.** is enabled. + - A file restored during the final deletion check remains in the receiving Vault. +- We can resolve configuration mismatches when **Encrypt internal file Properties** is enabled but its encryption prerequisites are inactive. (#1180) + - Compatible chunk settings can be aligned automatically, or reviewed through `Sync now`. Applying those settings preserves our inactive encryption preference. +- We can receive synchronised updates to unchanged files even after the local database has discarded their recorded revision, avoiding unnecessary conflicts. (#1207) + - Saved content evidence survives revision-history pruning. Genuine local edits remain protected, and older records are upgraded only after their content is verified. + +### Maintenance + +- The Review Harness can compare SHA-256 and xxHash64 file-content hashing on the current device, with fixed inputs from 4 KiB to 50 MiB and separate initialisation and warm-up measurements. + +## 1.0.37 + +16th October, 2026 + +### Security + +- Fixed an issue where synchronised changes could reach files outside the intended synchronisation scope under certain conditions. Updating is recommended. + +### CLI + +#### Fixed + +- We can edit files through the CLI without resetting their creation time where the filesystem supplies it. (#1243) + - Filesystems without birth-time support retain the existing change-time fallback. File replacement and restoration of a synchronised creation time remain subject to filesystem limitations. +- `p2p-host` restores configured P2P automatic synchronisation, following, and change announcements after initialisation. (#1247) + - Headless peer acceptance follows the configured policy. Single-operation commands retain their automatic synchronisation suspension. +- We can select CLI-only Docker patches by their full release tag, such as `1.0.36-patch1-cli`, while keeping the Obsidian plug-in version. + - Patch and RC images receive version and SHA-qualified tags. The `latest` and major-minor tags remain reserved for ordinary stable releases. + +### Acknowledgements + +Thank you for your contributions! + +- @kimjansheden, for reporting the issue and contributing the fix and regression tests. + +## 1.0.36 + +13th October, 2026 + +### Configuration Encryption + +#### Fixed + +- We can change the protection of sensitive configuration with Apply and keep our saved settings readable after a restart. (#1238, #1245) + - The selected protection mode and passphrase are saved together. If Apply fails, our edits remain available to retry. +- We can enter a missing or unusable custom configuration passphrase at start-up to unlock our protected settings. (#1239, #1245) + - Cancelling the recovery dialogue preserves the encrypted configuration and stops that start-up attempt. Headless command-line start-up cannot complete the dialogue when recovery input is required. + +### Localisation + +#### Fixed + +- We can read more settings, maintenance controls, and Fast Setup instructions in Spanish. (#1213, #1214) + - Remote database lock warnings now use the existing translations consistently. + - Fast Setup follows the selected display language while preserving saved choices across restarts and language changes. +- We can read more settings, setup instructions, and messages in Simplified Chinese. (#1242) + +### Maintenance + +- Development tooling now uses compatible dependency versions with available security fixes. (#1245) + - Repository-owned build and import alias helpers remove the vulnerable `braces` dependency. + +### Acknowledgements + +Thank you for your contributions! + +- @zeedif (#1213, #1214) +- @lingting0 (#1242) + ## 1.0.35 5th October, 2026 diff --git a/docs/specs_data_compression.md b/docs/specs_data_compression.md index 119bc879..ed419bcf 100644 --- a/docs/specs_data_compression.md +++ b/docs/specs_data_compression.md @@ -33,7 +33,7 @@ The current benchmark measures the Node CLI process. It does not establish Obsid ## Reproducible benchmark -The benchmark is implemented under `src/apps/cli/testdeno` and packaged by `test/bench-network`. It exercises the real CLI mirror and synchronisation path. The recorded three-repeat result below used Commonlib 0.1.0-rc.4, CouchDB 3.5.0, the V3 Rabin–Karp splitter, optional Data Compression, and E2EE V2. +The benchmark is implemented under `test/e2e-cli` and packaged by `test/bench-network`. It exercises the real CLI mirror and synchronisation path. The recorded three-repeat result below used Commonlib 0.1.0-rc.4, CouchDB 3.5.0, the V3 Rabin–Karp splitter, optional Data Compression, and E2EE V2. The current Compose runner installs the checked-out repository's package and lockfile with `npm ci`; Self-hosted LiveSync 1.0.42 selects Commonlib 0.2.2. Running the command below measures that checkout, rather than recreating the older package environment automatically. Keep the recorded results associated with their original measurement date and dependency version. diff --git a/docs/tips/hidden-file-sync.md b/docs/tips/hidden-file-sync.md index 4bf0ca85..e5b74c55 100644 --- a/docs/tips/hidden-file-sync.md +++ b/docs/tips/hidden-file-sync.md @@ -72,6 +72,12 @@ Hidden File Sync must be enabled independently after ordinary setup on each devi If you later run Rebuild or Fetch as a recovery operation, expect optional features to be disabled again. Complete ordinary recovery first, then repeat this guide deliberately. +## Resolve a JSON conflict + +When automatic merging cannot combine conflicting JSON changes and no overwrite pattern applies, the **Conflicted Setting** dialogue compares two revisions. Review the proposed contents before choosing **Apply**. Closing the dialogue leaves the conflict unresolved. + +A later conflict check for the same file cancels the earlier comparison and checks the latest revisions again. If manual resolution is still needed, a replacement dialogue opens. A cancelled dialogue cannot apply a later selection. A resolution received through replication closes the comparison for that file; a remaining conflict is checked again. Dialogues for other files wait their turn. An **Apply** operation which has already started finishes before the replacement proceeds. + ## Automatic in Customisation Sync **Automatic** delegates an item's files to Hidden File Sync. Enable Hidden File Sync on this device through the steps above before choosing Automatic. If Hidden File Sync is disabled, the choice displays a Notice without changing the item's mode or starting initialisation. Previously saved Automatic assignments remain saved, so this prerequisite also applies after importing settings from another device. diff --git a/esbuild.config.mjs b/esbuild.config.mjs index e2e16e8e..ab7684a6 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -194,7 +194,7 @@ const context = await esbuild.context({ platform: "browser", metafile: true, sourcemap: prod ? false : "inline", - treeShaking: false, + treeShaking: true, outfile: "main_org.js", mainFields: ["browser", "module", "main"], minifyWhitespace: false, diff --git a/eslint.community.config.mjs b/eslint.community.config.mjs index e81aeb31..64447714 100644 --- a/eslint.community.config.mjs +++ b/eslint.community.config.mjs @@ -20,7 +20,7 @@ export default defineConfig( "**/*.unit.spec.ts", "**/test/**", "src/apps/_test/**", - "src/apps/cli/testdeno/**", + "test/e2e-cli/**", ]), { languageOptions: { diff --git a/eslint.config.mjs b/eslint.config.mjs index bcbd8688..12067646 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -48,7 +48,7 @@ export default defineConfig([ "test/**", "**/test/**", "src/apps/_test/**", - "src/apps/cli/testdeno/**", + "test/e2e-cli/**", "**/*.test.ts", "**/*.unit.spec.ts", "**/test.ts", @@ -120,6 +120,13 @@ export default defineConfig([ "@typescript-eslint/restrict-template-expressions": "warn", }, }, + { + files: ["src/apps/cli/vite.config.ts", "src/apps/webapp/vite.config.ts"], + rules: { + // Vite loads these configurations before the application source alias is available. + "livesync-imports/prefer-source-alias": "off", + }, + }, { files: ["src/apps/browser/**/*.{ts,svelte}", "src/apps/webapp/**/*.ts"], rules: { diff --git a/manifest.json b/manifest.json index e1929275..24d4a637 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "obsidian-livesync", "name": "Self-hosted LiveSync", - "version": "1.0.42", + "version": "1.0.44", "minAppVersion": "1.7.2", "description": "Community implementation of self-hosted livesync. Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "author": "vorotamoroz", diff --git a/package-lock.json b/package-lock.json index 717ba6f1..911e5037 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "obsidian-livesync", - "version": "1.0.42", + "version": "1.0.44", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "obsidian-livesync", - "version": "1.0.42", + "version": "1.0.44", "license": "MIT", "workspaces": [ "src/apps/cli", @@ -23,8 +23,8 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.2", - "@vrtmrz/obsidian-plugin-kit": "0.1.4", + "@vrtmrz/livesync-commonlib": "0.2.3", + "@vrtmrz/obsidian-plugin-kit": "0.1.7", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", "diff-match-patch": "^1.0.5", @@ -32,7 +32,7 @@ "markdown-it": "^14.3.3", "minimatch": "^10.2.5", "obsidian": "^1.13.1", - "octagonal-wheels": "^0.1.55", + "octagonal-wheels": "0.1.56", "qrcode-generator": "^1.4.4", "xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2" }, @@ -57,6 +57,7 @@ "@typescript-eslint/parser": "8.69.0", "@vitest/coverage-v8": "^4.1.12", "@vrtmrz/obsidian-test-session": "0.2.6", + "c8": "^12.0.0", "dotenv-cli": "^11.0.0", "esbuild": "0.28.1", "esbuild-plugin-inline-worker": "^0.1.1", @@ -1822,6 +1823,16 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", + "integrity": "sha512-+Sg6GCR/wy1oSmQDFq4LQDAhm3ETKnorxN+y5nbLULOR3P0c14f2Wurzj3/xqPXtasLFfHd5iRFQ7AJt4KH2cw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -3427,6 +3438,13 @@ "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", "license": "MIT" }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -4103,9 +4121,9 @@ } }, "node_modules/@vrtmrz/livesync-commonlib": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.2.tgz", - "integrity": "sha512-+nl4ntmRtUmSUuh9mOpicE0U/wpzhSPSQQXy9eg/2xdTBL58jkrQ6bQjbVKEhqUj0bUvg0ESjZm8YX/x+4zXiA==", + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@vrtmrz/livesync-commonlib/-/livesync-commonlib-0.2.3.tgz", + "integrity": "sha512-n1BaoegBVGDqerZ9EcNbgd4KFtRPJ/lPp5ADsAsHTkPzSwev4SAFxxyFgFKLpTLpPv7McFao4oGBHGvRvoXWEg==", "license": "MIT", "dependencies": { "@aws-crypto/sha256-js": "5.2.0", @@ -4123,7 +4141,7 @@ "idb": "^8.0.3", "markdown-it": "^14.3.3", "minimatch": "^10.2.6", - "octagonal-wheels": "^0.1.55", + "octagonal-wheels": "0.1.56", "pouchdb-adapter-http": "^9.0.0", "pouchdb-adapter-idb": "^9.0.0", "pouchdb-adapter-indexeddb": "^9.0.0", @@ -4170,12 +4188,14 @@ } }, "node_modules/@vrtmrz/obsidian-plugin-kit": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/@vrtmrz/obsidian-plugin-kit/-/obsidian-plugin-kit-0.1.4.tgz", - "integrity": "sha512-MxZgd7UOr8DXk0e+JsAXJmYj/8bfIoEliI4ruvp1Cu4U+mmVI/p63nZ5we3dDW+EaCgRAgF+ztuVDkcMUk1DwA==", + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/@vrtmrz/obsidian-plugin-kit/-/obsidian-plugin-kit-0.1.7.tgz", + "integrity": "sha512-Ax6u6CEHVrY+DJJtaCcVkSGzf9fia1ypzGblQ1uG4AvMjgc2o4STs6l2P7D7ZB81pryE4MfkSpBLvENxyTbWLw==", "license": "MIT", "dependencies": { - "@vrtmrz/ui-interactions": "0.1.2" + "@vrtmrz/secret-store": "0.1.0", + "@vrtmrz/ui-interactions": "0.1.2", + "@vrtmrz/value-store": "0.1.0" }, "peerDependencies": { "obsidian": ">=1.8.7" @@ -4195,6 +4215,15 @@ "playwright": ">=1.50.0" } }, + "node_modules/@vrtmrz/secret-store": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@vrtmrz/secret-store/-/secret-store-0.1.0.tgz", + "integrity": "sha512-9qkJQdt5IRxhPIIQr55bRMaKZjDpF9fLpTtIOsR0Ab90Wl52hj3wsY28EUtgCCRZ1qbXHAgDuplbg0NNMjAolQ==", + "license": "MIT", + "dependencies": { + "@vrtmrz/value-store": "0.1.0" + } + }, "node_modules/@vrtmrz/ui-interactions": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/@vrtmrz/ui-interactions/-/ui-interactions-0.1.2.tgz", @@ -4264,6 +4293,19 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-regex": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.5.0.tgz", + "integrity": "sha512-zhvbbbdt1kbw7EJ9OQoxmQcuaNR9P5kTj7e/YdmOSU10aHWbycofJxA6jKfoEy3vlLdctOF9ZHaxLtmSm998wQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -4606,6 +4648,40 @@ "dev": true, "license": "MIT" }, + "node_modules/c8": { + "version": "12.0.0", + "resolved": "https://registry.npmjs.org/c8/-/c8-12.0.0.tgz", + "integrity": "sha512-4zpJvrd1nKWutnnKC2pXkFmb6iM1l+ffN//o1CzlTNwW7GSOs9a1xrLqkC48nU8oEkjmPZLPiwMsIaOvoF4Pqg==", + "dev": true, + "license": "ISC", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.1", + "@istanbuljs/schema": "^0.1.3", + "find-up": "^5.0.0", + "foreground-child": "^3.1.1", + "istanbul-lib-coverage": "^3.2.0", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.1.6", + "test-exclude": "^8.0.0", + "v8-to-istanbul": "^9.0.0", + "yargs": "^18.0.0", + "yargs-parser": "^21.1.1" + }, + "bin": { + "c8": "bin/c8.js" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + }, + "peerDependencies": { + "monocart-coverage-reports": "^2" + }, + "peerDependenciesMeta": { + "monocart-coverage-reports": { + "optional": true + } + } + }, "node_modules/call-bind": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", @@ -4717,6 +4793,39 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/cliui/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -5082,6 +5191,13 @@ "node": ">= 0.4" } }, + "node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "dev": true, + "license": "MIT" + }, "node_modules/empathic": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/empathic/-/empathic-2.0.1.tgz", @@ -5418,6 +5534,16 @@ "node": ">=18" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -6518,6 +6644,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -6599,6 +6742,29 @@ "node": ">= 0.4" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.7.0.tgz", + "integrity": "sha512-XjH1AECxf0giL2V1aU8vKyRR2ppRUb5c0EvT7zuJTokQ74bNo52zOtghqdWIqrhUD79fo3x0WfKZdOqxF6LG1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -6669,6 +6835,24 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -8195,6 +8379,16 @@ "dev": true, "license": "MIT" }, + "node_modules/lru-cache": { + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz", + "integrity": "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, "node_modules/ltgt": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", @@ -8360,6 +8554,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/module-replacements": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/module-replacements/-/module-replacements-2.11.0.tgz", @@ -8646,9 +8850,9 @@ "license": "MIT" }, "node_modules/octagonal-wheels": { - "version": "0.1.55", - "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.55.tgz", - "integrity": "sha512-mHsQv/h4Tw4ZGJCmTxXFlgeRvPuL9s47C0arD8Njqld+DKr3ppaWjVteEBP+9bWFSOpv/a1u2yY3LYNFGmuexg==", + "version": "0.1.56", + "resolved": "https://registry.npmjs.org/octagonal-wheels/-/octagonal-wheels-0.1.56.tgz", + "integrity": "sha512-c4xQqdeEGE//ZExMJDEsqPsUHsRVHMNuxudux6scYI46eqA/54ao3WJmEZj3fbSsUhR7uCRmJ5ddd1cfRybIfg==", "license": "MIT", "dependencies": { "idb": "^8.0.3" @@ -8787,6 +8991,23 @@ "dev": true, "license": "MIT" }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -9881,6 +10102,19 @@ "dev": true, "license": "ISC" }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -9975,6 +10209,23 @@ ], "license": "MIT" }, + "node_modules/string-width": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.3.0.tgz", + "integrity": "sha512-ZbmZM0JCihQN91dWnxoipT2KOEyHqEyfRXUyjuRhW8b/xnqPDoq4gWEVApTVa9db2wN8mmoikgFBbjh71+cGeQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/string.prototype.matchall": { "version": "4.0.12", "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.12.tgz", @@ -10073,6 +10324,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", @@ -10397,6 +10664,21 @@ "node": ">=10" } }, + "node_modules/test-exclude": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-8.0.0.tgz", + "integrity": "sha512-ZOffsNrXYggvU1mDGHk54I96r26P8SyMjO5slMKSc7+IWmtB/MQKnEC2fP51imB3/pT6YK5cT5E8f+Dd9KdyOQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^13.0.6", + "minimatch": "^10.2.2" + }, + "engines": { + "node": "20 || >=22" + } + }, "node_modules/through2": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz", @@ -10833,6 +11115,21 @@ "uuid": "dist/esm/bin/uuid" } }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, "node_modules/vite": { "version": "8.1.6", "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.6.tgz", @@ -11269,6 +11566,55 @@ "node": ">=0.10.0" } }, + "node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/write-stream": { "version": "0.4.3", "resolved": "https://registry.npmjs.org/write-stream/-/write-stream-0.4.3.tgz", @@ -11314,6 +11660,16 @@ "integrity": "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==", "license": "MIT" }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, "node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", @@ -11347,6 +11703,44 @@ "url": "https://github.com/sponsors/ota-meshi" } }, + "node_modules/yargs": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.2.0.tgz", + "integrity": "sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^8.2.1", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -11369,11 +11763,12 @@ }, "src/apps/cli": { "name": "self-hosted-livesync-cli", - "version": "1.0.42-cli", + "version": "1.0.44-cli", "dependencies": { + "@aws-crypto/sha256-js": "5.2.0", "chokidar": "^4.0.0", "minimatch": "^10.2.5", - "octagonal-wheels": "^0.1.55", + "octagonal-wheels": "0.1.56", "pouchdb-adapter-http": "^9.0.0", "pouchdb-adapter-leveldb": "^9.0.0", "pouchdb-core": "^9.0.0", @@ -11394,9 +11789,9 @@ }, "src/apps/webapp": { "name": "livesync-webapp", - "version": "1.0.42-webapp", + "version": "1.0.44-webapp", "dependencies": { - "octagonal-wheels": "^0.1.55" + "octagonal-wheels": "0.1.56" }, "devDependencies": { "@sveltejs/vite-plugin-svelte": "^7.1.2", @@ -11406,9 +11801,9 @@ } }, "src/apps/webpeer": { - "version": "1.0.42-webpeer", + "version": "1.0.44-webpeer", "dependencies": { - "octagonal-wheels": "^0.1.55" + "octagonal-wheels": "0.1.56" }, "devDependencies": { "@sveltejs/vite-plugin-svelte": "^7.1.2", diff --git a/package.json b/package.json index 84e21ce5..182e7ee3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsidian-livesync", - "version": "1.0.42", + "version": "1.0.44", "description": "Reflect your vault changes to some other devices immediately. Please make sure to disable other synchronize solutions to avoid content corruption or duplication.", "main": "main.js", "type": "module", @@ -15,7 +15,7 @@ "lint:community:tools": "eslint --config eslint.community.config.mjs --concurrency off --max-warnings 0 _tools", "svelte-check": "svelte-check --tsconfig ./tsconfig.json --fail-on-warnings", "tsc-check": "tsc --noEmit", - "tsc-check:apps": "tsc --noEmit -p src/apps/browser/tsconfig.json && tsc --noEmit -p src/apps/cli/tsconfig.json && tsc --noEmit -p src/apps/webapp/tsconfig.json && tsc --noEmit -p src/apps/webpeer/tsconfig.app.json && tsc --noEmit -p src/apps/webpeer/tsconfig.node.json", + "tsc-check:apps": "tsc --noEmit -p src/apps/browser/tsconfig.json && tsc --noEmit -p src/apps/cli/tsconfig.json && tsc --noEmit -p test/e2e-cli/tsconfig.json && tsc --noEmit -p src/apps/webapp/tsconfig.json && tsc --noEmit -p src/apps/webpeer/tsconfig.app.json && tsc --noEmit -p src/apps/webpeer/tsconfig.node.json", "pretty:importpath": "cd utilsdeno && deno run -A ./normalise-imports.ts", "pretty:json": "prettier --config ./.prettierrc.mjs \"**/*.json\" --write --log-level error", "pretty": "npm run prettyNoWrite -- --write --log-level error", @@ -41,7 +41,7 @@ "test:setup-tools": "bash utils/flyio/setenv.test.sh && deno test -A --config=utils/flyio/deno.jsonc --frozen --lock=utils/flyio/deno.lock utils/livesync-commonlib-version.test.ts utils/couchdb/provision.test.ts utils/setup/generate_setup_uri.test.ts utils/flyio/generate_setupuri.test.ts", "test:contract:contexts": "vitest run --config vitest.config.unit.ts src/modules/services/ObsidianServiceContext.unit.spec.ts src/apps/cli/services/NodeServiceContext.unit.spec.ts src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts", "test:contract:context:webapp": "vitest run --config vitest.config.unit.ts src/apps/browser/createLiveSyncBrowserServiceHub.unit.spec.ts", - "test:contract:context:cli": "npm run build --workspace self-hosted-livesync-cli && deno task --cwd src/apps/cli/testdeno test:setup-put-cat", + "test:contract:context:cli": "npm run build --workspace self-hosted-livesync-cli && npm run test:e2e:task --workspace self-hosted-livesync-cli -- test:setup-put-cat", "test:contract:context:obsidian": "npm run build && npm run test:e2e:obsidian:smoke", "test:e2e:cli": "npm run test:e2e:ci --workspace self-hosted-livesync-cli", "test:e2e:cli:p2p": "npm run test:e2e:p2p --workspace self-hosted-livesync-cli", @@ -100,8 +100,10 @@ "test:e2e:obsidian:customisation-sync": "tsx test/e2e-obsidian/scripts/customisation-sync.ts", "test:e2e:obsidian:customisation-catalogue-reload": "tsx test/e2e-obsidian/scripts/customisation-catalogue-reload.ts", "test:e2e:obsidian:optional-file-save-completion": "tsx test/e2e-obsidian/scripts/optional-file-save-completion.ts", + "test:e2e:obsidian:log-lifecycle": "tsx test/e2e-obsidian/scripts/log-lifecycle.ts", "test:e2e:obsidian:livesync-batch": "tsx test/e2e-obsidian/scripts/issue1207-livesync-batch.ts", "test:e2e:obsidian:received-change-readiness": "tsx test/e2e-obsidian/scripts/received-change-readiness.ts", + "test:e2e:obsidian:sync-guard": "tsx test/e2e-obsidian/scripts/sync-guard.ts", "test:e2e:obsidian:remote-feature-change": "tsx test/e2e-obsidian/scripts/remote-feature-change.ts", "test:e2e:obsidian:internal-metadata-migration": "tsx test/e2e-obsidian/scripts/internal-metadata-migration.ts", "test:e2e:obsidian:internal-metadata-doctor": "tsx test/e2e-obsidian/scripts/internal-metadata-migration.ts --doctor", @@ -131,7 +133,9 @@ "version": "node version-bump.mjs && node update-workspaces.mjs && npm run pretty:json && git add manifest.json versions.json src/apps/cli/package.json src/apps/webpeer/package.json src/apps/webapp/package.json", "test:e2e:obsidian:startup-integrity": "tsx test/e2e-obsidian/scripts/startup-integrity.ts", "test:e2e:obsidian:third-party-chunks": "tsx test/e2e-obsidian/scripts/third-party-chunks.ts", - "test:e2e:obsidian:upgrade-livesync-batching": "tsx test/e2e-obsidian/scripts/upgrade-livesync-batching.ts" + "test:e2e:obsidian:upgrade-livesync-batching": "tsx test/e2e-obsidian/scripts/upgrade-livesync-batching.ts", + "pretest:browser-ui-lifecycle": "svelte-check --tsconfig test/browser-apps/ui-lifecycle/tsconfig.json --fail-on-warnings && vite build --config test/browser-apps/ui-lifecycle/vite.config.ts", + "test:browser-ui-lifecycle": "deno test -A --no-check --frozen --config test/browser-apps/deno.json --lock test/browser-apps/deno.lock test/browser-apps/ui-lifecycle/browser-ui-lifecycle.test.ts" }, "keywords": [], "author": "vorotamoroz", @@ -157,6 +161,7 @@ "@typescript-eslint/parser": "8.69.0", "@vitest/coverage-v8": "^4.1.12", "@vrtmrz/obsidian-test-session": "0.2.6", + "c8": "^12.0.0", "dotenv-cli": "^11.0.0", "esbuild": "0.28.1", "esbuild-plugin-inline-worker": "^0.1.1", @@ -204,8 +209,8 @@ "@smithy/types": "^4.14.3", "@smithy/util-retry": "^4.4.5", "@vrtmrz/browser-ui-kit": "0.1.0", - "@vrtmrz/livesync-commonlib": "0.2.2", - "@vrtmrz/obsidian-plugin-kit": "0.1.4", + "@vrtmrz/livesync-commonlib": "0.2.3", + "@vrtmrz/obsidian-plugin-kit": "0.1.7", "@vrtmrz/ui-interactions": "0.1.2", "@vrtmrz/value-store": "0.1.0", "diff-match-patch": "^1.0.5", @@ -213,7 +218,7 @@ "markdown-it": "^14.3.3", "minimatch": "^10.2.5", "obsidian": "^1.13.1", - "octagonal-wheels": "^0.1.55", + "octagonal-wheels": "0.1.56", "qrcode-generator": "^1.4.4", "xxhash-wasm-102": "npm:xxhash-wasm@^1.0.2" }, diff --git a/src/LiveSyncBaseCore.ts b/src/LiveSyncBaseCore.ts index 9870a448..6255cedf 100644 --- a/src/LiveSyncBaseCore.ts +++ b/src/LiveSyncBaseCore.ts @@ -31,6 +31,7 @@ import { useReplicationScheduling, type ReplicationSchedulingControl } from "./s import { createCentralReplicatorProviderDefinitions } from "./common/replicatorProviders"; import { useReplicationFeature } from "./serviceFeatures/replication"; import { useConflictResolutionFeature } from "./serviceFeatures/conflictResolution"; +import { createInstanceLogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; import { useBasicCommandsFeature } from "./serviceFeatures/basicCommands"; /** Focused views returned by serviceFeatures which the host may consume during composition. */ @@ -290,7 +291,11 @@ export class LiveSyncBaseCore< * (Please refer `serviceFeatures` for more details) */ initialiseServiceFeatures(): LiveSyncCoreFeatureViews { - useConflictResolutionFeature(this); + useConflictResolutionFeature({ + services: this.services, + serviceModules: this.serviceModules, + log: createInstanceLogFunction("SF:ConflictResolution", this.services.API), + }); useTargetFilters(this); // enable target filter feature. usePrepareDatabaseForUse(this); @@ -303,7 +308,7 @@ export class LiveSyncBaseCore< } export interface IMinimumLiveSyncCommands { - onunload(): void; + onunload(): void | Promise; onload(): void | Promise; constructor: { name: string }; } diff --git a/src/apps/browser/BrowserConfirm.ts b/src/apps/browser/BrowserConfirm.ts index a7cbc5a7..8f875777 100644 --- a/src/apps/browser/BrowserConfirm.ts +++ b/src/apps/browser/BrowserConfirm.ts @@ -1,11 +1,10 @@ import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; -import { _activeDocument } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; -import { EVENT_PLUGIN_UNLOADED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; import { BrowserUiNotifications, createBrowserUi } from "@vrtmrz/browser-ui-kit"; import { createNativeElement } from "@/apps/browserDom"; import { renderMessageMarkdownInto } from "./ui/renderMessageMarkdown"; import { UiInteractionsConfirm } from "./UiInteractionsConfirm"; +import { createBrowserUiScope, type BrowserUiScope } from "./BrowserUiScope"; /** * Compatibility facade consumed by Commonlib while browser presentation is @@ -14,26 +13,26 @@ import { UiInteractionsConfirm } from "./UiInteractionsConfirm"; export class BrowserConfirm extends UiInteractionsConfirm { readonly context: T; - constructor(context: T) { - const dialogueController = new AbortController(); + constructor(context: T, ui: BrowserUiScope = createBrowserUiScope(context)) { const notifications = new BrowserUiNotifications({ - document: _activeDocument, + document: ui.document, }); super({ ui: createBrowserUi({ - document: _activeDocument, - signal: dialogueController.signal, + document: ui.document, + signal: ui.signal, renderMarkdown: ({ container, markdown }) => { renderMessageMarkdownInto(container, markdown); }, }), notifications, - createActionAnchor: () => createNativeElement(_activeDocument, "a"), + createActionAnchor: () => createNativeElement(ui.document, "a"), }); this.context = context; - context.events.onceEvent(EVENT_PLUGIN_UNLOADED, () => { - dialogueController.abort(); + if (ui.signal.aborted) { notifications.dispose(); - }); + } else { + ui.signal.addEventListener("abort", () => notifications.dispose(), { once: true }); + } } } diff --git a/src/apps/browser/BrowserKeyValueDatabase.ts b/src/apps/browser/BrowserKeyValueDatabase.ts index f1f17a1f..02acf3dd 100644 --- a/src/apps/browser/BrowserKeyValueDatabase.ts +++ b/src/apps/browser/BrowserKeyValueDatabase.ts @@ -3,14 +3,14 @@ import type { KeyValueDatabaseFactory, } from "@vrtmrz/livesync-commonlib/compat/interfaces/KeyValueDatabase"; import { deleteDB, openDB, type IDBPDatabase } from "idb"; -import { serialized } from "octagonal-wheels/concurrency/lock"; +import { withLock } from "octagonal-wheels/concurrency/lock"; /** Creates an application-owned IndexedDB key-value factory for browser runtimes. */ export function createBrowserKeyValueDatabaseFactory(): KeyValueDatabaseFactory { const cache = new Map(); return async (databaseKey) => - await serialized(`OpenBrowserKeyValueDatabase-${databaseKey}`, async () => { + await withLock(`OpenBrowserKeyValueDatabase-${databaseKey}`, async () => { const cached = cache.get(databaseKey); if (cached && !cached.isDestroyed) { return cached; diff --git a/src/apps/browser/BrowserMenu.ts b/src/apps/browser/BrowserMenu.ts index 55af8921..7b9bd8db 100644 --- a/src/apps/browser/BrowserMenu.ts +++ b/src/apps/browser/BrowserMenu.ts @@ -1,9 +1,19 @@ -import { promiseWithResolvers, type PromiseWithResolvers } from "octagonal-wheels/promises"; +import { fireAndForget, promiseWithResolvers, type PromiseWithResolvers } from "octagonal-wheels/promises"; import { createNativeElement } from "@/apps/browserDom"; -import { mount } from "svelte"; +import type { BrowserUiOptions } from "./BrowserUiScope"; +import { mount, unmount } from "svelte"; import MenuView from "./ui/MenuView.svelte"; import { _activeDocument } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; +type MountedMenu = ReturnType; +type MenuSession = { + waitingForClose: PromiseWithResolvers; + element: HTMLElement; + component?: MountedMenu; + abortListener?: () => void; + closed: boolean; +}; + export class MenuItem { type = "item"; title = ""; @@ -29,7 +39,15 @@ export class Menu { type = "menu"; items: (MenuItem | MenuSeparator)[] = []; - constructor() {} + private readonly document: Document; + private readonly signal?: AbortSignal; + private activeSession?: MenuSession; + + constructor(options: BrowserUiOptions = {}) { + this.document = options.document ?? _activeDocument; + this.signal = options.signal; + } + addItem(callback: (item: MenuItem) => void) { const item = new MenuItem(); callback(item); @@ -41,32 +59,75 @@ export class Menu { return this; } waitingForClose?: PromiseWithResolvers; - showAtPosition(pos: { x: number; y: number }) { - const el = createNativeElement(_activeDocument, "div"); - if (this.waitingForClose) { - this.waitingForClose.resolve(); + + showAtPosition(pos: { x: number; y: number }): Promise { + if (this.activeSession) { + this.closeSession(this.activeSession); } - this.waitingForClose = promiseWithResolvers(); - mount(MenuView, { - target: el, - props: { - items: this.items, - closeMenu: () => { - this.waitingForClose?.resolve(); - this.waitingForClose = undefined; + if (this.signal?.aborted) { + this.waitingForClose = undefined; + return Promise.resolve(); + } + + const waitingForClose = promiseWithResolvers(); + const element = createNativeElement(this.document, "div"); + const session: MenuSession = { + waitingForClose, + element, + closed: false, + }; + this.activeSession = session; + this.waitingForClose = waitingForClose; + + const closeMenu = () => this.closeSession(session); + session.abortListener = closeMenu; + this.signal?.addEventListener("abort", closeMenu, { once: true }); + + try { + session.component = mount(MenuView, { + target: element, + props: { + items: this.items, + closeMenu, + x: pos.x, + y: pos.y, }, - x: pos.x, - y: pos.y, - }, - }); - _activeDocument.body.appendChild(el); - void this.waitingForClose.promise.finally(() => { - el.remove(); - }); - return this.waitingForClose.promise; + }); + if (!session.closed) { + this.document.body.appendChild(element); + } else if (session.component) { + fireAndForget(() => unmount(session.component!)); + } + } catch (error) { + this.closeSession(session); + throw error; + } + + return waitingForClose.promise; } + + private closeSession(session: MenuSession): void { + if (session.closed) return; + session.closed = true; + if (session.abortListener) { + this.signal?.removeEventListener("abort", session.abortListener); + session.abortListener = undefined; + } + if (this.activeSession === session) { + this.activeSession = undefined; + this.waitingForClose = undefined; + } + + session.element.remove(); + if (session.component) { + fireAndForget(() => unmount(session.component!)); + } + session.waitingForClose.resolve(); + } + hide() { - this.waitingForClose?.resolve(); - this.waitingForClose = undefined; + if (this.activeSession) { + this.closeSession(this.activeSession); + } } } diff --git a/src/apps/browser/BrowserSvelteDialogManager.ts b/src/apps/browser/BrowserSvelteDialogManager.ts index b0e89a0d..17ae4f8d 100644 --- a/src/apps/browser/BrowserSvelteDialogManager.ts +++ b/src/apps/browser/BrowserSvelteDialogManager.ts @@ -8,6 +8,7 @@ import type { SvelteDialogManagerDependencies } from "@vrtmrz/livesync-commonlib import { _activeDocument } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import DialogHost from "@/modules/services/LiveSyncUI/DialogHost.svelte"; import { SvelteDialogSession } from "@/modules/services/SvelteDialogSession"; +import { createBrowserUiScope, type BrowserUiScope } from "./BrowserUiScope"; export class BrowserModal { contentEl: HTMLElement; @@ -15,14 +16,14 @@ export class BrowserModal { modalEl: HTMLElement; isOpen: boolean = false; baseEl: HTMLElement; - constructor() { - const baseEl = createNativeElement(_activeDocument, "popup"); + constructor(document: Document = _activeDocument) { + const baseEl = createNativeElement(document, "popup"); this.baseEl = baseEl; - this.contentEl = createNativeElement(_activeDocument, "div"); + this.contentEl = createNativeElement(document, "div"); this.contentEl.className = "modal-content"; - this.titleEl = createNativeElement(_activeDocument, "div"); + this.titleEl = createNativeElement(document, "div"); this.titleEl.className = "modal-title"; - this.modalEl = createNativeElement(_activeDocument, "div"); + this.modalEl = createNativeElement(document, "div"); this.modalEl.className = "modal"; this.modalEl.hidden = true; this.modalEl.appendChild(this.titleEl); @@ -33,7 +34,7 @@ export class BrowserModal { this.isOpen = true; this.modalEl.hidden = false; if (!this.baseEl.parentElement) { - _activeDocument.body.appendChild(this.baseEl); + this.baseEl.ownerDocument.body.appendChild(this.baseEl); } this.onOpen(); } @@ -57,9 +58,10 @@ export class LiveSyncBrowserDialog, component: ComponentHasResult, - initialData?: U + initialData?: U, + private readonly ui?: BrowserUiScope ) { - super(); + super(ui?.document ?? _activeDocument); this.session = new SvelteDialogSession({ surface: this, context, @@ -67,9 +69,15 @@ export class LiveSyncBrowserDialog extends SvelteDialogManagerBase { + constructor( + context: T, + dependencies: SvelteDialogManagerDependencies, + private readonly ui: BrowserUiScope = createBrowserUiScope(context) + ) { + super(context, dependencies); + } + override async openSvelteDialog( component: ComponentHasResult, initialData?: TU ): Promise { - const dialog = new LiveSyncBrowserDialog(this.context, this.dependents, component, initialData); + if (this.ui.signal.aborted) throw this.ui.signal.reason ?? new Error("Plug-in unloaded"); + const dialog = new LiveSyncBrowserDialog( + this.context, + this.dependents, + component, + initialData, + this.ui + ); dialog.open(); return await dialog.waitForClose(); } diff --git a/src/apps/browser/BrowserUiScope.ts b/src/apps/browser/BrowserUiScope.ts new file mode 100644 index 00000000..3fbc4439 --- /dev/null +++ b/src/apps/browser/BrowserUiScope.ts @@ -0,0 +1,38 @@ +import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; +import { EVENT_PLUGIN_UNLOADED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; +import { _activeDocument } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; + +export interface BrowserUiOptions { + readonly document?: Document; + readonly signal?: AbortSignal; +} + +/** Browser presentation target and lifetime owned by one application context. */ +export interface BrowserUiScope { + readonly document: Document; + readonly signal: AbortSignal; +} + +export function createBrowserUiScope( + context: Pick, + options: BrowserUiOptions = {} +): BrowserUiScope { + const controller = new AbortController(); + const parent = options.signal; + let unregisterUnload: (() => void) | undefined; + const end = (reason: unknown): void => { + if (controller.signal.aborted) return; + unregisterUnload?.(); + unregisterUnload = undefined; + parent?.removeEventListener("abort", onParentAbort); + controller.abort(reason); + }; + const onParentAbort = () => end(parent?.reason); + if (parent?.aborted) { + controller.abort(parent.reason); + } else { + unregisterUnload = context.events.onceEvent(EVENT_PLUGIN_UNLOADED, () => end(new Error("Plug-in unloaded"))); + parent?.addEventListener("abort", onParentAbort, { once: true }); + } + return { document: options.document ?? _activeDocument, signal: controller.signal }; +} diff --git a/src/apps/browser/LiveSyncBrowserUIService.ts b/src/apps/browser/LiveSyncBrowserUIService.ts index c0ab9d00..e8be307d 100644 --- a/src/apps/browser/LiveSyncBrowserUIService.ts +++ b/src/apps/browser/LiveSyncBrowserUIService.ts @@ -7,6 +7,7 @@ import type { InjectableAPIService } from "@vrtmrz/livesync-commonlib/compat/ser import { UIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/base/UIService"; import DialogToCopy from "@/modules/services/LiveSyncUI/dialogues/DialogueToCopy.svelte"; import { BrowserSvelteDialogManager } from "./BrowserSvelteDialogManager"; +import type { BrowserUiScope } from "./BrowserUiScope"; export interface LiveSyncBrowserUIServiceDependencies { API: InjectableAPIService; @@ -20,15 +21,19 @@ export class LiveSyncBrowserUIService extends UIServic override get dialogToCopy() { return DialogToCopy; } - constructor(context: T, dependents: LiveSyncBrowserUIServiceDependencies) { + constructor(context: T, dependents: LiveSyncBrowserUIServiceDependencies, ui?: BrowserUiScope) { const browserConfirm = dependents.API.confirm; - const obsidianSvelteDialogManager = new BrowserSvelteDialogManager(context, { - appLifecycle: dependents.appLifecycle, - config: dependents.config, - replicator: dependents.replicator, - confirm: browserConfirm, - control: dependents.control, - }); + const obsidianSvelteDialogManager = new BrowserSvelteDialogManager( + context, + { + appLifecycle: dependents.appLifecycle, + config: dependents.config, + replicator: dependents.replicator, + confirm: browserConfirm, + control: dependents.control, + }, + ui + ); super(context, { dialogManager: obsidianSvelteDialogManager, APIService: dependents.API, diff --git a/src/apps/browser/createLiveSyncBrowserServiceHub.ts b/src/apps/browser/createLiveSyncBrowserServiceHub.ts index 3197114a..cb0e355d 100644 --- a/src/apps/browser/createLiveSyncBrowserServiceHub.ts +++ b/src/apps/browser/createLiveSyncBrowserServiceHub.ts @@ -22,6 +22,7 @@ import { InjectableVaultServiceCompat } from "@vrtmrz/livesync-commonlib/compat/ import { setLang, translateLiveSyncMessage } from "@/common/translation"; import { BrowserConfirm } from "./BrowserConfirm"; +import { createBrowserUiScope, type BrowserUiOptions, type BrowserUiScope } from "./BrowserUiScope"; import { createBrowserKeyValueDatabaseFactory } from "./BrowserKeyValueDatabase"; import { LiveSyncBrowserAPIService, type LiveSyncBrowserAPIServiceOptions } from "./LiveSyncBrowserAPIService"; import { LiveSyncBrowserUIService } from "./LiveSyncBrowserUIService"; @@ -41,6 +42,7 @@ export interface LiveSyncBrowserRestartPolicy { export interface LiveSyncBrowserServiceHubOptions { deviceValues: ValueStore & ValueDeleter; context?: T; + ui?: BrowserUiOptions; getSystemVaultName?: () => string; settings?: LiveSyncBrowserSettingsPersistence; restart?: LiveSyncBrowserRestartPolicy; @@ -56,15 +58,18 @@ class LiveSyncBrowserKeyValueDBService extends KeyValu /** LiveSync-owned service composition shared by WebApp and WebPeer. */ export class LiveSyncBrowserServiceHub extends InjectableServiceHub { + readonly browserUi: BrowserUiScope; + constructor(options: LiveSyncBrowserServiceHubOptions) { const context = options.context ?? (new ServiceContext({ translate: translateLiveSyncMessage, }) as T); + const browserUi = createBrowserUiScope(context, options.ui); const API = new LiveSyncBrowserAPIService(context, { ...options.API, - confirm: new BrowserConfirm(context), + confirm: new BrowserConfirm(context, browserUi), getSystemVaultName: options.getSystemVaultName ?? (() => "livesync-browser"), }); const conflict = new InjectableConflictService(context); @@ -147,13 +152,17 @@ export class LiveSyncBrowserServiceHub extends Injecta APIService: API, replicatorService: replicator, }); - const ui = new LiveSyncBrowserUIService(context, { - API, - appLifecycle, - config, - control, - replicator, - }); + const ui = new LiveSyncBrowserUIService( + context, + { + API, + appLifecycle, + config, + control, + replicator, + }, + browserUi + ); super(context, { API, @@ -175,6 +184,7 @@ export class LiveSyncBrowserServiceHub extends Injecta ui, vault, }); + this.browserUi = browserUi; } } diff --git a/src/apps/caseInsensitiveFileLookup.ts b/src/apps/caseInsensitiveFileLookup.ts new file mode 100644 index 00000000..9be18f36 --- /dev/null +++ b/src/apps/caseInsensitiveFileLookup.ts @@ -0,0 +1,23 @@ +export async function findCaseInsensitiveFile( + requestedPath: string, + findExact: () => Promise, + cachedFiles: ReadonlyMap, + scan: () => Promise +): Promise { + const exact = await findExact(); + if (exact) return exact; + + const lowerPath = requestedPath.toLowerCase(); + const findCached = (): T | null => { + for (const [cachedPath, cachedFile] of cachedFiles) { + if (cachedPath.toLowerCase() === lowerPath) return cachedFile; + } + return null; + }; + + const cached = findCached(); + if (cached) return cached; + + await scan(); + return findCached(); +} diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index fb2a7827..0000a92e 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -70,6 +70,7 @@ livesync-cli [database-path] [command] [args...] - `--force` / `-f`: Required by `rebuild` and `fetch` because they replace a database. It also retains its existing `init-settings` overwrite behaviour. - `--write-settings`: Write setting migrations and other lasting changes after the command succeeds. - `init-settings` writes its target file. `setup`, `remote-add`, `remote-rm`, `remote-set`, and `remote-activate` write their settings changes without this option. + - `fetch` and `rebuild` always retain the selected local database suffix after success. Other lasting changes from these commands still require this option. - All remaining commands leave the settings file unchanged by default. - Temporary values used to suspend synchronisation or select a remote for one command are never written. - `--compat-remote-admin-exit-zero`: Preserve the former zero exit code when `mark-resolved`, `lock-remote`, or `unlock-remote` returns a provider verification failure. @@ -84,8 +85,10 @@ livesync-cli [database-path] [command] [args...] - If `vault-path` is omitted, it defaults to `database-path` (compatibility mode). - Use this command to keep your local `.md` files in sync with the database. - `rebuild`: Rebuild the local and remote databases from the selected Vault. This replaces existing local and remote database state and requires `--force`. - - This command may select a new local database suffix. Add `--write-settings` if later CLI commands should use the resulting database selection; otherwise the existing settings file remains unchanged. + - The selected local database suffix is saved automatically, so later commands use the reconstructed database. - `fetch`: Replace the local database with the configured remote state, then reflect fetched files to the selected Vault. Existing local database state is discarded, so this command requires `--force`. + - The selected local database suffix is saved automatically, including when `--write-settings` is omitted. + - Differing local files can remain and be preserved as conflict branches alongside the fetched content. For recovery from an intact remote, preserve the existing Vault separately and Fetch into an empty Vault; `--force` does not force file overwriting. - `ls [prefix]`: List files currently stored in the local database. - `push `: Push a local file `` into the database at path ``. - `pull `: Pull a file `` from the database into local file ``. @@ -105,6 +108,39 @@ livesync-cli [database-path] [command] [args...] Remote-administration commands verify the resulting milestone state through the selected provider. The existing `[Verification]` lines remain suitable for scripts which inspect command output, while the default exit code now reflects whether that verification succeeded. +### Interrupted reconstruction and older runtime state + +An interrupted `fetch` or `rebuild` leaves a record in +`/.livesync/reconstruction.json`. Ordinary synchronisation and +mutating commands stop while reconstruction is incomplete. Preserve the Vault, +database directory, and settings, check the reported failure, then retry the +same command with `--force` and the same database, Vault, and settings paths. +`rebuild` replaces remote state as well: confirm that the Vault remains the +authoritative copy before retrying it. Do not remove the record merely to +restart `daemon`. + +If reconstruction finished but its settings update was interrupted, the next +invocation completes that update before starting the core. If the settings +file was independently edited, the CLI stops for review. The pending record +can contain settings and credentials; protect it like the settings file. + +Older CLI versions stored runtime state without recording its database and +Vault owner. If unowned state is present, the CLI preserves it and stops +ordinary mutating commands. Read commands remain available for inspection. +Back up unique edits and establish the authoritative copy before explicitly +using `fetch --force` or `rebuild --force`. The old runtime files remain +available for diagnosis; they are not automatically imported into a new scope. + +This check runs before scanning, event replay, and synchronisation. Its message +identifies each affected file and distinguishes an unknown owner from a file +which cannot be read or parsed. Read commands display the warning and continue +inspection. After successful reconstruction for the selected database and Vault, +retained legacy files are no longer inspected, even if they are malformed. +Corruption in the currently selected runtime KV remains an initialisation error. + +Use one CLI process at a time for each database directory. Separate runtime +scopes do not make concurrent writers to shared device settings supported. + ### Examples ```bash diff --git a/src/apps/cli/adapters/NodeFileSystemAdapter.ts b/src/apps/cli/adapters/NodeFileSystemAdapter.ts index 6cb8f458..9ba123e4 100644 --- a/src/apps/cli/adapters/NodeFileSystemAdapter.ts +++ b/src/apps/cli/adapters/NodeFileSystemAdapter.ts @@ -8,6 +8,7 @@ import { NodeVaultAdapter } from "./NodeVaultAdapter"; import type { NodeFile, NodeFolder, NodeStat } from "./NodeTypes"; import { path } from "@vrtmrz/livesync-commonlib/node"; import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput"; +import { findCaseInsensitiveFile } from "@/apps/caseInsensitiveFileLookup"; /** * Complete file system adapter implementation for Node.js @@ -87,27 +88,12 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter this.getAbstractFileByPath(pathStr), + this.fileCache, + () => this.scanDirectory() + ); } async getFiles(): Promise { diff --git a/src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts b/src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts index 7d0fbaae..1791f6d5 100644 --- a/src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts +++ b/src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts @@ -124,6 +124,83 @@ describe("NodeFileSystemAdapter file enumeration", () => { }); }); +describe("NodeFileSystemAdapter case-insensitive path lookup", () => { + const tempDirs: string[] = []; + + async function createVault(files: string[] = []) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-cli-insensitive-")); + tempDirs.push(directory); + for (const file of files) { + await fs.mkdir(path.dirname(path.join(directory, file)), { recursive: true }); + await fs.writeFile(path.join(directory, file), `content of ${file}`); + } + return { directory, adapter: new NodeFileSystemAdapter(directory) }; + } + + afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true }))); + }); + + it("returns an exact path without scanning the directory", async () => { + const { adapter } = await createVault(["Calculus.md"]); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("Calculus.md")).resolves.toMatchObject({ + path: "Calculus.md", + }); + + expect(scanDirectory).not.toHaveBeenCalled(); + }); + + it("returns a cached case-insensitive match without rescanning", async () => { + const { adapter } = await createVault(["Calculus.md"]); + await adapter.scanDirectory(); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("calculus.md")).resolves.toMatchObject({ + path: "Calculus.md", + }); + + expect(scanDirectory).not.toHaveBeenCalled(); + }); + + it("rescans once and returns a newly cached case-insensitive match", async () => { + const { adapter } = await createVault(["Calculus.md"]); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("calculus.md")).resolves.toMatchObject({ + path: "Calculus.md", + }); + + expect(scanDirectory).toHaveBeenCalledOnce(); + await expect(adapter.getAbstractFileByPath("Calculus.md")).resolves.toMatchObject({ path: "Calculus.md" }); + }); + + it("returns null after one scan when no cached path matches", async () => { + const { adapter } = await createVault(["other.md"]); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("missing.md")).resolves.toBeNull(); + + expect(scanDirectory).toHaveBeenCalledOnce(); + }); + + it("returns null and reports a directory scan failure", async () => { + const missingDirectory = path.join(os.tmpdir(), `livesync-cli-insensitive-missing-${Date.now()}`); + const reportDiagnostic = vi.fn(); + const adapter = new NodeFileSystemAdapter(missingDirectory, reportDiagnostic); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("missing.md")).resolves.toBeNull(); + + expect(scanDirectory).toHaveBeenCalledOnce(); + expect(reportDiagnostic).toHaveBeenCalledWith( + expect.stringContaining("Error scanning directory"), + expect.any(Error) + ); + }); +}); + describe("NodeFileSystemAdapter Unicode path resolution", () => { const tempDirs: string[] = []; diff --git a/src/apps/cli/main.bootstrap.unit.spec.ts b/src/apps/cli/main.bootstrap.unit.spec.ts index 033771a1..676c776a 100644 --- a/src/apps/cli/main.bootstrap.unit.spec.ts +++ b/src/apps/cli/main.bootstrap.unit.spec.ts @@ -3,6 +3,7 @@ import { appendFile, mkdtemp, mkdir, readFile, rm, stat, writeFile } from "node: import { tmpdir } from "node:os"; import { join } from "node:path"; import * as chokidar from "chokidar"; +import { path } from "@vrtmrz/livesync-commonlib/node"; import * as offlineScanner from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner"; import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService"; import { @@ -20,6 +21,11 @@ import { main, type CliCommandRunner } from "./main"; import { cloneSettings } from "./settingsPersistence"; import type { CLICommand } from "./commands/types"; +vi.mock("@vrtmrz/livesync-commonlib/node", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, path: { ...actual.path } }; +}); + vi.mock("chokidar", { spy: true }); vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", { spy: true }); @@ -72,12 +78,20 @@ describe("CLI database preparation", () => { await rm(directory, { recursive: true, force: true }); }); - async function start(command: CLICommand, runner: CliCommandRunner, exitCode = 1, force = false) { + async function start( + command: CLICommand, + runner: CliCommandRunner, + exitCode = 1, + force = false, + writeSettings = false + ) { + process.exitCode = undefined; process.argv = [ "node", "livesync-cli", directory, ...(force ? ["--force"] : []), + ...(writeSettings ? ["--write-settings"] : []), "--vault", vaultPath, "--settings", @@ -88,6 +102,36 @@ describe("CLI database preparation", () => { await expect(main(standardIo, runner)).rejects.toThrow(`__EXIT__:${exitCode}`); } + it.each([ + { separator: "/", platform: "POSIX" }, + { separator: "\\", platform: "Windows" }, + ])("excludes internal Vault paths with the $platform host separator", async ({ separator }) => { + const targets = [ + "headless-vault-livesync-v2/LOCK", + ".livesync/settings.json", + "notes/.private/secret.md", + "notes/local.md", + ]; + const observed: boolean[] = []; + const runner = vi.fn(async (_options, { core }) => { + const originalSeparator = path.sep; + try { + Object.defineProperty(path, "sep", { value: separator }); + for (const target of targets) { + observed.push(await core.services.vault.isTargetFile(target as FilePathWithPrefix)); + } + } finally { + Object.defineProperty(path, "sep", { value: originalSeparator }); + } + return false; + }); + + await start("ls", runner); + + expect(runner).toHaveBeenCalledOnce(); + expect(observed).toEqual([false, false, false, true]); + }); + it("loads legacy device values and persists configuration through the service-owned store", async () => { const localStoragePath = join(directory, ".livesync", "runtime", "local-storage.json"); await mkdir(join(directory, ".livesync", "runtime"), { recursive: true }); @@ -350,6 +394,194 @@ describe("CLI database preparation", () => { expect(await readFile(join(vaultPath, "notes/local.md"), "utf-8")).toBe("local content"); }); + it.each(["fetch", "rebuild"] as const)( + "persists the selected database after %s without other command settings", + async (command) => { + const before = JSON.parse(await readFile(settingsPath, "utf-8")); + const runner = vi.fn(async (_options, { core }) => { + await core.services.setting.applyPartial({ + additionalSuffixOfDatabaseName: "headless-app", + customChunkSize: before.customChunkSize + 10, + }); + return true; + }); + + await start(command, runner, 0, true); + + const after = JSON.parse(await readFile(settingsPath, "utf-8")); + expect(after.additionalSuffixOfDatabaseName).toBe("headless-app"); + expect(after.customChunkSize).toBe(before.customChunkSize); + } + ); + + it.each(["suffix", "vault"] as const)("isolates persistent state by selected %s", async (selection) => { + await start("ls", async (_options, { core }) => { + await core.services.keyValueDB.kvDB.set("scope-probe", "original"); + await core.services.config.setValue("scope-probe", "original"); + return false; + }); + if (selection === "suffix") { + const settings = JSON.parse(await readFile(settingsPath, "utf-8")); + settings.additionalSuffixOfDatabaseName = "other"; + await writeFile(settingsPath, JSON.stringify(settings)); + } else { + vaultPath = join(directory, "other-vault"); + await mkdir(vaultPath); + } + const observed: unknown[] = []; + await start("ls", async (_options, { core }) => { + observed.push(await core.services.keyValueDB.kvDB.get("scope-probe")); + observed.push(await core.services.config.getValue("scope-probe")); + return false; + }); + expect(observed).toEqual([undefined, null]); + }); + + it.each(["sync", "daemon", "mirror", "put"] as const)( + "blocks %s after failed reconstruction before command preparation", + async (command) => { + await start("fetch", async () => false, 1, true); + const runner = vi.fn(async () => false); + const ready = vi.spyOn(ControlService.prototype, "onReady"); + + await start(command, runner); + + expect(runner).not.toHaveBeenCalled(); + expect(ready).not.toHaveBeenCalled(); + } + ); + + it("retains legacy queued work and blocks a scanner until explicit reconstruction succeeds", async () => { + const legacy = join(directory, ".livesync/runtime/keyvalue-db.json"); + const legacyContents = JSON.stringify({ + "queued-files": ["notes/unique.md"], + fileStatusMap: { "notes/local.md": 1 }, + }); + await mkdir(join(directory, ".livesync/runtime"), { recursive: true }); + await writeFile(legacy, legacyContents); + const runner = vi.fn(async () => false); + const scan = vi.spyOn(ServiceFileAccessCLI.prototype, "getFiles"); + + await start("mirror", runner); + expect(runner).not.toHaveBeenCalled(); + expect(scan).not.toHaveBeenCalled(); + expect(await readFile(legacy, "utf-8")).toBe(legacyContents); + + await start("fetch", async () => true, 0, true); + const later = vi.fn(async () => false); + await start("sync", later); + expect(later).toHaveBeenCalledOnce(); + expect(await readFile(legacy, "utf-8")).toBe(legacyContents); + }); + + it("permits normal commands after reconstruction while preserving malformed legacy JSON", async () => { + const legacy = join(directory, ".livesync/runtime/keyvalue-db.json"); + await mkdir(join(directory, ".livesync/runtime"), { recursive: true }); + await writeFile(legacy, '{"unfinished":'); + await start("fetch", async () => true, 0, true); + const runner = vi.fn(async () => false); + await start("sync", runner); + expect(runner).toHaveBeenCalledOnce(); + expect(await readFile(legacy, "utf-8")).toBe('{"unfinished":'); + }); + + it.each([ + ["kv", '{"unfinished":', "cannot be parsed"], + ["snapshot", "[", "cannot be parsed"], + ["kv", '{"queued-files":["unique.md"]}', "owner is unknown"], + ["snapshot", '[{"path":"unique.md"}]', "owner is unknown"], + ])("diagnoses unowned %s (%s) before readiness and ignores it after recovery", async (kind, contents, reason) => { + const legacy = + kind === "kv" + ? join(directory, ".livesync/runtime/keyvalue-db.json") + : join(vaultPath, ".livesync-snapshot.json"); + await mkdir(join(directory, ".livesync/runtime"), { recursive: true }); + await writeFile(legacy, contents); + const messages = () => standardIo.writeStderr.mock.calls.flat().join(""); + const reader = vi.fn(async () => false); + await start("ls", reader); + expect(reader).toHaveBeenCalledOnce(); + expect(messages()).toContain(reason); + expect(messages()).toContain(legacy); + expect(messages()).toContain("inspection may continue"); + expect(messages().split("[Legacy state]")).toHaveLength(2); + expect(standardIo.writeStdout.mock.calls.flat().join("")).not.toContain("[Legacy state]"); + + standardIo.writeStderr.mockClear(); + const writer = vi.fn(async () => false); + const ready = vi.spyOn(ControlService.prototype, "onReady"); + await start("mirror", writer); + expect(writer).not.toHaveBeenCalled(); + expect(ready).not.toHaveBeenCalled(); + expect(messages()).toContain("stopped before scanning"); + expect(messages()).toContain(reason); + expect(messages().split("[Legacy state]")).toHaveLength(2); + ready.mockRestore(); + + await start("fetch", async () => true, 0, true); + standardIo.writeStderr.mockClear(); + const later = vi.fn(async () => false); + await start("sync", later); + expect(later).toHaveBeenCalledOnce(); + expect(messages()).not.toContain("[Legacy state]"); + expect(await readFile(legacy, "utf-8")).toBe(contents); + }); + + it("blocks read commands which request settings writes while reconstruction is interrupted", async () => { + await start("fetch", async () => false, 1, true); + const runner = vi.fn(async () => false); + const load = vi.spyOn(ControlService.prototype, "onLoad"); + + await start("ls", runner, 1, false, true); + + expect(runner).not.toHaveBeenCalled(); + expect(load).not.toHaveBeenCalled(); + }); + + it("invalidates every Vault scope for the reset database and retains other database state", async () => { + const originalVault = vaultPath; + const otherVault = join(directory, "other-vault"); + await mkdir(otherVault); + const select = async (suffix: string, selectedVault: string) => { + vaultPath = selectedVault; + const settings = JSON.parse(await readFile(settingsPath, "utf-8")); + settings.additionalSuffixOfDatabaseName = suffix; + await writeFile(settingsPath, JSON.stringify(settings)); + }; + for (const [suffix, selectedVault] of [ + ["", originalVault], + ["", otherVault], + ["other", originalVault], + ]) { + await select(suffix, selectedVault); + await start("ls", async (_options, { core }) => { + await core.services.keyValueDB.kvDB.set("scope-probe", "stored"); + return false; + }); + } + await select("", originalVault); + let reset: boolean | undefined; + await start("ls", async (_options, { core }) => { + reset = await core.services.database.resetDatabase(); + return false; + }); + expect(reset).toBe(true); + + const observed: unknown[] = []; + for (const [suffix, selectedVault] of [ + ["", originalVault], + ["", otherVault], + ["other", originalVault], + ]) { + await select(suffix, selectedVault); + await start("ls", async (_options, { core }) => { + observed.push(await core.services.keyValueDB.kvDB.get("scope-probe")); + return false; + }); + } + expect(observed).toEqual([undefined, undefined, "stored"]); + }); + it("does not import vault files for standalone database commands", async () => { const storedPaths: string[] = []; const runner = vi.fn(async (_options, { core }) => { diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index cea0ec35..fcb66916 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -41,6 +41,14 @@ import { reconcileDurableSettings, settingsEqual, } from "./settingsPersistence"; +import { + inspectLegacyState, + isMutatingCommand, + isReconstructionCommand, + readOptionalFile, + ReconstructionState, + writeFileAtomically, +} from "./runtimeState"; const SETTINGS_FILE = ".livesync/settings.json"; @@ -381,7 +389,7 @@ export async function main( } // Resolve database path - const databasePath = path.resolve(options.databasePath!); + let databasePath = path.resolve(options.databasePath!); // Check if database directory exists try { const stat = await fs.stat(databasePath); @@ -389,18 +397,19 @@ export async function main( writeStderrLine(standardIo, `Error: ${databasePath} is not a directory`); process.exit(1); } + databasePath = await fs.realpath(databasePath); } catch { writeStderrLine(standardIo, `Error: Database directory ${databasePath} does not exist`); process.exit(1); } // Resolve settings path - const settingsPath = options.settingsPath + let settingsPath = options.settingsPath ? path.resolve(options.settingsPath) : path.join(databasePath, SETTINGS_FILE); // Resolve vault path: mirror positional argument takes priority, // then --vault flag, otherwise fall back to databasePath. - const vaultPath = + let vaultPath = options.command === "mirror" && options.commandArgs[0] ? path.resolve(options.commandArgs[0]) : options.vaultPath @@ -414,11 +423,22 @@ export async function main( writeStderrLine(standardIo, `Error: Vault path ${vaultPath} is not a directory`); process.exit(1); } + vaultPath = await fs.realpath(vaultPath); } catch { writeStderrLine(standardIo, `Error: Vault directory ${vaultPath} does not exist`); process.exit(1); } + settingsPath = await fs.realpath(settingsPath).catch(() => settingsPath); + const reconstruction = new ReconstructionState(databasePath, vaultPath, settingsPath); + try { + reconstruction.prepare(options.command, options.force ?? false, options.writeSettings ?? false); + } catch (error) { + writeStderrLine(standardIo, "[Error] Reconstruction recovery is required:", error); + process.exit(1); + return; + } + infoLog(`Self-hosted LiveSync CLI`); infoLog(`Database Path: ${databasePath}`); infoLog(`Vault Path: ${vaultPath}`); @@ -437,7 +457,7 @@ export async function main( } // Create service context and hub - const context = new NodeServiceContext(databasePath, standardIo); + const context = new NodeServiceContext(databasePath, standardIo, vaultPath); const serviceHubInstance = new NodeServiceHub(databasePath, context); serviceHubInstance.API.addLog.setHandler((message: unknown, level: LOG_LEVEL) => { let levelStr = ""; @@ -479,7 +499,7 @@ export async function main( // Setup settings handlers const settingService = serviceHubInstance.setting; - const originalSettingsText = await fs.readFile(settingsPath, "utf-8").catch(() => undefined); + const originalSettingsText = readOptionalFile(settingsPath); let latestPreparedSettingsText: string | undefined; let preparedSettingsRevision = 0; let commandIsRunning = false; @@ -550,7 +570,7 @@ export async function main( // Add target filter to prevent internal files are handled core.services.vault.isTargetFile.addHandler(async (target) => { const targetPath = stripAllPrefixes(getPathFromUXFileInfo(target)); - const parts = targetPath.split(path.sep); + const parts = targetPath.split("/"); // if some part of the path starts with dot, treat it as internal file and ignore. if (parts.some((part) => part.startsWith("."))) { return await Promise.resolve(false); @@ -600,21 +620,19 @@ export async function main( const writeSettingsAtomically = async (content: string | undefined): Promise => { if (content === undefined || content === originalSettingsText) return; - const temporaryPath = `${settingsPath}.${process.pid}.tmp`; - await fs.mkdir(path.dirname(settingsPath), { recursive: true }); - try { - await fs.writeFile(temporaryPath, content, "utf-8"); - await fs.rename(temporaryPath, settingsPath); - if (options.verbose) { - writeStderrLine(standardIo, `[Settings] Saved to ${settingsPath}`); - } - } catch (error) { - await fs.unlink(temporaryPath).catch(() => {}); - throw error; + writeFileAtomically(settingsPath, content); + if (options.verbose) { + writeStderrLine(standardIo, `[Settings] Saved to ${settingsPath}`); } }; // Start the core + let unloaded = false; + const unload = async () => { + if (unloaded) return; + unloaded = true; + await core.services.control.onUnload(); + }; try { infoLog(`[Starting] Initializing LiveSync...`); @@ -623,6 +641,31 @@ export async function main( writeStderrLine(standardIo, `[Error] Failed to initialize LiveSync`); process.exit(1); } + if ((await core.services.keyValueDB.kvDB.get("cli-legacy-state-reviewed")) !== true) { + const legacyIssues = inspectLegacyState(databasePath, vaultPath); + if (legacyIssues.length > 0) { + const reasons = { + unowned: "database and Vault owner is unknown", + invalid: "state cannot be parsed in its expected format; owner and contents cannot be established", + unreadable: "state cannot be read; owner and contents cannot be established", + }; + const details = legacyIssues.map((issue) => `${issue.path}: ${reasons[issue.reason]}`).join("; "); + const advice = + "Preserve these files, back up unique edits, and verify the authoritative source before choosing fetch --force or rebuild --force."; + const blocked = + !isReconstructionCommand(options.command) && + (isMutatingCommand(options.command) || options.writeSettings); + if (blocked) { + throw new Error( + `[Legacy state] ${details}. Command '${options.command}' stopped before scanning, event replay, or synchronisation. ${advice}` + ); + } + const action = isReconstructionCommand(options.command) + ? "Explicit reconstruction will proceed; the legacy files will be retained." + : "Read-only inspection may continue; the legacy files will be retained."; + writeStderrLine(standardIo, `[Legacy state] ${details}. ${action} ${advice}`); + } + } const settingsAfterLoadText = latestPreparedSettingsText ? preserveStoredSetting(latestPreparedSettingsText, originalSettingsText, "useIndexedDBAdapter") : originalSettingsText; @@ -655,7 +698,7 @@ export async function main( try { readyResult = await core.services.control.onReady(); } finally { - if (!readyResult) await core.services.control.onUnload(); + if (!readyResult) await unload(); } if (!readyResult) { throw new Error("Failed to initialise LiveSync."); @@ -751,6 +794,21 @@ export async function main( settingsTextToCommit = settingsAfterLoadText; } } + if (result && isReconstructionCommand(options.command)) { + if (!options.writeSettings) { + const durable = + originalSettingsText === undefined + ? cloneSettings(durableSettingsBeforeSuspend) + : (JSON.parse(originalSettingsText) as ObsidianLiveSyncSettings); + durable.additionalSuffixOfDatabaseName = + core.services.setting.currentSettings().additionalSuffixOfDatabaseName; + settingsTextToCommit = JSON.stringify(durable, null, 2); + } + settingsTextToCommit ??= originalSettingsText; + if (settingsTextToCommit === undefined) + throw new Error("Reconstruction completed without database selection settings."); + await core.services.keyValueDB.kvDB.set("cli-legacy-state-reviewed", true); + } if (!result) { writeStderrLine(standardIo, `[Error] Command '${options.command}' failed`); @@ -764,10 +822,16 @@ export async function main( // Keep the process running await new Promise(() => {}); } else { - await core.services.control.onUnload(); - await writeSettingsAtomically(settingsTextToCommit); + await unload(); + if (result && isReconstructionCommand(options.command)) { + reconstruction.complete(settingsTextToCommit!); + reconstruction.commit(); + } else { + await writeSettingsAtomically(settingsTextToCommit); + } } } catch (error) { + await unload().catch(() => {}); writeStderrLine(standardIo, `[Error] Failed to start:`, error); process.exit(1); } diff --git a/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts b/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts index e6369fce..94e26165 100644 --- a/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts +++ b/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts @@ -2,7 +2,6 @@ import type { FilePath, UXFileInfoStub, UXInternalFileInfoStub } from "@vrtmrz/l import type { FileEventItem } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { IStorageEventManagerAdapter } from "@vrtmrz/livesync-commonlib/compat/managers/adapters"; import type { - IStorageEventTypeGuardAdapter, IStorageEventPersistenceAdapter, IStorageEventWatchAdapter, IStorageEventStatusAdapter, @@ -15,47 +14,26 @@ import { watch as chokidarWatch, type FSWatcher } from "chokidar"; import type { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules"; import { fsPromises as fs, getNodeFileCreationTime, path, type Stats } from "@vrtmrz/livesync-commonlib/node"; import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput"; - -/** - * CLI-specific type guard adapter - */ -class CLITypeGuardAdapter implements IStorageEventTypeGuardAdapter { - isFile(file: unknown): file is NodeFile { - return !!( - file && - typeof file === "object" && - "path" in file && - "stat" in file && - !(file as { isFolder?: boolean }).isFolder - ); - } - - isFolder(item: unknown): item is NodeFolder { - return !!( - item && - typeof item === "object" && - "path" in item && - (item as { isFolder?: boolean }).isFolder === true - ); - } -} +import { NodeTypeGuardAdapter } from "@/apps/cli/adapters/NodeTypeGuardAdapter"; +import { writeFileAtomically } from "@/apps/cli/runtimeState"; /** * CLI-specific persistence adapter (file-based snapshot) */ class CLIPersistenceAdapter implements IStorageEventPersistenceAdapter { - private snapshotPath: string; + private snapshotPath: () => string; constructor( basePath: string, - private reportDiagnostic: CliDiagnosticReporter = () => undefined + private reportDiagnostic: CliDiagnosticReporter = () => undefined, + snapshotPath?: () => string ) { - this.snapshotPath = path.join(basePath, ".livesync-snapshot.json"); + this.snapshotPath = snapshotPath ?? (() => path.join(basePath, ".livesync-snapshot.json")); } async saveSnapshot(snapshot: (FileEventItem | FileEventItemSentinel)[]): Promise { try { - await fs.writeFile(this.snapshotPath, JSON.stringify(snapshot, null, 2), "utf-8"); + writeFileAtomically(this.snapshotPath(), JSON.stringify(snapshot, null, 2)); } catch (error) { this.reportDiagnostic("Failed to save snapshot:", error); } @@ -63,7 +41,7 @@ class CLIPersistenceAdapter implements IStorageEventPersistenceAdapter { async loadSnapshot(): Promise<(FileEventItem | FileEventItemSentinel)[] | null> { try { - const content = await fs.readFile(this.snapshotPath, "utf-8"); + const content = await fs.readFile(this.snapshotPath(), "utf-8"); return JSON.parse(content) as (FileEventItem | FileEventItemSentinel)[]; } catch { return null; @@ -211,7 +189,7 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter { * Composite adapter for CLI StorageEventManager */ export class CLIStorageEventManagerAdapter implements IStorageEventManagerAdapter { - readonly typeGuard: CLITypeGuardAdapter; + readonly typeGuard: NodeTypeGuardAdapter; readonly persistence: CLIPersistenceAdapter; readonly watch: CLIWatchAdapter; readonly status: CLIStatusAdapter; @@ -221,10 +199,11 @@ export class CLIStorageEventManagerAdapter implements IStorageEventManagerAdapte basePath: string, ignoreRules?: IgnoreRules, watchEnabled: boolean = false, - reportDiagnostic: CliDiagnosticReporter = () => undefined + reportDiagnostic: CliDiagnosticReporter = () => undefined, + snapshotPath?: () => string ) { - this.typeGuard = new CLITypeGuardAdapter(); - this.persistence = new CLIPersistenceAdapter(basePath, reportDiagnostic); + this.typeGuard = new NodeTypeGuardAdapter(); + this.persistence = new CLIPersistenceAdapter(basePath, reportDiagnostic, snapshotPath); this.watch = new CLIWatchAdapter(basePath, ignoreRules, watchEnabled, reportDiagnostic); this.status = new CLIStatusAdapter(); this.converter = new CLIConverterAdapter(); diff --git a/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts b/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts index bf8f6712..9437e74b 100644 --- a/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts +++ b/src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts @@ -1,6 +1,8 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; import type { IStorageEventWatchHandlers } from "@vrtmrz/livesync-commonlib/compat/managers/adapters"; import type { NodeFile } from "@/apps/cli/adapters/NodeTypes"; +import { fs, os, path } from "@vrtmrz/livesync-commonlib/node"; +import { runtimeScopeDirectory } from "../runtimeState"; // ── chokidar mock ────────────────────────────────────────────────────────────── // Must be hoisted before imports that pull in chokidar. @@ -44,6 +46,27 @@ describe("CLIStorageEventManagerAdapter", () => { }); }); + it("keeps pending snapshots with the selected database and Vault pair", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-snapshot-")); + let database = "first"; + let vault = "vault-a"; + const adapter = new CLIStorageEventManagerAdapter(directory, undefined, false, undefined, () => + path.join(runtimeScopeDirectory(directory, database, vault), "storage-events.json") + ); + try { + await adapter.persistence.saveSnapshot([{ type: "SENTINEL_FLUSH" }]); + database = "second"; + expect(await adapter.persistence.loadSnapshot()).toBeNull(); + database = "first"; + vault = "vault-b"; + expect(await adapter.persistence.loadSnapshot()).toBeNull(); + vault = "vault-a"; + expect(await adapter.persistence.loadSnapshot()).toEqual([{ type: "SENTINEL_FLUSH" }]); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + it("beginWatch is no-op when watchEnabled=false", async () => { const adapter = new CLIStorageEventManagerAdapter("/base", undefined, false); const handlers = makeHandlers(); @@ -201,3 +224,27 @@ describe("CLIStorageEventManagerAdapter", () => { processExitSpy.mockRestore(); }); }); + +describe("CLIStorageEventManagerAdapter type guard", () => { + const typeGuard = new CLIStorageEventManagerAdapter("/base").typeGuard; + + it.each([ + { label: "null", value: null, expected: false }, + { label: "an object without a path", value: { stat: {} }, expected: false }, + { label: "an object without a stat", value: { path: "note.md" }, expected: false }, + { label: "a file-shaped object", value: { path: "note.md", stat: {} }, expected: true }, + { label: "a folder-shaped object", value: { path: "folder", isFolder: true }, expected: false }, + ])("classifies $label as a file", ({ value, expected }) => { + expect(typeGuard.isFile(value)).toBe(expected); + }); + + it.each([ + { label: "null", value: null, expected: false }, + { label: "an object without a path", value: { isFolder: true }, expected: false }, + { label: "an object without the folder marker", value: { path: "folder" }, expected: false }, + { label: "a folder-shaped object", value: { path: "folder", isFolder: true }, expected: true }, + { label: "a file-shaped object", value: { path: "note.md", stat: {} }, expected: false }, + ])("classifies $label as a folder", ({ value, expected }) => { + expect(typeGuard.isFolder(value)).toBe(expected); + }); +}); diff --git a/src/apps/cli/managers/StorageEventManagerCLI.ts b/src/apps/cli/managers/StorageEventManagerCLI.ts index e96fdeee..90bba6af 100644 --- a/src/apps/cli/managers/StorageEventManagerCLI.ts +++ b/src/apps/cli/managers/StorageEventManagerCLI.ts @@ -1,9 +1,15 @@ -import { StorageEventManagerBase, type StorageEventManagerBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/managers/StorageEventManager"; +import { + StorageEventManagerBase, + type StorageEventManagerBaseDependencies, +} from "@vrtmrz/livesync-commonlib/compat/managers/StorageEventManager"; import { CLIStorageEventManagerAdapter } from "./CLIStorageEventManagerAdapter"; import type { IMinimumLiveSyncCommands, LiveSyncBaseCore } from "@/LiveSyncBaseCore"; import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; import type { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules"; import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger"; +import { runtimeScopeDirectory } from "@/apps/cli/runtimeState"; +import type { NodeServiceContext } from "@/apps/cli/services/NodeServiceContext"; +import { path as nodePath } from "@vrtmrz/livesync-commonlib/node"; // import type { IMinimumLiveSyncCommands } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; export class StorageEventManagerCLI extends StorageEventManagerBase { @@ -16,12 +22,23 @@ export class StorageEventManagerCLI extends StorageEventManagerBase { - dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE); - if (detail !== undefined) { - dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE); - } - }); + const context = core.services.context as NodeServiceContext; + const adapter = new CLIStorageEventManagerAdapter( + basePath, + ignoreRules, + watchEnabled, + (message, detail) => { + dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE); + if (detail !== undefined) { + dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE); + } + }, + () => + nodePath.join( + runtimeScopeDirectory(context.databasePath, dependencies.vaultService.getVaultName(), basePath), + "storage-events.json" + ) + ); super(adapter, dependencies); this.core = core; } diff --git a/src/apps/cli/package.json b/src/apps/cli/package.json index 43e21f1c..0cbdd4dc 100644 --- a/src/apps/cli/package.json +++ b/src/apps/cli/package.json @@ -1,7 +1,7 @@ { "name": "self-hosted-livesync-cli", "private": true, - "version": "1.0.42-cli", + "version": "1.0.44-cli", "main": "dist/index.cjs", "type": "module", "scripts": { @@ -12,7 +12,7 @@ "buildRun": "npm run build && npm run cli --", "build:docker": "docker build -f Dockerfile -t livesync-cli ../../..", "check": "tsc -p tsconfig.json", - "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", + "test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/main.bootstrap.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/runtimeState.unit.spec.ts src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts src/apps/cli/services/NodeLocalStorage.unit.spec.ts src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/daemonCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts src/apps/cli/deploy/install.unit.spec.ts src/apps/cli/adapters/NodeFileSystemAdapter.unit.spec.ts", "test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh", "test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh", "test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh", @@ -20,24 +20,27 @@ "test:e2e:setup-put-cat": "bash test/test-setup-put-cat-linux.sh", "test:e2e:sync-two-local": "bash test/test-sync-two-local-databases-linux.sh", "pretest:e2e:ci": "npm run build", - "test:e2e:ci": "deno task --cwd testdeno test:ci", - "test:e2e:p2p": "deno task --cwd testdeno test:p2p:compose", + "test:e2e:ci": "node ../../../test/e2e-cli/run-ci-suite.ts", + "test:e2e:p2p": "node ../../../test/e2e-cli/run-compose-p2p.ts", "test:e2e:mirror": "bash test/test-mirror-linux.sh", "test:e2e:remote-commands": "bash test/test-remote-commands-linux.sh", "pretest:e2e:all": "npm run build", - "test:e2e:all": "deno task --cwd testdeno test:ci && deno task --cwd testdeno test:p2p:compose", + "test:e2e:all": "node ../../../test/e2e-cli/run-ci-suite.ts && node ../../../test/e2e-cli/run-compose-p2p.ts", "pretest:e2e:docker:all": "npm run build:docker", "test:e2e:docker:push-pull": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-push-pull-linux.sh", "test:e2e:docker:setup-put-cat": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-setup-put-cat-linux.sh", "test:e2e:docker:mirror": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-mirror-linux.sh", "test:e2e:docker:remote-commands": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-remote-commands-linux.sh", "test:e2e:docker:sync-two-local": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-sync-two-local-databases-linux.sh", - "test:e2e:docker:all": "export RUN_BUILD=0 && npm run test:e2e:docker:setup-put-cat && npm run test:e2e:docker:push-pull && npm run test:e2e:docker:sync-two-local && npm run test:e2e:docker:mirror && npm run test:e2e:docker:remote-commands" + "test:e2e:docker:all": "export RUN_BUILD=0 && npm run test:e2e:docker:setup-put-cat && npm run test:e2e:docker:push-pull && npm run test:e2e:docker:sync-two-local && npm run test:e2e:docker:mirror && npm run test:e2e:docker:remote-commands", + "test:e2e:task": "node ../../../test/e2e-cli/run-task.ts", + "test:e2e:coverage": "node ../../../test/e2e-cli/run-coverage.ts" }, "dependencies": { + "@aws-crypto/sha256-js": "5.2.0", "chokidar": "^4.0.0", "minimatch": "^10.2.5", - "octagonal-wheels": "^0.1.55", + "octagonal-wheels": "0.1.56", "pouchdb-adapter-http": "^9.0.0", "pouchdb-adapter-leveldb": "^9.0.0", "pouchdb-core": "^9.0.0", diff --git a/src/apps/cli/runtimeState.ts b/src/apps/cli/runtimeState.ts new file mode 100644 index 00000000..4c1564d5 --- /dev/null +++ b/src/apps/cli/runtimeState.ts @@ -0,0 +1,222 @@ +// This module belongs exclusively to the Node.js CLI host. +import { Sha256 } from "@aws-crypto/sha256-js"; +import { fs, path } from "@vrtmrz/livesync-commonlib/node"; +import type { CLICommand } from "./commands/types"; + +function sha256Hex(value: string): string { + const hash = new Sha256(); + hash.update(new TextEncoder().encode(value)); + return Array.from(hash.digestSync(), (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +export function runtimeScopeDirectory(databasePath: string, databaseName: string, vaultPath: string): string { + const scope = sha256Hex(vaultPath); + return path.join(runtimeDatabaseDirectory(databasePath, databaseName), scope); +} + +export function runtimeDatabaseDirectory(databasePath: string, databaseName: string): string { + const database = sha256Hex(databaseName); + return path.join(databasePath, ".livesync", "runtime", "scopes", database); +} + +export function writeFileAtomically(filePath: string, content: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const temporaryPath = `${filePath}.${process.pid}.${crypto.randomUUID()}.tmp`; + try { + const descriptor = fs.openSync(temporaryPath, "wx", 0o600); + try { + fs.writeFileSync(descriptor, content, "utf-8"); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + fs.renameSync(temporaryPath, filePath); + syncDirectory(path.dirname(filePath)); + } finally { + fs.rmSync(temporaryPath, { force: true }); + } +} + +function syncDirectory(directory: string): void { + // Windows does not support opening a directory for fsync. + if (process.platform === "win32") return; + const descriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +export function readOptionalFile(filePath: string): string | undefined { + try { + return fs.readFileSync(filePath, "utf-8"); + } catch (error) { + if ((error as { code?: string }).code === "ENOENT") return undefined; + throw error; + } +} + +export function isReconstructionCommand(command: CLICommand): command is "fetch" | "rebuild" { + return command === "fetch" || command === "rebuild"; +} + +const READ_COMMANDS = new Set([ + "cat", + "cat-rev", + "ls", + "info", + "remote-ls", + "remote-export", + "remote-status", + "p2p-peers", +]); + +export function isMutatingCommand(command: CLICommand): boolean { + return !READ_COMMANDS.has(command); +} + +interface ReconstructionMarker { + version: 1; + command: "fetch" | "rebuild"; + databasePath: string; + vaultPath: string; + settingsPath: string; + settingsBeforeHash: string; + phase: "running" | "completed"; + settingsText?: string; +} + +function hashSettings(content: string | undefined): string { + return sha256Hex(JSON.stringify(content ?? null)); +} + +export class ReconstructionState { + readonly markerPath: string; + private marker: ReconstructionMarker | undefined; + + constructor( + private readonly databasePath: string, + private readonly vaultPath: string, + private readonly settingsPath: string + ) { + this.markerPath = path.join(databasePath, ".livesync", "reconstruction.json"); + } + + prepare(command: CLICommand, force: boolean, writeSettings = false): void { + const contents = readOptionalFile(this.markerPath); + if (contents !== undefined) { + const parsed: unknown = JSON.parse(contents); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new Error(`Invalid reconstruction state: ${this.markerPath}. Preserve it for recovery.`); + } + const marker = parsed as ReconstructionMarker; + if ( + marker.version !== 1 || + !isReconstructionCommand(marker.command) || + (marker.phase !== "running" && marker.phase !== "completed") || + typeof marker.settingsPath !== "string" || + typeof marker.vaultPath !== "string" || + typeof marker.settingsBeforeHash !== "string" || + marker.databasePath !== this.databasePath || + (marker.phase === "completed" && typeof marker.settingsText !== "string") + ) { + throw new Error(`Invalid reconstruction state: ${this.markerPath}. Preserve it for recovery.`); + } + this.marker = marker; + if (marker.phase === "completed") { + this.commit(); + } else if ( + (isMutatingCommand(command) || writeSettings) && + !( + command === marker.command && + force && + marker.vaultPath === this.vaultPath && + marker.settingsPath === this.settingsPath + ) + ) { + throw new Error( + `An interrupted ${marker.command} requires recovery. Retry ${marker.command} --force with the same database, Vault, and settings paths. State: ${this.markerPath}` + ); + } + } + if (isReconstructionCommand(command)) { + this.marker = { + version: 1, + command, + databasePath: this.databasePath, + vaultPath: this.vaultPath, + settingsPath: this.settingsPath, + phase: "running", + settingsBeforeHash: hashSettings(readOptionalFile(this.settingsPath)), + }; + this.save(); + } + } + + complete(settingsText: string): void { + if (!this.marker || this.marker.phase !== "running") throw new Error("Reconstruction was not started."); + this.marker = { ...this.marker, phase: "completed", settingsText }; + this.save(); + } + + commit(): void { + const marker = this.marker; + if (!marker || marker.phase !== "completed" || marker.settingsText === undefined) { + throw new Error("Reconstruction has no completed settings update."); + } + const current = readOptionalFile(marker.settingsPath); + if (current !== marker.settingsText && hashSettings(current) !== marker.settingsBeforeHash) { + throw new Error( + `The settings file changed after reconstruction: ${marker.settingsPath}. Preserve the reconstruction state and review both settings versions before continuing.` + ); + } + if (current !== marker.settingsText) writeFileAtomically(marker.settingsPath, marker.settingsText); + this.clear(); + } + + clear(): void { + fs.rmSync(this.markerPath, { force: true }); + syncDirectory(path.dirname(this.markerPath)); + this.marker = undefined; + } + + private save(): void { + writeFileAtomically(this.markerPath, JSON.stringify(this.marker, null, 2)); + } +} + +export interface LegacyStateIssue { + path: string; + reason: "unowned" | "invalid" | "unreadable"; +} + +export function inspectLegacyState(databasePath: string, vaultPath: string): LegacyStateIssue[] { + const issues: LegacyStateIssue[] = []; + for (const [filePath, expectArray] of [ + [path.join(databasePath, ".livesync", "runtime", "keyvalue-db.json"), false], + [path.join(vaultPath, ".livesync-snapshot.json"), true], + ] as const) { + let contents: string | undefined; + try { + contents = readOptionalFile(filePath); + } catch { + issues.push({ path: filePath, reason: "unreadable" }); + continue; + } + if (contents === undefined) continue; + let value: unknown; + try { + value = JSON.parse(contents); + } catch { + issues.push({ path: filePath, reason: "invalid" }); + continue; + } + if (value === null || typeof value !== "object" || Array.isArray(value) !== expectArray) { + issues.push({ path: filePath, reason: "invalid" }); + } else if (Object.keys(value).length > 0) { + issues.push({ path: filePath, reason: "unowned" }); + } + } + return issues; +} diff --git a/src/apps/cli/runtimeState.unit.spec.ts b/src/apps/cli/runtimeState.unit.spec.ts new file mode 100644 index 00000000..b6c1e561 --- /dev/null +++ b/src/apps/cli/runtimeState.unit.spec.ts @@ -0,0 +1,160 @@ +import { fs, os, path } from "@vrtmrz/livesync-commonlib/node"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + ReconstructionState, + inspectLegacyState, + readOptionalFile, + runtimeDatabaseDirectory, + runtimeScopeDirectory, +} from "./runtimeState"; + +describe("CLI runtime scope compatibility", () => { + it("retains SHA-256 directory names for database and Unicode Vault paths", () => { + expect(runtimeScopeDirectory("state", "headless-app", "/vault/日本語😀")).toBe( + path.join( + "state", + ".livesync", + "runtime", + "scopes", + "511afda5613706862fa4add801b3f9b19e0db2acacc07b9218f2d01ff41ca61d", + "542d31d6918ad8bf915e204e41e479fdbd7251690f182fd802378323b4c6de23" + ) + ); + expect(path.basename(runtimeDatabaseDirectory("state", ""))).toBe( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + }); +}); + +describe("CLI reconstruction state", () => { + let directory: string; + let settingsPath: string; + let state: ReconstructionState; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-reconstruction-")); + settingsPath = path.join(directory, "settings.json"); + fs.writeFileSync(settingsPath, JSON.stringify({ additionalSuffixOfDatabaseName: "original" })); + state = new ReconstructionState(directory, path.join(directory, "vault"), settingsPath); + }); + + afterEach(() => fs.rmSync(directory, { recursive: true, force: true })); + + it("retains the persisted settings fingerprint across hash implementations", () => { + state.prepare("fetch", true); + expect(JSON.parse(fs.readFileSync(state.markerPath, "utf-8"))).toMatchObject({ + settingsBeforeHash: "29e04e3dd1d10b352678a922ff5a03fbe529d3505e9cf46c2ecc860f44478ae8", + }); + }); + + it("persists progress before returning and permits only the matching forced retry", () => { + state.prepare("fetch", true); + expect(JSON.parse(fs.readFileSync(state.markerPath, "utf-8"))).toMatchObject({ + phase: "running", + command: "fetch", + }); + if (process.platform !== "win32") expect(fs.statSync(state.markerPath).mode & 0o777).toBe(0o600); + expect(() => state.prepare("sync", false)).toThrow("interrupted fetch"); + expect(() => state.prepare("ls", false, true)).toThrow("interrupted fetch"); + expect(() => state.prepare("fetch", false)).toThrow("interrupted fetch"); + expect(() => state.prepare("rebuild", true)).toThrow("interrupted fetch"); + expect(() => + new ReconstructionState(directory, path.join(directory, "other"), settingsPath).prepare("fetch", true) + ).toThrow("interrupted fetch"); + expect(() => + new ReconstructionState( + directory, + path.join(directory, "vault"), + path.join(directory, "other.json") + ).prepare("fetch", true) + ).toThrow("interrupted fetch"); + expect(() => state.prepare("ls", false)).not.toThrow(); + expect(() => state.prepare("fetch", true)).not.toThrow(); + }); + + it("recovers completed settings before permitting another command", () => { + state.prepare("fetch", true); + const completed = JSON.stringify({ additionalSuffixOfDatabaseName: "headless-app", retained: true }); + state.complete(completed); + + new ReconstructionState(directory, path.join(directory, "vault"), settingsPath).prepare("sync", false); + + expect(fs.readFileSync(settingsPath, "utf-8")).toBe(completed); + expect(readOptionalFile(state.markerPath)).toBeUndefined(); + }); + + it.each(["normal completion", "restart"] as const)( + "preserves user settings changed during reconstruction at %s", + (phase) => { + state.prepare("rebuild", true); + state.complete(JSON.stringify({ additionalSuffixOfDatabaseName: "headless-app" })); + const edited = JSON.stringify({ additionalSuffixOfDatabaseName: "user-selection", unique: "edit" }); + fs.writeFileSync(settingsPath, edited); + + const commit = + phase === "restart" + ? () => + new ReconstructionState(directory, path.join(directory, "vault"), settingsPath).prepare( + "sync", + false + ) + : () => state.commit(); + + expect(commit).toThrow("settings file changed"); + expect(fs.readFileSync(settingsPath, "utf-8")).toBe(edited); + expect(JSON.parse(fs.readFileSync(state.markerPath, "utf-8"))).toMatchObject({ phase: "completed" }); + } + ); + + it("clears a completed marker after an interruption following the settings rename", () => { + state.prepare("fetch", true); + const completed = JSON.stringify({ additionalSuffixOfDatabaseName: "headless-app" }); + state.complete(completed); + fs.writeFileSync(settingsPath, completed); + state.prepare("ls", false); + expect(readOptionalFile(state.markerPath)).toBeUndefined(); + }); + + it.each(["null", "[]", "{}", '{"version":2}', '{"version":1,"phase":"completed"}'])( + "refuses malformed state %s without clearing it", + (contents) => { + fs.mkdirSync(path.dirname(state.markerPath), { recursive: true }); + fs.writeFileSync(state.markerPath, contents); + expect(() => state.prepare("sync", false)).toThrow("Invalid reconstruction state"); + expect(fs.readFileSync(state.markerPath, "utf-8")).toBe(contents); + } + ); + + it.each(["null", "[]", '"text"', "{", ""])("reports invalid legacy KV %s without changing it", (contents) => { + const legacy = path.join(directory, ".livesync/runtime/keyvalue-db.json"); + fs.mkdirSync(path.dirname(legacy), { recursive: true }); + fs.writeFileSync(legacy, contents); + expect(inspectLegacyState(directory, directory)).toEqual([{ path: legacy, reason: "invalid" }]); + expect(fs.readFileSync(legacy, "utf-8")).toBe(contents); + }); + + it("reports both unreadable KV and invalid snapshot instead of hiding the second problem", () => { + const legacy = path.join(directory, ".livesync/runtime/keyvalue-db.json"); + const snapshot = path.join(directory, ".livesync-snapshot.json"); + fs.mkdirSync(legacy, { recursive: true }); + fs.writeFileSync(snapshot, "{}"); + expect(inspectLegacyState(directory, directory)).toEqual([ + { path: legacy, reason: "unreadable" }, + { path: snapshot, reason: "invalid" }, + ]); + expect(fs.statSync(legacy).isDirectory()).toBe(true); + expect(fs.readFileSync(snapshot, "utf-8")).toBe("{}"); + }); + + it("preserves non-empty unowned legacy state and ignores empty legacy containers", () => { + const legacy = path.join(directory, ".livesync", "runtime", "keyvalue-db.json"); + fs.mkdirSync(path.dirname(legacy), { recursive: true }); + fs.writeFileSync(legacy, "{}"); + fs.writeFileSync(path.join(directory, ".livesync-snapshot.json"), "[]"); + expect(inspectLegacyState(directory, directory)).toEqual([]); + const queued = JSON.stringify({ "queued-files": ["unique-edit.md"] }); + fs.writeFileSync(legacy, queued); + expect(inspectLegacyState(directory, directory)).toEqual([{ path: legacy, reason: "unowned" }]); + expect(fs.readFileSync(legacy, "utf-8")).toBe(queued); + }); +}); diff --git a/src/apps/cli/serviceModules/CLIServiceModules.ts b/src/apps/cli/serviceModules/CLIServiceModules.ts index 29b9df90..53cdf12f 100644 --- a/src/apps/cli/serviceModules/CLIServiceModules.ts +++ b/src/apps/cli/serviceModules/CLIServiceModules.ts @@ -1,6 +1,4 @@ import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub"; -import { ServiceRebuilder } from "@vrtmrz/livesync-commonlib/compat/serviceModules/Rebuilder"; -import { ServiceFileHandler } from "@/serviceModules/FileHandler"; import { StorageAccessManager } from "@vrtmrz/livesync-commonlib/compat/managers/StorageProcessingManager"; import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; @@ -10,7 +8,7 @@ import { ServiceDatabaseFileAccessCLI } from "./DatabaseFileAccess"; import { StorageEventManagerCLI } from "@/apps/cli/managers/StorageEventManagerCLI"; import type { ServiceModules } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; import type { IgnoreRules } from "./IgnoreRules"; -import { createFileReflectionProvenance } from "@/serviceModules/FileReflectionProvenance"; +import { composeSharedServiceModules } from "@/apps/serviceModules/composeSharedServiceModules"; /** * Initialize service modules for CLI version @@ -82,40 +80,7 @@ export function initialiseServiceModulesCLI( vault: services.vault, }); - // File handler (platform-independent) - const fileHandler = new ServiceFileHandler({ - events: services.context.events, - API: services.API, - databaseFileAccess: databaseFileAccess, - conflict: services.conflict, - setting: services.setting, - fileProcessing: services.fileProcessing, - vault: services.vault, - path: services.path, - replication: services.replication, - storageAccess: storageAccess, - fileReflectionProvenance: createFileReflectionProvenance(services.keyValueDB), - }); - - // Rebuilder (platform-independent) - const rebuilder = new ServiceRebuilder({ - path: services.path, - events: services.context.events, - API: services.API, - database: services.database, - appLifecycle: services.appLifecycle, - setting: services.setting, - remote: services.remote, - databaseEvents: services.databaseEvents, - replication: services.replication, - replicator: services.replicator, - UI: services.UI, - vault: services.vault, - fileHandler: fileHandler, - fileProcessing: services.fileProcessing, - storageAccess: storageAccess, - control: services.control, - }); + const { fileHandler, rebuilder } = composeSharedServiceModules(services, databaseFileAccess, storageAccess); return { rebuilder, diff --git a/src/apps/cli/services/NodeKeyValueDBService.ts b/src/apps/cli/services/NodeKeyValueDBService.ts index 421fa0f5..759b3da8 100644 --- a/src/apps/cli/services/NodeKeyValueDBService.ts +++ b/src/apps/cli/services/NodeKeyValueDBService.ts @@ -9,6 +9,7 @@ import type { IVaultService } from "@vrtmrz/livesync-commonlib/compat/services/b import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase"; import { createInstanceLogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; import { fs as nodeFs, path as nodePath } from "@vrtmrz/livesync-commonlib/node"; +import { readOptionalFile, writeFileAtomically } from "@/apps/cli/runtimeState"; const NODE_KV_TYPED_KEY = "__nodeKvType"; const NODE_KV_VALUES_KEY = "values"; @@ -103,12 +104,11 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase { } private load() { - try { - const loaded = JSON.parse(nodeFs.readFileSync(this.filePath, "utf-8")) as Record; - this.data = new Map(Object.entries(loaded).map(([key, value]) => [key, deserializeFromNodeKV(value)])); - } catch { - this.data = new Map(); - } + const contents = readOptionalFile(this.filePath); + if (contents === undefined) return; + const loaded: unknown = JSON.parse(contents); + if (!isRecord(loaded) || Array.isArray(loaded)) throw new Error("Invalid Node key-value database"); + this.data = new Map(Object.entries(loaded).map(([key, value]) => [key, deserializeFromNodeKV(value)])); } private flush() { @@ -116,7 +116,7 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase { const serializable = Object.fromEntries( [...this.data.entries()].map(([key, value]) => [key, serializeForNodeKV(value)]) ); - nodeFs.writeFileSync(this.filePath, JSON.stringify(serializable, null, 2), "utf-8"); + writeFileAtomically(this.filePath, JSON.stringify(serializable, null, 2)); } async get(key: IDBValidKey): Promise { @@ -163,9 +163,7 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase { return filtered; } - async close(): Promise { - this.flush(); - } + async close(): Promise {} async destroy(): Promise { this.data.clear(); @@ -185,7 +183,8 @@ export class NodeKeyValueDBService { private _kvDB: KeyValueDatabase | undefined; private _simpleStore: SimpleStore | undefined; - private filePath: string; + private filePath: string | (() => string); + private openedFilePath: string | undefined; private _log = createInstanceLogFunction("NodeKeyValueDBService"); get simpleStore() { @@ -202,7 +201,7 @@ export class NodeKeyValueDBService return this._kvDB; } - constructor(context: T, dependencies: NodeKeyValueDBDependencies, filePath: string) { + constructor(context: T, dependencies: NodeKeyValueDBDependencies, filePath: string | (() => string)) { super(context); this.filePath = filePath; @@ -215,7 +214,13 @@ export class NodeKeyValueDBService private async openKeyValueDB(): Promise { try { - this._kvDB = new NodeFileKeyValueDatabase(this.filePath); + const filePath = typeof this.filePath === "string" ? this.filePath : this.filePath(); + if (this._kvDB && this.openedFilePath === filePath) return true; + await this._kvDB?.close(); + this._kvDB = undefined; + this.openedFilePath = undefined; + this._kvDB = new NodeFileKeyValueDatabase(filePath); + this.openedFilePath = filePath; return true; } catch (ex) { this._log("Failed to open Node key-value database", LOG_LEVEL_NOTICE); @@ -226,8 +231,10 @@ export class NodeKeyValueDBService private async _everyOnResetDatabase(): Promise { try { - await this._kvDB?.del("queued-files"); + if (!(await this.openKeyValueDB())) return false; await this._kvDB?.destroy(); + this._kvDB = undefined; + this.openedFilePath = undefined; return await this.openKeyValueDB(); } catch (ex) { this._log("Failed to reset Node key-value database", LOG_LEVEL_NOTICE); diff --git a/src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts b/src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts index a8d3d1d4..a938e620 100644 --- a/src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts +++ b/src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts @@ -1,4 +1,5 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { fs, os, path } from "@vrtmrz/livesync-commonlib/node"; import { createServiceContext } from "@vrtmrz/livesync-commonlib/compat/services/base/ServiceBase"; import type { NodeKeyValueDBDependencies } from "./NodeKeyValueDBService"; import { NodeKeyValueDBService } from "./NodeKeyValueDBService"; @@ -45,3 +46,70 @@ describe("NodeKeyValueDBService.openSimpleStore", () => { await expect(store.get("key")).rejects.toThrow("KeyValueDB is not initialized yet"); }); }); + +describe("NodeKeyValueDBService persistence", () => { + let directory: string; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "livesync-node-kv-")); + }); + afterEach(() => fs.rmSync(directory, { recursive: true, force: true })); + + function compose(filePath: string | (() => string)) { + const handlers: Record Promise> = {}; + const event = (name: string) => ({ + addHandler: (handler: () => Promise) => { + handlers[name] = handler; + }, + }); + const dependencies = { + appLifecycle: { onSettingLoaded: event("load") }, + databaseEvents: { + onResetDatabase: event("reset"), + onDatabaseInitialisation: event("initialise"), + onUnloadDatabase: event("unload"), + onCloseDatabase: event("close"), + }, + vault: {}, + } as unknown as NodeKeyValueDBDependencies; + return { service: new NodeKeyValueDBService(createServiceContext(), dependencies, filePath), handlers }; + } + + it("retargets existing SimpleStore handles after database selection and resets only the selected file", async () => { + let selected = path.join(directory, "first.json"); + const { service, handlers } = compose(() => selected); + const store = service.openSimpleStore("journal"); + expect(await handlers.load()).toBe(true); + await store.set("checkpoint", new Set(["first"])); + selected = path.join(directory, "second.json"); + expect(await handlers.initialise()).toBe(true); + expect(await store.get("checkpoint")).toBeUndefined(); + await store.set("checkpoint", "second"); + expect(await handlers.reset()).toBe(true); + expect(await store.get("checkpoint")).toBeUndefined(); + selected = path.join(directory, "first.json"); + expect(await handlers.initialise()).toBe(true); + expect(await store.get("checkpoint")).toEqual(new Set(["first"])); + }); + + it("does not overwrite newer data when a stale handle closes", async () => { + const filePath = path.join(directory, "state.json"); + const first = compose(filePath); + await first.handlers.load(); + await first.service.kvDB.set("queued-files", ["original.md"]); + const second = compose(filePath); + await second.handlers.load(); + await first.service.kvDB.set("queued-files", ["unique-edit.md"]); + await second.handlers.close(); + expect(JSON.parse(fs.readFileSync(filePath, "utf-8"))["queued-files"]).toEqual(["unique-edit.md"]); + }); + + it.each(["{", "null", "[]"])("refuses corrupt state %s without replacing it", async (contents) => { + const filePath = path.join(directory, "state.json"); + fs.writeFileSync(filePath, contents); + const { service, handlers } = compose(filePath); + expect(await handlers.load()).toBe(false); + expect(() => service.kvDB).toThrow("not initialized"); + expect(fs.readFileSync(filePath, "utf-8")).toBe(contents); + }); +}); diff --git a/src/apps/cli/services/NodeServiceContext.ts b/src/apps/cli/services/NodeServiceContext.ts index fef43d91..31420180 100644 --- a/src/apps/cli/services/NodeServiceContext.ts +++ b/src/apps/cli/services/NodeServiceContext.ts @@ -6,7 +6,8 @@ import { ServiceContext, type StandardIo } from "@vrtmrz/livesync-commonlib/cont export class NodeServiceContext extends ServiceContext { constructor( readonly databasePath: string, - readonly standardIo: StandardIo + readonly standardIo: StandardIo, + readonly runtimeVaultPath: string = databasePath ) { super({ events: eventHub, translate: translateLiveSyncMessage }); } diff --git a/src/apps/cli/services/NodeServiceHub.ts b/src/apps/cli/services/NodeServiceHub.ts index 484861c2..ad436209 100644 --- a/src/apps/cli/services/NodeServiceHub.ts +++ b/src/apps/cli/services/NodeServiceHub.ts @@ -24,11 +24,14 @@ import { NodeKeyValueDBService } from "./NodeKeyValueDBService"; import { createNodeLocalStorage } from "./NodeLocalStorage"; import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService"; import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; -import { path as nodePath } from "@vrtmrz/livesync-commonlib/node"; +import { fs, path as nodePath } from "@vrtmrz/livesync-commonlib/node"; import type { KeyValueDBService } from "@vrtmrz/livesync-commonlib/compat/services/base/KeyValueDBService"; import { PouchDB } from "@/apps/cli/lib/pouchdb-node"; import { NodeServiceContext } from "./NodeServiceContext"; import { setLang } from "@/common/translation"; +import { runtimeDatabaseDirectory, runtimeScopeDirectory } from "@/apps/cli/runtimeState"; +import { createDeletableValueStore } from "@vrtmrz/value-store"; +import type { SettingServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/SettingService"; export { NodeServiceContext } from "./NodeServiceContext"; @@ -86,17 +89,31 @@ class NodeUIService extends UIService { } } +class NodeSettingService extends InjectableSettingService { + declare readonly config: InjectableSettingService["config"]; + + constructor(context: T, dependencies: SettingServiceDependencies) { + super(context, dependencies); + const databaseConfig = this.config; + const vaultScope = nodePath.basename(runtimeScopeDirectory(context.databasePath, "", context.runtimeVaultPath)); + this.config = createDeletableValueStore({ + get: (key) => databaseConfig.getValue(`${key}-${vaultScope}`), + set: (key, value) => databaseConfig.setValue(`${key}-${vaultScope}`, value), + delete: (key) => databaseConfig.delete(`${key}-${vaultScope}`), + }); + } +} + export class NodeServiceHub extends InjectableServiceHub { constructor(basePath: string, context: T) { const runtimeDir = nodePath.join(basePath, ".livesync", "runtime"); const localStoragePath = nodePath.join(runtimeDir, "local-storage.json"); - const keyValueDBPath = nodePath.join(runtimeDir, "keyvalue-db.json"); const API = new HeadlessAPIService(context); const conflict = new InjectableConflictService(context); const fileProcessing = new InjectableFileProcessingService(context); - const setting = new InjectableSettingService(context, { + const setting = new NodeSettingService(context, { APIService: API, onDisplayLanguageChanged: setLang, deviceValues: createNodeLocalStorage(localStoragePath), @@ -118,8 +135,13 @@ export class NodeServiceHub extends InjectableServ settingService: setting, APIService: API, }); + const scopeDirectory = () => runtimeScopeDirectory(basePath, vault.getVaultName(), context.runtimeVaultPath); const test = new InjectableTestService(context); const databaseEvents = new InjectableDatabaseEventService(context); + databaseEvents.onResetDatabase.addHandler(async () => { + fs.rmSync(runtimeDatabaseDirectory(basePath, vault.getVaultName()), { force: true, recursive: true }); + return true; + }); const path = new PathServiceCompat(context, { settingService: setting, }); @@ -156,7 +178,7 @@ export class NodeServiceHub extends InjectableServ databaseEvents, vault, }, - keyValueDBPath + () => nodePath.join(scopeDirectory(), "keyvalue-db.json") ); const control = new ControlService(context, { diff --git a/src/apps/cli/test-support/p2p-automatic-host.test.ts b/src/apps/cli/test-support/p2p-automatic-host.test.ts index 229fcc57..f4e03930 100644 --- a/src/apps/cli/test-support/p2p-automatic-host.test.ts +++ b/src/apps/cli/test-support/p2p-automatic-host.test.ts @@ -82,7 +82,7 @@ export async function runP2PAutomaticHostFixture(options: CLIOptions, context: C } const commandOutcome = runCommand(options, context).then( (result) => ({ type: "result" as const, result }), - (error) => ({ type: "error" as const, error }) + (error: unknown) => ({ type: "error" as const, error }) ); const connectedWaitStarted = Date.now(); while (!service.transportLifecycle.isConnected) { diff --git a/src/apps/cli/testdeno/CONTRIBUTING_TESTS.md b/src/apps/cli/testdeno/CONTRIBUTING_TESTS.md index 051cfa9c..2bb1999a 100644 --- a/src/apps/cli/testdeno/CONTRIBUTING_TESTS.md +++ b/src/apps/cli/testdeno/CONTRIBUTING_TESTS.md @@ -1,151 +1,3 @@ -# Writing CLI Tests on Deno +# Moved -This guide explains how to add or update tests under `src/apps/cli/testdeno/`. -Note that new tests should be added to the Deno suite rather than the existing bash suite due to the cross-platform execution and TypeScript benefits. - -## Scope - -The Deno suite is designed for cross-platform execution, with a strong focus on Windows compatibility while keeping behaviour equivalent to existing bash tests. - -## Principles - -- Keep one scenario per file when practical. -- Reuse helpers from `helpers/` rather than duplicating process, Docker, or settings logic. -- Prefer deterministic data over random inputs unless randomness is explicitly required. -- Ensure every test can clean up automatically. -- Keep assertions actionable with clear failure messages. - -## Directory structure - -``` -src/apps/cli/testdeno/ - helpers/ - backgroundCli.ts - cli.ts - docker.ts - env.ts - p2p.ts - settings.ts - temp.ts - test-*.ts - deno.json -``` - -## Test file naming - -- Use `test-.ts`. -- Use names aligned with existing bash tests when porting, for example: - - `test-sync-locked-remote.ts` - - `test-p2p-sync.ts` - -## Core helper usage - -### Temporary workspace - -Use `TempDir` and `await using` so cleanup is automatic: - -```ts -await using workDir = await TempDir.create("livesync-cli-my-test"); -``` - -### CLI execution - -- `runCli(...)`: returns code and combined output. -- `runCliOrFail(...)`: throws on non-zero exit. -- `runCliWithInputOrFail(input, ...)`: for `put` and stdin-driven commands. - -### Settings - -- `initSettingsFile(...)`: creates a baseline settings file. -- `applyCouchdbSettings(...)`: applies CouchDB fields. -- `applyRemoteSyncSettings(...)`: applies remote and encryption fields. -- `applyP2pSettings(...)`: applies P2P fields. -- `applyP2pTestTweaks(...)`: enables P2P-only test profile. - -### Docker services - -- `startCouchdb(...)`, `stopCouchdb()` -- `startP2pRelay()`, `stopP2pRelay()` - -### P2P discovery - -- `discoverPeer(...)` -- `maybeStartLocalRelay(...)` -- `stopLocalRelayIfStarted(...)` - -### Background host process - -Use `startCliInBackground(...)` for long-running host mode such as `p2p-host`. - -## Recommended test structure - -1. Arrange -2. Act -3. Assert -4. Cleanup in `finally` - -Example skeleton: - -```ts -Deno.test("feature: behaviour", async () => { - await using workDir = await TempDir.create("example"); - // Arrange - - try { - // Act - - // Assert - } finally { - // Optional explicit cleanup - } -}); -``` - -## Reliability guidelines - -- Use explicit waits only when needed for eventual consistency. -- Re-run sync operations where the protocol is eventually consistent. -- For network-sensitive commands, use `LIVESYNC_CLI_RETRY` during debugging. -- Keep Docker container reuse disabled by default unless debugging. -- Use `npm run test:e2e:cli:p2p` for canonical P2P validation. It runs the Deno scenario in Compose because host networking and WebRTC candidate selection are not reproducible across environments. Individual `deno task test:p2p-*` tasks remain available when explicitly invoked for cross-platform diagnostics, but are not selected by the default suite or CI. - -## Environment variables - -Common variables: - -- `LIVESYNC_DOCKER_MODE` -- `LIVESYNC_DOCKER_COMMAND` -- `LIVESYNC_TEST_TEE` -- `LIVESYNC_DOCKER_TEE` -- `LIVESYNC_CLI_DEBUG` -- `LIVESYNC_CLI_VERBOSE` -- `LIVESYNC_CLI_RETRY` -- `LIVESYNC_DEBUG_KEEP_DOCKER` - -P2P variables: - -- `RELAY` -- `ROOM_ID` -- `PASSPHRASE` -- `APP_ID` -- `PEERS_TIMEOUT` -- `SYNC_TIMEOUT` -- `USE_INTERNAL_RELAY` - -## Adding a new test task - -1. Add the test file under `src/apps/cli/testdeno/`. -2. Add a task in `src/apps/cli/testdeno/deno.json`. -3. Update `src/apps/cli/testdeno/test_dev_deno.md`. -4. Run the new task locally. - -## Validation checklist - -- The test passes on a clean workspace. -- The test does not leave persistent artefacts unless explicitly requested. -- Failure messages identify both expected and actual behaviour. -- The corresponding task is documented. - -## Out of scope for this suite - -- One-off reproduction scripts that are not intended as stable regression tests. +The CLI test authoring guide is now [test/e2e-cli/CONTRIBUTING_TESTS.md](../../../../test/e2e-cli/CONTRIBUTING_TESTS.md). diff --git a/src/apps/cli/testdeno/deno.json b/src/apps/cli/testdeno/deno.json deleted file mode 100644 index 4d114f82..00000000 --- a/src/apps/cli/testdeno/deno.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "tasks": { - "test": "deno task test:ci", - "test:ci": "deno run -A --no-check run-ci-suite.ts", - "test:p2p:compose": "deno run -A --no-check run-compose-p2p.ts", - "test:local": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts test-mirror.ts test-daemon.ts", - "test:daemon": "deno test --env-file=.test.env -A --no-check test-daemon.ts", - "test:daemon-startup": "deno test --env-file=.test.env -A --no-check test-daemon-startup.ts", - "test:decoupled-vault": "deno test --env-file=.test.env -A --no-check test-decoupled-vault.ts", - "test:remote-commands": "deno test --env-file=.test.env -A --no-check test-remote-commands.ts", - "test:settings-writeback": "deno test -A --no-check test-settings-writeback.ts", - "test:remote-administration-exit-codes": "deno test -A --no-check test-remote-administration-exit-codes.ts", - "test:push-pull": "deno test --env-file=.test.env -A --no-check test-push-pull.ts", - "test:setup-put-cat": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts", - "test:mirror": "deno test --env-file=.test.env -A --no-check test-mirror.ts", - "test:sync-two-local": "deno test --env-file=.test.env -A --no-check test-sync-two-local-databases.ts", - "test:sync-locked-remote": "deno test --env-file=.test.env -A --no-check test-sync-locked-remote.ts", - "test:p2p-host": "deno test --env-file=.test.env -A --no-check test-p2p-host.ts", - "test:p2p-peers": "deno test --env-file=.test.env -A --no-check test-p2p-peers-local-relay.ts", - "test:p2p-sync": "deno test --env-file=.test.env -A --no-check test-p2p-sync.ts", - "test:p2p-automatic-host": "deno test --env-file=.test.env -A --no-check test-p2p-automatic-host.ts", - "test:p2p-replacement": "deno test --env-file=.test.env -A --no-check test-p2p-replicator-replacement.ts", - "test:p2p-relay-disconnect": "deno test --env-file=.test.env -A --no-check test-p2p-relay-disconnect.ts", - "test:p2p:ci": "deno test --env-file=.test.env -A --no-check test-p2p-sync.ts test-p2p-automatic-host.ts test-p2p-replicator-replacement.ts test-p2p-relay-disconnect.ts", - "test:p2p-three-nodes": "deno test --env-file=.test.env -A --no-check test-p2p-three-nodes-conflict.ts", - "test:p2p-upload-download": "deno test --env-file=.test.env -A --no-check test-p2p-upload-download-repro.ts", - "test:benchmark-contract": "deno test --env-file=.test.env -A --no-check test-benchmark-contract.ts", - "bench:p2p": "deno run --env-file=.test.env -A --no-check bench-p2p.ts", - "bench:couchdb": "deno run --env-file=.test.env -A --no-check bench-couchdb.ts", - "bench:compression": "deno run --env-file=.test.env -A --no-check bench-compression.ts", - "bench:cases": "deno run --env-file=.test.env -A --no-check bench-network-cases.ts", - "bench:latency-sweep": "deno run --env-file=.test.env -A --no-check bench-latency-sweep.ts", - "bench:p2p-split-node": "deno run --env-file=.test.env -A --no-check bench-p2p-split-node.ts", - "bench:item1": "bash ./bench-run-item1.sh", - "bench:item1:full": "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh", - "test:e2e-couchdb": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-couchdb.ts", - "test:e2e-matrix": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-matrix.ts", - "test:e2e-matrix:couchdb-enc0": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: COUCHDB-enc0' test-e2e-two-vaults-matrix.ts", - "test:e2e-matrix:couchdb-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: COUCHDB-enc1' test-e2e-two-vaults-matrix.ts", - "test:e2e-matrix:minio-enc0": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc0' test-e2e-two-vaults-matrix.ts", - "test:e2e-matrix:minio-enc1": "deno test --env-file=.test.env -A --no-check --filter='e2e matrix: MINIO-enc1' test-e2e-two-vaults-matrix.ts" - }, - "imports": { - "@std/assert": "jsr:@std/assert@^1.0.13", - "@std/path": "jsr:@std/path@^1.0.9" - } -} diff --git a/src/apps/cli/testdeno/deno.lock b/src/apps/cli/testdeno/deno.lock deleted file mode 100644 index a65fb541..00000000 --- a/src/apps/cli/testdeno/deno.lock +++ /dev/null @@ -1,31 +0,0 @@ -{ - "version": "5", - "specifiers": { - "jsr:@std/assert@^1.0.13": "1.0.19", - "jsr:@std/internal@^1.0.12": "1.0.12", - "jsr:@std/path@^1.0.9": "1.1.4" - }, - "jsr": { - "@std/assert@1.0.19": { - "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e", - "dependencies": [ - "jsr:@std/internal" - ] - }, - "@std/internal@1.0.12": { - "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027" - }, - "@std/path@1.1.4": { - "integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5", - "dependencies": [ - "jsr:@std/internal" - ] - } - }, - "workspace": { - "dependencies": [ - "jsr:@std/assert@^1.0.13", - "jsr:@std/path@^1.0.9" - ] - } -} diff --git a/src/apps/cli/testdeno/helpers/net.ts b/src/apps/cli/testdeno/helpers/net.ts deleted file mode 100644 index 35a9d473..00000000 --- a/src/apps/cli/testdeno/helpers/net.ts +++ /dev/null @@ -1,68 +0,0 @@ -type WaitForPortOptions = { - timeoutMs?: number; - intervalMs?: number; - connectTimeoutMs?: number; -}; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function connectWithTimeout(hostname: string, port: number, timeoutMs: number): Promise { - let timer: ReturnType | undefined; - try { - const connPromise = Deno.connect({ hostname, port }); - const timeoutPromise = new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error(`connect timeout after ${timeoutMs}ms`)), timeoutMs); - }); - const conn = await Promise.race([connPromise, timeoutPromise]); - conn.close(); - } finally { - if (timer !== undefined) { - clearTimeout(timer); - } - } -} - -export async function waitForPort(hostname: string, port: number, options: WaitForPortOptions = {}): Promise { - const timeoutMs = options.timeoutMs ?? 15000; - const intervalMs = options.intervalMs ?? 250; - const connectTimeoutMs = options.connectTimeoutMs ?? 1000; - - const started = Date.now(); - let lastError: unknown; - - while (Date.now() - started < timeoutMs) { - try { - await connectWithTimeout(hostname, port, connectTimeoutMs); - return; - } catch (error) { - lastError = error; - await sleep(intervalMs); - } - } - - throw new Error( - `Port ${hostname}:${port} did not become ready within ${timeoutMs}ms` + - (lastError ? ` (last error: ${String(lastError)})` : "") - ); -} - -export async function getOptimalLoopbackIp(): Promise { - const ipv4 = "127.0.0.1"; - const ipv6 = "::1"; - - try { - const l = Deno.listen({ hostname: ipv4, port: 0 }); - l.close(); - return ipv4; - } catch { - try { - const l = Deno.listen({ hostname: ipv6, port: 0 }); - l.close(); - return ipv6; - } catch { - return ipv4; // fallback to default - } - } -} diff --git a/src/apps/cli/testdeno/helpers/temp.ts b/src/apps/cli/testdeno/helpers/temp.ts deleted file mode 100644 index 655d18d7..00000000 --- a/src/apps/cli/testdeno/helpers/temp.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { join } from "@std/path"; - -/** - * A temporary directory that cleans itself up via `await using`. - * Requires TypeScript 5.2+ / Deno 1.40+ for the AsyncDisposable protocol. - * - * @example - * ```ts - * await using tmp = await TempDir.create(); - * const file = tmp.join("data.json"); - * ``` - */ -export class TempDir implements AsyncDisposable { - readonly path: string; - - private constructor(path: string) { - this.path = path; - } - - static async create(prefix = "livesync-deno-test"): Promise { - const path = await Deno.makeTempDir({ prefix: `${prefix}.` }); - return new TempDir(path); - } - - /** Return an OS path joined to the temp directory root. */ - join(...parts: string[]): string { - return join(this.path, ...parts); - } - - async [Symbol.asyncDispose](): Promise { - await Deno.remove(this.path, { recursive: true }).catch(() => {}); - } -} diff --git a/src/apps/cli/testdeno/run-compose-p2p.ts b/src/apps/cli/testdeno/run-compose-p2p.ts deleted file mode 100644 index 66ed898f..00000000 --- a/src/apps/cli/testdeno/run-compose-p2p.ts +++ /dev/null @@ -1,50 +0,0 @@ -const repositoryRoot = await Deno.realPath(new URL("../../../../", import.meta.url)); -const composeArgs = ["compose", "-f", "test/bench-network/compose.yml"]; -const p2pEnvironment = { - CLI_E2E_TASK: Deno.env.get("CLI_E2E_TASK") ?? "test:p2p:ci", - RELAY: Deno.env.get("RELAY") ?? "ws://nostr-relay:7777/", - PEERS_TIMEOUT: Deno.env.get("PEERS_TIMEOUT") ?? "20", - SYNC_TIMEOUT: Deno.env.get("SYNC_TIMEOUT") ?? "60", - LIVESYNC_USE_COTURN: Deno.env.get("LIVESYNC_USE_COTURN") ?? "0", - TURN_SERVERS: Deno.env.get("TURN_SERVERS") ?? "none", - LIVESYNC_P2P_PEERS_RETRY: Deno.env.get("LIVESYNC_P2P_PEERS_RETRY") ?? "1", - LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "60000", - BENCH_LIVESYNC_TEST_TEE: Deno.env.get("BENCH_LIVESYNC_TEST_TEE") ?? "0", - LIVESYNC_CLI_DEBUG: Deno.env.get("LIVESYNC_CLI_DEBUG") ?? "0", - LIVESYNC_CLI_VERBOSE: Deno.env.get("LIVESYNC_CLI_VERBOSE") ?? "0", -}; - -async function runDocker(args: string[], env?: Record): Promise { - return await new Deno.Command("docker", { - args, - cwd: repositoryRoot, - env, - stdin: "inherit", - stdout: "inherit", - stderr: "inherit", - }).spawn().status; -} - -let testStatus: Deno.CommandStatus | undefined; -try { - testStatus = await runDocker( - [...composeArgs, "run", "--build", "--rm", "bench-runner", "run-livesync-cli-e2e"], - p2pEnvironment - ); -} finally { - const cleanupStatus = await runDocker([...composeArgs, "down", "-v", "--remove-orphans"]); - if (!cleanupStatus.success) { - console.error(`[CLI E2E] Compose cleanup failed with exit code ${cleanupStatus.code}.`); - if (testStatus?.success) { - Deno.exit(cleanupStatus.code); - } - } -} - -if (!testStatus?.success) { - const code = testStatus?.code ?? 1; - console.error(`[CLI E2E] Compose P2P suite failed with exit code ${code}.`); - Deno.exit(code); -} - -console.log("\n[CLI E2E] Compose P2P suite passed."); diff --git a/src/apps/cli/testdeno/test-p2p-relay-disconnect.ts b/src/apps/cli/testdeno/test-p2p-relay-disconnect.ts deleted file mode 100644 index 1dc00778..00000000 --- a/src/apps/cli/testdeno/test-p2p-relay-disconnect.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { assert, assertEquals } from "@std/assert"; - -Deno.test("p2p lifecycle: explicit disconnect closes and pauses relay WebSockets", async () => { - const command = new Deno.Command("node", { - args: [new URL("./relay-disconnect-probe.mjs", import.meta.url).pathname], - env: { - RELAY: Deno.env.get("RELAY") ?? "ws://nostr-relay:7777/", - RELAY_TIMEOUT_MS: Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000", - }, - stdout: "piped", - stderr: "piped", - }); - const result = await command.output(); - const stdout = new TextDecoder().decode(result.stdout).trim(); - const stderr = new TextDecoder().decode(result.stderr).trim(); - - assert(result.success, `Relay disconnect probe failed\nstdout: ${stdout}\nstderr: ${stderr}`); - const report = JSON.parse(stdout.split("\n").at(-1) ?? "{}") as { - socketsClosed?: number; - stayedDisconnectedWhilePaused?: boolean; - reconnectedAfterResume?: boolean; - }; - assert((report.socketsClosed ?? 0) > 0, "The probe did not observe an open relay WebSocket"); - assertEquals(report.stayedDisconnectedWhilePaused, true); - assertEquals(report.reconnectedAfterResume, true); -}); diff --git a/src/apps/cli/testdeno/test_dev_deno.md b/src/apps/cli/testdeno/test_dev_deno.md index bdd67142..ca5065f8 100644 --- a/src/apps/cli/testdeno/test_dev_deno.md +++ b/src/apps/cli/testdeno/test_dev_deno.md @@ -1,351 +1,3 @@ -# CLI Deno Test Development Notes +# Moved -This document provides an overview of the Deno-based compatibility tests under `src/apps/cli/testdeno/`. -The Deno suite is the canonical CLI E2E entry point. P2P scenarios run through the repository Compose entry point so that networking, signalling, and the runner environment are reproducible. Existing Bash tests under `src/apps/cli/test/` remain as legacy implementation references, but are not exposed as supported P2P entry points. - ---- - -## Goals - -- Keep the existing Bash tests as migration references while using Deno and Compose for supported execution. -- Provide direct execution from Windows PowerShell. -- Establish a TypeScript (Deno) foundation for core end-to-end and integration scenarios. - ---- - -## Directory structure - -``` -src/apps/cli/testdeno/ - deno.json - run-ci-suite.ts - run-compose-p2p.ts - CONTRIBUTING_TESTS.md - helpers/ - backgroundCli.ts - cli.ts - docker.ts - env.ts - p2p.ts - settings.ts - temp.ts - test-e2e-two-vaults-couchdb.ts - test-push-pull.ts - test-p2p-host.ts - test-p2p-peers-local-relay.ts - test-p2p-sync.ts - test-p2p-automatic-host.ts - test-p2p-three-nodes-conflict.ts - test-p2p-upload-download-repro.ts - test-e2e-two-vaults-matrix.ts - test-setup-put-cat.ts - test-mirror.ts - test-sync-two-local-databases.ts - test-sync-locked-remote.ts - test-daemon.ts - test-decoupled-vault.ts - test-remote-commands.ts -``` - ---- - -## Key files - -### `deno.json` - -- Defines Deno tasks. -- Defines import maps for `@std/assert` and `@std/path`. - -Main tasks: - -- `deno task test` -- `deno task test:ci` -- `deno task test:p2p:compose` -- `deno task test:local` -- `deno task test:daemon` -- `deno task test:decoupled-vault` -- `deno task test:remote-commands` -- `deno task test:push-pull` -- `deno task test:setup-put-cat` -- `deno task test:mirror` -- `deno task test:sync-two-local` -- `deno task test:sync-locked-remote` -- `deno task test:p2p-host` -- `deno task test:p2p-peers` -- `deno task test:p2p-sync` -- `deno task test:p2p-automatic-host` -- `deno task test:p2p-three-nodes` -- `deno task test:p2p-upload-download` -- `deno task test:e2e-couchdb` -- `deno task test:e2e-matrix` - -`deno task test` is an alias for the non-P2P `test:ci` suite. The individual P2P tasks are explicit host-direct entry points for cross-platform diagnostics; they are never selected by the default suite or CI. Use `test:p2p:compose` for canonical P2P verification. - -### `helpers/cli.ts` - -- CLI execution wrappers. -- `runCli`, `runCliOrFail`, `runCliWithInput`. -- Output normalisation via `sanitiseCatStdout`. -- Comparison utilities, including `assertFilesEqual`. - -This file corresponds to `run_cli` and common assertions in `test-helpers.sh`. - -### `helpers/settings.ts` - -- Executes `init-settings --force`. -- Marks `isConfigured = true`. -- Applies CouchDB and P2P settings. -- Applies remote synchronisation settings and P2P test tweaks. - -This file corresponds to settings helpers in `test-helpers.sh`. - -### `helpers/docker.ts` - -- Starts, stops, and initialises CouchDB and RustFS directly from Deno. -- Configures CouchDB via `fetch + retry`. -- Initialises S3 buckets using the RustFS `rc` client, including CORS for signed browser requests. -- Starts and stops the P2P relay through the same Docker runner. - -These flows do not require Bash on the host. The S3 matrix tasks, environment variables, and container name retain their existing `minio` names for compatibility; RustFS provides the test backend. The RustFS server and `rc` client images are pinned by version and digest. - -### `helpers/backgroundCli.ts` - -- Starts long-running commands such as `p2p-host` in the background. -- Waits for readiness logs and handles termination. - -### `helpers/p2p.ts` - -- Determines whether a local relay should be started. -- Parses `p2p-peers` output. -- Discovers peer IDs with a fallback based on advertisement logs. - -### `helpers/env.ts` - -- Loads `.test.env`. -- Supports `KEY=value`, single-quoted values, and double-quoted values. - -### `helpers/temp.ts` - -- Provides `TempDir`. -- Uses `await using` to auto-clean temporary directories. - ---- - -## Implemented tests - -### `test-push-pull.ts` - -- Verifies push and pull round trips. -- Uses environment variables or `.test.env` for CouchDB values. - -### `test-setup-put-cat.ts` - -- Verifies `setup` with full setup URI generation via `encodeSettingsToSetupURI`. -- Verifies `push`, `cat`, `ls`, `info`, `rm`, `resolve`, `cat-rev`, and `pull-rev`. -- Does not require an external remote. - -### `test-mirror.ts` - -- Verifies seven core mirror scenarios, including an unrecognised revision origin. -- Does not require an external remote. - -### `test-sync-two-local-databases.ts` - -- Verifies sync between two vaults and CouchDB. -- Verifies conflict detection and resolve propagation. -- Starts Docker CouchDB by default when `LIVESYNC_START_DOCKER != 0`. - -### `test-sync-locked-remote.ts` - -- Updates the CouchDB milestone `locked` flag. -- Verifies sync success when unlocked. -- Verifies actionable CLI error when locked. - -### `test-p2p-host.ts` - -- Verifies that `p2p-host` starts and emits readiness output. - -### `test-p2p-peers-local-relay.ts` - -- Verifies peer discovery through a local relay. - -### `test-p2p-sync.ts` - -- Verifies that `p2p-sync` completes after peer discovery. - -### `test-p2p-automatic-host.ts` - -- Verifies that `p2p-host` restores its configured P2P automation policy and stays connected while multiple clients synchronise through it. -- Covers automatic baseline transfer, watched updates, client disconnection, and host restart persistence without manual P2P sync commands. - -### `test-p2p-three-nodes-conflict.ts` - -- Uses one host and two clients. -- Verifies conflict creation, detection via `info`, and resolution via `resolve`. - -### `test-p2p-upload-download-repro.ts` - -- Uses host, upload, and download nodes. -- Verifies transfer of text files and binary files, including larger files. - -### `test-e2e-two-vaults-couchdb.ts` - -- Verifies two-vault end-to-end scenarios on CouchDB. -- Runs both encryption-off and encryption-on cases. -- Includes conflict marker checks in `ls` and resolve propagation checks. - -### `test-e2e-two-vaults-matrix.ts` - -- Verifies the matrix equivalent of the bash script. -- Runs four combinations: - - `COUCHDB-enc0` - - `COUCHDB-enc1` - - `MINIO-enc0` - - `MINIO-enc1` - -### `test-daemon.ts` - -- Verifies daemon-related ignore rules behaviour. -- Exercises scenarios with `.livesync/ignore` wildcard rules, missing ignore rules, and imported `.gitignore` rules. - -### `test-decoupled-vault.ts` - -- Verifies push, pull, and mirror command behaviour when the vault directory is decoupled from the database directory. - -### `test-remote-commands.ts` - -- Verifies remote database management commands: `remote-status`, `lock-remote`, `unlock-remote`, and `mark-resolved`. - ---- - -## Running tests (PowerShell) - -From the repository root, use the canonical package scripts. `test:e2e:cli` runs the same non-P2P task set selected by the default CLI CI workflow. P2P validation runs in Compose so peer discovery does not depend on host loopback, firewall, or WebRTC candidate behaviour. - -```powershell -npm run test:e2e:cli -npm run test:e2e:cli:p2p -npm run test:e2e:cli:all -``` - -From `src/apps/cli/testdeno`: - -```powershell -cd src/apps/cli/testdeno - -# Canonical suites -deno task test:ci -deno task test:p2p:compose - -# Local-only set -deno task test:local - -# Individual tests -deno task test:setup-put-cat -deno task test:mirror -deno task test:daemon -deno task test:push-pull -deno task test:sync-locked-remote - -# CouchDB and Object Storage tests -deno task test:sync-two-local -deno task test:decoupled-vault -deno task test:remote-commands -deno task test:e2e-couchdb -deno task test:e2e-matrix - -# Explicit host-direct P2P diagnostics for cross-platform investigations. -# These are not part of the default suite or release evidence. -deno task test:p2p-host -deno task test:p2p-peers -deno task test:p2p-sync -deno task test:p2p-three-nodes -deno task test:p2p-upload-download -``` - ---- - -## Environment variables - -### CouchDB - -- `COUCHDB_URI` -- `COUCHDB_USER` -- `COUCHDB_PASSWORD` -- `COUCHDB_DBNAME` - -Equivalent keys in `src/apps/cli/.test.env`: - -- `hostname` -- `username` -- `password` -- `dbname` - -### Behaviour switches - -- `LIVESYNC_START_DOCKER=0`: use existing CouchDB. -- `REMOTE_PATH`: override target path for selected tests. -- `LIVESYNC_TEST_TEE=1`: stream CLI stdout and stderr during execution. -- `LIVESYNC_DOCKER_TEE=1`: stream Docker stdout and stderr. -- `LIVESYNC_CLI_RETRY=`: retry transient network failures. -- `LIVESYNC_DEBUG_KEEP_DOCKER=1`: keep `couchdb-test` after test completion. - -### Docker command selection - -`helpers/docker.ts` supports command selection via environment variables. - -- `LIVESYNC_DOCKER_MODE=auto` (default) - - Windows: tries `wsl docker` first, then `docker`. - - Non-Windows: tries `docker` first, then `wsl docker`. -- `LIVESYNC_DOCKER_MODE=native`: always uses `docker`. -- `LIVESYNC_DOCKER_MODE=wsl`: always uses `wsl docker`. -- `LIVESYNC_DOCKER_COMMAND="..."`: custom command, for example `wsl docker`. - -`LIVESYNC_DOCKER_COMMAND` has priority over `LIVESYNC_DOCKER_MODE`. - -PowerShell examples: - -```powershell -# Use Docker in WSL explicitly -$env:LIVESYNC_DOCKER_MODE = "wsl" -deno task test:sync-two-local - -# Full custom command -$env:LIVESYNC_DOCKER_COMMAND = "wsl docker" -deno task test:sync-two-local -``` - -### P2P - -- `RELAY` -- `ROOM_ID` -- `PASSPHRASE` -- `APP_ID` -- `PEERS_TIMEOUT` -- `SYNC_TIMEOUT` -- `USE_INTERNAL_RELAY=0|1` -- `TIMEOUT_SECONDS` - ---- - -## Continuous Integration - -The GitHub Actions workflow `.github/workflows/cli-deno-tests.yml` runs automatically on pushes and pull requests affecting the CLI or its shared runtime inputs. It builds the CLI once, then distributes the bundle to the non-P2P test jobs. The local `test:ci` runner and the automatic matrix both read `ci-tasks.json`. During a manual dispatch, select a test subset and optionally enable 'Enable verbose and debug logging' for detailed troubleshooting traces. - -The separate `.github/workflows/cli-p2p-tests.yml` workflow runs the canonical Compose P2P suite automatically for changes to the CLI, its shared runtime inputs, or `test/bench-network/`. A change confined to the Compose fixtures does not start the non-P2P matrix. Host-direct `test:p2p` tasks remain available for explicit diagnostics. - -The `.github/workflows/cli-docker-validation.yml` workflow builds one production JavaScript bundle with Node.js 22, then tests Docker images in parallel on native amd64 and arm64 runners. Each image uses the shared bundle and prepares its own runtime dependencies. Validation runs without registry credentials, and both architecture results are collected before publication. The publishing workflow, `.github/workflows/cli-docker.yml`, imports the validated OCI image from each architecture, collects the image digests, and publishes the combined manifest. It retains the existing publication conditions for main, release tags, and manual dispatch. A dry run builds both images and checks the digest pair while skipping registry uploads and manifest publication. - ---- - -## Current limitations - -- S3 startup and matrix coverage use RustFS. Current limits are elsewhere, not setup URI generation. - ---- - -## Maintenance policy - -- Existing bash tests remain available. -- Deno tests are expanded in parallel for cross-platform usage. -- New scenarios should be added through reusable helpers in `helpers/`. +The CLI test development guide is now [test/e2e-cli/test_dev_node.md](../../../../test/e2e-cli/test_dev_node.md). diff --git a/src/apps/cli/tsconfig.json b/src/apps/cli/tsconfig.json index 0d9fbb57..9828ca59 100644 --- a/src/apps/cli/tsconfig.json +++ b/src/apps/cli/tsconfig.json @@ -27,5 +27,5 @@ } }, "include": ["*.ts", "**/*.ts", "**/*.tsx"], - "exclude": ["node_modules", "dist", "test", "testdeno"] + "exclude": ["node_modules", "dist", "test"] } diff --git a/src/apps/cli/vite.config.ts b/src/apps/cli/vite.config.ts index a3212a34..713683ba 100644 --- a/src/apps/cli/vite.config.ts +++ b/src/apps/cli/vite.config.ts @@ -1,18 +1,11 @@ import { defaultServerConditions, defaultServerMainFields, defineConfig } from "vite"; import { svelte } from "@sveltejs/vite-plugin-svelte"; -import { fileURLToPath, fs, isBuiltin, path } from "@vrtmrz/livesync-commonlib/node"; +import { fileURLToPath, isBuiltin, path } from "@vrtmrz/livesync-commonlib/node"; +import { readVersion } from "../readVersion"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const resolve = (...args: string[]) => path.resolve(...args).replace(/\\/g, "/"); const repoRoot = path.resolve(__dirname, "../../.."); -function readVersion(filePath: string): string | undefined { - const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); - if (typeof parsed !== "object" || parsed === null || !("version" in parsed)) { - return undefined; - } - return typeof parsed.version === "string" ? parsed.version : undefined; -} - const packageVersion = readVersion(path.resolve(repoRoot, "package.json")); const manifestVersion = readVersion(path.resolve(repoRoot, "manifest.json")); // https://vite.dev/config/ @@ -49,26 +42,6 @@ if (typeof globalThis.FileReader === "undefined") { } `; -function injectBanner(): import("vite").Plugin { - return { - name: "inject-banner", - generateBundle(_options, bundle) { - for (const chunk of Object.values(bundle)) { - if (chunk.type === "chunk" && chunk.isEntry) { - // Insert after the shebang line if present, otherwise at the top. - if (chunk.code.startsWith("#!")) { - const newline = chunk.code.indexOf("\n"); - chunk.code = - chunk.code.slice(0, newline + 1) + fileReaderPolyfillBanner + chunk.code.slice(newline + 1); - } else { - chunk.code = fileReaderPolyfillBanner + chunk.code; - } - } - } - }, - }; -} - const buildInputs: Record = { index: resolve(__dirname, "entrypoint.ts"), }; @@ -77,7 +50,7 @@ if (process.env.LIVESYNC_CLI_TEST_SUPPORT === "1") { } export default defineConfig({ - plugins: [svelte(), injectBanner()], + plugins: [svelte()], resolve: { // This bundle runs in Node. Vite's client defaults include the `browser` // export condition, which would select Commonlib's inline Web Worker. @@ -98,7 +71,16 @@ export default defineConfig({ outDir: "dist", emptyOutDir: true, minify: false, + sourcemap: process.env.LIVESYNC_CLI_COVERAGE === "1", rollupOptions: { + output: { + banner: (chunk) => (chunk.isEntry ? fileReaderPolyfillBanner : ""), + // c8 filters mapped ranges before resolving relative source paths. + sourcemapPathTransform: + process.env.LIVESYNC_CLI_COVERAGE === "1" + ? (source: string, mapPath: string) => path.resolve(path.dirname(mapPath), source) + : undefined, + }, input: buildInputs, external: (id) => { if (isBuiltin(id)) return true; diff --git a/src/apps/readVersion.ts b/src/apps/readVersion.ts new file mode 100644 index 00000000..d6bc09cc --- /dev/null +++ b/src/apps/readVersion.ts @@ -0,0 +1,10 @@ +import { fs } from "@vrtmrz/livesync-commonlib/node"; + +/** Read a string-valued version field from a JSON file during Node builds. */ +export function readVersion(filePath: string): string | undefined { + const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); + if (typeof parsed !== "object" || parsed === null || !("version" in parsed)) { + return undefined; + } + return typeof parsed.version === "string" ? parsed.version : undefined; +} diff --git a/src/apps/readVersion.unit.spec.ts b/src/apps/readVersion.unit.spec.ts new file mode 100644 index 00000000..3de40238 --- /dev/null +++ b/src/apps/readVersion.unit.spec.ts @@ -0,0 +1,36 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { readVersion } from "./readVersion"; + +describe("readVersion", () => { + let directory: string; + + beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), "livesync-read-version-")); + }); + + afterEach(() => { + rmSync(directory, { recursive: true, force: true }); + }); + + it("returns only string version values", () => { + const filePath = join(directory, "version.json"); + writeFileSync(filePath, JSON.stringify({ version: "1.2.3" })); + expect(readVersion(filePath)).toBe("1.2.3"); + + for (const contents of ["null", "[]", "{}", '{"version": 42}', '{"version": null}']) { + writeFileSync(filePath, contents); + expect(readVersion(filePath)).toBeUndefined(); + } + }); + + it("preserves file read and JSON parse errors", () => { + expect(() => readVersion(join(directory, "missing.json"))).toThrow(); + + const filePath = join(directory, "invalid.json"); + writeFileSync(filePath, "not JSON"); + expect(() => readVersion(filePath)).toThrow(SyntaxError); + }); +}); diff --git a/src/apps/serviceModules/composeSharedServiceModules.ts b/src/apps/serviceModules/composeSharedServiceModules.ts new file mode 100644 index 00000000..c5698106 --- /dev/null +++ b/src/apps/serviceModules/composeSharedServiceModules.ts @@ -0,0 +1,50 @@ +import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub"; +import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; +import type { DatabaseFileAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/DatabaseFileAccess"; +import type { ServiceModules } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; +import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess"; +import { ServiceRebuilder } from "@vrtmrz/livesync-commonlib/compat/serviceModules/Rebuilder"; +import { ServiceFileHandler } from "@/serviceModules/FileHandler"; +import { createFileReflectionProvenance } from "@/serviceModules/FileReflectionProvenance"; + +/** Compose the service modules shared by app hosts after their storage setup. */ +export function composeSharedServiceModules( + services: InjectableServiceHub, + databaseFileAccess: DatabaseFileAccess, + storageAccess: StorageAccess +): Pick { + const fileHandler = new ServiceFileHandler({ + events: services.context.events, + API: services.API, + databaseFileAccess, + conflict: services.conflict, + setting: services.setting, + fileProcessing: services.fileProcessing, + vault: services.vault, + path: services.path, + replication: services.replication, + storageAccess, + fileReflectionProvenance: createFileReflectionProvenance(services.keyValueDB), + }); + + const rebuilder = new ServiceRebuilder({ + path: services.path, + events: services.context.events, + API: services.API, + database: services.database, + appLifecycle: services.appLifecycle, + setting: services.setting, + remote: services.remote, + databaseEvents: services.databaseEvents, + replication: services.replication, + replicator: services.replicator, + UI: services.UI, + vault: services.vault, + fileHandler, + fileProcessing: services.fileProcessing, + storageAccess, + control: services.control, + }); + + return { rebuilder, fileHandler }; +} diff --git a/src/apps/serviceModules/serviceComposition.unit.spec.ts b/src/apps/serviceModules/serviceComposition.unit.spec.ts new file mode 100644 index 00000000..0d81413b --- /dev/null +++ b/src/apps/serviceModules/serviceComposition.unit.spec.ts @@ -0,0 +1,379 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; +import { initialiseServiceModulesCLI } from "@/apps/cli/serviceModules/CLIServiceModules"; +import { initialiseServiceModulesFSAPI } from "@/apps/webapp/serviceModules/FSAPIServiceModules"; +import { FILE_REFLECTION_PROVENANCE_STORE } from "@/serviceModules/FileReflectionProvenance"; +import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; +import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub"; + +const composition = vi.hoisted(() => ({ + calls: [] as string[], + dependencies: {} as Record, + instances: {} as Record, + capture(name: string, dependencies: unknown, instance: unknown) { + this.calls.push(name); + this.dependencies[name] = dependencies; + this.instances[name] = instance; + }, +})); + +vi.mock("@vrtmrz/livesync-commonlib/compat/managers/StorageProcessingManager", () => ({ + StorageAccessManager: class { + constructor() { + composition.capture("StorageAccessManager", undefined, this); + } + }, +})); + +vi.mock("@vrtmrz/livesync-commonlib/compat/serviceModules/Rebuilder", () => ({ + ServiceRebuilder: class { + constructor(dependencies: unknown) { + composition.capture("ServiceRebuilder", dependencies, this); + } + }, +})); + +vi.mock("@/serviceModules/FileHandler", () => ({ + ServiceFileHandler: class { + constructor(dependencies: unknown) { + composition.capture("ServiceFileHandler", dependencies, this); + } + }, +})); + +vi.mock("@/apps/cli/serviceModules/FileAccessCLI", () => ({ + FileAccessCLI: class { + constructor(...args: unknown[]) { + composition.capture("FileAccessCLI", args, this); + } + }, +})); + +vi.mock("@/apps/cli/serviceModules/ServiceFileAccessImpl", () => ({ + ServiceFileAccessCLI: class { + constructor(dependencies: unknown) { + composition.capture("ServiceFileAccessCLI", dependencies, this); + } + }, +})); + +vi.mock("@/apps/cli/serviceModules/DatabaseFileAccess", () => ({ + ServiceDatabaseFileAccessCLI: class { + constructor(dependencies: unknown) { + composition.capture("ServiceDatabaseFileAccessCLI", dependencies, this); + } + }, +})); + +vi.mock("@/apps/cli/managers/StorageEventManagerCLI", () => ({ + StorageEventManagerCLI: class { + close = vi.fn(async () => undefined); + + constructor(...args: unknown[]) { + composition.capture("StorageEventManagerCLI", args, this); + } + }, +})); + +vi.mock("@/apps/webapp/serviceModules/FileAccessFSAPI", () => ({ + FileAccessFSAPI: class { + constructor(...args: unknown[]) { + composition.capture("FileAccessFSAPI", args, this); + } + }, +})); + +vi.mock("@/apps/webapp/serviceModules/ServiceFileAccessImpl", () => ({ + ServiceFileAccessFSAPI: class { + constructor(dependencies: unknown) { + composition.capture("ServiceFileAccessFSAPI", dependencies, this); + } + }, +})); + +vi.mock("@/apps/webapp/serviceModules/DatabaseFileAccess", () => ({ + ServiceDatabaseFileAccessFSAPI: class { + constructor(dependencies: unknown) { + composition.capture("ServiceDatabaseFileAccessFSAPI", dependencies, this); + } + }, +})); + +vi.mock("@/apps/webapp/managers/StorageEventManagerFSAPI", () => ({ + StorageEventManagerFSAPI: class { + constructor(...args: unknown[]) { + composition.capture("StorageEventManagerFSAPI", args, this); + } + }, +})); + +function createServices() { + const hostServices = Object.fromEntries( + [ + "API", + "conflict", + "control", + "database", + "databaseEvents", + "fileProcessing", + "path", + "remote", + "replication", + "replicator", + "setting", + "UI", + "vault", + ].map((name) => [name, { name }]) + ); + const events = { label: "host event hub" }; + const addUnloadHandler = vi.fn<(handler: () => Promise) => void>(); + const openSimpleStore = vi.fn((name: string) => { + composition.calls.push("openSimpleStore"); + composition.dependencies.openSimpleStore = name; + return { name }; + }); + + return { + ...hostServices, + appLifecycle: { onUnload: { addHandler: addUnloadHandler } }, + context: { events }, + events, + keyValueDB: { openSimpleStore }, + addUnloadHandler, + }; +} + +function createCore() { + return {} as LiveSyncBaseCore; +} + +beforeEach(() => { + composition.calls.length = 0; + composition.dependencies = {}; + composition.instances = {}; +}); + +describe("app service module composition", () => { + it("preserves CLI host construction, shared wiring, provenance, and watcher shutdown", async () => { + const services = createServices(); + const core = createCore(); + + const modules = initialiseServiceModulesCLI( + "/vault", + core, + services as unknown as InjectableServiceHub + ); + + expect(composition.calls).toEqual([ + "StorageAccessManager", + "FileAccessCLI", + "StorageEventManagerCLI", + "ServiceFileAccessCLI", + "ServiceDatabaseFileAccessCLI", + "openSimpleStore", + "ServiceFileHandler", + "ServiceRebuilder", + ]); + expect(composition.dependencies.openSimpleStore).toBe(FILE_REFLECTION_PROVENANCE_STORE); + + const [basePath, fileAccessOptions] = composition.dependencies.FileAccessCLI as [ + string, + Record, + ]; + const [eventBasePath, eventCore, eventDependencies, ignoreRules, watchEnabled] = composition.dependencies + .StorageEventManagerCLI as [string, unknown, Record, unknown, boolean]; + const storageAccessDependencies = composition.dependencies.ServiceFileAccessCLI as Record; + const databaseAccessDependencies = composition.dependencies.ServiceDatabaseFileAccessCLI as Record< + string, + unknown + >; + const fileHandlerDependencies = composition.dependencies.ServiceFileHandler as Record; + const rebuilderDependencies = composition.dependencies.ServiceRebuilder as Record; + + expect(basePath).toBe("/vault"); + expect(fileAccessOptions).toEqual({ + storageAccessManager: composition.instances.StorageAccessManager, + vaultService: services.vault, + settingService: services.setting, + APIService: services.API, + pathService: services.path, + }); + expect(eventBasePath).toBe("/vault"); + expect(eventCore).toBe(core); + expect(eventDependencies).toEqual({ + fileProcessing: services.fileProcessing, + setting: services.setting, + vaultService: services.vault, + storageAccessManager: composition.instances.StorageAccessManager, + APIService: services.API, + }); + expect(ignoreRules).toBeUndefined(); + expect(watchEnabled).toBe(false); + expect(storageAccessDependencies).toEqual({ + API: services.API, + setting: services.setting, + fileProcessing: services.fileProcessing, + vault: services.vault, + appLifecycle: services.appLifecycle, + storageEventManager: composition.instances.StorageEventManagerCLI, + storageAccessManager: composition.instances.StorageAccessManager, + vaultAccess: composition.instances.FileAccessCLI, + }); + expect(databaseAccessDependencies).toEqual({ + events: services.events, + API: services.API, + database: services.database, + path: services.path, + storageAccess: modules.storageAccess, + vault: services.vault, + }); + expect(modules.databaseFileAccess).toBe(composition.instances.ServiceDatabaseFileAccessCLI); + expect(modules.fileHandler).toBe(composition.instances.ServiceFileHandler); + expect(modules.rebuilder).toBe(composition.instances.ServiceRebuilder); + expect(fileHandlerDependencies).toEqual({ + events: services.events, + API: services.API, + databaseFileAccess: modules.databaseFileAccess, + conflict: services.conflict, + setting: services.setting, + fileProcessing: services.fileProcessing, + vault: services.vault, + path: services.path, + replication: services.replication, + storageAccess: modules.storageAccess, + fileReflectionProvenance: expect.any(Object), + }); + expect(rebuilderDependencies).toEqual({ + path: services.path, + events: services.events, + API: services.API, + database: services.database, + appLifecycle: services.appLifecycle, + setting: services.setting, + remote: services.remote, + databaseEvents: services.databaseEvents, + replication: services.replication, + replicator: services.replicator, + UI: services.UI, + vault: services.vault, + fileHandler: modules.fileHandler, + fileProcessing: services.fileProcessing, + storageAccess: modules.storageAccess, + control: services.control, + }); + + const unloadHandler = services.addUnloadHandler.mock.calls[0]?.[0]; + expect(unloadHandler).toBeDefined(); + await expect(unloadHandler?.()).resolves.toBe(true); + expect( + (composition.instances.StorageEventManagerCLI as { close: ReturnType }).close + ).toHaveBeenCalledOnce(); + }); + + it("preserves WebApp host construction and returns its vault access and event manager", () => { + const services = createServices(); + const core = createCore(); + const rootHandle = {} as FileSystemDirectoryHandle; + + const modules = initialiseServiceModulesFSAPI( + rootHandle, + core, + services as unknown as InjectableServiceHub + ); + + expect(composition.calls).toEqual([ + "StorageAccessManager", + "FileAccessFSAPI", + "StorageEventManagerFSAPI", + "ServiceFileAccessFSAPI", + "ServiceDatabaseFileAccessFSAPI", + "openSimpleStore", + "ServiceFileHandler", + "ServiceRebuilder", + ]); + expect(composition.dependencies.openSimpleStore).toBe(FILE_REFLECTION_PROVENANCE_STORE); + const [fileAccessRoot, fileAccessOptions] = composition.dependencies.FileAccessFSAPI as [ + FileSystemDirectoryHandle, + Record, + ]; + const [eventRoot, eventCore, eventDependencies] = composition.dependencies.StorageEventManagerFSAPI as [ + FileSystemDirectoryHandle, + unknown, + Record, + ]; + const storageAccessDependencies = composition.dependencies.ServiceFileAccessFSAPI as Record; + const databaseAccessDependencies = composition.dependencies.ServiceDatabaseFileAccessFSAPI as Record< + string, + unknown + >; + + expect(fileAccessRoot).toBe(rootHandle); + expect(fileAccessOptions).toEqual({ + storageAccessManager: composition.instances.StorageAccessManager, + vaultService: services.vault, + settingService: services.setting, + APIService: services.API, + pathService: services.path, + }); + expect(eventRoot).toBe(rootHandle); + expect(eventCore).toBe(core); + expect(eventDependencies).toEqual({ + fileProcessing: services.fileProcessing, + setting: services.setting, + vaultService: services.vault, + storageAccessManager: composition.instances.StorageAccessManager, + APIService: services.API, + }); + expect(storageAccessDependencies).toEqual({ + API: services.API, + setting: services.setting, + fileProcessing: services.fileProcessing, + vault: services.vault, + appLifecycle: services.appLifecycle, + storageEventManager: composition.instances.StorageEventManagerFSAPI, + storageAccessManager: composition.instances.StorageAccessManager, + vaultAccess: composition.instances.FileAccessFSAPI, + }); + expect(databaseAccessDependencies).toEqual({ + events: services.events, + API: services.API, + database: services.database, + path: services.path, + storageAccess: modules.storageAccess, + vault: services.vault, + }); + expect(modules.vaultAccess).toBe(composition.instances.FileAccessFSAPI); + expect(modules.storageEventManager).toBe(composition.instances.StorageEventManagerFSAPI); + expect(composition.dependencies.ServiceFileHandler).toEqual({ + events: services.events, + API: services.API, + databaseFileAccess: modules.databaseFileAccess, + conflict: services.conflict, + setting: services.setting, + fileProcessing: services.fileProcessing, + vault: services.vault, + path: services.path, + replication: services.replication, + storageAccess: modules.storageAccess, + fileReflectionProvenance: expect.any(Object), + }); + expect(composition.dependencies.ServiceRebuilder).toEqual({ + path: services.path, + events: services.events, + API: services.API, + database: services.database, + appLifecycle: services.appLifecycle, + setting: services.setting, + remote: services.remote, + databaseEvents: services.databaseEvents, + replication: services.replication, + replicator: services.replicator, + UI: services.UI, + vault: services.vault, + fileHandler: modules.fileHandler, + fileProcessing: services.fileProcessing, + storageAccess: modules.storageAccess, + control: services.control, + }); + }); +}); diff --git a/src/apps/webapp/adapters/FSAPIFileSystemAdapter.ts b/src/apps/webapp/adapters/FSAPIFileSystemAdapter.ts index 432ca0f4..fbf5cfbd 100644 --- a/src/apps/webapp/adapters/FSAPIFileSystemAdapter.ts +++ b/src/apps/webapp/adapters/FSAPIFileSystemAdapter.ts @@ -8,6 +8,7 @@ import { FSAPIVaultAdapter } from "./FSAPIVaultAdapter"; import type { FSAPIFile, FSAPIFolder, FSAPIStat } from "./FSAPITypes"; import { shareRunningResult } from "octagonal-wheels/concurrency/lock_v2"; import type { WebAppLog } from "@/apps/webapp/WebAppLog"; +import { findCaseInsensitiveFile } from "@/apps/caseInsensitiveFileLookup"; /** * Complete file system adapter implementation for FileSystem API @@ -85,28 +86,12 @@ export class FSAPIFileSystemAdapter implements IFileSystemAdapter { const pathStr = this.normalisePath(p); - const exact = await this.getAbstractFileByPath(pathStr); - if (exact) { - return exact; - } - // TODO: Refactor: Very, Very heavy. - - const lowerPath = pathStr.toLowerCase(); - for (const [cachedPath, cachedFile] of this.fileCache.entries()) { - if (cachedPath.toLowerCase() === lowerPath) { - return cachedFile; - } - } - - await this.scanDirectory(); - - for (const [cachedPath, cachedFile] of this.fileCache.entries()) { - if (cachedPath.toLowerCase() === lowerPath) { - return cachedFile; - } - } - - return null; + return await findCaseInsensitiveFile( + pathStr, + () => this.getAbstractFileByPath(pathStr), + this.fileCache, + () => this.scanDirectory() + ); } async getFiles(): Promise { diff --git a/src/apps/webapp/adapters/FSAPIStorageAdapter.unit.spec.ts b/src/apps/webapp/adapters/FSAPIStorageAdapter.unit.spec.ts index 2859023f..399ca7e7 100644 --- a/src/apps/webapp/adapters/FSAPIStorageAdapter.unit.spec.ts +++ b/src/apps/webapp/adapters/FSAPIStorageAdapter.unit.spec.ts @@ -110,6 +110,83 @@ describe("FileSystemAccessStorageAdapter", () => { } }); +describe("FSAPIVaultAdapter creation", () => { + it.each(["text", "binary"] as const)( + "creates nested %s files with exact content and stored metadata", + async (kind) => { + const root = new MemoryDirectoryHandle("root") as unknown as FileSystemDirectoryHandle; + const adapter = new FSAPIVaultAdapter(root); + const data = new Uint8Array([0, 255, 1, 128, 42]).buffer; + const path = "/nested//folder/file.dat"; + const file = + kind === "text" ? await adapter.create(path, "日本語😀") : await adapter.createBinary(path, data); + const expected = kind === "text" ? new TextEncoder().encode("日本語😀") : new Uint8Array(data); + const nested = await root.getDirectoryHandle("nested"); + const folder = await nested.getDirectoryHandle("folder"); + const stored = await folder.getFileHandle("file.dat"); + expect(file.path).toBe(path); + expect(file.handle).toBe(stored); + expect(new Uint8Array(await adapter.readBinary(file))).toEqual(expected); + expect(file.stat).toEqual({ size: expected.byteLength, mtime: 1, ctime: 1, type: "file" }); + } + ); + + it.each(["text", "binary"] as const)("waits for the %s writer to close before reading metadata", async (kind) => { + let releaseClose!: () => void; + const closed = new Promise((resolve) => { + releaseClose = resolve; + }); + const events: string[] = []; + const getFile = vi.fn(async () => { + events.push("metadata"); + return new File(["result"], "file.dat", { lastModified: 123 }); + }); + const close = vi.fn(async () => { + events.push("close-start"); + await closed; + events.push("closed"); + }); + const write = vi.fn(async () => { + events.push("write"); + }); + const fileHandle = { createWritable: vi.fn(async () => ({ write, close })), getFile }; + const root = { getFileHandle: vi.fn(async () => fileHandle) } as unknown as FileSystemDirectoryHandle; + const adapter = new FSAPIVaultAdapter(root); + const data = new Uint8Array([0, 255]).buffer; + const pending = kind === "text" ? adapter.create("file.dat", "text") : adapter.createBinary("file.dat", data); + try { + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); + expect(getFile).not.toHaveBeenCalled(); + } finally { + releaseClose(); + } + const file = await pending; + expect(events).toEqual(["write", "close-start", "closed", "metadata"]); + expect(write).toHaveBeenCalledWith(kind === "text" ? "text" : data); + expect(file.stat).toEqual({ size: 6, ctime: 123, mtime: 123, type: "file" }); + }); + + it.each(["text", "binary"] as const)("propagates a %s write failure without returning metadata", async (kind) => { + const failure = new Error("fixture write failed"); + const getFile = vi.fn(); + const fileHandle = { + createWritable: vi.fn(async () => ({ + write: vi.fn(async () => { + throw failure; + }), + close: vi.fn(), + })), + getFile, + }; + const root = { getFileHandle: vi.fn(async () => fileHandle) } as unknown as FileSystemDirectoryHandle; + const adapter = new FSAPIVaultAdapter(root); + const pending = + kind === "text" ? adapter.create("file.dat", "text") : adapter.createBinary("file.dat", new ArrayBuffer(2)); + await expect(pending).rejects.toBe(failure); + expect(getFile).not.toHaveBeenCalled(); + }); +}); + describe("FSAPIVaultAdapter.rename", () => { it("moves the file through a temporary copy while preserving content", async () => { const memoryRoot = new MemoryDirectoryHandle("root"); @@ -142,6 +219,20 @@ describe("FSAPIVaultAdapter.rename", () => { }); describe("FSAPIFileSystemAdapter path case", () => { + it("returns an exact path without scanning the directory", async () => { + const root = new MemoryDirectoryHandle("root") as unknown as FileSystemDirectoryHandle; + const vault = new FSAPIVaultAdapter(root); + await vault.create("Calculus.md", "content"); + const adapter = new FSAPIFileSystemAdapter(root, vi.fn()); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("Calculus.md")).resolves.toMatchObject({ + path: "Calculus.md", + }); + + expect(scanDirectory).not.toHaveBeenCalled(); + }); + it("returns the stored path case from a case-insensitive file system", async () => { const memoryRoot = new MemoryDirectoryHandle("root", true); const root = memoryRoot as unknown as FileSystemDirectoryHandle; @@ -155,6 +246,48 @@ describe("FSAPIFileSystemAdapter path case", () => { ); }); + it("returns a cached case-insensitive match without rescanning", async () => { + const root = new MemoryDirectoryHandle("root") as unknown as FileSystemDirectoryHandle; + const vault = new FSAPIVaultAdapter(root); + await vault.create("Calculus.md", "content"); + const adapter = new FSAPIFileSystemAdapter(root, vi.fn()); + await adapter.scanDirectory(); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("calculus.md")).resolves.toMatchObject({ + path: "Calculus.md", + }); + + expect(scanDirectory).not.toHaveBeenCalled(); + }); + + it("rescans once and returns a newly cached case-insensitive match", async () => { + const root = new MemoryDirectoryHandle("root") as unknown as FileSystemDirectoryHandle; + const vault = new FSAPIVaultAdapter(root); + await vault.create("Calculus.md", "content"); + const adapter = new FSAPIFileSystemAdapter(root, vi.fn()); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("calculus.md")).resolves.toMatchObject({ + path: "Calculus.md", + }); + + expect(scanDirectory).toHaveBeenCalledOnce(); + await expect(adapter.getAbstractFileByPath("Calculus.md")).resolves.toMatchObject({ path: "Calculus.md" }); + }); + + it("returns null after one scan when no cached path matches", async () => { + const root = new MemoryDirectoryHandle("root") as unknown as FileSystemDirectoryHandle; + const vault = new FSAPIVaultAdapter(root); + await vault.create("other.md", "content"); + const adapter = new FSAPIFileSystemAdapter(root, vi.fn()); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("missing.md")).resolves.toBeNull(); + + expect(scanDirectory).toHaveBeenCalledOnce(); + }); + it("reports scan failures through the injected Webapp log", async () => { const root = { name: "root", @@ -173,4 +306,25 @@ describe("FSAPIFileSystemAdapter path case", () => { "fsapi-scan" ); }); + + it("returns null and reports scan failure during case-insensitive lookup", async () => { + const root = { + name: "root", + async *entries(): AsyncIterableIterator<[string, FileSystemHandle]> { + throw new Error("scan failed"); + }, + } as unknown as FileSystemDirectoryHandle; + const addLog = vi.fn(); + const adapter = new FSAPIFileSystemAdapter(root, addLog); + const scanDirectory = vi.spyOn(adapter, "scanDirectory"); + + await expect(adapter.getAbstractFileByPathInsensitive("missing.md")).resolves.toBeNull(); + + expect(scanDirectory).toHaveBeenCalledOnce(); + expect(addLog).toHaveBeenCalledWith( + "Error scanning directory '.': Error: scan failed", + LOG_LEVEL_NOTICE, + "fsapi-scan" + ); + }); }); diff --git a/src/apps/webapp/adapters/FSAPIVaultAdapter.ts b/src/apps/webapp/adapters/FSAPIVaultAdapter.ts index 9e2d8e27..c31e9f27 100644 --- a/src/apps/webapp/adapters/FSAPIVaultAdapter.ts +++ b/src/apps/webapp/adapters/FSAPIVaultAdapter.ts @@ -36,37 +36,14 @@ export class FSAPIVaultAdapter implements IVaultAdapter { } async create(p: string, data: string, options?: UXDataWriteOptions): Promise { - const parts = p.split("/").filter((part) => part !== ""); - const fileName = parts[parts.length - 1]; - - // Navigate to parent directory, creating as needed - let currentHandle = this.rootHandle; - for (let i = 0; i < parts.length - 1; i++) { - currentHandle = await currentHandle.getDirectoryHandle(parts[i], { create: true }); - } - - // Create the file - const fileHandle = await currentHandle.getFileHandle(fileName, { create: true }); - const writable = await fileHandle.createWritable(); - await writable.write(data); - await writable.close(); - - // Get file metadata - const fileObject = await fileHandle.getFile(); - - return { - path: p as FilePath, - stat: { - size: fileObject.size, - mtime: fileObject.lastModified, - ctime: fileObject.lastModified, - type: "file", - }, - handle: fileHandle, - }; + return await this.createFile(p, data); } async createBinary(p: string, data: ArrayBuffer, options?: UXDataWriteOptions): Promise { + return await this.createFile(p, data); + } + + private async createFile(p: string, data: string | ArrayBuffer): Promise { const parts = p.split("/").filter((part) => part !== ""); const fileName = parts[parts.length - 1]; diff --git a/src/apps/webapp/managers/FSAPIStorageEventManagerAdapter.ts b/src/apps/webapp/managers/FSAPIStorageEventManagerAdapter.ts index 6594b61d..139ebb7d 100644 --- a/src/apps/webapp/managers/FSAPIStorageEventManagerAdapter.ts +++ b/src/apps/webapp/managers/FSAPIStorageEventManagerAdapter.ts @@ -2,7 +2,6 @@ import type { FilePath, UXFileInfoStub, UXInternalFileInfoStub } from "@vrtmrz/l import type { FileEventItem } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { IStorageEventManagerAdapter } from "@vrtmrz/livesync-commonlib/compat/managers/adapters"; import type { - IStorageEventTypeGuardAdapter, IStorageEventPersistenceAdapter, IStorageEventWatchAdapter, IStorageEventStatusAdapter, @@ -14,6 +13,7 @@ import type { FSAPIFile, FSAPIFolder } from "@/apps/webapp/adapters/FSAPITypes"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { WebAppLog } from "@/apps/webapp/WebAppLog"; +import { FSAPITypeGuardAdapter } from "@/apps/webapp/adapters/FSAPITypeGuardAdapter"; type FileSystemObserverRecord = { changedHandle?: FileSystemFileHandle | FileSystemDirectoryHandle; @@ -34,32 +34,6 @@ type GlobalWithFileSystemObserver = typeof compatGlobal & { FileSystemObserver?: FileSystemObserverConstructor; }; -/** - * FileSystem API-specific type guard adapter - */ -class FSAPITypeGuardAdapter implements IStorageEventTypeGuardAdapter { - isFile(file: unknown): file is FSAPIFile { - return !!( - file && - typeof file === "object" && - "path" in file && - "stat" in file && - "handle" in file && - !(file as { isFolder?: boolean }).isFolder - ); - } - - isFolder(item: unknown): item is FSAPIFolder { - return !!( - item && - typeof item === "object" && - "path" in item && - (item as { isFolder?: boolean }).isFolder === true && - "handle" in item - ); - } -} - /** * FileSystem API-specific persistence adapter (IndexedDB-based snapshot) */ diff --git a/src/apps/webapp/managers/FSAPIStorageEventManagerAdapter.unit.spec.ts b/src/apps/webapp/managers/FSAPIStorageEventManagerAdapter.unit.spec.ts new file mode 100644 index 00000000..881c5b61 --- /dev/null +++ b/src/apps/webapp/managers/FSAPIStorageEventManagerAdapter.unit.spec.ts @@ -0,0 +1,40 @@ +import { describe, expect, it, vi } from "vitest"; +import { FSAPIStorageEventManagerAdapter } from "./FSAPIStorageEventManagerAdapter"; + +describe("FSAPIStorageEventManagerAdapter type guard", () => { + const typeGuard = new FSAPIStorageEventManagerAdapter({} as FileSystemDirectoryHandle, vi.fn()).typeGuard; + + it.each([ + { label: "null", value: null, expected: false }, + { label: "an object without a path", value: { stat: {}, handle: undefined }, expected: false }, + { label: "an object without a stat", value: { path: "note.md", handle: undefined }, expected: false }, + { label: "an object without a handle", value: { path: "note.md", stat: {} }, expected: false }, + { + label: "a file-shaped object with a handle property", + value: { path: "note.md", stat: {}, handle: undefined }, + expected: true, + }, + { + label: "a folder-shaped object", + value: { path: "folder", handle: undefined, isFolder: true }, + expected: false, + }, + ])("classifies $label as a file", ({ value, expected }) => { + expect(typeGuard.isFile(value)).toBe(expected); + }); + + it.each([ + { label: "null", value: null, expected: false }, + { label: "an object without a path", value: { isFolder: true, handle: undefined }, expected: false }, + { label: "an object without the folder marker", value: { path: "folder", handle: undefined }, expected: false }, + { label: "an object without a handle", value: { path: "folder", isFolder: true }, expected: false }, + { + label: "a folder-shaped object with a handle property", + value: { path: "folder", isFolder: true, handle: undefined }, + expected: true, + }, + { label: "a file-shaped object", value: { path: "note.md", stat: {}, handle: undefined }, expected: false }, + ])("classifies $label as a folder", ({ value, expected }) => { + expect(typeGuard.isFolder(value)).toBe(expected); + }); +}); diff --git a/src/apps/webapp/package.json b/src/apps/webapp/package.json index ccda9b89..d6e8095c 100644 --- a/src/apps/webapp/package.json +++ b/src/apps/webapp/package.json @@ -1,7 +1,7 @@ { "name": "livesync-webapp", "private": true, - "version": "1.0.42-webapp", + "version": "1.0.44-webapp", "type": "module", "description": "Browser-based Self-hosted LiveSync using FileSystem API", "scripts": { @@ -15,7 +15,7 @@ "test:browser": "deno test -A --no-check --frozen --config ../../../test/browser-apps/deno.json --lock ../../../test/browser-apps/deno.lock ../../../test/browser-apps/webapp/browser-smoke.test.ts" }, "dependencies": { - "octagonal-wheels": "^0.1.55" + "octagonal-wheels": "0.1.56" }, "devDependencies": { "@sveltejs/vite-plugin-svelte": "^7.1.2", diff --git a/src/apps/webapp/serviceModules/FSAPIServiceModules.ts b/src/apps/webapp/serviceModules/FSAPIServiceModules.ts index bda8059d..fd70825e 100644 --- a/src/apps/webapp/serviceModules/FSAPIServiceModules.ts +++ b/src/apps/webapp/serviceModules/FSAPIServiceModules.ts @@ -1,5 +1,4 @@ import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub"; -import { ServiceRebuilder } from "@vrtmrz/livesync-commonlib/compat/serviceModules/Rebuilder"; import { StorageAccessManager } from "@vrtmrz/livesync-commonlib/compat/managers/StorageProcessingManager"; import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; @@ -9,8 +8,7 @@ import { ServiceFileAccessFSAPI } from "./ServiceFileAccessImpl"; import { ServiceDatabaseFileAccessFSAPI } from "./DatabaseFileAccess"; import { StorageEventManagerFSAPI } from "@/apps/webapp/managers/StorageEventManagerFSAPI"; import type { ServiceModules } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; -import { ServiceFileHandler } from "@/serviceModules/FileHandler"; -import { createFileReflectionProvenance } from "@/serviceModules/FileReflectionProvenance"; +import { composeSharedServiceModules } from "@/apps/serviceModules/composeSharedServiceModules"; export interface FSAPIServiceModules extends ServiceModules { vaultAccess: FileAccessFSAPI; @@ -73,40 +71,7 @@ export function initialiseServiceModulesFSAPI( vault: services.vault, }); - // File handler (platform-independent) - const fileHandler = new ServiceFileHandler({ - events: services.context.events, - API: services.API, - databaseFileAccess: databaseFileAccess, - conflict: services.conflict, - setting: services.setting, - fileProcessing: services.fileProcessing, - vault: services.vault, - path: services.path, - replication: services.replication, - storageAccess: storageAccess, - fileReflectionProvenance: createFileReflectionProvenance(services.keyValueDB), - }); - - // Rebuilder (platform-independent) - const rebuilder = new ServiceRebuilder({ - path: services.path, - events: services.context.events, - API: services.API, - database: services.database, - appLifecycle: services.appLifecycle, - setting: services.setting, - remote: services.remote, - databaseEvents: services.databaseEvents, - replication: services.replication, - replicator: services.replicator, - UI: services.UI, - vault: services.vault, - fileHandler: fileHandler, - fileProcessing: services.fileProcessing, - storageAccess: storageAccess, - control: services.control, - }); + const { fileHandler, rebuilder } = composeSharedServiceModules(services, databaseFileAccess, storageAccess); return { rebuilder, diff --git a/src/apps/webapp/vite.config.ts b/src/apps/webapp/vite.config.ts index e48b74d1..4564fd31 100644 --- a/src/apps/webapp/vite.config.ts +++ b/src/apps/webapp/vite.config.ts @@ -1,17 +1,10 @@ import { defineConfig } from "vite"; import { svelte } from "@sveltejs/vite-plugin-svelte"; -import { fileURLToPath, fs, path } from "@vrtmrz/livesync-commonlib/node"; +import { fileURLToPath, path } from "@vrtmrz/livesync-commonlib/node"; +import { readVersion } from "../readVersion"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(__dirname, "../../.."); -function readVersion(filePath: string): string | undefined { - const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8")); - if (typeof parsed !== "object" || parsed === null || !("version" in parsed)) { - return undefined; - } - return typeof parsed.version === "string" ? parsed.version : undefined; -} - const packageVersion = readVersion(path.resolve(repoRoot, "package.json")); const manifestVersion = readVersion(path.resolve(repoRoot, "manifest.json")); // https://vite.dev/config/ diff --git a/src/apps/webpeer/package.json b/src/apps/webpeer/package.json index c9047af9..d3ceea52 100644 --- a/src/apps/webpeer/package.json +++ b/src/apps/webpeer/package.json @@ -1,7 +1,7 @@ { "name": "webpeer", "private": true, - "version": "1.0.42-webpeer", + "version": "1.0.44-webpeer", "type": "module", "scripts": { "dev": "vite", @@ -15,7 +15,7 @@ "test:browser": "deno test -A --no-check --frozen --config ../../../test/browser-apps/deno.json --lock ../../../test/browser-apps/deno.lock ../../../test/browser-apps/webpeer/browser-smoke.test.ts" }, "dependencies": { - "octagonal-wheels": "^0.1.55" + "octagonal-wheels": "0.1.56" }, "devDependencies": { "eslint-plugin-svelte": "^3.19.0", diff --git a/src/apps/webpeer/src/WebPeerRuntime.ts b/src/apps/webpeer/src/WebPeerRuntime.ts index fd29947a..2b317d6b 100644 --- a/src/apps/webpeer/src/WebPeerRuntime.ts +++ b/src/apps/webpeer/src/WebPeerRuntime.ts @@ -145,7 +145,7 @@ export class WebPeerRuntime { private showPeerMenu(peer: PeerStatus, event: MouseEvent): void { this.menu?.hide(); - this.menu = new Menu() + this.menu = new Menu(this.services.browserUi) .addItem((item) => item.setTitle("📥 Only fetch").onClick(async () => { await this.p2p.targetedTransfer.pullFromPeer(peer.peerId); diff --git a/src/common/KeyValueDB.ts b/src/common/KeyValueDB.ts index 47a75c70..a571274f 100644 --- a/src/common/KeyValueDB.ts +++ b/src/common/KeyValueDB.ts @@ -1,6 +1,6 @@ import { deleteDB, type IDBPDatabase, openDB } from "idb"; import type { KeyValueDatabase } from "@vrtmrz/livesync-commonlib/compat/interfaces/KeyValueDatabase"; -import { serialized } from "octagonal-wheels/concurrency/lock"; +import { withLock } from "octagonal-wheels/concurrency/lock"; import { Logger } from "octagonal-wheels/common/logger"; const databaseCache: { [key: string]: IDBPDatabase } = {}; export { OpenKeyValueDatabase } from "./KeyValueDBv2.ts"; @@ -13,7 +13,7 @@ export const _OpenKeyValueDatabase = async (dbKey: string): Promise | null = null; const _openDB = () => { - return serialized("keyvaluedb-" + dbKey, async () => { + return withLock("keyvaluedb-" + dbKey, async () => { const dbInstance = await openDB(dbKey, 1, { upgrade(db, _oldVersion, _newVersion, _transaction, _event) { return db.createObjectStore(storeKey); diff --git a/src/common/KeyValueDBv2.ts b/src/common/KeyValueDBv2.ts index d21c15bd..cf4c3598 100644 --- a/src/common/KeyValueDBv2.ts +++ b/src/common/KeyValueDBv2.ts @@ -1,12 +1,12 @@ import { LOG_LEVEL_VERBOSE, Logger } from "@vrtmrz/livesync-commonlib/compat/common/logger"; import type { KeyValueDatabase } from "@vrtmrz/livesync-commonlib/compat/interfaces/KeyValueDatabase"; import { deleteDB, openDB, type IDBPDatabase } from "idb"; -import { serialized } from "octagonal-wheels/concurrency/lock"; +import { withLock } from "octagonal-wheels/concurrency/lock"; const databaseCache = new Map(); export async function OpenKeyValueDatabase(dbKey: string): Promise { - return await serialized(`OpenKeyValueDatabase-${dbKey}`, async () => { + return await withLock(`OpenKeyValueDatabase-${dbKey}`, async () => { const cachedDB = databaseCache.get(dbKey); if (cachedDB) { if (!cachedDB.isDestroyed) { diff --git a/src/common/diffNavigation.ts b/src/common/diffNavigation.ts new file mode 100644 index 00000000..543e8490 --- /dev/null +++ b/src/common/diffNavigation.ts @@ -0,0 +1,25 @@ +import { stepCyclicIndex } from "octagonal-wheels/collection"; + +export function navigateDiffNodes( + root: ParentNode, + selector: string, + navigation: { + currentDiffIndex: number; + diffNavIndicator: { setText(text: string): void }; + }, + direction: "prev" | "next" +): void { + const diffElements = root.querySelectorAll(selector); + if (diffElements.length === 0) return; + + const previousFocused = root.querySelector(".diff-focused"); + if (previousFocused) { + previousFocused.classList.remove("diff-focused"); + } + + navigation.currentDiffIndex = stepCyclicIndex(navigation.currentDiffIndex, diffElements.length, direction); + const target = diffElements[navigation.currentDiffIndex]; + target.classList.add("diff-focused"); + target.scrollIntoView({ behavior: "smooth", block: "center" }); + navigation.diffNavIndicator.setText(`${navigation.currentDiffIndex + 1}/${diffElements.length}`); +} diff --git a/src/common/ownedRemoteResource.ts b/src/common/ownedRemoteResource.ts index 60483414..19eaa1a3 100644 --- a/src/common/ownedRemoteResource.ts +++ b/src/common/ownedRemoteResource.ts @@ -1,3 +1,5 @@ +import { withOwnedResource, type DisposableResource } from "octagonal-wheels/resource"; + /** * Run a finite operation with a flow-owned remote resource and release it * after either success or failure. @@ -6,13 +8,9 @@ * caller's ownership boundary explicit and prevents finite flows from leaking * a provider-owned resource when their operation rejects. */ -export async function withOwnedRemoteResource }, TResult>( +export function withOwnedRemoteResource }, TResult>( resource: TResource, operation: (ownedResource: TResource) => Promise ): Promise { - try { - return await operation(resource); - } finally { - await resource.dispose(); - } + return withOwnedResource(resource, operation); } diff --git a/src/common/ownedRemoteResource.unit.spec.ts b/src/common/ownedRemoteResource.unit.spec.ts index 840acf7e..2eb042f9 100644 --- a/src/common/ownedRemoteResource.unit.spec.ts +++ b/src/common/ownedRemoteResource.unit.spec.ts @@ -26,3 +26,47 @@ describe("flow-owned remote resources", () => { expect(dispose).toHaveBeenCalledOnce(); }); }); + +describe("owned remote resource completion", () => { + it("waits for cleanup and preserves its receiver", async () => { + let finish!: () => void; + let started!: () => void; + const disposing = new Promise((resolve) => { + started = resolve; + }); + const cleanup = new Promise((resolve) => { + finish = resolve; + }); + const resource = { + dispose: vi.fn(function (this: unknown) { + expect(this).toBe(resource); + started(); + return cleanup; + }), + }; + let settled = false; + const result = withOwnedRemoteResource(resource, async () => "done"); + void result.then(() => { + settled = true; + }); + await disposing; + expect(settled).toBe(false); + finish(); + await expect(result).resolves.toBe("done"); + expect(resource.dispose).toHaveBeenCalledOnce(); + }); + + it("keeps finally precedence when both the operation and cleanup fail", async () => { + const operationError = new Error("operation"); + const disposalError = new Error("cleanup"); + const dispose = vi.fn(async () => { + throw disposalError; + }); + await expect( + withOwnedRemoteResource({ dispose }, async () => { + throw operationError; + }) + ).rejects.toBe(disposalError); + expect(dispose).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/common/pathCompatibility.ts b/src/common/pathCompatibility.ts index 69ad7254..b9f3b1ff 100644 --- a/src/common/pathCompatibility.ts +++ b/src/common/pathCompatibility.ts @@ -1,3 +1,5 @@ +import { findPathComponentsExceedingUtf8Limit as findKitPathComponentsExceedingUtf8Limit } from "octagonal-wheels/path"; + export const ANDROID_LINUX_PATH_COMPONENT_UTF8_WARNING_BOUNDARY = 255; export interface OversizedPathComponent { @@ -5,8 +7,6 @@ export interface OversizedPathComponent { utf8Bytes: number; } -const utf8Encoder = new TextEncoder(); - /** * Return path components which exceed the conservative Android/Linux * compatibility boundary. @@ -18,9 +18,5 @@ export function findPathComponentsExceedingUtf8Limit( path: string, maxBytes: number = ANDROID_LINUX_PATH_COMPONENT_UTF8_WARNING_BOUNDARY ): OversizedPathComponent[] { - return path - .split("/") - .filter((component) => component.length > 0) - .map((component) => ({ component, utf8Bytes: utf8Encoder.encode(component).byteLength })) - .filter(({ utf8Bytes }) => utf8Bytes > maxBytes); + return findKitPathComponentsExceedingUtf8Limit(path, maxBytes); } diff --git a/src/common/replicatorProviders.ts b/src/common/replicatorProviders.ts index 55fa13b0..19b13d38 100644 --- a/src/common/replicatorProviders.ts +++ b/src/common/replicatorProviders.ts @@ -73,7 +73,7 @@ async function runOneShotWithOutcome( // Manual and unattended wrappers share the provider transfer operation, but // keep interaction authority and result presentation explicit at this boundary. -const couchDBUserInitiatedOneShot: UserInitiatedOneShotRunner = async (instance, setting, request) => { +const userInitiatedOneShot: UserInitiatedOneShotRunner = async (instance, setting, request) => { return await runOneShotWithOutcome( instance, setting, @@ -81,20 +81,7 @@ const couchDBUserInitiatedOneShot: UserInitiatedOneShotRunner = async (instance, ); }; -const couchDBUnattendedOneShot: UnattendedOneShotRunner = async (instance, setting, request) => { - if (request.interaction.kind !== NO_INTERACTION.kind) return replicationBlocked("interaction-required"); - return await runOneShotWithOutcome(instance, setting, false); -}; - -const objectStorageUserInitiatedOneShot: UserInitiatedOneShotRunner = async (instance, setting, request) => { - return await runOneShotWithOutcome( - instance, - setting, - request.progressPresentation === REPLICATION_PROGRESS_PRESENTATIONS.NOTICE - ); -}; - -const objectStorageUnattendedOneShot: UnattendedOneShotRunner = async (instance, setting, request) => { +const unattendedOneShot: UnattendedOneShotRunner = async (instance, setting, request) => { if (request.interaction.kind !== NO_INTERACTION.kind) return replicationBlocked("interaction-required"); return await runOneShotWithOutcome(instance, setting, false); }; @@ -136,8 +123,8 @@ export function createCentralReplicatorProviderDefinitions( ), }, centralRemoteAdministration: COUCHDB_CENTRAL_REMOTE_ADMINISTRATION_CAPABILITY, - userInitiatedOneShot: supportedCapability(couchDBUserInitiatedOneShot), - unattendedOneShot: supportedCapability(couchDBUnattendedOneShot), + userInitiatedOneShot: supportedCapability(userInitiatedOneShot), + unattendedOneShot: supportedCapability(unattendedOneShot), continuous: supportedOpenReplicationContinuous(), stopActiveTransfer: supportedStopActiveTransfer(), }, @@ -166,8 +153,8 @@ export function createCentralReplicatorProviderDefinitions( [REMOTE_RESOURCE_KINDS.INITIAL_FETCH_FAST_FORWARD]: CAPABILITY_NOT_APPLICABLE, }, centralRemoteAdministration: OBJECT_STORAGE_CENTRAL_REMOTE_ADMINISTRATION_CAPABILITY, - userInitiatedOneShot: supportedCapability(objectStorageUserInitiatedOneShot), - unattendedOneShot: supportedCapability(objectStorageUnattendedOneShot), + userInitiatedOneShot: supportedCapability(userInitiatedOneShot), + unattendedOneShot: supportedCapability(unattendedOneShot), continuous: CAPABILITY_NOT_APPLICABLE, stopActiveTransfer: supportedStopActiveTransfer(), }, diff --git a/src/common/replicatorProviders.unit.spec.ts b/src/common/replicatorProviders.unit.spec.ts index 0354537d..8bf02b6d 100644 --- a/src/common/replicatorProviders.unit.spec.ts +++ b/src/common/replicatorProviders.unit.spec.ts @@ -209,6 +209,37 @@ describe("central Replicator provider definitions", () => { expect(openOneShotReplicationWithOutcome).toHaveBeenNthCalledWith(2, setting, true); }); + it.each([REMOTE_COUCHDB, REMOTE_MINIO] as const)( + "keeps unattended %s work quiet and rejects interaction before opening replication", + async (remoteType) => { + const definition = createCentralReplicatorProviderDefinitions({} as never).get(remoteType)!; + if (definition.unattendedOneShot.kind !== CAPABILITY_SUPPORT_KINDS.SUPPORTED) { + throw new Error("Unattended OneShot is unavailable"); + } + const setting = Object.assign(createNewVaultSettings(), { remoteType }); + const instance = { + initializeDatabaseForReplication: vi.fn(async () => true), + openReplication: vi.fn(async () => true), + terminateSync: vi.fn(), + closeReplication: vi.fn(), + openOneShotReplicationWithOutcome: vi.fn(async () => REPLICATION_COMPLETED), + }; + const blocked = await definition.unattendedOneShot.run(instance, setting, { + trigger: "resume", + interaction: USER_INITIATED_REPLICATION_AUTHORITY, + } as never); + expect(blocked).toEqual({ status: "blocked", reason: "interaction-required" }); + expect(instance.openOneShotReplicationWithOutcome).not.toHaveBeenCalled(); + await expect( + definition.unattendedOneShot.run(instance, setting, { + trigger: "resume", + interaction: NO_INTERACTION, + }) + ).resolves.toBe(REPLICATION_COMPLETED); + expect(instance.openOneShotReplicationWithOutcome).toHaveBeenCalledExactlyOnceWith(setting, false); + } + ); + it("dispatches central finite work through the declared operation rather than constructor identity", async () => { const definitions = createCentralReplicatorProviderDefinitions({} as never); const couchDB = definitions.get(REMOTE_COUCHDB)!; diff --git a/src/common/replicatorResources/shared.ts b/src/common/replicatorResources/shared.ts index a88f070a..e2b4b8a1 100644 --- a/src/common/replicatorResources/shared.ts +++ b/src/common/replicatorResources/shared.ts @@ -1,3 +1,4 @@ +import { createOwnedResource } from "octagonal-wheels/resource"; import type { RemoteDBSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; /** @@ -13,13 +14,8 @@ export interface ResourceReplicator { /** Create one idempotent asynchronous disposer for a private Replicator. */ export function createReplicatorDisposer(replicator: ResourceReplicator): () => Promise { - let disposal: Promise | undefined; - return () => { - if (disposal === undefined) { - disposal = Promise.resolve().then(() => replicator.closeReplication()); - } - return disposal; - }; + const resource = createOwnedResource(replicator, (owned) => Promise.resolve().then(() => owned.closeReplication())); + return () => Promise.resolve(resource.dispose()); } /** Fence a finite resource from later edits to its source settings object. */ diff --git a/src/common/replicatorResources/shared.unit.spec.ts b/src/common/replicatorResources/shared.unit.spec.ts new file mode 100644 index 00000000..e2705440 --- /dev/null +++ b/src/common/replicatorResources/shared.unit.spec.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from "vitest"; +import { createReplicatorDisposer } from "./shared"; + +describe("Replicator disposal", () => { + it("defers closing, preserves the receiver, and retains one completion promise", async () => { + let finish!: () => void; + const closed = new Promise((resolve) => { + finish = resolve; + }); + const replicator = { + closeReplication: vi.fn(function (this: unknown) { + expect(this).toBe(replicator); + return closed; + }), + }; + const dispose = createReplicatorDisposer(replicator); + const pending = dispose(); + expect(replicator.closeReplication).not.toHaveBeenCalled(); + expect(dispose()).toBe(pending); + await Promise.resolve(); + expect(replicator.closeReplication).toHaveBeenCalledOnce(); + finish(); + await pending; + expect(dispose()).toBe(pending); + expect(replicator.closeReplication).toHaveBeenCalledOnce(); + }); + + it("retains synchronous closing failures as one rejected promise", async () => { + const error = new Error("close"); + const replicator = { + closeReplication: vi.fn(() => { + throw error; + }), + }; + const dispose = createReplicatorDisposer(replicator); + const pending = dispose(); + await expect(pending).rejects.toBe(error); + expect(dispose()).toBe(pending); + await expect(dispose()).rejects.toBe(error); + expect(replicator.closeReplication).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/common/utils.ts b/src/common/utils.ts index f4bc621d..fc248ff1 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -389,13 +389,4 @@ export async function autosaveCache(db: KeyValueDatabase, mapKey: string): }; } -export function onlyInNTimes(n: number, proc: (progress: number) => unknown) { - let counter = 0; - return function () { - if (counter++ % n == 0) { - proc(counter); - } - }; -} - export { displayRev } from "@vrtmrz/livesync-commonlib/compat/common/utils"; diff --git a/src/features/ConfigSync/CmdConfigSync.applyContent.unit.spec.ts b/src/features/ConfigSync/CmdConfigSync.applyContent.unit.spec.ts new file mode 100644 index 00000000..95e7cab2 --- /dev/null +++ b/src/features/ConfigSync/CmdConfigSync.applyContent.unit.spec.ts @@ -0,0 +1,275 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + addIcon: vi.fn(), + diff_match_patch: class DiffMatchPatch {}, + normalizePath: vi.fn((path: string) => path), + parseYaml: vi.fn(), + Platform: {}, +})); +vi.mock("./PluginDialogModal.ts", () => ({ + PluginDialogModal: class PluginDialogModal {}, +})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ + JsonResolveModal: class JsonResolveModal {}, +})); +vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({ + ConflictResolveModal: class ConflictResolveModal {}, +})); +vi.mock("@/features/LiveSyncCommands.ts", () => ({ + LiveSyncCommands: class LiveSyncCommands { + core!: { services: unknown }; + get services() { + return this.core.services; + } + }, +})); +vi.mock("@/common/types.ts", () => ({ + ICXHeader: "ix:", + PERIODIC_PLUGIN_SWEEP: 60, +})); +vi.mock("@/common/utils.ts", () => ({ + cancelTask: vi.fn(), + EVEN: Symbol("even"), + isCustomisationSyncMetadata: vi.fn(), + isPluginMetadata: vi.fn(), + scheduleTask: vi.fn(), +})); +vi.mock("@/common/PeriodicProcessor.ts", () => ({ + PeriodicProcessor: class PeriodicProcessor {}, +})); +vi.mock("@/common/events.ts", () => ({ + EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync", + eventHub: { + onEvent: vi.fn(), + }, +})); +vi.mock("@/common/translation", () => ({ + $msg: vi.fn((message: string) => message), +})); +vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ + getObsidianCommunityPluginManager: vi.fn(), +})); + +import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { ConfigSync, type PluginDataExDisplayV2 } from "./CmdConfigSync"; + +const BASE_DIR = ".obsidian"; +const DEVICE_NAME = "test-device"; +const STAT = { mtime: 123456, ctime: 123000 }; + +type ApplyFile = { + filename: string; + datatype: "newnote" | "plain"; + data: string[]; + mtime: number; + ctime: number; +}; + +function makeApplyFile(datatype: ApplyFile["datatype"], data: string[]): ApplyFile { + return { + filename: "plugins/example/data.json", + datatype, + data, + mtime: STAT.mtime, + ctime: STAT.ctime, + }; +} + +function setup(file: ApplyFile, currentText = "before", currentBinary = new ArrayBuffer(0)) { + const path = `${BASE_DIR}/${file.filename}`; + const events: string[] = []; + const storageAccess = { + ensureDir: vi.fn(async (_path: string) => { + events.push("ensure"); + }), + readHiddenFileText: vi.fn(async (_path: string) => { + events.push("read-text"); + return currentText; + }), + readHiddenFileBinary: vi.fn(async (_path: string) => { + events.push("read-binary"); + return currentBinary; + }), + writeHiddenFileAuto: vi.fn( + async (_path: string, _content: string | ArrayBuffer, _stat?: { mtime: number; ctime: number }) => { + events.push("write"); + } + ), + }; + const storeCustomisationFileV2 = vi.fn(async (_path: string, _term: string) => { + events.push("store"); + return true; + }); + const log = vi.fn(); + const configSync = Object.create(ConfigSync.prototype) as ConfigSync; + Object.assign(configSync, { + core: { + services: { + API: { getSystemConfigDir: () => BASE_DIR }, + setting: { getDeviceAndVaultName: () => DEVICE_NAME }, + }, + storageAccess, + }, + _log: log, + storeCustomisationFileV2, + }); + const data = { + displayName: "Example plug-in", + name: "example", + files: [file], + } as unknown as PluginDataExDisplayV2; + return { configSync, data, events, log, path, storageAccess, storeCustomisationFileV2 }; +} + +const changedInputs = [ + { + kind: "text", + file: makeApplyFile("plain", ["replacement text"]), + currentText: "previous text", + expectedText: "replacement text", + }, + { + kind: "binary", + file: makeApplyFile("newnote", ["AQID"]), + currentBinary: new Uint8Array([9]).buffer, + expectedBytes: [1, 2, 3], + }, +] as const; + +const unchangedInputs = [ + { + kind: "text", + file: makeApplyFile("plain", ["unchanged text"]), + currentText: "unchanged text", + }, + { + kind: "binary", + file: makeApplyFile("newnote", ["AQID"]), + currentBinary: new Uint8Array([1, 2, 3]).buffer, + }, +] as const; + +describe("ConfigSync V2 content application", () => { + it.each(changedInputs)("writes changed $kind content, then stores it", async (input) => { + const fixture = setup( + input.file, + input.kind === "text" ? input.currentText : undefined, + input.kind === "binary" ? input.currentBinary : undefined + ); + + await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(true); + + expect(fixture.storageAccess.ensureDir).toHaveBeenCalledWith(fixture.path); + expect(fixture.storageAccess.writeHiddenFileAuto).toHaveBeenCalledOnce(); + expect(fixture.storageAccess.writeHiddenFileAuto).toHaveBeenCalledWith( + fixture.path, + input.kind === "text" ? input.expectedText : expect.any(ArrayBuffer), + STAT + ); + if (input.kind === "binary") { + const written = fixture.storageAccess.writeHiddenFileAuto.mock.calls[0]?.[1]; + expect(written).toBeInstanceOf(ArrayBuffer); + expect(Array.from(new Uint8Array(written as ArrayBuffer))).toEqual(input.expectedBytes); + } + expect(fixture.events).toEqual([ + "ensure", + input.kind === "text" ? "read-text" : "read-binary", + "write", + "store", + ]); + expect(fixture.storeCustomisationFileV2).toHaveBeenCalledWith(fixture.path, DEVICE_NAME); + }); + + it("writes the file timestamps captured before reading existing content", async () => { + const file = makeApplyFile("plain", ["replacement text"]); + const fixture = setup(file, "previous text"); + fixture.storageAccess.readHiddenFileText.mockImplementation(async () => { + fixture.events.push("read-text"); + file.mtime = STAT.mtime + 1; + file.ctime = STAT.ctime + 1; + return "previous text"; + }); + + await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(true); + + expect(fixture.storageAccess.writeHiddenFileAuto).toHaveBeenCalledWith(fixture.path, "replacement text", STAT); + }); + + it.each(unchangedInputs)("skips writing and storing unchanged $kind content", async (input) => { + const fixture = setup( + input.file, + input.kind === "text" ? input.currentText : undefined, + input.kind === "binary" ? input.currentBinary : undefined + ); + + await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(true); + + expect(fixture.storageAccess.writeHiddenFileAuto).not.toHaveBeenCalled(); + expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled(); + expect(fixture.events).toEqual(["ensure", input.kind === "text" ? "read-text" : "read-binary"]); + expect(fixture.log).toHaveBeenCalledWith( + `The file ${input.file.filename} is already up-to-date`, + LOG_LEVEL_VERBOSE + ); + }); + + it.each([ + { kind: "text", file: makeApplyFile("plain", ["replacement text"]) }, + { kind: "binary", file: makeApplyFile("newnote", ["AQID"]) }, + ])("uses empty $kind content when reading the existing file fails", async ({ kind, file }) => { + const fixture = setup(file); + const readError = new Error("Could not read existing content"); + if (kind === "text") { + fixture.storageAccess.readHiddenFileText.mockRejectedValueOnce(readError); + } else { + fixture.storageAccess.readHiddenFileBinary.mockRejectedValueOnce(readError); + } + + await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(true); + + expect(fixture.storageAccess.writeHiddenFileAuto).toHaveBeenCalledOnce(); + expect(fixture.storeCustomisationFileV2).toHaveBeenCalledWith(fixture.path, DEVICE_NAME); + expect(fixture.log).toHaveBeenCalledWith(`Could not read the file ${file.filename}`, LOG_LEVEL_VERBOSE); + expect(fixture.log).toHaveBeenCalledWith(readError, LOG_LEVEL_VERBOSE); + }); + + it("awaits the write before storing the applied content", async () => { + const file = makeApplyFile("plain", ["replacement text"]); + const fixture = setup(file, "previous text"); + let finishWrite!: () => void; + let signalWriteStarted!: () => void; + const writeGate = new Promise((resolve) => { + finishWrite = resolve; + }); + const writeStarted = new Promise((resolve) => { + signalWriteStarted = resolve; + }); + fixture.storageAccess.writeHiddenFileAuto.mockImplementation(async () => { + fixture.events.push("write-start"); + signalWriteStarted(); + await writeGate; + fixture.events.push("write-finished"); + }); + + const applying = fixture.configSync.applyDataV2(fixture.data); + await writeStarted; + expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled(); + + finishWrite(); + await expect(applying).resolves.toBe(true); + expect(fixture.events.slice(-3)).toEqual(["write-start", "write-finished", "store"]); + }); + + it("returns false after a write failure and does not store the file", async () => { + const fixture = setup(makeApplyFile("plain", ["replacement text"]), "previous text"); + const writeError = new Error("Write failed"); + fixture.storageAccess.writeHiddenFileAuto.mockRejectedValueOnce(writeError); + + await expect(fixture.configSync.applyDataV2(fixture.data)).resolves.toBe(false); + + expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled(); + expect(fixture.log).toHaveBeenCalledWith("Applying Example plug-in.. Failed", LOG_LEVEL_NOTICE); + expect(fixture.log).toHaveBeenCalledWith(writeError, LOG_LEVEL_VERBOSE); + }); +}); diff --git a/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts b/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts index 06b04919..ddfce2b0 100644 --- a/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts +++ b/src/features/ConfigSync/CmdConfigSync.catalogue.unit.spec.ts @@ -84,7 +84,7 @@ async function catalogue() { pluginListGeneration: 0, loadedManifest_mTime: new Map(), updatingV2Count: 0, - pluginScanProcessor: { clearQueue: vi.fn(), enqueue: vi.fn() }, + pluginScanProcessor: { clearPending: vi.fn(), enqueue: vi.fn() }, _log: vi.fn(), storeCustomisationFileV2: vi.fn(async () => true), }); diff --git a/src/features/ConfigSync/CmdConfigSync.command.unit.spec.ts b/src/features/ConfigSync/CmdConfigSync.command.unit.spec.ts index 5f524f8a..6124c4bd 100644 --- a/src/features/ConfigSync/CmdConfigSync.command.unit.spec.ts +++ b/src/features/ConfigSync/CmdConfigSync.command.unit.spec.ts @@ -91,7 +91,7 @@ describe("ConfigSync commands", () => { expect(showPluginSyncModal).toHaveBeenCalledOnce(); }); - it("cancels the pending configuration Notice before releasing its owned UI", () => { + it("cancels the pending configuration Notice before releasing its owned UI", async () => { const notices = { hide: vi.fn() }; const periodicPluginSweepProcessor = { disable: vi.fn() }; const configSync = Object.create(ConfigSync.prototype) as ConfigSync; @@ -102,9 +102,13 @@ describe("ConfigSync commands", () => { }, }, periodicPluginSweepProcessor, + pluginList: [], + pluginListGeneration: 0, + loadedManifest_mTime: new Map(), + pluginScanProcessor: { clearPending: vi.fn(), stop: vi.fn(async () => {}) }, }); - configSync.onunload(); + await configSync.onunload(); expect(cancelTask).toHaveBeenCalledWith("config-sync:updated-configuration"); expect(notices.hide).toHaveBeenCalledWith("config-sync:updated-configuration"); diff --git a/src/features/ConfigSync/CmdConfigSync.queue.unit.spec.ts b/src/features/ConfigSync/CmdConfigSync.queue.unit.spec.ts new file mode 100644 index 00000000..98f4000d --- /dev/null +++ b/src/features/ConfigSync/CmdConfigSync.queue.unit.spec.ts @@ -0,0 +1,591 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + addIcon: vi.fn(), + diff_match_patch: class DiffMatchPatch {}, + normalizePath: (path: string) => path, + parseYaml: vi.fn(), + Platform: {}, +})); +vi.mock("./PluginDialogModal.ts", () => ({ PluginDialogModal: class PluginDialogModal {} })); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} })); +vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({ + ConflictResolveModal: class ConflictResolveModal {}, +})); +vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" })); +vi.mock("@/common/utils.ts", () => ({ + cancelTask: vi.fn(), + EVEN: Symbol("even"), + isCustomisationSyncMetadata: vi.fn(), + isPluginMetadata: vi.fn(), + scheduleTask: vi.fn(), +})); +vi.mock("@/common/PeriodicProcessor.ts", () => ({ + PeriodicProcessor: class PeriodicProcessor { + disable = vi.fn(); + }, +})); +vi.mock("@/common/events.ts", () => ({ + EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync", + eventHub: { onEvent: vi.fn() }, +})); +vi.mock("@/common/translation", () => ({ $msg: (message: string) => message })); +vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ getObsidianCommunityPluginManager: vi.fn() })); + +import { get } from "svelte/store"; +import { getDocData } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { + DEFAULT_SETTINGS, + type AnyEntry, + type FilePathWithPrefix, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { createOptionalFileSyncDatabase, optionalFileEntry } from "../../../test/fixtures/optionalFileSyncDatabase"; +import { ConfigSync, pluginIsEnumerating, pluginList } from "./CmdConfigSync"; + +const queueOwners: ConfigSync[] = []; + +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function waitForCondition(predicate: () => boolean, message: string) { + for (let attempt = 0; attempt < 100; attempt++) { + if (predicate()) return; + await Promise.resolve(); + } + throw new Error(message); +} + +function queueEntry(path: string, data: unknown = {}): AnyEntry { + const documentPath = path as FilePathWithPrefix; + return { + _id: documentPath, + path: documentPath, + mtime: 2_000, + data: [JSON.stringify(data)], + } as unknown as AnyEntry; +} + +function legacyCatalogueEntry(path: string, name: string, mtime = 2_000): AnyEntry { + return queueEntry(path, { + category: "PLUGIN_MAIN", + name, + term: "device", + mtime, + files: [ + { + filename: `plugins/${name}/main.js`, + data: [`legacy source for ${name}`], + mtime, + size: name.length, + hash: `hash-${name}`, + }, + ], + }); +} + +function createQueueConfigSync( + settings: typeof DEFAULT_SETTINGS, + localDatabase: unknown, + services: unknown = { + API: { addLog: vi.fn(), addRibbonIcon: vi.fn() }, + path: { getPath: (entry: { path: string }) => entry.path }, + context: { notices: { hide: vi.fn() } }, + } +) { + vi.spyOn(ConfigSync.prototype, "onBindFunction").mockImplementation(() => {}); + const configSync = new ConfigSync({ + settings, + localDatabase, + services, + } as unknown as ConstructorParameters[0]); + queueOwners.push(configSync); + return configSync; +} + +afterEach(async () => { + for (const configSync of queueOwners.splice(0)) { + await configSync.pluginScanProcessor.stop(); + } + vi.restoreAllMocks(); + pluginList.set([]); + pluginIsEnumerating.set(false); + vi.useRealTimers(); +}); + +describe("ConfigSync legacy catalogue queue", () => { + it("starts duplicate entries in FIFO order with ten concurrent items", async () => { + const settings = { ...DEFAULT_SETTINGS, usePluginSync: true, usePluginSyncV2: true }; + const configSync = createQueueConfigSync(settings, {}); + const paths = [ + "ix:device/PLUGIN_MAIN/first.md", + "ix:device/PLUGIN_MAIN/second.md", + "ix:device/PLUGIN_MAIN/duplicate.md", + "ix:device/PLUGIN_MAIN/third.md", + "ix:device/PLUGIN_MAIN/duplicate.md", + "ix:device/PLUGIN_MAIN/fourth.md", + "ix:device/PLUGIN_MAIN/fifth.md", + "ix:device/PLUGIN_MAIN/sixth.md", + "ix:device/PLUGIN_MAIN/seventh.md", + "ix:device/PLUGIN_MAIN/eighth.md", + "ix:device/PLUGIN_MAIN/ninth.md", + "ix:device/PLUGIN_MAIN/tenth.md", + ]; + const gate = deferred(); + const started: string[] = []; + let active = 0; + let maximumActive = 0; + vi.spyOn(configSync, "migrateV1ToV2").mockImplementation(async (_showMessage, entry) => { + started.push(entry.path); + active++; + maximumActive = Math.max(maximumActive, active); + await gate.promise; + active--; + }); + + try { + for (const path of paths) + configSync.pluginScanProcessor.enqueue({ entry: queueEntry(path), generation: 0 }); + await waitForCondition(() => started.length === 10, "The first ten queue items did not start"); + + expect(started).toEqual(paths.slice(0, 10)); + expect(maximumActive).toBe(10); + + gate.resolve(); + await expect(configSync.pluginScanProcessor.waitForIdle()).resolves.toBeUndefined(); + expect(started).toEqual(paths); + } finally { + gate.resolve(); + } + }); + + it("continues with queued items after a migration throws", async () => { + const settings = { ...DEFAULT_SETTINGS, usePluginSync: true, usePluginSyncV2: true }; + const configSync = createQueueConfigSync(settings, {}); + const paths = Array.from({ length: 11 }, (_, index) => `ix:device/PLUGIN_MAIN/item-${index}.md`); + const gate = deferred(); + const started: string[] = []; + vi.spyOn(configSync, "migrateV1ToV2").mockImplementation(async (_showMessage, entry) => { + started.push(entry.path); + if (entry.path === paths[0]) throw new Error("migration failed"); + if (paths.slice(1, 10).includes(entry.path)) await gate.promise; + }); + + try { + for (const path of paths) + configSync.pluginScanProcessor.enqueue({ entry: queueEntry(path), generation: 0 }); + await waitForCondition(() => started.length === 10, "The first ten queue items did not start"); + await waitForCondition(() => started.length === 11, "The item after the failed migration did not start"); + + gate.resolve(); + await expect(configSync.pluginScanProcessor.waitForIdle()).resolves.toBeUndefined(); + expect(started).toEqual(paths); + } finally { + gate.resolve(); + } + }); + + it("does not restore an old read result after reloading a newer catalogue", async () => { + vi.useFakeTimers(); + const oldPath = "ix:device/PLUGIN_MAIN/old" as FilePathWithPrefix; + const newerPath = "ix:device/PLUGIN_MAIN/newer" as FilePathWithPrefix; + const oldEntry = legacyCatalogueEntry(oldPath, "old"); + const newerEntry = legacyCatalogueEntry(newerPath, "newer", 3_000); + const oldRead = deferred(); + let scanCount = 0; + let oldReadStarted = false; + const localDatabase = { + findEntries: vi.fn(async function* () { + for (const entry of scanCount++ === 0 ? [oldEntry] : [newerEntry]) yield entry; + }), + getDBEntry: vi.fn(async (path: FilePathWithPrefix) => { + if (path === oldPath) { + oldReadStarted = true; + await oldRead.promise; + return oldEntry; + } + return newerEntry; + }), + }; + const settings = { ...DEFAULT_SETTINGS, usePluginSync: true, usePluginSyncV2: false }; + const configSync = createQueueConfigSync(settings, localDatabase); + + try { + await configSync.updatePluginList(false); + await vi.advanceTimersByTimeAsync(100); + await waitForCondition(() => oldReadStarted, "The old catalogue read did not start"); + + await configSync.reloadPluginList(false); + await vi.advanceTimersByTimeAsync(100); + expect(configSync.pluginList.map((entry) => entry.name)).toEqual(["newer"]); + + oldRead.resolve(); + await expect(configSync.pluginScanProcessor.waitForIdle()).resolves.toBeUndefined(); + expect(configSync.pluginList.map((entry) => entry.name)).toEqual(["newer"]); + expect(get(pluginList).map((entry) => entry.name)).toEqual(["newer"]); + } finally { + oldRead.resolve(); + await vi.advanceTimersByTimeAsync(100); + } + }); + + it.each(["catalogue update", "settings application", "temporary suspension"] as const)( + "does not restore an active legacy read after disabling through %s and re-enabling", + async (route) => { + vi.useFakeTimers(); + const oldPath = "ix:device/PLUGIN_MAIN/old" as FilePathWithPrefix; + const oldEntry = legacyCatalogueEntry(oldPath, "old"); + const oldRead = deferred(); + let oldReadStarted = false; + let scanCount = 0; + const localDatabase = { + findEntries: vi.fn(async function* () { + if (scanCount++ === 0) yield oldEntry; + }), + getDBEntry: vi.fn(async () => { + oldReadStarted = true; + await oldRead.promise; + return oldEntry; + }), + }; + const settings = { ...DEFAULT_SETTINGS, usePluginSync: true, usePluginSyncV2: false }; + const configSync = createQueueConfigSync(settings, localDatabase); + + try { + await configSync.updatePluginList(false); + await vi.advanceTimersByTimeAsync(100); + await waitForCondition(() => oldReadStarted, "The active legacy read did not start"); + + settings.usePluginSync = false; + if (route === "catalogue update") { + await configSync.updatePluginList(false); + } else if (route === "settings application") { + await configSync._everyRealizeSettingSyncMode(); + } else { + await ( + configSync as unknown as { _allSuspendExtraSync(): Promise } + )._allSuspendExtraSync(); + } + settings.usePluginSync = true; + await configSync.updatePluginList(false); + + oldRead.resolve(); + await expect(configSync.pluginScanProcessor.waitForIdle()).resolves.toBeUndefined(); + expect(configSync.pluginList).toEqual([]); + expect(get(pluginList)).toEqual([]); + } finally { + oldRead.resolve(); + await vi.advanceTimersByTimeAsync(100); + } + } + ); + + it("stops admission on unload and joins a read without changing the replacement instance's list", async () => { + vi.useFakeTimers(); + const entry = legacyCatalogueEntry("ix:device/PLUGIN_MAIN/old", "old"); + const read = deferred(); + const localDatabase = { + findEntries: vi.fn(async function* () { + yield entry; + }), + getDBEntry: vi.fn(async () => { + await read.promise; + return entry; + }), + }; + const settings = { ...DEFAULT_SETTINGS, usePluginSync: true, usePluginSyncV2: false }; + const configSync = createQueueConfigSync(settings, localDatabase); + try { + await configSync.updatePluginList(false); + await vi.advanceTimersByTimeAsync(100); + expect(localDatabase.getDBEntry).toHaveBeenCalledOnce(); + let stopped = false; + const done = configSync.onunload().then(() => { + stopped = true; + }); + const replacement = { name: "replacement" } as (typeof configSync.pluginList)[number]; + pluginList.set([replacement]); + await configSync.updatePluginList(false); + await configSync.reloadPluginList(false); + await vi.advanceTimersByTimeAsync(100); + expect(localDatabase.findEntries).toHaveBeenCalledOnce(); + expect(stopped).toBe(false); + expect(configSync.pluginScanProcessor.remaining).toBe(1); + read.resolve(); + await done; + expect(configSync.pluginList).toEqual([]); + expect(get(pluginList)).toEqual([replacement]); + expect(configSync.pluginScanProcessor.remaining).toBe(0); + } finally { + read.resolve(); + await vi.advanceTimersByTimeAsync(100); + } + }); + + it("cancels unstarted legacy reads when the first ten are blocked and the module is disabled", async () => { + const paths = Array.from({ length: 12 }, (_, index) => `ix:device/PLUGIN_MAIN/item-${index}`); + const entries = paths.map((path, index) => legacyCatalogueEntry(path, `item-${index}`)); + const gates = entries.slice(0, 10).map(() => deferred()); + const started: string[] = []; + const localDatabase = { + findEntries: vi.fn(async function* () { + for (const entry of entries) yield entry; + }), + getDBEntry: vi.fn(async (path: FilePathWithPrefix) => { + const index = paths.indexOf(path); + started.push(path); + if (index >= 0 && index < gates.length) await gates[index].promise; + return entries[index]; + }), + }; + const settings = { ...DEFAULT_SETTINGS, usePluginSync: true, usePluginSyncV2: false }; + const configSync = createQueueConfigSync(settings, localDatabase); + + try { + await configSync.updatePluginList(false); + await waitForCondition(() => started.length === 10, "The first ten legacy reads did not start"); + + settings.usePluginSync = false; + await configSync.updatePluginList(false); + for (const gate of gates) gate.resolve(); + await expect(configSync.pluginScanProcessor.waitForIdle()).resolves.toBeUndefined(); + + expect(started).toEqual(paths.slice(0, 10)); + } finally { + for (const gate of gates) gate.resolve(); + } + }); + + it("finishes a started V1-to-V2 migration but does not start pending migrations after disable", async () => { + const fixture = await createOptionalFileSyncDatabase(); + const v1Path = "ix:device/PLUGIN_MAIN/migrate.md" as FilePathWithPrefix; + const v2Path = "ix:device/PLUGIN_MAIN/migrate%main.js" as FilePathWithPrefix; + const source = "module.exports = 'migrated';"; + const data = { + category: "PLUGIN_MAIN", + name: "migrate", + term: "device", + mtime: 2_000, + files: [ + { + filename: "plugins/migrate/main.js", + data: [source], + mtime: 2_000, + size: source.length, + hash: "migration-source-hash", + }, + ], + }; + const settings = fixture.settings; + const configSync = createQueueConfigSync(settings, fixture.localDatabase, { + ...fixture.services, + API: { ...fixture.services.API, addRibbonIcon: vi.fn() }, + }); + const pendingGate = deferred(); + const putGate = deferred(); + const migrationFinished = deferred(); + const started: string[] = []; + let v2PutStarted = false; + + try { + await expect( + fixture.localDatabase.putDBEntry(optionalFileEntry(v1Path, JSON.stringify(data))) + ).resolves.toMatchObject({ ok: true }); + + const loadedLegacy = { + ...optionalFileEntry(v1Path, JSON.stringify(data)), + data: [JSON.stringify(data)], + } as unknown as Awaited>; + let migrationRead = false; + const read = fixture.localDatabase.getDBEntry.bind(fixture.localDatabase); + vi.spyOn(fixture.localDatabase, "getDBEntry").mockImplementation(async (path, ...args) => { + if (path === v1Path && !migrationRead) { + migrationRead = true; + return loadedLegacy; + } + return await read(path, ...args); + }); + const migrate = configSync.migrateV1ToV2.bind(configSync); + vi.spyOn(configSync, "migrateV1ToV2").mockImplementation(async (showMessage, entry) => { + started.push(entry.path); + if (entry.path === v1Path) { + try { + await migrate(showMessage, entry); + } finally { + migrationFinished.resolve(); + } + return; + } + await pendingGate.promise; + }); + const put = fixture.localDatabase.putDBEntry.bind(fixture.localDatabase); + vi.spyOn(fixture.localDatabase, "putDBEntry").mockImplementation(async (...args) => { + if ((args[0] as { path?: string }).path === v2Path) { + v2PutStarted = true; + await putGate.promise; + } + return await put(...args); + }); + + const paths = [ + v1Path, + ...Array.from( + { length: 10 }, + (_, index) => `ix:device/PLUGIN_MAIN/pending-${index}.md` as FilePathWithPrefix + ), + ]; + for (const path of paths) + configSync.pluginScanProcessor.enqueue({ entry: queueEntry(path), generation: 0 }); + await waitForCondition( + () => started.length === 10, + `The first ten migrations did not start: ${started.join(", ")}` + ); + await waitForCondition(() => v2PutStarted, "The migration did not reach its write"); + + settings.usePluginSync = false; + await configSync.updatePluginList(false); + putGate.resolve(); + await migrationFinished.promise; + expect.soft(started).toEqual(paths.slice(0, 10)); + + pendingGate.resolve(); + await expect(configSync.pluginScanProcessor.waitForIdle()).resolves.toBeUndefined(); + expect.soft(started).toEqual(paths.slice(0, 10)); + + const migrated = await fixture.localDatabase.getDBEntry(v2Path, undefined, false, false); + expect(migrated).not.toBe(false); + if (migrated !== false) expect(getDocData(migrated.data)).toContain(source); + await expect(fixture.localDatabase.getDBEntry(v1Path)).resolves.toBe(false); + } finally { + putGate.resolve(); + pendingGate.resolve(); + await configSync.pluginScanProcessor.waitForIdle(); + await fixture.dispose(); + } + }); + + it("loads legacy catalogue contents through the database with native timers", async () => { + const fixture = await createOptionalFileSyncDatabase(); + const path = "ix:device/PLUGIN_MAIN/native-timers" as FilePathWithPrefix; + const data = { + category: "PLUGIN_MAIN", + name: "native-timers", + term: "device", + mtime: 2_000, + files: [{ filename: "main.js", data: ["stored content"], mtime: 2_000, size: 14, hash: "stored-hash" }], + }; + let configSync: ConfigSync | undefined; + try { + fixture.settings.usePluginSyncV2 = false; + await expect( + fixture.localDatabase.putDBEntry(optionalFileEntry(path, JSON.stringify(data))) + ).resolves.toMatchObject({ ok: true }); + vi.spyOn(ConfigSync.prototype, "onBindFunction").mockImplementation(() => {}); + configSync = new ConfigSync({ + settings: fixture.settings, + localDatabase: fixture.localDatabase, + services: { + ...fixture.services, + API: { ...fixture.services.API, addRibbonIcon: vi.fn() }, + }, + } as unknown as ConstructorParameters[0]); + queueOwners.push(configSync); + await configSync.updatePluginList(false); + await expect(configSync.pluginScanProcessor.waitForIdle()).resolves.toBeUndefined(); + expect(configSync.pluginList).toEqual([ + { + ...data, + documentPath: path, + files: [{ ...data.files[0], data: ["stored-hash"] }], + }, + ]); + expect(get(pluginList)).toEqual(configSync.pluginList); + expect(configSync.pluginScanProcessor.remaining).toBe(0); + } finally { + await configSync?.pluginScanProcessor.stop(); + await fixture.dispose(); + } + }); + + it("returns after enumeration, then loads a queued legacy catalogue item after its configured delay", async () => { + vi.useFakeTimers(); + pluginList.set([]); + pluginIsEnumerating.set(false); + + const path = "ix:device/PLUGIN_MAIN/example" as FilePathWithPrefix; + const catalogueData = { + category: "PLUGIN_MAIN", + name: "example", + term: "device", + mtime: 2_000, + files: [ + { filename: "main.js", data: ["legacy plugin source"], mtime: 2_000, size: 20, hash: "legacy-hash" }, + ], + }; + const entry = { _id: path, path, mtime: 2_000, data: [JSON.stringify(catalogueData)] }; + const readGate = deferred(); + let loadHasStarted = false; + const localDatabase = { + findEntries: vi.fn(async function* () { + yield entry; + }), + getDBEntry: vi.fn(async () => { + loadHasStarted = true; + await readGate.promise; + return entry; + }), + }; + vi.spyOn(ConfigSync.prototype, "onBindFunction").mockImplementation(() => {}); + const configSync = new ConfigSync({ + settings: { ...DEFAULT_SETTINGS, usePluginSync: true, usePluginSyncV2: false }, + localDatabase, + services: { + API: { addLog: vi.fn(), addRibbonIcon: vi.fn() }, + path: { getPath: (document: { path: string }) => document.path }, + }, + } as unknown as ConstructorParameters[0]); + queueOwners.push(configSync); + + let drained = false; + try { + await configSync.updatePluginList(false); + + expect(configSync.pluginScanProcessor.remaining).toBe(1); + expect(configSync.pluginList).toEqual([]); + expect(get(pluginIsEnumerating)).toBe(false); + + await vi.advanceTimersByTimeAsync(99); + expect(loadHasStarted).toBe(false); + + await vi.advanceTimersByTimeAsync(1); + expect(loadHasStarted).toBe(true); + expect(configSync.pluginScanProcessor.remaining).toBe(1); + const drain = configSync.pluginScanProcessor.waitForIdle(); + void drain.then(() => { + drained = true; + }); + + readGate.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + await waitForCondition(() => drained, "The legacy catalogue queue did not drain"); + + expect(configSync.pluginList).toEqual([ + { + ...catalogueData, + documentPath: path, + files: [{ ...catalogueData.files[0], data: ["legacy-hash"] }], + }, + ]); + expect(get(pluginList)).toEqual(configSync.pluginList); + expect(configSync.pluginScanProcessor.remaining).toBe(0); + expect(get(pluginIsEnumerating)).toBe(false); + } finally { + readGate.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + } + }); +}); diff --git a/src/features/ConfigSync/CmdConfigSync.scanAllConfigFiles.unit.spec.ts b/src/features/ConfigSync/CmdConfigSync.scanAllConfigFiles.unit.spec.ts new file mode 100644 index 00000000..85e763f3 --- /dev/null +++ b/src/features/ConfigSync/CmdConfigSync.scanAllConfigFiles.unit.spec.ts @@ -0,0 +1,279 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + addIcon: vi.fn(), + diff_match_patch: class DiffMatchPatch {}, + normalizePath: vi.fn((path: string) => path), + parseYaml: vi.fn(), + Platform: {}, +})); +vi.mock("./PluginDialogModal.ts", () => ({ + PluginDialogModal: class PluginDialogModal {}, +})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ + JsonResolveModal: class JsonResolveModal {}, +})); +vi.mock("@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", () => ({ + ConflictResolveModal: class ConflictResolveModal {}, +})); +vi.mock("@/features/LiveSyncCommands.ts", () => ({ + LiveSyncCommands: class LiveSyncCommands { + core!: { services: unknown; localDatabase: unknown }; + get localDatabase() { + return this.core.localDatabase; + } + get services() { + return this.core.services; + } + }, +})); +vi.mock("@/common/types.ts", () => ({ + ICXHeader: "ix:", + PERIODIC_PLUGIN_SWEEP: 60, +})); +vi.mock("@/common/utils.ts", () => ({ + cancelTask: vi.fn(), + EVEN: Symbol("even"), + isCustomisationSyncMetadata: vi.fn(), + isPluginMetadata: vi.fn(), + scheduleTask: vi.fn(), +})); +vi.mock("@/common/PeriodicProcessor.ts", () => ({ + PeriodicProcessor: class PeriodicProcessor {}, +})); +vi.mock("@/common/events.ts", () => ({ + EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG: "open-plugin-sync", + eventHub: { + onEvent: vi.fn(), + }, +})); +vi.mock("@/common/translation", () => ({ + $msg: vi.fn((message: string) => message), +})); +vi.mock("@/common/obsidianCommunityPlugins.ts", () => ({ + getObsidianCommunityPluginManager: vi.fn(), +})); + +vi.mock("octagonal-wheels/concurrency/scoped.iteration", { spy: true }); + +import { LOG_LEVEL_VERBOSE, type FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import * as iterationModule from "octagonal-wheels/concurrency/scoped.iteration"; +import { promiseWithResolvers } from "octagonal-wheels/promises"; +import { ConfigSync } from "./CmdConfigSync"; + +const term = "device"; +const prefix = `ix:${term}/`; +const keyOf = (path: string) => prefix + path; +type Row = { _id: string; path: string }; +const row = (name: string, path = name): Row => ({ _id: keyOf(name), path }); + +function setup(files: string[] = [], rows: Row[] = []) { + const events: string[] = []; + const findEntries = vi.fn(async function* () { + yield* rows; + }); + const scanInternalFiles = vi.fn(async () => files as FilePath[]); + const storeCustomisationFileV2 = vi.fn(async (path: string, _term: string) => { + events.push(`store:${path}`); + }); + const deleteConfigOnDatabase = vi.fn(async (key: string) => { + events.push(`delete:${key}`); + }); + const updatePluginList = vi.fn(async (_showMessage: boolean) => {}); + const log = vi.fn(); + const configSync = Object.create(ConfigSync.prototype) as ConfigSync; + Object.assign(configSync, { + core: { + settings: { usePluginSyncV2: true }, + localDatabase: { findEntries }, + services: { setting: { getDeviceAndVaultName: () => term } }, + }, + scanInternalFiles, + isTargetPath: (path: string) => path !== "ignored", + filenameWithUnifiedKey: (path: string, _term: string) => keyOf(path), + unifiedKeyPrefixOfTerminal: (_term: string) => prefix, + storeCustomisationFileV2, + deleteConfigOnDatabase, + updatePluginList, + _log: log, + }); + return { + configSync, + events, + findEntries, + scanInternalFiles, + storeCustomisationFileV2, + deleteConfigOnDatabase, + updatePluginList, + log, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("ConfigSync V2 scan behaviour", () => { + it("finishes existing database entries before storing extra files and retains target filtering", async () => { + const fixture = setup( + ["matched", "extra", "ignored"], + [row("matched"), row("stale"), row("encoded", "encoded%path")] + ); + await fixture.configSync.scanAllConfigFiles(false); + + expect(fixture.events).toEqual(["store:matched", `delete:${keyOf("stale")}`, "store:extra"]); + expect(fixture.storeCustomisationFileV2.mock.calls).toEqual([ + ["matched", term], + ["extra", term], + ]); + expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledExactlyOnceWith(keyOf("stale")); + expect(fixture.findEntries).toHaveBeenCalledExactlyOnceWith(prefix, `${prefix}\u{10ffff}`, { + include_docs: true, + }); + expect(fixture.updatePluginList).toHaveBeenCalledExactlyOnceWith(false); + }); + + it("finishes database enumeration before dispatching database work", async () => { + const fixture = setup(); + const enumerationPaused = promiseWithResolvers(); + const continueEnumeration = promiseWithResolvers(); + fixture.findEntries.mockImplementation(async function* () { + yield row("stale"); + enumerationPaused.resolve(); + await continueEnumeration.promise; + }); + + const scan = fixture.configSync.scanAllConfigFiles(false); + try { + await enumerationPaused.promise; + expect(fixture.deleteConfigOnDatabase).not.toHaveBeenCalled(); + expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled(); + } finally { + continueEnumeration.resolve(); + await scan; + } + + expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledExactlyOnceWith(keyOf("stale")); + }); + + it("limits actual database operations to ten and completes that batch before the extra batch", async () => { + const fixture = setup( + ["extra"], + Array.from({ length: 11 }, (_, index) => row(String(index))) + ); + const gates = Array.from({ length: 11 }, () => promiseWithResolvers()); + const tenStarted = promiseWithResolvers(); + const lastStarted = promiseWithResolvers(); + let active = 0; + let maximum = 0; + let started = 0; + fixture.deleteConfigOnDatabase.mockImplementation(async (key) => { + const index = Number(key.slice(prefix.length)); + active++; + maximum = Math.max(maximum, active); + started++; + if (started === 10) tenStarted.resolve(); + if (index === 10) lastStarted.resolve(); + try { + await gates[index].promise; + } finally { + active--; + } + }); + const scan = fixture.configSync.scanAllConfigFiles(false); + try { + await tenStarted.promise; + expect(active).toBe(10); + expect(started).toBe(10); + expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled(); + gates[0].resolve(); + await lastStarted.promise; + expect(active).toBe(10); + expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled(); + } finally { + gates.forEach((gate) => gate.resolve()); + await scan; + } + expect(maximum).toBe(10); + expect(active).toBe(0); + expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledTimes(11); + expect(fixture.storeCustomisationFileV2).toHaveBeenCalledExactlyOnceWith("extra", term); + }); + + it("logs a database processing error and continues through the extra batch", async () => { + const fixture = setup(["extra"], [row("bad"), row("other")]); + const error = new Error("delete"); + fixture.deleteConfigOnDatabase.mockRejectedValueOnce(error); + await fixture.configSync.scanAllConfigFiles(false); + + expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledTimes(2); + expect(fixture.log).toHaveBeenCalledWith(`scanAllConfigFiles - Error: ${keyOf("bad")}`, LOG_LEVEL_VERBOSE); + expect(fixture.log).toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE); + expect(fixture.storeCustomisationFileV2).toHaveBeenCalledExactlyOnceWith("extra", term); + expect(fixture.updatePluginList).toHaveBeenCalledOnce(); + }); + + it("retains a matched local file for the extra batch when its first store fails", async () => { + const fixture = setup(["matched"], [row("matched")]); + const error = new Error("first store"); + fixture.storeCustomisationFileV2.mockRejectedValueOnce(error); + await fixture.configSync.scanAllConfigFiles(false); + + expect(fixture.storeCustomisationFileV2.mock.calls).toEqual([ + ["matched", term], + ["matched", term], + ]); + expect(fixture.log).toHaveBeenCalledWith(`scanAllConfigFiles - Error: ${keyOf("matched")}`, LOG_LEVEL_VERBOSE); + expect(fixture.log).toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE); + expect(fixture.events).toEqual(["store:matched"]); + expect(fixture.updatePluginList).toHaveBeenCalledOnce(); + }); + + it("logs an extra-file processing error and continues with the remaining files", async () => { + const fixture = setup(["bad", "other"]); + const error = new Error("store"); + fixture.storeCustomisationFileV2.mockRejectedValueOnce(error); + await fixture.configSync.scanAllConfigFiles(false); + + expect(fixture.storeCustomisationFileV2.mock.calls).toEqual([ + ["bad", term], + ["other", term], + ]); + expect(fixture.log).toHaveBeenCalledWith("scanAllConfigFiles - Error: bad", LOG_LEVEL_VERBOSE); + expect(fixture.log).toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE); + expect(fixture.updatePluginList).toHaveBeenCalledOnce(); + }); +}); + +describe("ConfigSync scoped iteration composition", () => { + it("uses bounded child scopes for both batches with the same shared values", async () => { + const fixture = setup(["matched", "extra"], [row("matched"), row("stale")]); + await fixture.configSync.scanAllConfigFiles(false); + + expect(iterationModule.forEachInScope).toHaveBeenCalledTimes(2); + const [database, extra] = vi.mocked(iterationModule.forEachInScope).mock.calls; + expect(database[0]).toBe(extra[0]); + expect(database[0].value).toEqual({ term, localFileMap: new Map([[keyOf("extra"), "extra"]]) }); + expect(database[2]).toEqual({ concurrency: 10 }); + expect(extra[2]).toEqual({ concurrency: 10 }); + expect(fixture.events).toEqual(["store:matched", `delete:${keyOf("stale")}`, "store:extra"]); + }); + + it.each(["database", "extra"])("propagates a %s runner failure outside per-item recovery", async (batch) => { + const fixture = setup(["extra"], [row("stale")]); + const error = new Error("iteration"); + const actual = await vi.importActual( + "octagonal-wheels/concurrency/scoped.iteration" + ); + if (batch === "extra") { + vi.mocked(iterationModule.forEachInScope).mockImplementationOnce(actual.forEachInScope); + } + vi.mocked(iterationModule.forEachInScope).mockRejectedValueOnce(error); + await expect(fixture.configSync.scanAllConfigFiles(false)).rejects.toBe(error); + + expect(fixture.deleteConfigOnDatabase).toHaveBeenCalledTimes(batch === "extra" ? 1 : 0); + expect(fixture.storeCustomisationFileV2).not.toHaveBeenCalled(); + expect(fixture.log).not.toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE); + expect(fixture.updatePluginList).not.toHaveBeenCalled(); + }); +}); diff --git a/src/features/ConfigSync/CmdConfigSync.ts b/src/features/ConfigSync/CmdConfigSync.ts index 68ba9703..41219d05 100644 --- a/src/features/ConfigSync/CmdConfigSync.ts +++ b/src/features/ConfigSync/CmdConfigSync.ts @@ -49,17 +49,17 @@ import { decodeBinary, readString, } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/convert"; -import { serialized, shareRunningResult } from "octagonal-wheels/concurrency/lock"; +import { withLock, shareRunningResult } from "octagonal-wheels/concurrency/lock"; import { LiveSyncCommands } from "@/features/LiveSyncCommands.ts"; import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; import { cancelTask, EVEN, isCustomisationSyncMetadata, isPluginMetadata, scheduleTask } from "@/common/utils.ts"; import { PeriodicProcessor } from "@/common/PeriodicProcessor.ts"; import { JsonResolveModal } from "@/features/HiddenFileCommon/JsonResolveModal.ts"; -import { QueueProcessor } from "octagonal-wheels/concurrency/processor"; import { pluginScanningCount } from "@vrtmrz/livesync-commonlib/compat/mock_and_interop/stores"; import { base64ToArrayBuffer, base64ToString } from "octagonal-wheels/binary/base64"; import { ConflictResolveModal } from "@/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts"; -import { Semaphore } from "octagonal-wheels/concurrency/semaphore"; +import { forEachInScope } from "octagonal-wheels/concurrency/scoped.iteration"; +import { withScope } from "octagonal-wheels/resource/scope"; import { EVENT_REQUEST_OPEN_PLUGIN_SYNC_DIALOG, eventHub } from "@/common/events.ts"; import { PluginDialogModal } from "./PluginDialogModal.ts"; import { $msg } from "@/common/translation"; @@ -68,10 +68,12 @@ import type { LiveSyncCore } from "@/main.ts"; import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError"; import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts"; import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlugins.ts"; +import { createModificationHandler } from "octagonal-wheels/modification"; import type { HandlerOutcome } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"; import type { OptionalFileEventOptions } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import { PendingConfigSaves } from "./PendingConfigSaves"; +import { ConfigScanQueue } from "./ConfigScanQueue"; const d = "\u200b"; const d2 = "\n"; @@ -285,6 +287,10 @@ export type PluginDataExDisplay = { mtime: number; }; type LoadedEntryPluginDataExFile = LoadedEntry & PluginDataExFile; +type ConfigFileApplicationTarget = { + readonly path: FilePath; + readonly stat: { readonly mtime: number; readonly ctime: number }; +}; function categoryToFolder(category: string, configDir: string = ""): string { switch (category) { @@ -388,13 +394,6 @@ export type PluginDataEx = { }; export class ConfigSync extends LiveSyncCommands { - constructor(core: LiveSyncCore) { - super(core); - pluginScanningCount.onChanged((e) => { - const total = e.value; - pluginIsEnumerating.set(total != 0); - }); - } get configDir() { return this.core.services.API.getSystemConfigDir(); } @@ -417,6 +416,7 @@ export class ConfigSync extends LiveSyncCommands { pluginList: IPluginDataExDisplay[] = []; private pluginListGeneration = 0; + private unloaded = false; showPluginSyncModal() { if (!this.isThisModuleEnabled()) { return; @@ -435,11 +435,18 @@ export class ConfigSync extends LiveSyncCommands { this.pluginDialog = undefined; } } - onunload() { + onunload(): Promise { + this.unloaded = true; + this.resetPluginList(); + const stopped = this.pluginScanProcessor.stop(); + pluginScanningCount.value = 0; + pluginIsEnumerating.set(false); + cancelTask("updatePluginListV2"); cancelTask(UPDATED_CONFIGURATION_NOTICE_KEY); this.hidePluginSyncModal(); this.periodicPluginSweepProcessor?.disable(); this.services.context.notices.hide(UPDATED_CONFIGURATION_NOTICE_KEY); + return stopped; } addRibbonIcon = this.services.API.addRibbonIcon.bind(this.services.API); onload() { @@ -536,11 +543,21 @@ export class ConfigSync extends LiveSyncCommands { q?.toggleClass("sls-hidden", !this.isThisModuleEnabled()); return Promise.resolve(true); } - async reloadPluginList(showMessage: boolean) { + private isCurrentPluginList(generation: number): boolean { + return !this.unloaded && generation === this.pluginListGeneration && this.isThisModuleEnabled(); + } + + private resetPluginList(): void { this.pluginListGeneration++; + this.pluginScanProcessor.clearPending(); this.pluginList = []; this.loadedManifest_mTime.clear(); pluginList.set(this.pluginList); + } + + async reloadPluginList(showMessage: boolean) { + if (this.unloaded) return; + this.resetPluginList(); await this.updatePluginList(showMessage); } async loadPluginData(path: FilePathWithPrefix): Promise { @@ -575,77 +592,37 @@ export class ConfigSync extends LiveSyncCommands { return false; } - pluginScanProcessor = new QueueProcessor( - async (v: AnyEntry[]) => { - const plugin = v[0]; - if (this.useV2) { - await this.migrateV1ToV2(false, plugin); - return []; - } - const path = plugin.path || this.getPath(plugin); - const oldEntry = this.pluginList.find((e) => e.documentPath == path); - if (oldEntry && oldEntry.mtime == plugin.mtime) return []; + pluginScanProcessor = new ConfigScanQueue<{ entry: AnyEntry; generation: number }>({ + process: async ({ entry: plugin, generation }) => { + if (!this.isCurrentPluginList(generation)) return; try { + if (this.useV2) { + // Once started, migration owns its writes through completion. + await this.migrateV1ToV2(false, plugin); + return; + } + const path = plugin.path || this.getPath(plugin); + const oldEntry = this.pluginList.find((entry) => entry.documentPath == path); + if (oldEntry && oldEntry.mtime == plugin.mtime) return; const pluginData = await this.loadPluginData(path); + if (!this.isCurrentPluginList(generation)) return; if (pluginData) { - let newList = [...this.pluginList]; - newList = newList.filter((x) => x.documentPath != pluginData.documentPath); + const newList = this.pluginList.filter((entry) => entry.documentPath != pluginData.documentPath); newList.push(pluginData); this.pluginList = newList; pluginList.set(newList); } - // Failed to load - return []; } catch (ex) { - this._log(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE); + this._log(`Something happened at enumerating customization :${plugin.path}`, LOG_LEVEL_NOTICE); this._log(ex, LOG_LEVEL_VERBOSE); } - return []; }, - { - suspended: false, - batchSize: 1, - concurrentLimit: 10, - delay: 100, - yieldThreshold: 10, - maintainDelay: false, - totalRemainingReactiveSource: pluginScanningCount, - } - ).startPipeline(); - - pluginScanProcessorV2 = new QueueProcessor( - async (v: AnyEntry[]) => { - const plugin = v[0]; - const path = plugin.path || this.getPath(plugin); - const oldEntry = this.pluginList.find((e) => e.documentPath == path); - if (oldEntry && oldEntry.mtime == plugin.mtime) return []; - try { - const pluginData = await this.loadPluginData(path); - if (pluginData) { - let newList = [...this.pluginList]; - newList = newList.filter((x) => x.documentPath != pluginData.documentPath); - newList.push(pluginData); - this.pluginList = newList; - pluginList.set(newList); - } - // Failed to load - return []; - } catch (ex) { - this._log(`Something happened at enumerating customization :${path}`, LOG_LEVEL_NOTICE); - this._log(ex, LOG_LEVEL_VERBOSE); - } - return []; + onProgress: (remaining) => { + if (this.unloaded) return; + pluginScanningCount.value = remaining; + pluginIsEnumerating.set(remaining !== 0); }, - { - suspended: false, - batchSize: 1, - concurrentLimit: 10, - delay: 100, - yieldThreshold: 10, - maintainDelay: false, - totalRemainingReactiveSource: pluginScanningCount, - } - ).startPipeline(); + }); filenameToUnifiedKey(path: string, termOverRide?: string) { const term = termOverRide || this.services.setting.getDeviceAndVaultName(); @@ -768,14 +745,14 @@ export class ConfigSync extends LiveSyncCommands { async updatePluginListV2(showMessage: boolean, unifiedFilenameWithKey: FilePathWithPrefix): Promise { const generation = this.pluginListGeneration; - const isCurrent = () => generation === this.pluginListGeneration && this.isThisModuleEnabled(); + const isCurrent = () => this.isCurrentPluginList(generation); try { this.updatingV2Count++; pluginV2Progress.set(this.updatingV2Count); // const unifiedFilenameWithKey = this.filenameWithUnifiedKey(updatedDocumentPath); const { pathV1 } = this.parseUnifiedPath(unifiedFilenameWithKey); - await serialized(`config:catalogue:${pathV1}`, async () => { + await withLock(`config:catalogue:${pathV1}`, async () => { if (!isCurrent()) return; const oldEntry = this.pluginList.find((e) => e.documentPath == pathV1); let entry: PluginDataExDisplayV2 | undefined = undefined; @@ -883,11 +860,9 @@ export class ConfigSync extends LiveSyncCommands { } async updatePluginList(showMessage: boolean, updatedDocumentPath?: FilePathWithPrefix): Promise { + if (this.unloaded) return; if (!this.isThisModuleEnabled()) { - this.pluginListGeneration++; - this.pluginScanProcessor.clearQueue(); - this.pluginList = []; - pluginList.set(this.pluginList); + this.resetPluginList(); return; } const generation = this.pluginListGeneration; @@ -903,7 +878,7 @@ export class ConfigSync extends LiveSyncCommands { }) : this.localDatabase.findEntries(ICXHeader + "", `${ICXHeader}\u{10ffff}`, { include_docs: true }); for await (const v of plugins) { - if (generation !== this.pluginListGeneration || !this.isThisModuleEnabled()) return; + if (!this.isCurrentPluginList(generation)) return; if (v.deleted || v._deleted) continue; if (v.path.indexOf("%") !== -1) { fireAndForget(() => this.updatePluginListV2(showMessage, v.path)); @@ -912,7 +887,7 @@ export class ConfigSync extends LiveSyncCommands { const path = v.path || this.getPath(v); if (updatedDocumentPath && updatedDocumentPath != path) continue; - this.pluginScanProcessor.enqueue(v); + this.pluginScanProcessor.enqueue({ entry: v, generation }); } } finally { pluginIsEnumerating.set(false); @@ -950,7 +925,7 @@ export class ConfigSync extends LiveSyncCommands { path = path.split("%")[1] as FilePath; } if (fileA.path.endsWith(".json")) { - return serialized( + return withLock( "config:merge-data", () => new Promise((res) => { @@ -1027,44 +1002,63 @@ export class ConfigSync extends LiveSyncCommands { await this.storeCustomisationFileV2(path, this.services.setting.getDeviceAndVaultName()); } else { const files = data.files; + const applyBinaryContent = createModificationHandler< + ConfigFileApplicationTarget, + LoadedEntryPluginDataExFile, + ArrayBuffer, + ArrayBuffer + >({ + read: (target) => this.core.storageAccess.readHiddenFileBinary(target.path), + onReadError: (ex, _path, file) => { + this._log(`Could not read the file ${file.filename}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); + return new ArrayBuffer(0); + }, + decode: (file) => base64ToArrayBuffer(file.data), + isSame: (oldData, content) => isDocContentSame(oldData, content), + write: (target, content) => + this.core.storageAccess.writeHiddenFileAuto(target.path, content, target.stat), + }); + const applyTextContent = createModificationHandler< + ConfigFileApplicationTarget, + LoadedEntryPluginDataExFile, + string, + string + >({ + read: (target) => this.core.storageAccess.readHiddenFileText(target.path), + onReadError: (ex, _path, file) => { + this._log(`Could not read the file ${file.filename}`, LOG_LEVEL_VERBOSE); + this._log(ex, LOG_LEVEL_VERBOSE); + return ""; + }, + decode: (file) => getDocData(file.data), + isSame: (oldData, content) => isDocContentSame(oldData, content), + write: (target, content) => + this.core.storageAccess.writeHiddenFileAuto(target.path, content, target.stat), + }); for (const f of files) { // If files have applied, modified time will be updated to the current time. - const stat = { mtime: f.mtime, ctime: f.ctime }; const path = `${baseDir}/${f.filename}` as FilePath; + const target: ConfigFileApplicationTarget = { + path, + stat: { mtime: f.mtime, ctime: f.ctime }, + }; this._log(`Applying ${f.filename} of ${data.displayName || data.name}..`); // const contentEach = createBlob(f.data); await this.core.storageAccess.ensureDir(path); if (f.datatype == "newnote") { - let oldData; - try { - oldData = await this.core.storageAccess.readHiddenFileBinary(path); - } catch (ex) { - this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); - this._log(ex, LOG_LEVEL_VERBOSE); - oldData = new ArrayBuffer(0); - } - const content = base64ToArrayBuffer(f.data); - if (await isDocContentSame(oldData, content)) { + const result = await applyBinaryContent(target, f); + if (result == "unchanged") { this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); continue; } - await this.core.storageAccess.writeHiddenFileAuto(path, content, stat); } else { - let oldData; - try { - oldData = await this.core.storageAccess.readHiddenFileText(path); - } catch (ex) { - this._log(`Could not read the file ${f.filename}`, LOG_LEVEL_VERBOSE); - this._log(ex, LOG_LEVEL_VERBOSE); - oldData = ""; - } - const content = getDocData(f.data); - if (await isDocContentSame(oldData, content)) { + const result = await applyTextContent(target, f); + if (result == "unchanged") { this._log(`The file ${f.filename} is already up-to-date`, LOG_LEVEL_VERBOSE); continue; } - await this.core.storageAccess.writeHiddenFileAuto(path, content, stat); } this._log(`Applied ${f.filename} of ${data.displayName || data.name}..`); await this.storeCustomisationFileV2(path, this.services.setting.getDeviceAndVaultName()); @@ -1215,6 +1209,7 @@ export class ConfigSync extends LiveSyncCommands { return true; } _everyRealizeSettingSyncMode(): Promise { + if (!this.isThisModuleEnabled()) this.resetPluginList(); // applySettings calls onResuming after this handler when synchronisation is active. // That handler scans and restarts the sweep; doing so here would scan twice. this.periodicPluginSweepProcessor?.disable(); @@ -1281,7 +1276,7 @@ export class ConfigSync extends LiveSyncCommands { requireCompletion: boolean ) { const vf = this.filenameWithUnifiedKey(path, term); - return await serialized(`plugin-${vf}`, async () => { + return await withLock(`plugin-${vf}`, async () => { const prefixedFileName = vf; const id = await this.path2id(prefixedFileName); @@ -1384,7 +1379,7 @@ export class ConfigSync extends LiveSyncCommands { const vf = this.filenameToUnifiedKey(path, term); // console.warn(`Storing ${path} to ${bareVF} :--> ${keyedVF}`); - return await serialized(`plugin-${vf}`, async () => { + return await withLock(`plugin-${vf}`, async () => { const category = this.getFileCategory(path); let mtime = 0; let fileTargets = [] as FilePath[]; @@ -1613,49 +1608,35 @@ export class ConfigSync extends LiveSyncCommands { const entries = this.localDatabase.findEntries(prefix + "", `${prefix}\u{10ffff}`, { include_docs: true, }); - const tasks = [] as (() => Promise)[]; - const concurrency = 10; - const semaphore = Semaphore(concurrency); + const databaseEntries: { _id: string; path: FilePathWithPrefix }[] = []; for await (const item of entries) { - if (item.path.indexOf("%") !== -1) { - continue; - } - tasks.push(async () => { - const releaser = await semaphore.acquire(); + if (item.path.indexOf("%") === -1) databaseEntries.push(item); + } + await withScope({ term, localFileMap }, async (scope) => { + await forEachInScope(scope, databaseEntries, { concurrency: 10 }, async (item, ctx) => { try { const unifiedFilenameWithKey = `${item._id}` as FilePathWithPrefix; - const localPath = localFileMap.get(unifiedFilenameWithKey); + const localPath = ctx.value.localFileMap.get(unifiedFilenameWithKey); if (localPath) { - await this.storeCustomisationFileV2(localPath, term); - localFileMap.delete(unifiedFilenameWithKey); + await this.storeCustomisationFileV2(localPath, ctx.value.term); + ctx.value.localFileMap.delete(unifiedFilenameWithKey); } else { await this.deleteConfigOnDatabase(unifiedFilenameWithKey); } } catch (ex) { this._log(`scanAllConfigFiles - Error: ${item._id}`, LOG_LEVEL_VERBOSE); this._log(ex, LOG_LEVEL_VERBOSE); - } finally { - releaser(); } }); - } - await Promise.all(tasks.map((e) => e())); - // Extra files - const taskExtra = [] as (() => Promise)[]; - for (const [, filePath] of localFileMap) { - taskExtra.push(async () => { - const releaser = await semaphore.acquire(); + await forEachInScope(scope, localFileMap.values(), { concurrency: 10 }, async (filePath, ctx) => { try { - await this.storeCustomisationFileV2(filePath, term); + await this.storeCustomisationFileV2(filePath, ctx.value.term); } catch (ex) { this._log(`scanAllConfigFiles - Error: ${filePath}`, LOG_LEVEL_VERBOSE); this._log(ex, LOG_LEVEL_VERBOSE); - } finally { - releaser(); } }); - } - await Promise.all(taskExtra.map((e) => e())); + }); fireAndForget(() => this.updatePluginList(false)); } else { const files = filesAll @@ -1702,7 +1683,7 @@ export class ConfigSync extends LiveSyncCommands { ) { // const id = await this.path2id(prefixedFileName); const mtime = new Date().getTime(); - return await serialized("file-x-" + prefixedFileName, async () => { + return await withLock("file-x-" + prefixedFileName, async () => { try { const old = (await this.localDatabase.getDBEntryMeta(prefixedFileName, undefined, false)) as | InternalFileEntry @@ -1765,6 +1746,7 @@ export class ConfigSync extends LiveSyncCommands { this.core.settings.usePluginSync = false; this.core.settings.autoSweepPlugins = false; } + this.resetPluginList(); return Promise.resolve(true); } diff --git a/src/features/ConfigSync/ConfigScanQueue.ts b/src/features/ConfigSync/ConfigScanQueue.ts new file mode 100644 index 00000000..7982b1ca --- /dev/null +++ b/src/features/ConfigSync/ConfigScanQueue.ts @@ -0,0 +1,194 @@ +import { compatGlobal, type CompatTimeoutHandle } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; +import { createNotifier } from "octagonal-wheels/concurrency/notifier"; +import { forEachInScope } from "octagonal-wheels/concurrency/scoped.iteration"; +import { createAsyncQueue, type AsyncQueue } from "octagonal-wheels/iterable/asyncQueue"; +import { withScope, type Awaitable } from "octagonal-wheels/resource/scope"; + +interface ConfigScanQueueOptions { + /** The caller logs recoverable item failures and decides whether to continue. */ + process: (item: T) => Awaitable; + onProgress?: (remaining: number) => void; +} + +interface PendingQueue { + fifo: AsyncQueue; + iterator: AsyncIterator; +} + +interface PendingScan { + value: T; + queue: PendingQueue; +} + +/** Owns FIFO admission, the initial scan delay, and completion for one catalogue. */ +export class ConfigScanQueue { + private pending: PendingQueue; + private pendingCount = 0; + private active = 0; + private ready = false; + private stopped = false; + private stoppingAfterFailure = false; + private readonly failures: unknown[] = []; + private timer?: CompatTimeoutHandle; + private running?: Promise; + private readonly changed = createNotifier(); + + constructor(private readonly options: ConfigScanQueueOptions) { + this.pending = this.createPendingQueue(); + } + + private createPendingQueue(): PendingQueue { + const fifo = createAsyncQueue(); + return { fifo, iterator: fifo[Symbol.asyncIterator]() }; + } + + get remaining(): number { + return this.pendingCount + this.active; + } + + enqueue(value: T): void { + if (!this.canDispatch()) throw new Error("The customisation scan queue is closed."); + this.pending.fifo.enqueue(value); + this.pendingCount++; + this.reportProgress(); + this.schedule(); + } + + /** Discard unstarted scans; active reads and migrations retain their slots. */ + clearPending(): void { + const discarded = this.pending; + this.pending = this.createPendingQueue(); + this.pendingCount = 0; + this.ready = false; + this.clearTimer(); + discarded.fifo.dispose(); + this.reportProgress(); + } + + /** Stop admission and join active work. A processor must not await its own stop. */ + stop(): Promise { + this.stopped = true; + this.clearPending(); + return this.waitForIdle(); + } + + /** Include later arrivals until the queue is idle; this does not close admission. */ + async waitForIdle(): Promise { + this.schedule(); + while (this.running || this.timer !== undefined) await this.changed.wait(); + if (this.failures.length === 1) throw this.failures[0]; + if (this.failures.length > 1) { + throw Object.assign(new Error("Customisation scans failed."), { errors: [...this.failures] }); + } + } + + private canDispatch(): boolean { + return !this.stopped && !this.stoppingAfterFailure; + } + + private clearTimer(): void { + if (this.timer === undefined) return; + compatGlobal.clearTimeout(this.timer); + this.timer = undefined; + } + + private recordFailure(error: unknown): void { + this.stoppingAfterFailure = true; + this.failures.push(error); + this.clearTimer(); + this.changed.notify(); + } + + private reportProgress(): void { + if (this.failures.length === 0) { + try { + this.options.onProgress?.(this.remaining); + } catch (error) { + this.recordFailure(error); + } + } + this.changed.notify(); + } + + private claim(entry: PendingScan): boolean { + if (!this.canDispatch() || this.pending !== entry.queue) return false; + this.pendingCount--; + if (this.pendingCount === 0) this.ready = false; + return true; + } + + private async *availableWork() { + while (this.canDispatch()) { + const pending = this.pending; + if (!this.ready) { + if (this.active === 0 && this.pendingCount === 0) return; + await this.changed.wait(); + continue; + } + if (pending.fifo.size > 0) { + const entry = await pending.iterator.next(); + if (entry.done) { + if (pending !== this.pending) continue; + return; + } + // The callback checks queue identity so clearing a pulled item cancels it. + yield { value: entry.value, queue: pending }; + } else if (this.active > 0 || this.pendingCount > 0) { + // Keep free slots available to later arrivals while a slow scan finishes. + await this.changed.wait(); + } else { + return; + } + } + } + + private schedule(): void { + if (!this.canDispatch() || this.pendingCount === 0) return; + if (!this.ready) { + if (this.pendingCount > 10) { + this.ready = true; + this.clearTimer(); + } else if (this.timer === undefined) { + this.timer = compatGlobal.setTimeout(() => { + this.timer = undefined; + this.ready = true; + this.schedule(); + this.changed.notify(); + }, 100); + } + } + if (!this.ready || this.running) return; + this.running = Promise.resolve() + .then(() => + withScope(undefined, (parent) => + forEachInScope(parent, this.availableWork(), { concurrency: 10 }, async (entry, child) => { + if (!this.claim(entry)) return; + this.active++; + child.defer(() => { + this.active--; + this.reportProgress(); + }); + this.reportProgress(); + if (!this.canDispatch()) return; + try { + await this.options.process(entry.value); + } catch (error) { + this.stoppingAfterFailure = true; + this.clearTimer(); + this.changed.notify(); + throw error; + } + }) + ) + ) + .then( + (): void => undefined, + (error: unknown) => this.recordFailure(error) + ) + .finally(() => { + this.running = undefined; + this.schedule(); + this.changed.notify(); + }); + } +} diff --git a/src/features/ConfigSync/ConfigScanQueue.unit.spec.ts b/src/features/ConfigSync/ConfigScanQueue.unit.spec.ts new file mode 100644 index 00000000..4df2f64a --- /dev/null +++ b/src/features/ConfigSync/ConfigScanQueue.unit.spec.ts @@ -0,0 +1,400 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { ConfigScanQueue } from "./ConfigScanQueue"; + +const queues: { stop(): Promise }[] = []; +const releases: (() => void)[] = []; + +function gate() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + releases.push(resolve); + return { promise, resolve }; +} + +function queue(options: ConstructorParameters>[0]) { + const result = new ConfigScanQueue(options); + queues.push(result); + return result; +} + +beforeEach(() => vi.useFakeTimers()); +afterEach(async () => { + for (const release of releases.splice(0)) release(); + await Promise.all(queues.splice(0).map((item) => item.stop().catch(() => {}))); + vi.useRealTimers(); +}); + +describe("ConfigScanQueue", () => { + it("waits from the first arrival without extending the delay for later arrivals", async () => { + const seen: string[] = []; + const scans = queue({ + process: (item) => { + seen.push(item); + }, + }); + scans.enqueue("first"); + await vi.advanceTimersByTimeAsync(50); + scans.enqueue("second"); + await vi.advanceTimersByTimeAsync(49); + expect(seen).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + await scans.waitForIdle(); + expect(seen).toEqual(["first", "second"]); + expect(vi.getTimerCount()).toBe(0); + }); + + it("bypasses the delay only when pending work exceeds ten items", async () => { + const seen: number[] = []; + const scans = queue({ + process: (item) => { + seen.push(item); + }, + }); + for (let item = 0; item < 10; item++) scans.enqueue(item); + await vi.advanceTimersByTimeAsync(0); + expect(seen).toEqual([]); + scans.enqueue(10); + await vi.advanceTimersByTimeAsync(0); + await scans.waitForIdle(); + expect(seen).toEqual(Array.from({ length: 11 }, (_, index) => index)); + expect(vi.getTimerCount()).toBe(0); + }); + + it("keeps duplicate items in FIFO order and admits at most ten active scans", async () => { + const finish = Array.from({ length: 13 }, () => gate()); + const input = Array.from({ length: 13 }, (_, index) => index % 3); + const seen: number[] = []; + let active = 0; + let maximum = 0; + const progress: number[] = []; + const scans = queue({ + process: async (item) => { + const index = seen.length; + seen.push(item); + maximum = Math.max(maximum, ++active); + await finish[index].promise; + active--; + }, + onProgress: (remaining) => progress.push(remaining), + }); + input.forEach((item) => scans.enqueue(item)); + await vi.advanceTimersByTimeAsync(0); + expect(seen).toEqual(input.slice(0, 10)); + expect(scans.remaining).toBe(13); + finish[3].resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(seen).toEqual(input.slice(0, 11)); + finish.forEach((item) => item.resolve()); + await scans.waitForIdle(); + expect(seen).toEqual(input); + expect(maximum).toBe(10); + expect(scans.remaining).toBe(0); + expect(progress[progress.length - 1]).toBe(0); + }); + + it("starts a later arrival in a free slot before an earlier slow scan finishes", async () => { + const slow = gate(); + const seen: string[] = []; + const scans = queue({ + process: async (item) => { + seen.push(item); + if (item === "slow") await slow.promise; + }, + }); + scans.enqueue("slow"); + await vi.advanceTimersByTimeAsync(100); + let idle = false; + const done = scans.waitForIdle().then(() => { + idle = true; + }); + scans.enqueue("later"); + await vi.advanceTimersByTimeAsync(100); + expect(seen).toEqual(["slow", "later"]); + expect(idle).toBe(false); + slow.resolve(); + await done; + expect(scans.remaining).toBe(0); + }); + + it("clears delayed scans and gives replacement work its own delay", async () => { + const seen: string[] = []; + const scans = queue({ + process: (item) => { + seen.push(item); + }, + }); + scans.enqueue("old"); + await vi.advanceTimersByTimeAsync(50); + scans.clearPending(); + expect(scans.remaining).toBe(0); + expect(vi.getTimerCount()).toBe(0); + await scans.waitForIdle(); + scans.enqueue("new"); + await vi.advanceTimersByTimeAsync(99); + expect(seen).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + await scans.waitForIdle(); + expect(seen).toEqual(["new"]); + }); + + it("discards all unstarted work while active scans remain tracked", async () => { + const finish = gate(); + const seen: number[] = []; + const scans = queue({ + process: async (item) => { + seen.push(item); + await finish.promise; + }, + }); + for (let item = 0; item < 50; item++) scans.enqueue(item); + await vi.advanceTimersByTimeAsync(0); + expect(seen).toHaveLength(10); + scans.clearPending(); + expect(scans.remaining).toBe(10); + let idle = false; + const done = scans.waitForIdle().then(() => { + idle = true; + }); + await vi.advanceTimersByTimeAsync(0); + expect(idle).toBe(false); + finish.resolve(); + await done; + expect(seen).toEqual(Array.from({ length: 10 }, (_, index) => index)); + expect(scans.remaining).toBe(0); + }); + + it("invalidates prefetched work after clearing and admits replacements alongside active scans", async () => { + const finishFirst = gate(); + const finishReplacement = gate(); + let scans!: ConfigScanQueue; + const seen: string[] = []; + scans = queue({ + process: async (item) => { + seen.push(item); + if (item === "first") { + void Promise.resolve().then(() => Promise.resolve().then(() => scans.clearPending())); + await finishFirst.promise; + } else if (item === "replacement") { + await finishReplacement.promise; + } + }, + }); + scans.enqueue("first"); + scans.enqueue("stale"); + await vi.advanceTimersByTimeAsync(100); + expect(seen).toEqual(["first"]); + expect(scans.remaining).toBe(1); + + scans.enqueue("replacement"); + await vi.advanceTimersByTimeAsync(100); + expect(seen).toEqual(["first", "replacement"]); + expect(scans.remaining).toBe(2); + + finishFirst.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(scans.remaining).toBe(1); + finishReplacement.resolve(); + await scans.waitForIdle(); + expect(scans.remaining).toBe(0); + }); + + it("keeps the ten-scan limit across clear and replacement work", async () => { + const finish = Array.from({ length: 12 }, () => gate()); + const seen: number[] = []; + let active = 0; + let maximum = 0; + const scans = queue({ + process: async (item) => { + seen.push(item); + maximum = Math.max(maximum, ++active); + await finish[item].promise; + active--; + }, + }); + for (let item = 0; item < 10; item++) scans.enqueue(item); + await vi.advanceTimersByTimeAsync(100); + expect(seen).toEqual(Array.from({ length: 10 }, (_, index) => index)); + + scans.clearPending(); + scans.enqueue(10); + scans.enqueue(11); + await vi.advanceTimersByTimeAsync(100); + expect(seen).toEqual(Array.from({ length: 10 }, (_, index) => index)); + expect(maximum).toBe(10); + + finish[0].resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(seen).toEqual(Array.from({ length: 11 }, (_, index) => index)); + expect(maximum).toBe(10); + finish.forEach((item) => item.resolve()); + await scans.waitForIdle(); + expect(seen).toEqual(Array.from({ length: 12 }, (_, index) => index)); + expect(maximum).toBe(10); + }); + + it("stops while waiting for input, joins active work, and rejects later admission", async () => { + const finish = gate(); + const seen: number[] = []; + const scans = queue({ + process: async (item) => { + seen.push(item); + await finish.promise; + }, + }); + scans.enqueue(1); + await vi.advanceTimersByTimeAsync(100); + scans.enqueue(2); + let stopped = false; + const done = scans.stop().then(() => { + stopped = true; + }); + expect(() => scans.enqueue(3)).toThrow("closed"); + expect(vi.getTimerCount()).toBe(0); + await vi.advanceTimersByTimeAsync(0); + expect(stopped).toBe(false); + finish.resolve(); + await done; + expect(seen).toEqual([1]); + expect(scans.remaining).toBe(0); + await scans.stop(); + }); + + it("stops before the first scan without leaving a timer", async () => { + const process = vi.fn(); + const scans = queue({ process }); + scans.enqueue("pending"); + await scans.stop(); + await vi.advanceTimersByTimeAsync(100); + expect(process).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("observes an unhandled item failure after other started work settles", async () => { + const fail = gate(); + const finish = gate(); + const failure = new Error("read failed"); + const seen: number[] = []; + const scans = queue({ + process: async (item) => { + seen.push(item); + if (item === 0) { + await fail.promise; + throw failure; + } + await finish.promise; + }, + }); + for (let item = 0; item < 12; item++) scans.enqueue(item); + await vi.advanceTimersByTimeAsync(0); + let error: unknown; + const done = scans.waitForIdle().catch((reason: unknown) => { + error = reason; + }); + fail.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(error).toBeUndefined(); + expect(seen).toHaveLength(10); + expect(() => scans.enqueue(12)).toThrow("closed"); + finish.resolve(); + await done; + expect(error).toBe(failure); + expect(scans.remaining).toBe(2); + }); + + it("includes work enqueued by an active scan in an existing idle wait", async () => { + const seen: string[] = []; + const scans = queue({ + process: (item) => { + seen.push(item); + if (item === "first") scans.enqueue("follow-up"); + }, + }); + scans.enqueue("first"); + let idle = false; + const done = scans.waitForIdle().then(() => { + idle = true; + }); + await vi.advanceTimersByTimeAsync(100); + expect(seen).toEqual(["first"]); + expect(idle).toBe(false); + await vi.advanceTimersByTimeAsync(100); + await done; + expect(seen).toEqual(["first", "follow-up"]); + expect(vi.getTimerCount()).toBe(0); + }); + + it("wakes an input wait when its only active scan fails", async () => { + const finish = gate(); + const failure = new Error("read failed"); + const scans = queue({ + process: async () => { + await finish.promise; + throw failure; + }, + }); + scans.enqueue("only"); + await vi.advanceTimersByTimeAsync(100); + const done = expect(scans.waitForIdle()).rejects.toBe(failure); + finish.resolve(); + await done; + expect(scans.remaining).toBe(0); + }); + + it("rejects an existing idle wait if observing completion fails", async () => { + const finish = gate(); + const failure = new Error("observer failed"); + const scans = queue({ + process: () => finish.promise, + onProgress: (remaining) => { + if (remaining === 0) throw failure; + }, + }); + scans.enqueue("active"); + await vi.advanceTimersByTimeAsync(100); + const done = expect(scans.waitForIdle()).rejects.toBe(failure); + finish.resolve(); + await done; + expect(scans.remaining).toBe(0); + }); + + it("retains both an observer failure and a later failure from active work", async () => { + const finish = gate(); + const observerFailure = new Error("observer failed"); + const readFailure = new Error("read failed"); + const scans = queue({ + process: async () => { + await finish.promise; + throw readFailure; + }, + onProgress: (remaining) => { + if (remaining === 2) throw observerFailure; + }, + }); + scans.enqueue("active"); + await vi.advanceTimersByTimeAsync(100); + scans.enqueue("pending"); + const done = expect(scans.waitForIdle()).rejects.toMatchObject({ + message: "Customisation scans failed.", + errors: [observerFailure, readFailure], + }); + finish.resolve(); + await done; + }); + + it("wakes an idle waiter when the progress observer fails", async () => { + const failure = new Error("observer failed"); + const process = vi.fn(); + const scans = queue({ + process, + onProgress: () => { + throw failure; + }, + }); + scans.enqueue("pending"); + await expect(scans.waitForIdle()).rejects.toBe(failure); + expect(process).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/src/features/HiddenFileCommon/JsonResolveModal.ts b/src/features/HiddenFileCommon/JsonResolveModal.ts index 967645e2..6026702a 100644 --- a/src/features/HiddenFileCommon/JsonResolveModal.ts +++ b/src/features/HiddenFileCommon/JsonResolveModal.ts @@ -43,11 +43,12 @@ export class JsonResolveModal extends Modal { } async UICallback(keepRev?: string, mergedStr?: string) { - if (this.callback) { - await this.callback(keepRev, mergedStr); + const callback = this.callback; + this.callback = undefined; + if (callback) { + await callback(keepRev, mergedStr); } this.close(); - this.callback = undefined; } override onOpen() { @@ -78,8 +79,10 @@ export class JsonResolveModal extends Modal { const { contentEl } = this; contentEl.empty(); // contentEl.empty(); - if (this.callback != undefined) { - void this.callback(undefined); + const callback = this.callback; + this.callback = undefined; + if (callback) { + void callback(undefined); } if (this.component != undefined) { void unmount(this.component); diff --git a/src/features/HiddenFileCommon/JsonResolveModal.unit.spec.ts b/src/features/HiddenFileCommon/JsonResolveModal.unit.spec.ts new file mode 100644 index 00000000..c75f6451 --- /dev/null +++ b/src/features/HiddenFileCommon/JsonResolveModal.unit.spec.ts @@ -0,0 +1,67 @@ +import { describe, expect, it, vi } from "vitest"; +import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { sendSignal } from "@vrtmrz/livesync-commonlib/compat/common/utils"; +import { JsonResolveModal } from "./JsonResolveModal.ts"; + +vi.mock("@/deps.ts", () => ({ + App: class App {}, + Modal: class Modal { + contentEl = { empty: vi.fn() }; + close() { + (this as { onClose?: () => void }).onClose?.(); + } + }, +})); +vi.mock("./JsonResolvePane.svelte", () => ({ default: {} })); +vi.mock("svelte", () => ({ mount: vi.fn(), unmount: vi.fn() })); + +describe("JsonResolveModal completion", () => { + it("applies a selection once when closing the dialogue", async () => { + const filename = ".obsidian/selected-once.json" as FilePath; + const callback = vi.fn(async () => {}); + const modal = new JsonResolveModal({} as never, filename, [], callback); + + await modal.UICallback("2-kept"); + + expect(callback).toHaveBeenCalledExactlyOnceWith("2-kept", undefined); + sendSignal(`cancel-internal-conflict:${filename}`); + }); + + it("ignores a late selection after cancellation", async () => { + const filename = ".obsidian/cancelled-once.json" as FilePath; + const callback = vi.fn(async () => {}); + const modal = new JsonResolveModal({} as never, filename, [], callback); + + sendSignal(`cancel-internal-conflict:${filename}`); + await vi.waitFor(() => expect(callback).toHaveBeenCalledOnce()); + await modal.UICallback("2-stale"); + + expect(callback).toHaveBeenCalledExactlyOnceWith(undefined); + }); + + it("keeps an admitted write pending when cancellation closes its dialogue", async () => { + const filename = ".obsidian/writing-once.json" as FilePath; + let finish!: () => void; + const gate = new Promise((resolve) => { + finish = resolve; + }); + const callback = vi.fn(async () => await gate); + const modal = new JsonResolveModal({} as never, filename, [], callback); + const close = vi.spyOn(modal, "close"); + let completed = false; + const selected = modal.UICallback("2-kept").then(() => { + completed = true; + }); + + try { + sendSignal(`cancel-internal-conflict:${filename}`); + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); + + expect(callback).toHaveBeenCalledExactlyOnceWith("2-kept", undefined); + expect(completed).toBe(false); + } finally { + finish(); + await selected; + } + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.attemptProgress.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.attemptProgress.unit.spec.ts new file mode 100644 index 00000000..02b40969 --- /dev/null +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.attemptProgress.unit.spec.ts @@ -0,0 +1,143 @@ +import { describe, expect, it, vi } from "vitest"; +import { + type DocumentID, + type FilePath, + type FilePathWithPrefix, + type MetaEntry, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; + +vi.mock("@/deps.ts", () => ({})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ + JsonResolveModal: class JsonResolveModal {}, +})); +vi.mock("@/features/LiveSyncCommands.ts", () => ({ + LiveSyncCommands: class LiveSyncCommands { + plugin!: { app: unknown }; + core!: { services: unknown; settings: unknown }; + get app() { + return this.plugin.app; + } + get services() { + return this.core.services; + } + get settings() { + return this.core.settings; + } + }, +})); +vi.mock("./configureHiddenFileSyncMode.ts", () => ({ + configureHiddenFileSyncMode: vi.fn(), +})); + +import { HiddenFileSync } from "./CmdHiddenFileSync.ts"; + +function createProgress(events: string[] = []) { + return { + log: vi.fn((message: string) => { + if (/^(?:\d+\/\d+|Checking \d+\/\d+)$/.test(message)) events.push(message); + }), + once: vi.fn(), + done: vi.fn(), + }; +} + +function createStoragePaths(count: number): FilePath[] { + return Array.from({ length: count }, (_, index) => `.obsidian/plugins/example/${index}.json` as FilePath); +} + +function createDatabaseEntries(count: number): MetaEntry[] { + return Array.from( + { length: count }, + (_, index) => + ({ + _id: `i:.obsidian/plugins/example/${index}.json` as DocumentID, + _rev: "1-test", + path: `i:.obsidian/plugins/example/${index}.json` as FilePathWithPrefix, + ctime: 1, + mtime: 1, + size: 0, + type: "plain", + datatype: "plain", + children: [], + eden: {}, + deleted: false, + }) as MetaEntry + ); +} + +function createHiddenFileSync(count: number, events: string[] = []) { + const progress = createProgress(events); + const statHidden = vi.fn(async () => { + events.push(`stat:${statHidden.mock.calls.length}`); + return { mtime: 1 }; + }); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { + _progress: vi.fn(() => progress), + _log: vi.fn(), + _fileInfoLastProcessed: new Map(), + _databaseInfoLastProcessed: new Map(), + scanInternalFileNames: vi.fn(async () => createStoragePaths(count)), + getAllDatabaseFiles: vi.fn(async () => createDatabaseEntries(count)), + getPath: vi.fn((file: MetaEntry) => file.path), + updateLastProcessed: vi.fn(), + scanAllStorageChanges: vi.fn(async () => undefined), + scanAllDatabaseChanges: vi.fn(async () => undefined), + resetLastProcessedDatabase: vi.fn(), + resetLastProcessedFile: vi.fn(), + useStorageFiles: vi.fn(async () => false), + useDatabaseFiles: vi.fn(async () => false), + trackStorageFileModification: vi.fn(async () => false), + trackDatabaseFileModification: vi.fn(async () => false), + core: { storageAccess: { statHidden } }, + }); + return { hiddenFileSync, progress, statHidden }; +} + +describe("HiddenFileSync attempt progress", () => { + it("reports applyOfflineChanges attempts at 1 and each 25 items, without a final partial count", async () => { + const events: string[] = []; + const { hiddenFileSync, progress } = createHiddenFileSync(52, events); + + await hiddenFileSync.applyOfflineChanges(false); + + expect(progress.log.mock.calls.map(([message]) => message).filter((message) => /^\d+\/\d+$/.test(message))).toEqual([ + "1/52", + "26/52", + "51/52", + ]); + expect(events.indexOf("51/52")).toBeLessThan(events.indexOf("stat:1")); + }); + + it("reports rebuildMerging attempts at 1 and each 100 items, without a final partial count", async () => { + const events: string[] = []; + const { hiddenFileSync, progress } = createHiddenFileSync(202, events); + + await hiddenFileSync.rebuildMerging(false); + + expect(progress.log.mock.calls.map(([message]) => message).filter((message) => message.startsWith("Checking "))).toEqual([ + "Checking 1/202", + "Checking 101/202", + "Checking 201/202", + ]); + expect(events.indexOf("Checking 101/202")).toBeLessThan(events.indexOf("stat:101")); + }); + + it("preserves stat failures from applyOfflineChanges", async () => { + const { hiddenFileSync, progress, statHidden } = createHiddenFileSync(1); + const error = new Error("stat failed"); + statHidden.mockRejectedValueOnce(error); + + await expect(hiddenFileSync.applyOfflineChanges(false)).rejects.toBe(error); + expect(progress.done).not.toHaveBeenCalled(); + }); + + it("preserves stat failures from rebuildMerging", async () => { + const { hiddenFileSync, progress, statHidden } = createHiddenFileSync(1); + const error = new Error("stat failed"); + statHidden.mockRejectedValueOnce(error); + + await expect(hiddenFileSync.rebuildMerging(false)).rejects.toBe(error); + expect(progress.done).not.toHaveBeenCalled(); + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts index 5ef0e4de..d681ba74 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.completion.unit.spec.ts @@ -90,6 +90,39 @@ afterEach(async () => { for (const fixture of fixtureOwners.splice(0)) await fixture.dispose(); }); +describe("Hidden File Sync initial modification times", () => { + it.each([ + { label: "newer", localMtime: 30_000, expected: "local content" }, + { label: "older", localMtime: 10_000, expected: "database content" }, + { label: "equal", localMtime: 20_000, expected: "database content" }, + ])("stores only a newer local file when the local timestamp is $label", async ({ localMtime, expected }) => { + const { fixture, hiddenFileSync, storageAccess } = await createHiddenFileSync(); + const entry = optionalFileEntry(`i:${path}`, "database content"); + entry.mtime = 20_000; + await fixture.localDatabase.putDBEntry(entry); + storageAccess.setFile("local content", localMtime); + + await hiddenFileSync.trackStorageFileModification(path, true); + + const saved = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(saved).not.toBe(false); + if (saved) { + expect(await getText(saved)).toBe(expected); + expect(saved.mtime).toBe(expected === "local content" ? localMtime : 20_000); + } + }); + + it("stores a local-only file when only newer changes are requested", async () => { + const { fixture, hiddenFileSync } = await createHiddenFileSync(); + + await hiddenFileSync.trackStorageFileModification(path, true); + + const saved = await fixture.localDatabase.getDBEntry(`i:${path}` as FilePath); + expect(saved).not.toBe(false); + if (saved) expect(await getText(saved)).toBe(content); + }); +}); + describe("Hidden File Sync optional save outcomes", () => { it("retries a failed database write when the storage fingerprint is unchanged", async () => { const { fixture, hiddenFileSync } = await createHiddenFileSync(); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.progress.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.progress.unit.spec.ts new file mode 100644 index 00000000..dd301a80 --- /dev/null +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.progress.unit.spec.ts @@ -0,0 +1,219 @@ +import { describe, expect, it, vi } from "vitest"; +import { + LOG_LEVEL_VERBOSE, + type DocumentID, + type FilePath, + type FilePathWithPrefix, + type MetaEntry, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; + +vi.mock("@/deps.ts", () => ({})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ + JsonResolveModal: class JsonResolveModal {}, +})); +vi.mock("@/features/LiveSyncCommands.ts", () => ({ + LiveSyncCommands: class LiveSyncCommands { + plugin!: { app: unknown }; + core!: { services: unknown; settings: unknown }; + get app() { + return this.plugin.app; + } + get services() { + return this.core.services; + } + get settings() { + return this.core.settings; + } + }, +})); +vi.mock("./configureHiddenFileSyncMode.ts", () => ({ + configureHiddenFileSyncMode: vi.fn(), +})); + +import { HiddenFileSync } from "./CmdHiddenFileSync.ts"; + +function createProgress() { + return { + log: vi.fn(), + once: vi.fn(), + done: vi.fn(), + }; +} + +function createHiddenFileSync() { + const progress = createProgress(); + const log = vi.fn(); + const trackStorageFileModification = vi.fn(async (_file: FilePath) => false); + const trackDatabaseFileModification = vi.fn(async (_path: FilePath) => false); + const isTargetFile = vi.fn(async (_path: FilePath) => true); + const getPath = vi.fn((file: MetaEntry) => file.path); + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + + Object.assign(hiddenFileSync, { + _progress: vi.fn(() => progress), + _log: log, + trackStorageFileModification, + trackDatabaseFileModification, + isTargetFile, + getPath, + }); + + return { + hiddenFileSync, + progress, + log, + trackStorageFileModification, + trackDatabaseFileModification, + isTargetFile, + }; +} + +function createStoragePaths(count: number): FilePath[] { + return Array.from({ length: count }, (_, index) => `.obsidian/plugins/example/${index}.json` as FilePath); +} + +function createDatabaseEntries(count: number): MetaEntry[] { + return Array.from( + { length: count }, + (_, index) => + ({ + _id: `i:.obsidian/plugins/example/${index}.json` as DocumentID, + _rev: "1-test", + path: `i:.obsidian/plugins/example/${index}.json` as FilePathWithPrefix, + ctime: 1, + mtime: 1, + size: 0, + type: "plain", + datatype: "plain", + children: [], + eden: {}, + deleted: false, + }) as MetaEntry + ); +} + +describe("HiddenFileSync batch progress consumers", () => { + it("counts completed storage modifications even when they return false", async () => { + const { hiddenFileSync, progress, trackStorageFileModification } = createHiddenFileSync(); + const files = createStoragePaths(101); + + await hiddenFileSync.trackScannedStorageChanges(files); + + expect(trackStorageFileModification).toHaveBeenCalledTimes(101); + expect(progress.log.mock.calls).toEqual([["1/101"], ["101/101"]]); + expect(progress.done).toHaveBeenCalledTimes(1); + }); + + it("logs and skips failed storage items when counting progress", async () => { + const { hiddenFileSync, progress, log, trackStorageFileModification } = createHiddenFileSync(); + const files = createStoragePaths(100); + const failingFile = files[50]; + const error = new Error("Storage item failed"); + trackStorageFileModification.mockImplementation(async (file) => { + if (file === failingFile) throw error; + return false; + }); + + await hiddenFileSync.trackScannedStorageChanges(files); + + expect(progress.once).toHaveBeenCalledWith(`Failed to process storage change file:${failingFile}`); + expect(log).toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE); + expect(progress.log.mock.calls).toEqual([["1/100"]]); + expect(progress.done).toHaveBeenCalledTimes(1); + }); + + it("counts completed database items, including filtered items and false returns", async () => { + const { hiddenFileSync, progress, trackDatabaseFileModification, isTargetFile } = createHiddenFileSync(); + const files = createDatabaseEntries(101); + const filteredPath = ".obsidian/plugins/example/50.json" as FilePath; + isTargetFile.mockImplementation(async (path) => path !== filteredPath); + + await hiddenFileSync.trackScannedDatabaseChange(files); + + expect(isTargetFile).toHaveBeenCalledTimes(101); + expect(trackDatabaseFileModification).toHaveBeenCalledTimes(100); + expect(progress.log.mock.calls).toEqual([["1/101"], ["101/101"]]); + expect(progress.done).toHaveBeenCalledTimes(1); + }); + + it("logs and skips failed database scan items when counting progress", async () => { + const { hiddenFileSync, progress, log, trackDatabaseFileModification } = createHiddenFileSync(); + const files = createDatabaseEntries(101); + const failingPath = ".obsidian/plugins/example/50.json" as FilePath; + const error = new Error("Database scan item failed"); + trackDatabaseFileModification.mockImplementation(async (path) => { + if (path === failingPath) throw error; + return false; + }); + + await hiddenFileSync.trackScannedDatabaseChange(files); + + expect(log).toHaveBeenCalledWith( + expect.stringContaining("Failed to process storage change file:"), + expect.anything() + ); + expect(log).toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE); + expect(progress.log.mock.calls).toEqual([["1/101"]]); + expect(progress.done).toHaveBeenCalledTimes(1); + }); + + it("uses the database-file progress interval and counts false returns", async () => { + const { hiddenFileSync, progress, trackDatabaseFileModification } = createHiddenFileSync(); + const files = createDatabaseEntries(26); + + await expect(hiddenFileSync.useDatabaseFiles(files)).resolves.toBe(true); + + expect(trackDatabaseFileModification).toHaveBeenCalledTimes(26); + expect(progress.log.mock.calls).toEqual([["Scanning database files..."], ["1/26"], ["26/26"]]); + expect(progress.done).toHaveBeenCalledTimes(1); + }); + + it("logs and skips failed database-file items when counting progress", async () => { + const { hiddenFileSync, progress, log, trackDatabaseFileModification } = createHiddenFileSync(); + const files = createDatabaseEntries(26); + const failingPath = ".obsidian/plugins/example/12.json" as FilePath; + const error = new Error("Database file item failed"); + trackDatabaseFileModification.mockImplementation(async (path) => { + if (path === failingPath) throw error; + return false; + }); + + await expect(hiddenFileSync.useDatabaseFiles(files)).resolves.toBe(true); + + expect(log).toHaveBeenCalledWith(expect.stringContaining("Failed to process database changes:")); + expect(log).toHaveBeenCalledWith(error, LOG_LEVEL_VERBOSE); + expect(progress.log.mock.calls).toEqual([["Scanning database files..."], ["1/26"]]); + expect(progress.done).toHaveBeenCalledTimes(1); + }); + + it("starts all storage work eagerly and calls done only after all promises resolve", async () => { + const { hiddenFileSync, progress, trackStorageFileModification } = createHiddenFileSync(); + const files = createStoragePaths(101); + const resolveModifications: Array<(result: boolean) => void> = []; + trackStorageFileModification.mockImplementation( + () => new Promise((resolve) => resolveModifications.push(resolve)) + ); + + const operation = hiddenFileSync.trackScannedStorageChanges(files); + expect(trackStorageFileModification).toHaveBeenCalledTimes(files.length); + expect(progress.done).not.toHaveBeenCalled(); + + for (const resolveModification of resolveModifications) resolveModification(false); + await operation; + + expect(progress.done).toHaveBeenCalledTimes(1); + }); + + it("does not call done when the item error handler rejects the batch", async () => { + const { hiddenFileSync, progress, trackStorageFileModification } = createHiddenFileSync(); + const itemError = new Error("Storage item failed"); + const handlerError = new Error("Progress error handler failed"); + trackStorageFileModification.mockRejectedValueOnce(itemError); + progress.once.mockImplementation(() => { + throw handlerError; + }); + + await expect(hiddenFileSync.trackScannedStorageChanges(createStoragePaths(1))).rejects.toBe(handlerError); + expect(progress.done).not.toHaveBeenCalled(); + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.queue.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.queue.unit.spec.ts new file mode 100644 index 00000000..7460692c --- /dev/null +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.queue.unit.spec.ts @@ -0,0 +1,549 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({})); +vi.mock("@/features/HiddenFileCommon/JsonResolveModal.ts", () => ({ JsonResolveModal: class JsonResolveModal {} })); +vi.mock("@/modules/features/ModuleLog.ts", () => ({ MARK_DONE: "" })); +vi.mock("@/common/PeriodicProcessor.ts", () => ({ + PeriodicProcessor: class PeriodicProcessor { + disable = vi.fn(); + }, +})); +vi.mock("./configureHiddenFileSyncMode.ts", () => ({ configureHiddenFileSyncMode: vi.fn() })); + +import { + DEFAULT_SETTINGS, + type DocumentID, + type FilePathWithPrefix, + type LoadedEntry, + type MetaEntry, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { HiddenFileSync } from "./CmdHiddenFileSync"; +import { sendSignal, waitForSignal } from "@vrtmrz/livesync-commonlib/compat/common/utils"; + +const queueOwners: HiddenFileSync[] = []; + +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function waitForCondition(predicate: () => boolean, message: string) { + for (let attempt = 0; attempt < 30; attempt++) { + if (predicate()) return; + await Promise.resolve(); + } + throw new Error(message); +} + +function createHiddenFileSync(localDatabase: object) { + vi.spyOn(HiddenFileSync.prototype, "onBindFunction").mockImplementation(() => {}); + const hiddenFileSync = new HiddenFileSync({ + settings: { + ...DEFAULT_SETTINGS, + syncInternalFiles: true, + syncInternalFileOverwritePatterns: "", + }, + localDatabase, + services: { API: { addLog: vi.fn() }, context: { app: {} } }, + } as unknown as ConstructorParameters[0]); + hiddenFileSync._log = vi.fn(); + queueOwners.push(hiddenFileSync); + return { hiddenFileSync }; +} + +afterEach(() => { + for (const hiddenFileSync of queueOwners.splice(0)) { + hiddenFileSync.conflictResolutionProcessor.dispose(); + } + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +describe("HiddenFileSync conflict queue", () => { + it("keeps a self-requeued conflict check and waits for its follow-up", async () => { + vi.useFakeTimers(); + const path = "i:.obsidian/plugins/example/data.txt" as FilePathWithPrefix; + const id = String(path) as DocumentID; + const conflictDoc = { + _id: id, + _rev: "2-current", + path, + _conflicts: ["2-conflicted"], + } as MetaEntry; + const resolvedDoc = { _id: id, _rev: "3-resolved", path } as MetaEntry; + let rawRead = 0; + const localDatabase = { + findEntries: vi.fn(async function* () { + yield conflictDoc; + }), + getRaw: vi.fn(async () => (rawRead++ === 0 ? conflictDoc : resolvedDoc)), + }; + const { hiddenFileSync } = createHiddenFileSync(localDatabase); + hiddenFileSync.path2id = async () => id; + + let resolutionCalls = 0; + let requeued = false; + hiddenFileSync.resolveByNewerEntry = vi.fn(async () => { + resolutionCalls++; + hiddenFileSync.requeueConflictCheck(path); + requeued = true; + }); + + let completed = false; + hiddenFileSync.queueConflictCheck(path); + try { + await waitForCondition( + () => hiddenFileSync.conflictResolutionProcessor.remaining === 1, + "The initial conflict check was not queued" + ); + + await vi.advanceTimersByTimeAsync(9); + expect(localDatabase.getRaw).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1); + await waitForCondition( + () => + requeued && + hiddenFileSync.conflictResolutionProcessor.nowProcessing === 0 && + hiddenFileSync.conflictResolutionProcessor.remaining === 1, + "The conflict follow-up was not retained after the first pass" + ); + void hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed().then(() => { + completed = true; + }); + expect(completed).toBe(false); + expect(localDatabase.getRaw).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(9); + expect(localDatabase.getRaw).toHaveBeenCalledTimes(1); + expect(completed).toBe(false); + + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(1_000); + await waitForCondition(() => completed, "The conflict pipeline did not await its follow-up"); + + expect(localDatabase.getRaw).toHaveBeenCalledTimes(2); + expect(resolutionCalls).toBe(1); + expect(hiddenFileSync.pendingConflictChecks.has(path)).toBe(false); + expect(hiddenFileSync.conflictResolutionProcessor.remaining).toBe(0); + } finally { + await vi.runAllTimersAsync(); + } + }); + + it.each(["prefixed", "unprefixed"])( + "cancels the older same-file dialogue and presents the latest revisions for a %s request", + async (form) => { + vi.useFakeTimers(); + const filename = `.obsidian/replacement-${form}.json`; + const path = `i:${filename}` as FilePathWithPrefix; + const id = String(path) as DocumentID; + let generation = 2; + const localDatabase = { + getRaw: vi.fn(async (_id: DocumentID, options?: { revs_info?: boolean }) => + options?.revs_info + ? {} + : { _id: id, path, _rev: `${generation}-current`, _conflicts: [`${generation}-other`] } + ), + getDBEntry: vi.fn( + async (_path: FilePathWithPrefix, options: { rev: string }) => + ({ _id: id, path, _rev: options.rev }) as LoadedEntry + ), + managers: { conflictManager: { mergeObject: vi.fn(async () => false) } }, + }; + const { hiddenFileSync } = createHiddenFileSync(localDatabase); + hiddenFileSync.path2id = async () => id; + const finishFirst = deferred(); + let cancelled = false; + const opened: string[][] = []; + hiddenFileSync.showJSONMergeDialogAndMerge = vi.fn(async (left, right) => { + opened.push([left._rev!, right._rev!]); + if (opened.length > 1) return false; + return await Promise.race([ + finishFirst.promise, + waitForSignal(`cancel-internal-conflict:${filename}`).then(() => { + cancelled = true; + return false; + }), + ]); + }); + + try { + hiddenFileSync.queueConflictCheck(path); + await vi.advanceTimersByTimeAsync(100); + expect(opened).toEqual([["2-current", "2-other"]]); + + generation = 3; + hiddenFileSync.queueConflictCheck((form === "prefixed" ? path : filename) as FilePathWithPrefix); + await vi.advanceTimersByTimeAsync(100); + + expect(cancelled).toBe(true); + expect(opened).toEqual([ + ["2-current", "2-other"], + ["3-current", "3-other"], + ]); + expect(await hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed()).toBe(true); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(0); + } finally { + finishFirst.resolve(false); + sendSignal(`cancel-internal-conflict:${filename}`); + await vi.advanceTimersByTimeAsync(100); + } + } + ); + + it.each(["same", "different"])( + "closes only a dialogue for the %s file when a replicated resolution arrives", + async (target) => { + vi.useFakeTimers(); + const filename = `.obsidian/received-${target}.json`; + const path = `i:${filename}` as FilePathWithPrefix; + const incomingPath = (target === "same" ? path : "i:.obsidian/other-resolved.json") as FilePathWithPrefix; + const id = String(path) as DocumentID; + let resolved = false; + const localDatabase = { + getRaw: vi.fn(async (_id: DocumentID, options?: { revs_info?: boolean }) => + options?.revs_info + ? {} + : { + _id: id, + path, + _rev: resolved ? "3-resolved" : "2-current", + _conflicts: resolved ? [] : ["2-other"], + } + ), + getDBEntry: vi.fn(async () => ({ _id: id, path }) as LoadedEntry), + managers: { conflictManager: { mergeObject: vi.fn(async () => false) } }, + }; + const { hiddenFileSync } = createHiddenFileSync(localDatabase); + hiddenFileSync.path2id = async () => id; + hiddenFileSync.getPath = () => incomingPath; + hiddenFileSync.isTargetFile = vi.fn(async () => true); + hiddenFileSync.trackDatabaseFileModification = vi.fn(async () => true); + const finishFirst = deferred(); + let cancelled = false; + hiddenFileSync.showJSONMergeDialogAndMerge = vi.fn( + async () => + await Promise.race([ + finishFirst.promise, + waitForSignal(`cancel-internal-conflict:${filename}`).then(() => { + cancelled = true; + return false; + }), + ]) + ); + + try { + hiddenFileSync.queueConflictCheck(path); + await vi.advanceTimersByTimeAsync(100); + expect(hiddenFileSync.showJSONMergeDialogAndMerge).toHaveBeenCalledOnce(); + + resolved = target === "same"; + const incoming = { + _id: String(incomingPath) as DocumentID, + path: incomingPath, + _rev: "3-received", + } as LoadedEntry; + expect(await hiddenFileSync._anyProcessOptionalSyncFiles(incoming)).toBe(true); + await vi.advanceTimersByTimeAsync(100); + + expect(cancelled).toBe(target === "same"); + expect(hiddenFileSync.showJSONMergeDialogAndMerge).toHaveBeenCalledOnce(); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(target === "same" ? 0 : 1); + if (target === "same") { + expect(await hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed()).toBe(true); + } + } finally { + finishFirst.resolve(false); + sendSignal(`cancel-internal-conflict:${filename}`); + await vi.advanceTimersByTimeAsync(100); + } + } + ); + + it("waits for a cancelled manual merge dialogue and then finishes without requeueing", async () => { + vi.useFakeTimers(); + const path = "i:.obsidian/plugins/example/data.json" as FilePathWithPrefix; + const id = String(path) as DocumentID; + const conflictDoc = { + _id: id, + _rev: "2-current", + path, + _conflicts: ["2-conflicted"], + } as MetaEntry; + const historyDoc = {} as unknown as MetaEntry; + const loaded = { _id: id, _rev: "2-current", path } as LoadedEntry; + const dialogue = deferred(); + let dialogueOpened = false; + const localDatabase = { + findEntries: vi.fn(async function* () { + yield conflictDoc; + }), + getRaw: vi.fn(async (_id: DocumentID, options?: { revs_info?: boolean }) => + options?.revs_info ? historyDoc : conflictDoc + ), + getDBEntry: vi.fn(async () => loaded), + managers: { + conflictManager: { + mergeObject: vi.fn(async () => false), + }, + }, + }; + const { hiddenFileSync } = createHiddenFileSync(localDatabase); + hiddenFileSync.path2id = async () => id; + hiddenFileSync.showJSONMergeDialogAndMerge = vi.fn(async () => { + dialogueOpened = true; + return await dialogue.promise; + }); + + let completed = false; + hiddenFileSync.queueConflictCheck(path); + try { + await waitForCondition( + () => hiddenFileSync.conflictResolutionProcessor.remaining === 1, + "The initial conflict check was not queued" + ); + + await vi.advanceTimersByTimeAsync(100); + expect(dialogueOpened).toBe(true); + void hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed().then(() => { + completed = true; + }); + expect(completed).toBe(false); + expect(hiddenFileSync.conflictResolutionProcessor.totalRemaining).toBeGreaterThan(0); + expect(localDatabase.managers.conflictManager.mergeObject).toHaveBeenCalledOnce(); + expect(localDatabase.getDBEntry).toHaveBeenCalledTimes(2); + + dialogue.resolve(false); + await vi.advanceTimersByTimeAsync(1_000); + await waitForCondition(() => completed, "The cancelled dialogue did not release the pipeline"); + + expect(hiddenFileSync.pendingConflictChecks.has(path)).toBe(false); + expect(hiddenFileSync.conflictResolutionProcessor.totalRemaining).toBe(0); + } finally { + dialogue.resolve(false); + await vi.runAllTimersAsync(); + } + }); +}); + +describe("HiddenFileSync pipeline contracts", () => { + it("starts above the threshold, limits analysis to five, and retains FIFO input", async () => { + vi.useFakeTimers(); + const gates = Array.from({ length: 11 }, () => deferred()); + const paths = gates.map((_, index) => `i:.obsidian/${index}.txt` as FilePathWithPrefix); + const reads: string[] = []; + const getRaw = vi.fn((id: DocumentID) => { + reads.push(id); + return gates[paths.indexOf(id as unknown as FilePathWithPrefix)].promise; + }); + const { hiddenFileSync } = createHiddenFileSync({ getRaw }); + hiddenFileSync.path2id = async (path) => path as unknown as DocumentID; + for (const path of paths.slice(0, 10)) hiddenFileSync.queueConflictCheck(path); + await vi.advanceTimersByTimeAsync(0); + expect(reads).toEqual([]); + hiddenFileSync.queueConflictCheck(paths[10]); + await vi.advanceTimersByTimeAsync(0); + expect(reads).toEqual(paths.slice(0, 5)); + hiddenFileSync.queueConflictCheck(paths[0]); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(11); + gates[0].resolve({ path: paths[0] } as MetaEntry); + await vi.advanceTimersByTimeAsync(0); + expect(reads).toEqual(paths.slice(0, 6)); + gates.forEach((gate, index) => gate.resolve({ path: paths[index] } as MetaEntry)); + await vi.advanceTimersByTimeAsync(100); + expect(await hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed()).toBe(true); + expect(reads).toEqual([...paths, paths[0]]); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(0); + }); + + it("continues analysis while one manual dialogue is open and serialises the dialogues", async () => { + vi.useFakeTimers(); + const paths = Array.from({ length: 12 }, (_, index) => `i:.obsidian/${index}.json` as FilePathWithPrefix); + const gates = paths.map(() => deferred()); + const firstDialogue = deferred(); + let firstCancelled = false; + const localDatabase = { + getRaw: vi.fn(async (id: DocumentID, options?: { revs_info?: boolean }) => + options?.revs_info + ? {} + : { + _id: id, + path: id, + _rev: "2-current", + _conflicts: ["2-other"], + } + ), + getDBEntry: vi.fn(async (path: FilePathWithPrefix) => ({ path }) as LoadedEntry), + managers: { conflictManager: { mergeObject: vi.fn(async () => false) } }, + }; + const { hiddenFileSync } = createHiddenFileSync(localDatabase); + hiddenFileSync.path2id = async (path) => path as unknown as DocumentID; + const opened: FilePathWithPrefix[] = []; + hiddenFileSync.showJSONMergeDialogAndMerge = vi.fn((doc) => { + opened.push(doc.path); + if (opened.length === 1) { + return Promise.race([ + firstDialogue.promise, + waitForSignal(`cancel-internal-conflict:${doc.path.slice(2)}`).then(() => { + firstCancelled = true; + return false; + }), + ]); + } + return gates[paths.indexOf(doc.path)].promise; + }); + paths.forEach((path) => hiddenFileSync.queueConflictCheck(path)); + await vi.advanceTimersByTimeAsync(100); + expect(localDatabase.managers.conflictManager.mergeObject).toHaveBeenCalledTimes(12); + expect(opened).toEqual([paths[0]]); + paths.forEach((path) => hiddenFileSync.queueConflictCheck(path)); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(12); + await vi.advanceTimersByTimeAsync(100); + expect(firstCancelled).toBe(true); + expect(opened).toEqual([paths[0], paths[0]]); + expect(localDatabase.managers.conflictManager.mergeObject).toHaveBeenCalledTimes(24); + let completed = false; + void hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed().then(() => { + completed = true; + }); + for (let index = 0; index < paths.length; index++) { + expect(opened).toEqual([paths[0], ...paths.slice(0, index + 1)]); + expect(completed).toBe(false); + gates[index].resolve(false); + await vi.advanceTimersByTimeAsync(100); + } + expect(completed).toBe(true); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(0); + }); + + it("logs an analysis error and continues with later paths", async () => { + vi.useFakeTimers(); + const failure = new Error("Injected read failure"); + const getRaw = vi.fn().mockRejectedValueOnce(failure).mockResolvedValue({}); + const { hiddenFileSync } = createHiddenFileSync({ getRaw }); + hiddenFileSync.path2id = async (path) => path as unknown as DocumentID; + hiddenFileSync.queueConflictCheck("i:bad.txt" as FilePathWithPrefix); + hiddenFileSync.queueConflictCheck("i:good.txt" as FilePathWithPrefix); + await vi.advanceTimersByTimeAsync(100); + expect(await hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed()).toBe(true); + expect(getRaw).toHaveBeenCalledTimes(2); + expect(hiddenFileSync._log).toHaveBeenCalledWith(failure, expect.anything()); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(0); + }); + + it("collects a conflict enumeration before resuming analysis", async () => { + vi.useFakeTimers(); + const gate = deferred(); + const path = "i:.obsidian/scanned.txt" as FilePathWithPrefix; + const getRaw = vi.fn(async () => ({})); + const { hiddenFileSync } = createHiddenFileSync({ + getRaw, + findEntries: async function* () { + await gate.promise; + yield { _id: path, path, _conflicts: ["2-other"] }; + }, + }); + hiddenFileSync.path2id = async (value) => value as unknown as DocumentID; + const result = hiddenFileSync.resolveConflictOnInternalFiles(); + await vi.advanceTimersByTimeAsync(100); + hiddenFileSync.queueConflictCheck(path); + await vi.advanceTimersByTimeAsync(100); + expect(getRaw).not.toHaveBeenCalled(); + gate.resolve(); + await vi.advanceTimersByTimeAsync(100); + await result; + expect(getRaw).toHaveBeenCalledOnce(); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(0); + }); +}); + +describe("HiddenFileSync conflict queue ownership", () => { + it("stops delayed work on unload and rejects later conflict events", async () => { + vi.useFakeTimers(); + const getRaw = vi.fn(async () => ({})); + const { hiddenFileSync } = createHiddenFileSync({ getRaw }); + hiddenFileSync.path2id = async (path) => path as unknown as DocumentID; + const path = "i:.obsidian/queued.txt" as FilePathWithPrefix; + hiddenFileSync.queueConflictCheck(path); + const idle = hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed(); + hiddenFileSync.onunload(); + hiddenFileSync.queueConflictCheck(path); + await vi.advanceTimersByTimeAsync(100); + expect(await idle).toBe(false); + expect(getRaw).not.toHaveBeenCalled(); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("finishes an admitted analysis without handing its result to a new dialogue after unload", async () => { + vi.useFakeTimers(); + const gate = deferred(); + const path = "i:.obsidian/active.json" as FilePathWithPrefix; + const getRaw = vi + .fn() + .mockImplementationOnce(() => gate.promise) + .mockResolvedValue({}); + const getDBEntry = vi.fn(async () => ({ path })); + const mergeObject = vi.fn(async () => false); + const { hiddenFileSync } = createHiddenFileSync({ + getRaw, + getDBEntry, + managers: { conflictManager: { mergeObject } }, + }); + hiddenFileSync.path2id = async () => path as unknown as DocumentID; + hiddenFileSync.queueConflictCheck(path); + await vi.advanceTimersByTimeAsync(10); + expect(getRaw).toHaveBeenCalledOnce(); + hiddenFileSync.onunload(); + expect(await hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed()).toBe(false); + gate.resolve({ + _id: String(path) as DocumentID, + path, + _rev: "2-current", + _conflicts: ["2-other"], + } as MetaEntry); + await vi.advanceTimersByTimeAsync(100); + expect(mergeObject).toHaveBeenCalledOnce(); + expect(getDBEntry).not.toHaveBeenCalled(); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(0); + }); + + it("does not requeue a manual resolution which finishes after unload", async () => { + vi.useFakeTimers(); + const gate = deferred(); + const path = "i:.obsidian/dialogue.json" as FilePathWithPrefix; + const getRaw = vi.fn(async (_id: DocumentID, options?: { revs_info?: boolean }) => + options?.revs_info + ? {} + : { + _id: path, + path, + _rev: "2-current", + _conflicts: ["2-other"], + } + ); + const getDBEntry = vi.fn(async () => ({ path }) as LoadedEntry); + const { hiddenFileSync } = createHiddenFileSync({ + getRaw, + getDBEntry, + managers: { conflictManager: { mergeObject: async () => false } }, + }); + hiddenFileSync.path2id = async () => path as unknown as DocumentID; + hiddenFileSync.showJSONMergeDialogAndMerge = vi.fn(() => gate.promise); + hiddenFileSync.queueConflictCheck(path); + await vi.advanceTimersByTimeAsync(20); + expect(hiddenFileSync.showJSONMergeDialogAndMerge).toHaveBeenCalledOnce(); + hiddenFileSync.onunload(); + expect(await hiddenFileSync.conflictResolutionProcessor.waitForAllProcessed()).toBe(false); + gate.resolve(true); + await vi.advanceTimersByTimeAsync(100); + expect(getRaw).toHaveBeenCalledTimes(2); + expect(getDBEntry).toHaveBeenCalledTimes(2); + expect(hiddenFileSync.pendingConflictChecks.size).toBe(0); + expect(hiddenFileSync.conflictResolutionProcessor.totalRemaining).toBe(0); + }); +}); diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.ts index 04588bc1..c5d42bde 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.ts @@ -35,17 +35,17 @@ import { getLogLevel, autosaveCache, type MapLike, - onlyInNTimes, BASE_IS_NEW, EVEN, displayRev, } from "@/common/utils.ts"; import { PeriodicProcessor } from "@/common/PeriodicProcessor.ts"; -import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock"; +import { withLock, skipIfDuplicated } from "octagonal-wheels/concurrency/lock"; import { JsonResolveModal } from "@/features/HiddenFileCommon/JsonResolveModal.ts"; import { LiveSyncCommands } from "@/features/LiveSyncCommands.ts"; import { addPrefix, stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; -import { QueueProcessor } from "octagonal-wheels/concurrency/processor"; +import { createCoalescingNotifier } from "octagonal-wheels/concurrency/coalescingNotifier"; +import { HiddenFileConflictQueue } from "./HiddenFileConflictQueue.ts"; import { hiddenFilesEventCount, hiddenFilesProcessingCount, @@ -57,6 +57,7 @@ import { tryGetFilePath } from "@vrtmrz/livesync-commonlib/compat/common/utils.d import type { HandlerOutcome } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"; import type { OptionalFileEventOptions } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import { configureHiddenFileSyncMode, type ConfigureHiddenFileSyncResult } from "./configureHiddenFileSyncMode.ts"; +import { performBatchWithProgress } from "./performBatchWithProgress.ts"; import type { OptionalSyncFeatureMode } from "@/features/optionalSyncFeatures.ts"; import { getObsidianCommunityPluginManager } from "@/common/obsidianCommunityPlugins.ts"; import { $msg } from "@/common/translation"; @@ -107,6 +108,8 @@ export class HiddenFileSync extends LiveSyncCommands { } onunload() { this.periodicInternalFileScanProcessor?.disable(); + this.conflictResolutionProcessor.dispose(); + this.pendingConflictChecks.clear(); } onload() { this.services.API.addCommand({ @@ -544,7 +547,7 @@ export class HiddenFileSync extends LiveSyncCommands { hiddenFilesEventCount.value++; const rel = await this.semaphore.acquire(); try { - return await serialized(`hidden-file-event:${file}`, async () => { + return await withLock(`hidden-file-event:${file}`, async () => { hiddenFilesProcessingCount.value++; try { return await fn(); @@ -571,18 +574,15 @@ export class HiddenFileSync extends LiveSyncCommands { ) { const logLevel = getLogLevel(showNotice); const p = this._progress(`[⚙ Storage -> DB ]\n`, logLevel); - const notifyProgress = onlyInNTimes(100, (progress) => p.log(`${progress}/${processFiles.length}`)); - const processes = processFiles.map(async (file, i) => { - try { - await this.trackStorageFileModification(file, onlyNew, forceWriteAll, includeDeleted); - notifyProgress(); - } catch (ex) { + await performBatchWithProgress(processFiles, 100, { + processItem: (file) => this.trackStorageFileModification(file, onlyNew, forceWriteAll, includeDeleted), + onProgress: (progress) => p.log(`${progress}/${processFiles.length}`), + onItemError: (ex, file) => { p.once(`Failed to process storage change file:${file}`); this._log(ex, LOG_LEVEL_VERBOSE); - } + }, + onDone: () => p.done(), }); - await Promise.all(processes); - p.done(); } async scanAllStorageChanges( showNotice: boolean = false, @@ -688,7 +688,7 @@ Offline Changed files: ${processFiles.length}`; const dbMTime = getComparingMTime(filesOnDB, includeDeleted); const diff = compareMTime(storageMTime, dbMTime); - if (diff != TARGET_IS_NEW) { + if (diff != BASE_IS_NEW) { this._log(`Hidden file:${path} is not new.`, LOG_LEVEL_VERBOSE); if (filesOnDB && stat) { // OnlyNew not handles the deletion. @@ -741,21 +741,33 @@ Offline Changed files: ${processFiles.length}`; // --> Conflict processing - // Keep one in-flight conflict check per path so repeated sync events do not close the active merge dialogue. - pendingConflictChecks = new Set(); + // Coalesce unstarted checks; retain a replacement once processing has begun. + pendingConflictChecks = new Map(); queueConflictCheck(path: FilePathWithPrefix) { - if (this.pendingConflictChecks.has(path)) return; - this.pendingConflictChecks.add(path); - this.conflictResolutionProcessor.enqueue(path); + path = addPrefix(path, ICHeader); + const pending = this.pendingConflictChecks.get(path); + if (pending) { + if (pending.started) { + pending.recheck = true; + sendSignal(`cancel-internal-conflict:${stripAllPrefixes(path)}`); + } + return; + } + this.pendingConflictChecks.set(path, { started: false, recheck: false }); + if (!this.conflictResolutionProcessor.enqueue(path)) this.pendingConflictChecks.delete(path); } finishConflictCheck(path: FilePathWithPrefix) { + path = addPrefix(path, ICHeader); + const pending = this.pendingConflictChecks.get(path); this.pendingConflictChecks.delete(path); + if (pending?.recheck) this.queueConflictCheck(path); } requeueConflictCheck(path: FilePathWithPrefix) { - this.finishConflictCheck(path); + path = addPrefix(path, ICHeader); + this.pendingConflictChecks.delete(path); this.queueConflictCheck(path); } @@ -804,13 +816,18 @@ Offline Changed files: ${processFiles.length}`; } // check the file again } - conflictResolutionProcessor = new QueueProcessor( - async (paths: FilePathWithPrefix[]) => { - const path = paths[0]; + conflictResolutionProcessor = new HiddenFileConflictQueue( + async (path: FilePathWithPrefix) => { + const check = this.pendingConflictChecks.get(path); + if (check) check.started = true; try { // Retrieve data const id = await this.path2id(path, ICHeader); const doc = await this.localDatabase.getRaw(id, { conflicts: true }); + if (check?.recheck) { + this.requeueConflictCheck(path); + return []; + } if (doc._conflicts === undefined) { this.finishConflictCheck(path); return []; @@ -839,6 +856,10 @@ Offline Changed files: ${processFiles.length}`; doc._rev, conflictedRev ); + if (check?.recheck) { + this.requeueConflictCheck(path); + return []; + } if (result) { this._log(`Object merge:${path}`, LOG_LEVEL_INFO); const filename = stripAllPrefixes(path); @@ -862,7 +883,7 @@ Offline Changed files: ${processFiles.length}`; await this.resolveByNewerEntry(id, path, doc, revA, revB); return []; } - return [{ path, revA, revB, id, doc }]; + return [{ path, revA, revB, id, doc, check }]; } // When not JSON file, resolve conflicts by choosing a newer one. await this.resolveByNewerEntry(id, path, doc, revA, revB); @@ -874,46 +895,40 @@ Offline Changed files: ${processFiles.length}`; return []; } }, - { - suspended: false, - batchSize: 1, - concurrentLimit: 5, - delay: 10, - keepResultUntilDownstreamConnected: true, - yieldThreshold: 10, - pipeTo: new QueueProcessor( - async (results) => { - const { id, doc, path, revA, revB } = results[0]; - const prefixedPath = addPrefix(path, ICHeader); - const docAMerge = await this.localDatabase.getDBEntry(prefixedPath, { rev: revA }); - const docBMerge = await this.localDatabase.getDBEntry(prefixedPath, { rev: revB }); - try { - if (docAMerge != false && docBMerge != false) { - if (await this.showJSONMergeDialogAndMerge(docAMerge, docBMerge)) { - // Again for other conflicted revisions. - this.requeueConflictCheck(path); - } else { - this.finishConflictCheck(path); - } - return; - } else { - // If either revision could not read, force resolving by the newer one. - await this.resolveByNewerEntry(id, path, doc, revA, revB); - } - } catch (ex) { + async (result) => { + const { id, doc, path, revA, revB, check } = result; + if (check?.recheck) { + this.requeueConflictCheck(path); + return; + } + const prefixedPath = addPrefix(path, ICHeader); + const docAMerge = await this.localDatabase.getDBEntry(prefixedPath, { rev: revA }); + const docBMerge = await this.localDatabase.getDBEntry(prefixedPath, { rev: revB }); + if (check?.recheck) { + this.requeueConflictCheck(path); + return; + } + try { + if (docAMerge != false && docBMerge != false) { + if (await this.showJSONMergeDialogAndMerge(docAMerge, docBMerge)) { + // Again for other conflicted revisions. + this.requeueConflictCheck(path); + } else { this.finishConflictCheck(path); - throw ex; } - }, - { - suspended: false, - batchSize: 1, - concurrentLimit: 1, - delay: 10, - keepResultUntilDownstreamConnected: false, - yieldThreshold: 10, + return; + } else { + // If either revision could not read, force resolving by the newer one. + await this.resolveByNewerEntry(id, path, doc, revA, revB); } - ), + } catch (ex) { + this.finishConflictCheck(path); + throw ex; + } + }, + (error) => { + this._log("Processor error!"); + this._log(error, LOG_LEVEL_VERBOSE); } ); @@ -927,7 +942,7 @@ Offline Changed files: ${processFiles.length}`; const displayFilename = `${storeFilePath}`; // const path = this.prefixedConfigDir2configDir(stripAllPrefixes(docA.path)) || docA.path; // Cancel only when replacing an existing dialogue for the same path, not on every queue pass. - sendSignal(`cancel-internal-conflict:${docA.path}`); + sendSignal(`cancel-internal-conflict:${storageFilePath}`); const modal = new JsonResolveModal(this.app, storageFilePath, [docA, docB], async (keep, result) => { // modal.close(); try { @@ -1013,6 +1028,9 @@ Offline Changed files: ${processFiles.length}`; async processReplicationResult(doc: LoadedEntry): Promise { const info = this.getDocProps(doc); const path = info.path; + if (this.pendingConflictChecks.has(addPrefix(path, ICHeader))) { + this.queueConflictCheck(info.prefixedPath); + } const headerLine = `Tracking DB ${info.path} (${info.revDisplay}) :`; const ret = await this.trackDatabaseFileModification(path, headerLine); this._log(`${headerLine} Done: ${info.shortenedId})`, LOG_LEVEL_VERBOSE); @@ -1134,9 +1152,8 @@ Offline Changed files: ${processFiles.length}`; ) { const logLevel = getLogLevel(showNotice); const p = this._progress(`[⚙ DB -> Storage ]\n`, logLevel); - const notifyProgress = onlyInNTimes(100, (progress) => p.log(`${progress}/${processFiles.length}`)); - const processes = processFiles.map(async (file) => { - try { + await performBatchWithProgress(processFiles, 100, { + processItem: async (file) => { const path = stripAllPrefixes(this.getPath(file)); if (!(await this.isTargetFile(path))) { this._log( @@ -1153,19 +1170,19 @@ Offline Changed files: ${processFiles.length}`; includeDeletion ); } - notifyProgress(); - } catch (ex) { + }, + onProgress: (progress) => p.log(`${progress}/${processFiles.length}`), + onItemError: (ex, file) => { this._log(`Failed to process storage change file:${tryGetFilePath(file)}`, logLevel); this._log(ex, LOG_LEVEL_VERBOSE); - } + }, + onDone: () => p.done(), }); - await Promise.all(processes); - p.done(); } async applyOfflineChanges(showNotice: boolean) { const logLevel = getLogLevel(showNotice); - return await serialized("applyOfflineChanges", async () => { + return await withLock("applyOfflineChanges", async () => { const p = this._progress("[⚙ Apply untracked changes ]\n", logLevel); this._log(`Track changes.`, logLevel); p.log("Enumerating local files..."); @@ -1188,9 +1205,14 @@ Untracked database files: ${untrackedDatabase.length} Common untracked files: ${bothUntracked.length}`; p.once(stat); const semaphores = Semaphore(10); - const notifyProgress = onlyInNTimes(25, (progress) => p.log(`${progress}/${bothUntracked.length}`)); + const notifyProgress = createCoalescingNotifier({ + everyCount: 25, + leading: true, + onNotify: (progress) => p.log(`${progress}/${bothUntracked.length}`), + }); + let attempted = 0; const allProcesses = bothUntracked.map(async (file) => { - notifyProgress(); + notifyProgress.update(++attempted); const rel = await semaphores.acquire(); try { const fileStat = await this.core.storageAccess.statHidden(file); @@ -1260,20 +1282,18 @@ Offline Changed files: ${files.length}`; const logLevel = getLogLevel(showNotice); const p = this._progress(`[⚙ Scanning DB -> Storage ]\n`, logLevel); p.log("Scanning database files..."); - const notifyProgress = onlyInNTimes(25, (progress) => p.log(`${progress}/${files.length}`)); - const processFiles = files.map(async (file) => { - try { + await performBatchWithProgress(files, 25, { + processItem: async (file) => { const path = stripAllPrefixes(this.getPath(file)); await this.trackDatabaseFileModification(path, "[Scanning]", true, onlyNew, file); - notifyProgress(); - } catch (ex) { + }, + onProgress: (progress) => p.log(`${progress}/${files.length}`), + onItemError: (ex, file) => { this._log(`Failed to process database changes:${tryGetFilePath(file)}`); this._log(ex, LOG_LEVEL_VERBOSE); - } - return; + }, + onDone: () => p.done(), }); - await Promise.all(processFiles); - p.done(); return true; } @@ -1435,9 +1455,14 @@ Offline Changed files: ${files.length}`; const storageToDatabase = [] as FilePath[]; const databaseToStorage = [] as MetaEntry[]; - const eachProgress = onlyInNTimes(100, (progress) => p.log(`Checking ${progress}/${allFileNames.size}`)); + const eachProgress = createCoalescingNotifier({ + everyCount: 100, + leading: true, + onNotify: (progress) => p.log(`Checking ${progress}/${allFileNames.size}`), + }); + let attempted = 0; for (const file of allFileNames) { - eachProgress(); + eachProgress.update(++attempted); const storageMTime = await this.core.storageAccess.statHidden(file); const mtimeStorage = getComparingMTime(storageMTime); const dbEntry = allDatabaseMap.get(file)!; @@ -1678,7 +1703,7 @@ Offline Changed files: ${files.length}`; } const prefixedFileName = addPrefix(storeFilePath, ICHeader); - return await serialized("file-" + prefixedFileName, async () => { + return await withLock("file-" + prefixedFileName, async () => { try { const fileInfo = "stat" in file && "body" in file ? file : await this.loadFileWithInfo(storeFilePath); if (fileInfo.deleted) { @@ -1739,7 +1764,7 @@ Offline Changed files: ${files.length}`; } const prefixedFileName = addPrefix(storeFilePath, ICHeader); - return await serialized("file-" + prefixedFileName, async () => { + return await withLock("file-" + prefixedFileName, async () => { try { const baseData = await this.getLiveInternalRevision(prefixedFileName, baseRevision); if (baseData === false) { @@ -1817,7 +1842,7 @@ Offline Changed files: ${files.length}`; if (await this.services.vault.isIgnoredByIgnoreFile(storageFilePath)) { return undefined; } - return await serialized("file-" + prefixedFileName, async () => { + return await withLock("file-" + prefixedFileName, async () => { try { const baseData = strict ? await this.loadBaseSaveDataStrict(storeFilePath, false) @@ -1888,7 +1913,7 @@ Offline Changed files: ${files.length}`; if (await this.services.vault.isIgnoredByIgnoreFile(storageFilePath)) { return undefined; } - return await serialized("file-" + prefixedFileName, async () => { + return await withLock("file-" + prefixedFileName, async () => { try { // Check conflicted status const metaOnDB = requiredLiveRevision diff --git a/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts b/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts index 9018b222..4b3e5543 100644 --- a/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts +++ b/src/features/HiddenFileSync/CmdHiddenFileSync.unit.spec.ts @@ -98,6 +98,85 @@ function createHiddenRevisionOperation() { }; } +describe("HiddenFileSync storage loading", () => { + function createStorageReader(content: string | ArrayBuffer) { + const path = ".obsidian/plugins/example/data.json" as FilePath; + const stat = { + ctime: 123, + mtime: 456, + size: typeof content === "string" ? new TextEncoder().encode(content).byteLength : content.byteLength, + type: "file" as const, + }; + const storageAccess = { + statHidden: vi.fn(async (): Promise => stat), + readHiddenFileAuto: vi.fn(async () => content), + }; + const hiddenFileSync = Object.create(HiddenFileSync.prototype) as HiddenFileSync; + Object.assign(hiddenFileSync, { core: { storageAccess } }); + return { hiddenFileSync, storageAccess, path, stat }; + } + + it.each([ + { + kind: "text", + content: "first\r\n日本語 🐙", + expected: new TextEncoder().encode("first\r\n日本語 🐙"), + mime: "text/plain", + }, + { + kind: "binary", + content: new Uint8Array([0, 128, 255]).buffer, + expected: new Uint8Array([0, 128, 255]), + mime: "application/octet-stream", + }, + { kind: "empty text", content: "", expected: new Uint8Array(0), mime: "text/plain" }, + { + kind: "empty binary", + content: new ArrayBuffer(0), + expected: new Uint8Array(0), + mime: "application/octet-stream", + }, + ])("preserves $kind content and its file metadata", async ({ content, expected, mime }) => { + const { hiddenFileSync, storageAccess, path, stat } = createStorageReader(content); + + const file = await hiddenFileSync.loadFileWithInfo(path); + + expect(storageAccess.statHidden).toHaveBeenCalledWith(path); + expect(storageAccess.readHiddenFileAuto).toHaveBeenCalledWith(path); + expect(file).toMatchObject({ name: "data.json", path, stat, isInternal: true, deleted: false }); + expect(file.body).toBeInstanceOf(Blob); + expect(file.body.type).toBe(mime); + expect(new Uint8Array(await file.body.arrayBuffer())).toEqual(expected); + }); + + it("represents a missing file as a deletion without reading storage content", async () => { + const { hiddenFileSync, storageAccess, path } = createStorageReader("unavailable"); + storageAccess.statHidden.mockResolvedValue(null); + + const file = await hiddenFileSync.loadFileWithInfo(path); + + expect(storageAccess.readHiddenFileAuto).not.toHaveBeenCalled(); + expect(file).toMatchObject({ + name: "data.json", + path, + stat: { ctime: 0, mtime: 0, size: 0, type: "file" }, + isInternal: true, + deleted: true, + }); + expect(file.body.type).toBe("application/octet-stream"); + expect(file.body.size).toBe(0); + }); + + it("propagates a read failure for an existing file", async () => { + const { hiddenFileSync, storageAccess, path } = createStorageReader("unavailable"); + const error = new Error("Storage read failed"); + storageAccess.readHiddenFileAuto.mockRejectedValue(error); + + await expect(hiddenFileSync.loadFileWithInfo(path)).rejects.toBe(error); + expect(storageAccess.readHiddenFileAuto).toHaveBeenCalledWith(path); + }); +}); + describe("HiddenFileSync configuration-change notices", () => { it.each(["pushForce", "pullForce", "safe"] as const)( "refuses %s initialisation before processing files while Hidden File Sync is disabled", diff --git a/src/features/HiddenFileSync/HiddenFileConflictQueue.ts b/src/features/HiddenFileSync/HiddenFileConflictQueue.ts new file mode 100644 index 00000000..af933c37 --- /dev/null +++ b/src/features/HiddenFileSync/HiddenFileConflictQueue.ts @@ -0,0 +1,209 @@ +import { compatGlobal, type CompatTimeoutHandle } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; +import { withCancellation } from "octagonal-wheels/concurrency/cancellation"; +import { createNotifier } from "octagonal-wheels/concurrency/notifier"; +import { forEachInScope } from "octagonal-wheels/concurrency/scoped.iteration"; +import { createAsyncQueue } from "octagonal-wheels/iterable/asyncQueue"; +import { withOwnedResource } from "octagonal-wheels/resource"; + +/** The scheduling used by one hidden-file conflict stage. */ +class ConflictStage { + private readonly input = createAsyncQueue(); + private readonly reader = this.input[Symbol.asyncIterator](); + private readonly changed = createNotifier(); + private readonly cancellation = new AbortController(); + private timer: CompatTimeoutHandle | undefined; + private ready = false; + private suspended = false; + private stopped = false; + private outstanding = 0; + active = 0; + + constructor( + concurrency: number, + process: (item: T) => Promise, + private readonly notify: () => void, + onError: (error: unknown) => void, + onFailure: (error: unknown) => void + ) { + // The operation owns its input until started callbacks have settled. + // Abort also releases input immediately, before that final cleanup. + void withCancellation(undefined, { signal: this.cancellation.signal }, (scope) => { + const stopInput = () => { + this.input.dispose(); + this.clearTimer(); + this.changed.notify(); + }; + scope.signal.addEventListener("abort", stopInput, { once: true }); + scope.defer(() => scope.signal.removeEventListener("abort", stopInput)); + return withOwnedResource(this.input, () => + forEachInScope(scope, this.availableItems(), { concurrency }, async (entry) => { + while (this.suspended && !this.stopped) await this.changed.wait(); + if (this.stopped) return; + this.active++; + try { + await process(entry.value); + } catch (error) { + try { + onError(error); + } catch (reportingError) { + // Wake a pending input pull before the runner joins + // its other callbacks after an unexpected failure. + onFailure(reportingError); + throw reportingError; + } + } finally { + this.active--; + this.outstanding--; + this.notify(); + } + }) + ); + }).catch(onFailure); + } + + get remaining() { + return this.outstanding; + } + + enqueue(item: T): boolean { + if (this.stopped) return false; + this.input.enqueue(item); + this.outstanding++; + this.schedule(); + this.notify(); + return true; + } + + private clearTimer() { + if (this.timer === undefined) return; + compatGlobal.clearTimeout(this.timer); + this.timer = undefined; + } + + private schedule() { + if (this.stopped || this.suspended || this.input.size === 0) return; + if (this.ready || this.input.size > 10) { + this.clearTimer(); + this.ready = true; + this.changed.notify(); + } else if (this.timer === undefined) { + this.timer = compatGlobal.setTimeout(() => { + this.timer = undefined; + this.ready = true; + this.changed.notify(); + }, 10); + } + } + + private async *availableItems() { + while (!this.stopped) { + while (!this.stopped && (this.suspended || !this.ready || this.input.size === 0)) { + await this.changed.wait(); + } + if (this.stopped) return; + const entry = await this.reader.next(); + if (entry.done) return; + if (this.input.size === 0) { + this.ready = false; + this.clearTimer(); + } + // Outstanding includes this pulled item until its callback finishes. + yield { value: entry.value }; + } + } + + suspend() { + this.suspended = true; + this.ready = false; + this.clearTimer(); + } + + resume() { + if (!this.suspended || this.stopped) return; + this.suspended = false; + this.schedule(); + this.changed.notify(); + } + + dispose() { + if (this.stopped) return; + this.stopped = true; + this.outstanding = this.active; + this.cancellation.abort(); + this.notify(); + } +} + +/** Two hidden-file stages: five analyses, followed by one manual resolution. */ +export class HiddenFileConflictQueue { + private readonly changed = createNotifier(); + private readonly analysis: ConflictStage; + private readonly resolution: ConflictStage; + private stopped = false; + private failure: { error: unknown } | undefined; + + constructor( + analyse: (item: T) => Promise, + resolve: (item: U) => Promise, + onError: (error: unknown) => void + ) { + const notify = () => this.changed.notify(); + const onFailure = (error: unknown) => { + this.failure ??= { error }; + this.dispose(); + }; + this.resolution = new ConflictStage(1, resolve, notify, onError, onFailure); + this.analysis = new ConflictStage( + 5, + async (item) => { + const results = await analyse(item); + for (const result of results) this.resolution.enqueue(result); + }, + notify, + onError, + onFailure + ); + } + + enqueue(item: T): boolean { + return this.analysis.enqueue(item); + } + + get nowProcessing() { + return this.analysis.active; + } + + get remaining() { + return this.analysis.remaining; + } + + get totalRemaining() { + return this.analysis.remaining + this.resolution.remaining; + } + + suspend() { + this.analysis.suspend(); + } + + startPipeline(): this { + this.analysis.resume(); + return this; + } + + /** Includes handoff and self-requeues; disposal is never successful drainage. */ + async waitForAllProcessed(): Promise { + this.startPipeline(); + while (!this.stopped && this.totalRemaining > 0) await this.changed.wait(); + if (this.failure) throw this.failure.error; + return !this.stopped; + } + + /** Stop admission and release pending work without waiting for active dialogue or writes. */ + dispose() { + if (this.stopped) return; + this.stopped = true; + this.analysis.dispose(); + this.resolution.dispose(); + this.changed.notify(); + } +} diff --git a/src/features/HiddenFileSync/HiddenFileConflictQueue.unit.spec.ts b/src/features/HiddenFileSync/HiddenFileConflictQueue.unit.spec.ts new file mode 100644 index 00000000..d4a51f48 --- /dev/null +++ b/src/features/HiddenFileSync/HiddenFileConflictQueue.unit.spec.ts @@ -0,0 +1,194 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { HiddenFileConflictQueue } from "./HiddenFileConflictQueue.ts"; + +const owners: { dispose(): void }[] = []; +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} +function own(queue: HiddenFileConflictQueue) { + owners.push(queue); + return queue; +} + +afterEach(async () => { + owners.splice(0).forEach((owner) => owner.dispose()); + if (vi.isFakeTimers()) await vi.runAllTimersAsync(); + vi.useRealTimers(); +}); + +describe("HiddenFileConflictQueue", () => { + it("awaits repeated handoffs and self-requeues with native timers", async () => { + const seen: number[] = []; + let queue!: HiddenFileConflictQueue; + queue = own( + new HiddenFileConflictQueue( + async (value: number) => [value], + async (value) => { + seen.push(value); + if (value < 3) queue.enqueue(value + 1); + }, + vi.fn() + ) + ); + queue.enqueue(0); + expect(await queue.waitForAllProcessed()).toBe(true); + expect(seen).toEqual([0, 1, 2, 3]); + expect(queue.totalRemaining).toBe(0); + }); + + it("uses the first arrival's ten-millisecond delay in each stage", async () => { + vi.useFakeTimers(); + const analyse = vi.fn(async (value: number) => [value]); + const resolve = vi.fn(async (_value: number) => {}); + const queue = own(new HiddenFileConflictQueue(analyse, resolve, vi.fn())); + queue.enqueue(1); + await vi.advanceTimersByTimeAsync(9); + queue.enqueue(2); + expect(analyse).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(analyse.mock.calls).toEqual([[1], [2]]); + expect(resolve).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(9); + expect(resolve).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(resolve.mock.calls).toEqual([[1], [2]]); + expect(await queue.waitForAllProcessed()).toBe(true); + expect(queue.totalRemaining).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("starts both stages immediately above ten waiting items", async () => { + vi.useFakeTimers(); + const gate = deferred(); + const analyse = vi.fn(async (value: number) => [value]); + const resolve = vi.fn(async (_value: number) => { + await gate.promise; + }); + const queue = own(new HiddenFileConflictQueue(analyse, resolve, vi.fn())); + for (let value = 0; value < 11; value++) queue.enqueue(value); + await vi.advanceTimersByTimeAsync(0); + expect(analyse).toHaveBeenCalledTimes(11); + expect(resolve).toHaveBeenCalledTimes(1); + expect(queue.totalRemaining).toBe(11); + gate.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(resolve.mock.calls).toEqual(Array.from({ length: 11 }, (_, value) => [value])); + expect(await queue.waitForAllProcessed()).toBe(true); + }); + + it("logs item failures in both stages and continues without implicit retry", async () => { + vi.useFakeTimers(); + const first = new Error("analysis failed"); + const second = new Error("resolution failed"); + const analyse = vi.fn(async (value: number) => { + if (value === 1) throw first; + return [value]; + }); + const resolve = vi.fn(async (value: number) => { + if (value === 2) throw second; + }); + const onError = vi.fn(); + const queue = own(new HiddenFileConflictQueue(analyse, resolve, onError)); + [1, 2, 3].forEach((value) => queue.enqueue(value)); + await vi.advanceTimersByTimeAsync(20); + expect(onError.mock.calls).toEqual([[first], [second]]); + expect(analyse.mock.calls).toEqual([[1], [2], [3]]); + expect(resolve.mock.calls).toEqual([[2], [3]]); + expect(await queue.waitForAllProcessed()).toBe(true); + }); + + it("discards delayed input and releases idle waiters on disposal", async () => { + vi.useFakeTimers(); + const analyse = vi.fn(async (value: number) => [value]); + const resolve = vi.fn(async (_value: number) => {}); + const queue = own(new HiddenFileConflictQueue(analyse, resolve, vi.fn())); + queue.enqueue(1); + const idle = queue.waitForAllProcessed(); + queue.dispose(); + queue.dispose(); + expect(await idle).toBe(false); + expect(queue.enqueue(2)).toBe(false); + expect(queue.totalRemaining).toBe(0); + await vi.advanceTimersByTimeAsync(100); + expect(analyse).not.toHaveBeenCalled(); + expect(resolve).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("does not wait for active analysis or admit its late handoff and self-requeue after disposal", async () => { + vi.useFakeTimers(); + const gate = deferred(); + const attempted: boolean[] = []; + let finished = 0; + const analyse = vi.fn(async (value: number) => { + await gate.promise; + attempted.push(queue.enqueue(value)); + finished++; + return [value]; + }); + const resolve = vi.fn(async (_value: number) => {}); + const queue = own(new HiddenFileConflictQueue(analyse, resolve, vi.fn())); + for (let value = 0; value < 12; value++) queue.enqueue(value); + await vi.advanceTimersByTimeAsync(0); + expect(analyse).toHaveBeenCalledTimes(5); + const idle = queue.waitForAllProcessed(); + queue.dispose(); + expect(await idle).toBe(false); + expect(finished).toBe(0); + expect(queue.totalRemaining).toBe(5); + gate.resolve(); + await vi.advanceTimersByTimeAsync(100); + expect(finished).toBe(5); + expect(attempted).toEqual([false, false, false, false, false]); + expect(analyse).toHaveBeenCalledTimes(5); + expect(resolve).not.toHaveBeenCalled(); + expect(queue.totalRemaining).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("leaves an active dialogue running and discards the next dialogue", async () => { + vi.useFakeTimers(); + const gate = deferred(); + let finished = false; + const resolve = vi.fn(async (_value: number) => { + await gate.promise; + finished = true; + }); + const queue = own(new HiddenFileConflictQueue(async (value: number) => [value], resolve, vi.fn())); + queue.enqueue(1); + queue.enqueue(2); + await vi.advanceTimersByTimeAsync(20); + expect(resolve).toHaveBeenCalledTimes(1); + queue.dispose(); + expect(await queue.waitForAllProcessed()).toBe(false); + expect(finished).toBe(false); + gate.resolve(); + await vi.advanceTimersByTimeAsync(100); + expect(finished).toBe(true); + expect(resolve).toHaveBeenCalledTimes(1); + expect(queue.totalRemaining).toBe(0); + }); + + it("suspends analysis while already handed-off manual work continues", async () => { + vi.useFakeTimers(); + const analyse = vi.fn(async (value: number) => [value]); + const resolve = vi.fn(async (_value: number) => {}); + const queue = own(new HiddenFileConflictQueue(analyse, resolve, vi.fn())); + queue.enqueue(1); + await vi.advanceTimersByTimeAsync(10); + queue.suspend(); + queue.enqueue(2); + await vi.advanceTimersByTimeAsync(100); + expect(analyse.mock.calls).toEqual([[1]]); + expect(resolve.mock.calls).toEqual([[1]]); + queue.startPipeline(); + await vi.advanceTimersByTimeAsync(20); + expect(analyse.mock.calls).toEqual([[1], [2]]); + expect(resolve.mock.calls).toEqual([[1], [2]]); + expect(await queue.waitForAllProcessed()).toBe(true); + }); +}); diff --git a/src/features/HiddenFileSync/performBatchWithProgress.ts b/src/features/HiddenFileSync/performBatchWithProgress.ts new file mode 100644 index 00000000..7c0fd2dd --- /dev/null +++ b/src/features/HiddenFileSync/performBatchWithProgress.ts @@ -0,0 +1,49 @@ +import { createCoalescingNotifier } from "octagonal-wheels/concurrency/coalescingNotifier"; + +interface PerformBatchWithProgressCallbacks { + /** Runs one item. Its resolved value is ignored. */ + processItem: (item: T) => Promise; + /** Receives the number of successfully completed items. */ + onProgress: (completed: number) => void; + /** Handles an item or progress callback error. */ + onItemError: (error: unknown, item: T) => void; + /** Runs once after all per-item promises resolve. */ + onDone: () => void; +} + +/** + * Runs an eager batch and coalesces progress notifications by successful item count. + * @internal + * + * @param items - Items to process concurrently. + * @param eachCount - Number of completed items between progress notifications. + * @param callbacks - Item processing, progress, error, and completion callbacks. + * @returns A promise that resolves with no value after the batch and completion callback finish. + * @throws When notifier configuration is invalid, an error handler throws, or the completion callback throws. + * @remarks The first completed item triggers progress immediately. Work and the progress callback share + * one error boundary; errors are passed to `onItemError`. A rejected item operation does not advance + * the count. The batch has no concurrency limit or retry, and pending progress is not flushed when + * processing ends. + */ +export async function performBatchWithProgress( + items: readonly T[], + eachCount: number, + callbacks: PerformBatchWithProgressCallbacks +): Promise { + const notifier = createCoalescingNotifier({ + everyCount: eachCount, + leading: true, + onNotify: callbacks.onProgress, + }); + let completed = 0; + const processes = items.map(async (item) => { + try { + await callbacks.processItem(item); + notifier.update(++completed); + } catch (error) { + callbacks.onItemError(error, item); + } + }); + await Promise.all(processes); + callbacks.onDone(); +} diff --git a/src/features/HiddenFileSync/performBatchWithProgress.unit.spec.ts b/src/features/HiddenFileSync/performBatchWithProgress.unit.spec.ts new file mode 100644 index 00000000..ba0d7167 --- /dev/null +++ b/src/features/HiddenFileSync/performBatchWithProgress.unit.spec.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from "vitest"; +import { performBatchWithProgress } from "./performBatchWithProgress.ts"; + +describe("performBatchWithProgress", () => { + it("counts resolved false values, continues after item errors, and does not flush partial progress", async () => { + const error = new Error("Item failed"); + const processItem = vi.fn(async (item: number) => { + if (item === 2) throw error; + return false; + }); + const onProgress = vi.fn(); + const onItemError = vi.fn(); + const onDone = vi.fn(); + + const result = await performBatchWithProgress([1, 2, 3, 4, 5, 6, 7], 2, { + processItem, + onProgress, + onItemError, + onDone, + }); + + expect(result).toBeUndefined(); + expect(processItem).toHaveBeenCalledTimes(7); + expect(onItemError).toHaveBeenCalledWith(error, 2); + expect(onProgress.mock.calls).toEqual([[1], [3], [5]]); + expect(onDone).toHaveBeenCalledTimes(1); + }); + + it("starts every item eagerly and calls onDone only after all item promises resolve", async () => { + const items = [1, 2, 3]; + const resolvers: Array<() => void> = []; + const processItem = vi.fn( + () => + new Promise((resolve) => { + resolvers.push(resolve); + }) + ); + const onProgress = vi.fn(); + const onItemError = vi.fn(); + const onDone = vi.fn(); + + const operation = performBatchWithProgress(items, 100, { processItem, onProgress, onItemError, onDone }); + + expect(processItem).toHaveBeenCalledTimes(items.length); + expect(onDone).not.toHaveBeenCalled(); + for (const resolve of resolvers) resolve(); + await operation; + + expect(onDone).toHaveBeenCalledTimes(1); + }); + + it("routes a progress callback error through the item error handler", async () => { + const error = new Error("Progress callback failed"); + const onProgress = vi.fn(() => { + throw error; + }); + const onItemError = vi.fn(); + const onDone = vi.fn(); + + await performBatchWithProgress(["item"], 100, { + processItem: async () => false, + onProgress, + onItemError, + onDone, + }); + + expect(onItemError).toHaveBeenCalledWith(error, "item"); + expect(onDone).toHaveBeenCalledTimes(1); + }); + + it("rejects when the item error handler throws and skips onDone", async () => { + const itemError = new Error("Item failed"); + const handlerError = new Error("Error handler failed"); + const onItemError = vi.fn(() => { + throw handlerError; + }); + const onDone = vi.fn(); + + await expect( + performBatchWithProgress(["item"], 100, { + processItem: async () => { + throw itemError; + }, + onProgress: vi.fn(), + onItemError, + onDone, + }) + ).rejects.toBe(handlerError); + + expect(onItemError).toHaveBeenCalledWith(itemError, "item"); + expect(onDone).not.toHaveBeenCalled(); + }); +}); diff --git a/src/features/LiveSyncCommands.ts b/src/features/LiveSyncCommands.ts index 3f1ec783..d3467549 100644 --- a/src/features/LiveSyncCommands.ts +++ b/src/features/LiveSyncCommands.ts @@ -43,7 +43,7 @@ export abstract class LiveSyncCommands { this._log = createInstanceLogFunction(this.constructor.name, this.services.API); // __$checkInstanceBinding(this); } - abstract onunload(): void; + abstract onunload(): void | Promise; abstract onload(): void | Promise; _isMainReady() { diff --git a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts index 955813f3..5300ff03 100644 --- a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts +++ b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts @@ -12,7 +12,7 @@ import { } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { getNoFromRev } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; import { LiveSyncCommands } from "@/features/LiveSyncCommands"; -import { serialized } from "octagonal-wheels/concurrency/lock_v2"; +import { withLock } from "octagonal-wheels/concurrency/lock_v2"; import { arrayToChunkedArray } from "octagonal-wheels/collection"; import { EVENT_REQUEST_PERFORM_GC_V3, eventHub } from "@/common/events"; import type { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; @@ -435,7 +435,7 @@ Note: **Make sure to synchronise all devices before deletion.** chunkSet.add(doc._id); } else if ("children" in doc) { await processDoc(doc, isDeleted); - await serialized("x-process-doc", async () => await processDocRevisions(doc)); + await withLock("x-process-doc", async () => await processDocRevisions(doc)); } }; // Track changes diff --git a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts index 89796f4f..8fc5f546 100644 --- a/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts +++ b/src/features/LocalDatabaseMainte/CmdLocalDatabaseMainte.unit.spec.ts @@ -4,7 +4,7 @@ vi.mock("octagonal-wheels/number", () => ({ sizeToHumanReadable: vi.fn((value: number) => `${value} B`), })); vi.mock("octagonal-wheels/concurrency/lock_v2", () => ({ - serialized: vi.fn((_key: string, task: () => unknown) => task()), + withLock: vi.fn((_key: string, task: () => unknown) => task()), })); vi.mock("octagonal-wheels/collection", () => ({ arrayToChunkedArray: vi.fn((values: unknown[]) => [values]), diff --git a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.ts b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.ts index 7ff2d604..36c021df 100644 --- a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.ts +++ b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.ts @@ -1,4 +1,5 @@ import type { ReviewHarnessScenarioResult } from "./reviewHarnessTypes"; +import { createReviewHarnessHeapSampler } from "./reviewHarnessHeapSampling"; export const REVIEW_HARNESS_FILE_CONTENT_HASH_INPUTS = [ { label: "4 KiB", bytes: 4 * 1024 }, @@ -23,15 +24,6 @@ type BenchmarkPerformance = Pick & { type HashAlgorithm = "SHA-256" | "xxHash64"; -function readHeap(clock: BenchmarkPerformance): number | undefined { - try { - const bytes = clock.memory?.usedJSHeapSize; - return typeof bytes === "number" && Number.isFinite(bytes) && bytes >= 0 ? bytes : undefined; - } catch { - return undefined; - } -} - function summary(samples: readonly number[]): string { const sorted = [...samples].sort((a, b) => a - b); const middle = sorted[Math.floor(sorted.length / 2)]; @@ -63,19 +55,13 @@ export async function runReviewHarnessFileContentHashBenchmark( throw new Error("The xxHash64 initialisation time is invalid."); } - const before = readHeap(clock); - let highest = before; - const sampleHeap = () => { - const value = readHeap(clock); - if (value !== undefined) highest = Math.max(highest ?? value, value); - return value; - }; + const heap = createReviewHarnessHeapSampler(clock); const inputs: { readonly label: string; readonly bytes: number; readonly content: Blob }[] = []; for (const { label, bytes } of REVIEW_HARNESS_FILE_CONTENT_HASH_INPUTS) { await yieldControl(); inputs.push({ label, bytes, content: operations.createInput(bytes) }); - sampleHeap(); + heap.sample(); } const hash = (algorithm: HashAlgorithm, content: Blob) => @@ -99,7 +85,7 @@ export async function runReviewHarnessFileContentHashBenchmark( const elapsed = clock.now() - started; checkHash(algorithm, inputs[0].label, result); warmupTimes.set(algorithm, elapsed); - sampleHeap(); + heap.sample(); } const observations = [ @@ -124,7 +110,7 @@ export async function runReviewHarnessFileContentHashBenchmark( throw new Error(`The ${algorithm} measurement returned an invalid duration.`); } samples[algorithm].push(elapsed); - sampleHeap(); + heap.sample(); } } for (const algorithm of ["SHA-256", "xxHash64"] as const) { @@ -132,16 +118,7 @@ export async function runReviewHarnessFileContentHashBenchmark( } } - const after = sampleHeap(); - if (highest === undefined) { - observations.push("JavaScript heap: unavailable on this device."); - } else { - const mib = (bytes: number | undefined) => - bytes === undefined ? "unavailable" : `${(bytes / 1048576).toFixed(2)} MiB`; - observations.push( - `JavaScript heap: before=${mib(before)}; highest sampled=${mib(highest)}; after=${mib(after)}.` - ); - } + observations.push(heap.finish()); observations.push( "Heap samples are approximate, may include other Obsidian work, and are affected by garbage collection. They are neither total app RAM nor a true peak." ); diff --git a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.unit.spec.ts b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.unit.spec.ts index a3ffc39a..1f5e797e 100644 --- a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.unit.spec.ts +++ b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmark.unit.spec.ts @@ -74,6 +74,7 @@ describe("Review Harness file content hash measurements", () => { expect(result.status).toBe("passed"); expect(result.detail).toBe("File content hash measurements completed."); + expect(report).toContain("JavaScript heap: unavailable on this device."); expect([...f.inputs().contentBySize.keys()]).toEqual( REVIEW_HARNESS_FILE_CONTENT_HASH_INPUTS.map(({ bytes }) => bytes) ); @@ -107,6 +108,49 @@ describe("Review Harness file content hash measurements", () => { expect(report).not.toContain("4096n"); }); + it("tracks the sampled maximum and reports a lower final heap sample", async () => { + const f = fixture(); + let reads = 0; + const clock = { + now: f.now, + get memory() { + const sample = reads++; + const mebibytes = + sample === 0 ? 2 : sample === 1 ? 5 : sample === 31 ? 1 : 3; + return { usedJSHeapSize: mebibytes * 1048576 }; + }, + }; + const result = await runReviewHarnessFileContentHashBenchmark(f.operations, clock, f.yieldControl); + + expect(result.observations).toContain( + "JavaScript heap: before=2.00 MiB; highest sampled=5.00 MiB; after=1.00 MiB." + ); + expect(result.observations.join("\n")).toContain("neither total app RAM nor a true peak"); + expect(reads).toBe(32); + }); + + it.each([Number.NaN, Number.POSITIVE_INFINITY, -1, "throws"] as const)( + "keeps timings usable when the heap API returns %s", + async (value) => { + const f = fixture(); + const result = await runReviewHarnessFileContentHashBenchmark( + f.operations, + { + now: f.now, + get memory() { + if (value === "throws") throw new Error("Heap API unavailable"); + return { usedJSHeapSize: value as number }; + }, + }, + f.yieldControl + ); + + expect(result.status).toBe("passed"); + expect(result.observations).toContain("JavaScript heap: unavailable on this device."); + expect(result.observations.join("\n")).not.toMatch(/NaN|Infinity/u); + } + ); + it("uses the same deterministic byte sequence for short and long inputs", async () => { const shortInput = createFixedFileContentHashInput(4 * 1024); const longInput = createFixedFileContentHashInput(64 * 1024); diff --git a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts index 8d6b8831..12a49dbc 100644 --- a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts +++ b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.ts @@ -5,16 +5,41 @@ import { type FileContentHashBenchmarkOperations, } from "./reviewHarnessFileContentHashBenchmark"; +type XxHash64Engine = { + h64Raw(content: Uint8Array): unknown; +}; + +function isXxHashFactory(value: unknown): value is () => unknown { + return typeof value === "function"; +} + +function isXxHash64Engine(value: unknown): value is XxHash64Engine { + return typeof value === "object" && value !== null && "h64Raw" in value && typeof value.h64Raw === "function"; +} + export async function createFileContentHashBenchmarkOperations( clock: Pick = performance ): Promise { + const factory: unknown = xxhashNew; + if (!isXxHashFactory(factory)) { + throw new TypeError("The xxHash64 factory must be callable."); + } const started = clock.now(); - const xxHash = await xxhashNew(); + const xxHash: unknown = await factory(); const xxHashInitialisationMs = clock.now() - started; + if (!isXxHash64Engine(xxHash)) { + throw new TypeError("The xxHash64 engine must provide h64Raw."); + } return { xxHashInitialisationMs, createInput: createFixedFileContentHashInput, sha256: (content) => computeFileContentHash(content), - xxHash64: async (content) => xxHash.h64Raw(new Uint8Array(await content.arrayBuffer())), + xxHash64: async (content) => { + const hash = xxHash.h64Raw(new Uint8Array(await content.arrayBuffer())); + if (typeof hash !== "bigint") { + throw new TypeError("The xxHash64 result must be a bigint."); + } + return hash; + }, }; } diff --git a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.unit.spec.ts b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.unit.spec.ts index a5d6db38..c8f60005 100644 --- a/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.unit.spec.ts +++ b/src/features/ReviewHarness/reviewHarnessFileContentHashBenchmarkRuntime.unit.spec.ts @@ -1,6 +1,11 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { xxhashNew } from "octagonal-wheels/hash/xxhash.js"; import { createFileContentHashBenchmarkOperations } from "./reviewHarnessFileContentHashBenchmarkRuntime"; +vi.mock("octagonal-wheels/hash/xxhash.js", { spy: true }); + +afterEach(() => vi.restoreAllMocks()); + describe("Review Harness file content hash benchmark adapter", () => { it("uses the packaged SHA-256 helper and xxHash64 engine with fixed Blob content", async () => { const operations = await createFileContentHashBenchmarkOperations(); @@ -11,5 +16,19 @@ describe("Review Harness file content hash benchmark adapter", () => { expect(typeof (await operations.xxHash64(content))).toBe("bigint"); expect(await operations.sha256(content)).toBe(await operations.sha256(content)); expect(await operations.xxHash64(content)).toBe(await operations.xxHash64(content)); + expect(xxhashNew).toHaveBeenCalledOnce(); + }); + + it("rejects an engine without an xxHash64 operation", async () => { + vi.mocked(xxhashNew).mockResolvedValueOnce({}); + + await expect(createFileContentHashBenchmarkOperations()).rejects.toBeInstanceOf(TypeError); + }); + + it("rejects a hash result which is not a bigint", async () => { + vi.mocked(xxhashNew).mockResolvedValueOnce({ h64Raw: () => 123 }); + const operations = await createFileContentHashBenchmarkOperations(); + + await expect(operations.xxHash64(new Blob(["fixed content"]))).rejects.toBeInstanceOf(TypeError); }); }); diff --git a/src/features/ReviewHarness/reviewHarnessHeapSampling.ts b/src/features/ReviewHarness/reviewHarnessHeapSampling.ts new file mode 100644 index 00000000..1b5d286c --- /dev/null +++ b/src/features/ReviewHarness/reviewHarnessHeapSampling.ts @@ -0,0 +1,34 @@ +type HeapPerformance = { + readonly memory?: { readonly usedJSHeapSize: number }; +}; + +function readHeap(clock: HeapPerformance): number | undefined { + try { + const bytes = clock.memory?.usedJSHeapSize; + return typeof bytes === "number" && Number.isFinite(bytes) && bytes >= 0 ? bytes : undefined; + } catch { + return undefined; + } +} + +export function createReviewHarnessHeapSampler(clock: HeapPerformance): { + sample(): void; + finish(): string; +} { + const before = readHeap(clock); + let highest = before; + let after = before; + const sample = () => { + after = readHeap(clock); + if (after !== undefined) highest = Math.max(highest ?? after, after); + }; + const finish = () => { + sample(); + if (highest === undefined) return "JavaScript heap: unavailable on this device."; + const mib = (bytes: number | undefined) => + bytes === undefined ? "unavailable" : `${(bytes / 1048576).toFixed(2)} MiB`; + return `JavaScript heap: before=${mib(before)}; highest sampled=${mib(highest)}; after=${mib(after)}.`; + }; + + return { sample, finish }; +} diff --git a/src/features/ReviewHarness/reviewHarnessIdBenchmark.ts b/src/features/ReviewHarness/reviewHarnessIdBenchmark.ts index f9212724..6422380c 100644 --- a/src/features/ReviewHarness/reviewHarnessIdBenchmark.ts +++ b/src/features/ReviewHarness/reviewHarnessIdBenchmark.ts @@ -1,4 +1,5 @@ import type { ReviewHarnessScenarioResult } from "./reviewHarnessTypes"; +import { createReviewHarnessHeapSampler } from "./reviewHarnessHeapSampling"; export interface IdBenchmarkOperations { deriveKey(): Promise; @@ -15,15 +16,6 @@ const SAMPLES = 3; const BATCH_SIZE = 100; const WARMUP_COUNT = 32; -function readHeap(clock: BenchmarkPerformance): number | undefined { - try { - const bytes = clock.memory?.usedJSHeapSize; - return typeof bytes === "number" && Number.isFinite(bytes) && bytes >= 0 ? bytes : undefined; - } catch { - return undefined; - } -} - function summary(samples: readonly number[]): string { const sorted = [...samples].sort((a, b) => a - b); return `median=${sorted[1].toFixed(2)} ms; range=${sorted[0].toFixed(2)}–${sorted[2].toFixed(2)} ms`; @@ -34,13 +26,7 @@ export async function runReviewHarnessIdBenchmark( clock: BenchmarkPerformance = performance, yieldControl: () => Promise = () => new Promise((resolve) => window.setTimeout(resolve, 0)) ): Promise { - const before = readHeap(clock); - let highest = before; - const sampleHeap = () => { - const value = readHeap(clock); - if (value !== undefined) highest = Math.max(highest ?? value, value); - return value; - }; + const heap = createReviewHarnessHeapSampler(clock); const observations = [ "Fixed synthetic inputs; 3 samples, alternating legacy/independent order; 32 warm-up IDs per sample. Legacy Chunk algorithm: xxhash64.", "Compute timings include input construction and awaited ID generation. Initialisation, warm-up, and pauses between batches are excluded. This does not measure a Rebuild or remote transfer.", @@ -51,7 +37,7 @@ export async function runReviewHarnessIdBenchmark( const started = clock.now(); await operations.deriveKey(); derivationSamples.push(clock.now() - started); - sampleHeap(); + heap.sample(); } observations.push(`ID key derivation at save time: ${summary(derivationSamples)} per derivation.`); @@ -80,7 +66,7 @@ export async function runReviewHarnessIdBenchmark( const started = clock.now(); for (let i = batch; i < batch + BATCH_SIZE; i++) await scenario.run(i, independent); elapsed += clock.now() - started; - sampleHeap(); + heap.sample(); } samples[independent ? 1 : 0].push(elapsed); } @@ -92,16 +78,7 @@ export async function runReviewHarnessIdBenchmark( ); } } - const after = sampleHeap(); - if (highest === undefined) { - observations.push("JavaScript heap: unavailable on this device."); - } else { - const mib = (bytes: number | undefined) => - bytes === undefined ? "unavailable" : `${(bytes / 1048576).toFixed(2)} MiB`; - observations.push( - `JavaScript heap: before=${mib(before)}; highest sampled=${mib(highest)}; after=${mib(after)}.` - ); - } + observations.push(heap.finish()); observations.push( "Heap samples are approximate, may include other Obsidian work, and are affected by garbage collection. They are neither total app RAM nor a true peak." ); diff --git a/src/features/ReviewHarness/reviewHarnessIdBenchmark.unit.spec.ts b/src/features/ReviewHarness/reviewHarnessIdBenchmark.unit.spec.ts index 2e68837a..93bbdedc 100644 --- a/src/features/ReviewHarness/reviewHarnessIdBenchmark.unit.spec.ts +++ b/src/features/ReviewHarness/reviewHarnessIdBenchmark.unit.spec.ts @@ -73,6 +73,7 @@ describe("Review Harness ID measurements", () => { "JavaScript heap: before=2.00 MiB; highest sampled=5.00 MiB; after=1.00 MiB." ); expect(result.observations.join("\n")).toContain("neither total app RAM nor a true peak"); + expect(reads).toBe(245); }); it.each([Number.NaN, Number.POSITIVE_INFINITY, -1, "throws"])( diff --git a/src/main.onunload.unit.spec.ts b/src/main.onunload.unit.spec.ts new file mode 100644 index 00000000..af48724d --- /dev/null +++ b/src/main.onunload.unit.spec.ts @@ -0,0 +1,116 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createOwnedResource } from "octagonal-wheels/resource"; +import { LOG_LEVEL_URGENT } from "octagonal-wheels/common/logger"; +import { bindableFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"; + +const unusedStartupDependencies: readonly [string, readonly string[]][] = [ + ["@/serviceFeatures/useP2PSettingsPreparation", ["useP2PSettingsPreparation"]], + ["@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions", ["setGetLanguage"]], + ["./features/LiveSyncCommands.ts", ["LiveSyncCommands"]], + ["./features/HiddenFileSync/CmdHiddenFileSync.ts", ["HiddenFileSync"]], + ["./features/ConfigSync/CmdConfigSync.ts", ["ConfigSync"]], + ["./modules/features/ModuleLog.ts", ["ModuleLog"]], + ["./modules/essentialObsidian/ModuleObsidianEvents.ts", ["ModuleObsidianEvents"]], + ["./modules/features/ModuleObsidianSettingTab.ts", ["ModuleObsidianSettingDialogue"]], + ["./modules/features/ModuleObsidianDocumentHistory.ts", ["ModuleObsidianDocumentHistory"]], + ["./modules/features/ModuleGlobalHistory.ts", ["ModuleObsidianGlobalHistory"]], + ["./features/LocalDatabaseMainte/CmdLocalDatabaseMainte.ts", ["LocalDatabaseMaintenance"]], + ["./modules/services/ObsidianServiceHub.ts", ["ObsidianServiceHub"]], + ["@vrtmrz/livesync-commonlib/compat/serviceModules/Rebuilder", ["ServiceRebuilder"]], + ["@/serviceModules/DatabaseFileAccess.ts", ["ServiceDatabaseFileAccess"]], + ["@/serviceModules/ServiceFileAccessImpl.ts", ["ServiceFileAccessObsidian"]], + ["@vrtmrz/livesync-commonlib/compat/managers/StorageProcessingManager", ["StorageAccessManager"]], + ["./serviceModules/FileHandler.ts", ["ServiceFileHandler"]], + ["./serviceModules/FileAccessObsidian.ts", ["FileAccessObsidian"]], + ["./managers/StorageEventManagerObsidian.ts", ["StorageEventManagerObsidian"]], + ["@vrtmrz/livesync-commonlib/compat/mock_and_interop/wrapper", ["setNoticeClass"]], + ["./LiveSyncBaseCore.ts", ["LiveSyncBaseCore"]], + ["./modules/features/ModuleObsidianSettingAsMarkdown.ts", ["ModuleObsidianSettingsAsMarkdown"]], + ["./modules/features/SetupManager.ts", ["SetupManager"]], + ["./serviceFeatures/onLayoutReady/enablei18n.ts", ["enableI18nFeature"]], + ["@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", ["useOfflineScanner"]], + ["@vrtmrz/livesync-commonlib/compat/serviceFeatures/remoteConfig", ["useRemoteConfiguration"]], + ["@vrtmrz/livesync-commonlib/compat/serviceFeatures/checkRemoteSize", ["useCheckRemoteSize"]], + ["./serviceFeatures/redFlag.ts", ["useRedFlagFeatures"]], + ["./serviceFeatures/setupObsidian/setupProtocol.ts", ["useSetupProtocolFeature"]], + ["@/serviceFeatures/setupObsidian/qrCode", ["useSetupQRCodeFeature"]], + ["@/serviceFeatures/setupObsidian/setupUri", ["useSetupURIFeature"]], + [ + "./serviceFeatures/setupObsidian/setupManagerHandlers.ts", + ["showOnboardingInvitation", "useSetupManagerHandlersFeature"], + ], + ["@vrtmrz/livesync-commonlib/p2p", ["useP2PReplicatorCommands", "useP2PReplicatorFeature"]], + ["./serviceFeatures/useP2PReplicatorUI.ts", ["useP2PReplicatorUI"]], + ["./serviceFeatures/useReviewHarness.ts", ["useReviewHarness"]], + ["./features/P2PSync/P2PReplicator/P2PReplicationUI.ts", ["createOpenReplicationUI", "createOpenRebuildUI"]], + ["./serviceFeatures/compatibilityReview.ts", ["useCompatibilityReview"]], + ["./serviceFeatures/compatibilityReviewObsidian.ts", ["createObsidianCompatibilityReviewUi"]], + ["./serviceModules/FileReflectionProvenance.ts", ["createFileReflectionProvenance"]], + ["./serviceFeatures/interactiveConflictResolution", ["useInteractiveConflictResolutionFeature"]], + ["./modules/features/InteractiveConflictResolving/ConflictResolveModal.ts", ["ConflictResolveModal"]], + ["./serviceFeatures/obsidianReplicationRibbon.ts", ["useObsidianReplicationRibbonFeature"]], + ["./serviceFeatures/syncGuard", ["useSyncGuardFeature"]], + ["./serviceFeatures/syncGuard/SyncGuardOverlay", ["SyncGuardOverlay"]], + ["./serviceFeatures/startupLifecycle", ["useStartupLifecycleFeature"]], + ["./serviceFeatures/useJoinedDevices", ["useJoinedDevicesFeature"]], + ["./serviceFeatures/useDatabaseDiagnostics", ["useDatabaseDiagnosticsFeature"]], +]; + +beforeEach(() => { + vi.resetModules(); + vi.doMock("./deps", () => ({ Plugin: class {}, Notice: class {}, getLanguage: () => "en" })); + for (const [specifier, names] of unusedStartupDependencies) { + vi.doMock(specifier, () => Object.fromEntries(names.map((name) => [name, vi.fn()]))); + } +}); + +describe("Obsidian plug-in unload logging", () => { + it.each(["complete", "fail"] as const)( + "retains its log connection through shutdown and retires it afterwards: %s", + async (outcome) => { + const { default: LiveSyncPlugin } = await import("./main.ts"); + const events: string[] = []; + const sink = vi.fn((message: unknown) => events.push(String(message))); + const addLog = bindableFunction<(message: unknown, level?: number, key?: string) => void>("unload log"); + const unregister = addLog.setHandler(sink); + const dispose = vi.fn(() => { + unregister(); + events.push("disposed"); + }); + const logResource = createOwnedResource(sink, dispose); + let finish!: () => void; + let fail!: (reason: Error) => void; + const shutdown = new Promise((resolve, reject) => { + finish = resolve; + fail = reject; + }); + const onUnload = vi.fn(() => shutdown); + const core = { + getModule: vi.fn(() => ({ logResource })), + services: { API: { addLog }, control: { onUnload } }, + }; + + expect(LiveSyncPlugin.prototype.onunload.call({ core } as never)).toBeUndefined(); + expect(onUnload).toHaveBeenCalledTimes(1); + expect(dispose).not.toHaveBeenCalled(); + addLog("during shutdown"); + + const failure = new Error("database close failed"); + if (outcome === "fail") { + fail(failure); + } else { + finish(); + } + await vi.waitFor(() => expect(dispose).toHaveBeenCalledTimes(1)); + + if (outcome === "fail") { + const message = "Failed to unload Self-hosted LiveSync: " + String(failure); + expect(sink).toHaveBeenCalledWith(message, LOG_LEVEL_URGENT, "shutdown"); + expect(events).toEqual(["during shutdown", message, "disposed"]); + } else { + expect(events).toEqual(["during shutdown", "disposed"]); + } + expect(() => addLog("after shutdown")).toThrow("not assigned"); + } + ); +}); diff --git a/src/main.ts b/src/main.ts index e0f0bdc2..fb716949 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,6 +9,7 @@ import { ConfigSync } from "./features/ConfigSync/CmdConfigSync.ts"; // import { ModuleDev } from "./modules/extras/ModuleDev.ts"; import { ModuleLog } from "./modules/features/ModuleLog.ts"; +import { withOwnedResource } from "octagonal-wheels/resource"; import { ModuleObsidianEvents } from "./modules/essentialObsidian/ModuleObsidianEvents.ts"; import { ModuleObsidianSettingDialogue } from "./modules/features/ModuleObsidianSettingTab.ts"; import { ModuleObsidianDocumentHistory } from "./modules/features/ModuleObsidianDocumentHistory.ts"; @@ -237,12 +238,14 @@ export default class ObsidianLiveSyncPlugin extends Plugin { void this._startUp(); } override onunload(): void { - void this.core.services.control.onUnload().catch((error) => { - this.core.services.API.addLog( - "Failed to unload Self-hosted LiveSync: " + String(error), - LOG_LEVEL_URGENT, - "shutdown" - ); - }); + void withOwnedResource(this.core.getModule(ModuleLog).logResource, () => + this.core.services.control.onUnload().catch((error) => { + this.core.services.API.addLog( + "Failed to unload Self-hosted LiveSync: " + String(error), + LOG_LEVEL_URGENT, + "shutdown" + ); + }) + ); } } diff --git a/src/modules/features/DocumentHistory/DocumentHistoryModal.navigation.unit.spec.ts b/src/modules/features/DocumentHistory/DocumentHistoryModal.navigation.unit.spec.ts new file mode 100644 index 00000000..bef11ead --- /dev/null +++ b/src/modules/features/DocumentHistory/DocumentHistoryModal.navigation.unit.spec.ts @@ -0,0 +1,321 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + Modal: class Modal {}, + TFile: class TFile {}, +})); + +vi.mock("@/main.ts", () => ({ + default: class ObsidianLiveSyncPlugin {}, +})); + +vi.mock("@/common/utils.ts", () => ({ + getPathFromTFile: (file: { path: string }) => file.path, + isValidPath: vi.fn(() => true), +})); + +vi.mock("octagonal-wheels/concurrency/lock", () => ({ + scheduleOnceIfDuplicated: vi.fn((_key: string, callback: () => unknown) => callback()), +})); + +import { scheduleOnceIfDuplicated } from "octagonal-wheels/concurrency/lock"; +import { DocumentHistoryModal } from "./DocumentHistoryModal.ts"; + +type NavigationTarget = { + classList: { add: ReturnType; remove: ReturnType }; + scrollIntoView: ReturnType; +}; + +function createDiffView(count: number) { + const focused: { current: NavigationTarget | undefined } = { current: undefined }; + const createTargets = (targetCount: number) => + Array.from({ length: targetCount }, () => { + const target: NavigationTarget = { + classList: { + add: vi.fn(() => { + focused.current = target; + }), + remove: vi.fn(() => { + if (focused.current === target) focused.current = undefined; + }), + }, + scrollIntoView: vi.fn(), + }; + return target; + }); + let targets = createTargets(count); + const contentView = { + querySelectorAll: vi.fn(() => targets), + querySelector: vi.fn(() => (targets.includes(focused.current as NavigationTarget) ? focused.current : null)), + }; + return { + contentView, + targets, + replaceTargets(targetCount: number) { + focused.current = undefined; + targets = createTargets(targetCount); + return targets; + }, + }; +} + +function createHistoryModal() { + return Object.create(DocumentHistoryModal.prototype) as DocumentHistoryModal; +} + +describe("DocumentHistoryModal cyclic navigation", () => { + it("commits the selected index before scrolling, including when scrolling fails", () => { + const modal = createHistoryModal(); + const view = createDiffView(2); + const indicator = { setText: vi.fn() }; + const failure = new Error("fixture scroll failed"); + Object.assign(modal, { contentView: view.contentView, currentDiffIndex: -1, diffNavIndicator: indicator }); + view.targets[0].scrollIntoView.mockImplementation(() => { + expect(modal.currentDiffIndex).toBe(0); + throw failure; + }); + expect(() => modal.navigateDiff("next")).toThrow(failure); + expect(modal.currentDiffIndex).toBe(0); + expect(indicator.setText).not.toHaveBeenCalled(); + }); + + it("starts at the first diff for next and at the last diff for previous", () => { + const nextModal = createHistoryModal(); + const nextView = createDiffView(3); + const nextIndicator = { setText: vi.fn() }; + Object.assign(nextModal, { + contentView: nextView.contentView, + currentDiffIndex: -1, + diffNavIndicator: nextIndicator, + }); + + nextModal.navigateDiff("next"); + + expect(nextModal.currentDiffIndex).toBe(0); + expect(nextView.targets[0].classList.add).toHaveBeenCalledWith("diff-focused"); + expect(nextView.targets[0].scrollIntoView).toHaveBeenCalledWith({ behavior: "smooth", block: "center" }); + expect(nextIndicator.setText).toHaveBeenCalledWith("1/3"); + + const previousModal = createHistoryModal(); + const previousView = createDiffView(3); + const previousIndicator = { setText: vi.fn() }; + Object.assign(previousModal, { + contentView: previousView.contentView, + currentDiffIndex: -1, + diffNavIndicator: previousIndicator, + }); + + previousModal.navigateDiff("prev"); + + expect(previousModal.currentDiffIndex).toBe(2); + expect(previousView.targets[2].classList.add).toHaveBeenCalledWith("diff-focused"); + expect(previousView.targets[2].scrollIntoView).toHaveBeenCalledWith({ behavior: "smooth", block: "center" }); + expect(previousIndicator.setText).toHaveBeenCalledWith("3/3"); + }); + + it("wraps in both directions while moving focus, scrolling, and updating the indicator", () => { + const modal = createHistoryModal(); + const { contentView, targets } = createDiffView(3); + const indicator = { setText: vi.fn() }; + Object.assign(modal, { contentView, currentDiffIndex: -1, diffNavIndicator: indicator }); + + modal.navigateDiff("next"); + modal.navigateDiff("next"); + modal.navigateDiff("next"); + modal.navigateDiff("next"); + modal.navigateDiff("prev"); + + expect(modal.currentDiffIndex).toBe(2); + expect(targets.map((target) => target.classList.add.mock.calls.length)).toEqual([2, 1, 2]); + expect(targets.map((target) => target.classList.remove.mock.calls.length)).toEqual([2, 1, 1]); + expect(targets.map((target) => target.scrollIntoView.mock.calls.length)).toEqual([2, 1, 2]); + expect(indicator.setText.mock.calls.map(([text]) => text)).toEqual(["1/3", "2/3", "3/3", "1/3", "3/3"]); + }); + + it("leaves the current navigation index and content view untouched when no diff exists", () => { + const modal = createHistoryModal(); + const { contentView } = createDiffView(0); + const indicator = { setText: vi.fn() }; + Object.assign(modal, { contentView, currentDiffIndex: 1, diffNavIndicator: indicator }); + + modal.navigateDiff("next"); + + expect(modal.currentDiffIndex).toBe(1); + expect(contentView.querySelector).not.toHaveBeenCalled(); + expect(indicator.setText).not.toHaveBeenCalled(); + }); + + it("looks up diff nodes again after the content view is replaced", () => { + const modal = createHistoryModal(); + const view = createDiffView(2); + const indicator = { setText: vi.fn() }; + Object.assign(modal, { contentView: view.contentView, currentDiffIndex: -1, diffNavIndicator: indicator }); + + modal.navigateDiff("next"); + const replacementTargets = view.replaceTargets(3); + modal.navigateDiff("next"); + + expect(modal.currentDiffIndex).toBe(1); + expect(view.contentView.querySelectorAll).toHaveBeenCalledTimes(2); + expect(view.contentView.querySelector).toHaveBeenCalledTimes(2); + expect(replacementTargets[1].classList.add).toHaveBeenCalledWith("diff-focused"); + expect(replacementTargets[1].scrollIntoView).toHaveBeenCalledWith({ behavior: "smooth", block: "center" }); + expect(indicator.setText.mock.calls.map(([text]) => text)).toEqual(["1/2", "2/3"]); + }); + + it("keeps navigation indices and focus independent between modal instances", () => { + const firstModal = createHistoryModal(); + const secondModal = createHistoryModal(); + const firstView = createDiffView(2); + const secondView = createDiffView(2); + Object.assign(firstModal, { + contentView: firstView.contentView, + currentDiffIndex: -1, + diffNavIndicator: { setText: vi.fn() }, + }); + Object.assign(secondModal, { + contentView: secondView.contentView, + currentDiffIndex: -1, + diffNavIndicator: { setText: vi.fn() }, + }); + + firstModal.navigateDiff("next"); + secondModal.navigateDiff("next"); + firstModal.navigateDiff("next"); + + expect(firstModal.currentDiffIndex).toBe(1); + expect(secondModal.currentDiffIndex).toBe(0); + expect(firstView.targets[0].classList.remove).toHaveBeenCalledWith("diff-focused"); + expect(secondView.targets[0].classList.remove).not.toHaveBeenCalled(); + expect(secondView.targets[0].classList.add).toHaveBeenCalledWith("diff-focused"); + }); + + it("resets the indicator and visibility according to showDiff", () => { + const modal = createHistoryModal(); + const view = createDiffView(2); + const indicator = { setText: vi.fn() }; + const diffNavContainer = { setCssStyles: vi.fn() }; + const diffOnlyLabel = { setCssStyles: vi.fn() }; + Object.assign(modal, { + contentView: view.contentView, + currentDiffIndex: 1, + diffNavIndicator: indicator, + diffNavContainer, + diffOnlyLabel, + showDiff: true, + }); + + modal.resetDiffNavigation(); + + expect(modal.currentDiffIndex).toBe(-1); + expect(view.contentView.querySelectorAll).toHaveBeenCalledWith(".history-added, .history-deleted"); + expect(indicator.setText).toHaveBeenLastCalledWith("0/2"); + expect(diffNavContainer.setCssStyles).toHaveBeenLastCalledWith({ display: "flex" }); + expect(diffOnlyLabel.setCssStyles).toHaveBeenLastCalledWith({ display: "inline-block" }); + + view.replaceTargets(0); + modal.resetDiffNavigation(); + expect(indicator.setText).toHaveBeenLastCalledWith("\u2014"); + + Object.assign(modal, { showDiff: false }); + modal.resetDiffNavigation(); + expect(view.contentView.querySelectorAll).toHaveBeenCalledTimes(2); + expect(indicator.setText).toHaveBeenLastCalledWith("\u2014"); + expect(diffNavContainer.setCssStyles).toHaveBeenLastCalledWith({ display: "none" }); + expect(diffOnlyLabel.setCssStyles).toHaveBeenLastCalledWith({ display: "none" }); + }); + + it("updates visibility without an indicator", () => { + const modal = createHistoryModal(); + const { contentView } = createDiffView(2); + const diffNavContainer = { setCssStyles: vi.fn() }; + Object.assign(modal, { contentView, currentDiffIndex: 1, diffNavContainer, showDiff: true }); + + expect(() => modal.resetDiffNavigation()).not.toThrow(); + + expect(modal.currentDiffIndex).toBe(-1); + expect(contentView.querySelectorAll).not.toHaveBeenCalled(); + expect(diffNavContainer.setCssStyles).toHaveBeenCalledWith({ display: "flex" }); + }); + + it("maps selected search results to the revision slider and loads each wrapped selection", () => { + const modal = createHistoryModal(); + const scheduleLoad = vi.mocked(scheduleOnceIfDuplicated); + scheduleLoad.mockClear(); + const indicator = { setText: vi.fn() }; + Object.assign(modal, { + searchResults: [ + { rev: "newest", index: 0, matchType: "Content" }, + { rev: "middle", index: 2, matchType: "Diff" }, + { rev: "oldest", index: 4, matchType: "Content" }, + ], + currentSearchIndex: -1, + revs_info: Array.from({ length: 5 }, (_, index) => ({ rev: `rev-${index}` })), + range: { value: "" }, + loadRevs: vi.fn(), + searchResultIndicator: indicator, + searchPrevBtn: { disabled: true }, + searchNextBtn: { disabled: true }, + searchKeyword: "needle", + showDiff: false, + }); + + modal.navigateSearch("next"); + expect(modal.currentSearchIndex).toBe(0); + expect(modal.range.value).toBe("4"); + expect(modal.loadRevs).toHaveBeenCalledTimes(1); + expect(scheduleLoad).toHaveBeenCalledWith("loadRevs", expect.any(Function)); + expect(indicator.setText).toHaveBeenLastCalledWith("1/3 matches"); + + modal.navigateSearch("next"); + expect(modal.currentSearchIndex).toBe(1); + expect(modal.range.value).toBe("2"); + expect(modal.loadRevs).toHaveBeenCalledTimes(2); + + modal.navigateSearch("next"); + expect(modal.currentSearchIndex).toBe(2); + expect(modal.range.value).toBe("0"); + + modal.navigateSearch("next"); + expect(modal.currentSearchIndex).toBe(0); + expect(modal.range.value).toBe("4"); + + modal.navigateSearch("prev"); + expect(modal.currentSearchIndex).toBe(2); + expect(modal.range.value).toBe("0"); + expect(indicator.setText.mock.calls.map(([text]) => text)).toEqual([ + "1/3 matches", + "2/3 matches", + "3/3 matches", + "1/3 matches", + "3/3 matches", + ]); + }); + + it("leaves search state and controls untouched when there are no results", () => { + const modal = createHistoryModal(); + const indicator = { setText: vi.fn() }; + const searchPrevBtn = { disabled: true }; + const searchNextBtn = { disabled: true }; + Object.assign(modal, { + searchResults: [], + currentSearchIndex: -1, + range: { value: "unchanged" }, + loadRevs: vi.fn(), + searchResultIndicator: indicator, + searchPrevBtn, + searchNextBtn, + searchKeyword: "needle", + showDiff: false, + }); + + modal.navigateSearch("next"); + + expect(modal.currentSearchIndex).toBe(-1); + expect(modal.range.value).toBe("unchanged"); + expect(modal.loadRevs).not.toHaveBeenCalled(); + expect(indicator.setText).not.toHaveBeenCalled(); + expect(searchPrevBtn.disabled).toBe(true); + expect(searchNextBtn.disabled).toBe(true); + }); +}); diff --git a/src/modules/features/DocumentHistory/DocumentHistoryModal.ts b/src/modules/features/DocumentHistory/DocumentHistoryModal.ts index 57b734ba..1a9b106e 100644 --- a/src/modules/features/DocumentHistory/DocumentHistoryModal.ts +++ b/src/modules/features/DocumentHistory/DocumentHistoryModal.ts @@ -15,6 +15,8 @@ import { isErrorOfMissingDoc } from "@vrtmrz/livesync-commonlib/compat/pouchdb/u import { fireAndForget, getDocData } from "@vrtmrz/livesync-commonlib/compat/common/utils"; import { isPlainText, stripPrefix } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; import { scheduleOnceIfDuplicated } from "octagonal-wheels/concurrency/lock"; +import { stepCyclicIndex } from "octagonal-wheels/collection"; +import { navigateDiffNodes } from "@/common/diffNavigation.ts"; import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore.ts"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { @@ -375,26 +377,7 @@ export class DocumentHistoryModal extends Modal { * Only effective when diff highlighting is enabled. */ navigateDiff(direction: "prev" | "next") { - const diffElements = this.contentView.querySelectorAll(".history-added, .history-deleted"); - if (diffElements.length === 0) return; - - // Remove previous focus highlight - const prevFocused = this.contentView.querySelector(".diff-focused"); - if (prevFocused) { - prevFocused.classList.remove("diff-focused"); - } - - if (direction === "next") { - this.currentDiffIndex = (this.currentDiffIndex + 1) % diffElements.length; - } else { - this.currentDiffIndex = this.currentDiffIndex <= 0 ? diffElements.length - 1 : this.currentDiffIndex - 1; - } - - const target = diffElements[this.currentDiffIndex]; - target.classList.add("diff-focused"); - target.scrollIntoView({ behavior: "smooth", block: "center" }); - - this.diffNavIndicator.setText(`${this.currentDiffIndex + 1}/${diffElements.length}`); + navigateDiffNodes(this.contentView, ".history-added, .history-deleted", this, direction); } /** @@ -519,12 +502,7 @@ export class DocumentHistoryModal extends Modal { navigateSearch(direction: "prev" | "next") { if (this.searchResults.length === 0) return; - if (direction === "next") { - this.currentSearchIndex = (this.currentSearchIndex + 1) % this.searchResults.length; - } else { - this.currentSearchIndex = - this.currentSearchIndex <= 0 ? this.searchResults.length - 1 : this.currentSearchIndex - 1; - } + this.currentSearchIndex = stepCyclicIndex(this.currentSearchIndex, this.searchResults.length, direction); const match = this.searchResults[this.currentSearchIndex]; this.range.value = `${this.revs_info.length - 1 - match.index}`; diff --git a/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts b/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts index a5b22068..bc6674d7 100644 --- a/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts +++ b/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.ts @@ -7,6 +7,7 @@ import { type FilePathWithPrefix, } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { EVENT_CONFLICT_CANCELLED, EVENT_PLUGIN_UNLOADED, eventHub } from "@/common/events.ts"; +import { navigateDiffNodes } from "@/common/diffNavigation.ts"; import { promiseWithResolvers } from "octagonal-wheels/promises"; import { POSTPONED, type MergeDialogResult } from "@/serviceFeatures/interactiveConflictResolution/types"; @@ -83,24 +84,7 @@ export class ConflictResolveModal extends Modal { } navigateDiff(direction: "prev" | "next") { - const diffElements = this.diffView.querySelectorAll(".added, .deleted"); - if (diffElements.length === 0) return; - - const prevFocused = this.diffView.querySelector(".diff-focused"); - if (prevFocused) { - prevFocused.classList.remove("diff-focused"); - } - - if (direction === "next") { - this.currentDiffIndex = (this.currentDiffIndex + 1) % diffElements.length; - } else { - this.currentDiffIndex = this.currentDiffIndex <= 0 ? diffElements.length - 1 : this.currentDiffIndex - 1; - } - - const target = diffElements[this.currentDiffIndex]; - target.classList.add("diff-focused"); - target.scrollIntoView({ behavior: "smooth", block: "center" }); - this.diffNavIndicator.setText(`${this.currentDiffIndex + 1}/${diffElements.length}`); + navigateDiffNodes(this.diffView, ".added, .deleted", this, direction); } resetDiffNavigation() { diff --git a/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.unit.spec.ts b/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.unit.spec.ts index 8e4cfd94..b37fb8d1 100644 --- a/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.unit.spec.ts +++ b/src/modules/features/InteractiveConflictResolving/ConflictResolveModal.unit.spec.ts @@ -106,11 +106,7 @@ describe("ConflictResolveModal result lifecycle", () => { }); it("closes and completes its result when the plug-in unloads", async () => { - const modal = new ConflictResolveModal( - {} as never, - "open-during-unload.md" as FilePathWithPrefix, - conflict - ); + const modal = new ConflictResolveModal({} as never, "open-during-unload.md" as FilePathWithPrefix, conflict); modal.onOpen(); eventHub.emitEvent(EVENT_PLUGIN_UNLOADED); @@ -165,3 +161,186 @@ describe("ConflictResolveModal result lifecycle", () => { expect(previousState).toBe("still-open"); }); }); + +type NavigationTarget = { + classList: { add: ReturnType; remove: ReturnType }; + scrollIntoView: ReturnType; +}; + +function createDiffView(count: number) { + const focused: { current: NavigationTarget | undefined } = { current: undefined }; + const createTargets = (targetCount: number) => + Array.from({ length: targetCount }, () => { + const target: NavigationTarget = { + classList: { + add: vi.fn(() => { + focused.current = target; + }), + remove: vi.fn(() => { + if (focused.current === target) focused.current = undefined; + }), + }, + scrollIntoView: vi.fn(), + }; + return target; + }); + let targets = createTargets(count); + const diffView = { + querySelectorAll: vi.fn(() => targets), + querySelector: vi.fn(() => (targets.includes(focused.current as NavigationTarget) ? focused.current : null)), + }; + return { + diffView, + targets, + replaceTargets(targetCount: number) { + focused.current = undefined; + targets = createTargets(targetCount); + return targets; + }, + }; +} + +describe("ConflictResolveModal cyclic navigation", () => { + it("commits the selected index before scrolling, including when scrolling fails", () => { + const modal = Object.create(ConflictResolveModal.prototype) as ConflictResolveModal; + const view = createDiffView(2); + const indicator = { setText: vi.fn() }; + const failure = new Error("fixture scroll failed"); + Object.assign(modal, { diffView: view.diffView, currentDiffIndex: -1, diffNavIndicator: indicator }); + view.targets[0].scrollIntoView.mockImplementation(() => { + expect(modal.currentDiffIndex).toBe(0); + throw failure; + }); + expect(() => modal.navigateDiff("next")).toThrow(failure); + expect(modal.currentDiffIndex).toBe(0); + expect(indicator.setText).not.toHaveBeenCalled(); + }); + + it("starts at the first diff for next and at the last diff for previous", () => { + const nextModal = new ConflictResolveModal({} as never, "next.md" as FilePathWithPrefix, conflict); + const nextView = createDiffView(3); + const nextIndicator = { setText: vi.fn() }; + Object.assign(nextModal, { + diffView: nextView.diffView, + currentDiffIndex: -1, + diffNavIndicator: nextIndicator, + }); + + nextModal.navigateDiff("next"); + + expect(nextModal.currentDiffIndex).toBe(0); + expect(nextView.targets[0].classList.add).toHaveBeenCalledWith("diff-focused"); + expect(nextView.targets[0].scrollIntoView).toHaveBeenCalledWith({ behavior: "smooth", block: "center" }); + expect(nextIndicator.setText).toHaveBeenCalledWith("1/3"); + + const previousModal = new ConflictResolveModal({} as never, "previous.md" as FilePathWithPrefix, conflict); + const previousView = createDiffView(3); + const previousIndicator = { setText: vi.fn() }; + Object.assign(previousModal, { + diffView: previousView.diffView, + currentDiffIndex: -1, + diffNavIndicator: previousIndicator, + }); + + previousModal.navigateDiff("prev"); + + expect(previousModal.currentDiffIndex).toBe(2); + expect(previousView.targets[2].classList.add).toHaveBeenCalledWith("diff-focused"); + expect(previousView.targets[2].scrollIntoView).toHaveBeenCalledWith({ behavior: "smooth", block: "center" }); + expect(previousIndicator.setText).toHaveBeenCalledWith("3/3"); + }); + + it("wraps in both directions while moving focus, scrolling, and updating the indicator", () => { + const modal = new ConflictResolveModal({} as never, "wrap.md" as FilePathWithPrefix, conflict); + const { diffView, targets } = createDiffView(3); + const indicator = { setText: vi.fn() }; + Object.assign(modal, { diffView, currentDiffIndex: -1, diffNavIndicator: indicator }); + + modal.navigateDiff("next"); + modal.navigateDiff("next"); + modal.navigateDiff("next"); + modal.navigateDiff("next"); + modal.navigateDiff("prev"); + + expect(modal.currentDiffIndex).toBe(2); + expect(targets.map((target) => target.classList.add.mock.calls.length)).toEqual([2, 1, 2]); + expect(targets.map((target) => target.classList.remove.mock.calls.length)).toEqual([2, 1, 1]); + expect(targets.map((target) => target.scrollIntoView.mock.calls.length)).toEqual([2, 1, 2]); + expect(indicator.setText.mock.calls.map(([text]) => text)).toEqual(["1/3", "2/3", "3/3", "1/3", "3/3"]); + }); + + it("leaves the current navigation index and diff view untouched when no diff exists", () => { + const modal = new ConflictResolveModal({} as never, "empty.md" as FilePathWithPrefix, conflict); + const { diffView } = createDiffView(0); + const indicator = { setText: vi.fn() }; + Object.assign(modal, { diffView, currentDiffIndex: 1, diffNavIndicator: indicator }); + + modal.navigateDiff("next"); + + expect(modal.currentDiffIndex).toBe(1); + expect(diffView.querySelector).not.toHaveBeenCalled(); + expect(indicator.setText).not.toHaveBeenCalled(); + }); + + it("looks up diff nodes again after the diff view is replaced", () => { + const modal = new ConflictResolveModal({} as never, "replace.md" as FilePathWithPrefix, conflict); + const view = createDiffView(2); + const indicator = { setText: vi.fn() }; + Object.assign(modal, { diffView: view.diffView, currentDiffIndex: -1, diffNavIndicator: indicator }); + + modal.navigateDiff("next"); + const replacementTargets = view.replaceTargets(3); + modal.navigateDiff("next"); + + expect(modal.currentDiffIndex).toBe(1); + expect(view.diffView.querySelectorAll).toHaveBeenCalledTimes(2); + expect(view.diffView.querySelector).toHaveBeenCalledTimes(2); + expect(replacementTargets[1].classList.add).toHaveBeenCalledWith("diff-focused"); + expect(replacementTargets[1].scrollIntoView).toHaveBeenCalledWith({ behavior: "smooth", block: "center" }); + expect(indicator.setText.mock.calls.map(([text]) => text)).toEqual(["1/2", "2/3"]); + }); + + it("keeps navigation indices and focus independent between modal instances", () => { + const firstModal = new ConflictResolveModal({} as never, "first.md" as FilePathWithPrefix, conflict); + const secondModal = new ConflictResolveModal({} as never, "second.md" as FilePathWithPrefix, conflict); + const firstView = createDiffView(2); + const secondView = createDiffView(2); + Object.assign(firstModal, { + diffView: firstView.diffView, + currentDiffIndex: -1, + diffNavIndicator: { setText: vi.fn() }, + }); + Object.assign(secondModal, { + diffView: secondView.diffView, + currentDiffIndex: -1, + diffNavIndicator: { setText: vi.fn() }, + }); + + firstModal.navigateDiff("next"); + secondModal.navigateDiff("next"); + firstModal.navigateDiff("next"); + + expect(firstModal.currentDiffIndex).toBe(1); + expect(secondModal.currentDiffIndex).toBe(0); + expect(firstView.targets[0].classList.remove).toHaveBeenCalledWith("diff-focused"); + expect(secondView.targets[0].classList.remove).not.toHaveBeenCalled(); + expect(secondView.targets[0].classList.add).toHaveBeenCalledWith("diff-focused"); + }); + + it("always resets the indicator using the current diff count", () => { + const modal = new ConflictResolveModal({} as never, "reset.md" as FilePathWithPrefix, conflict); + const view = createDiffView(2); + const indicator = { setText: vi.fn() }; + Object.assign(modal, { diffView: view.diffView, currentDiffIndex: 1, diffNavIndicator: indicator }); + + modal.resetDiffNavigation(); + + expect(modal.currentDiffIndex).toBe(-1); + expect(view.diffView.querySelectorAll).toHaveBeenCalledWith(".added, .deleted"); + expect(indicator.setText).toHaveBeenLastCalledWith("0/2"); + + view.replaceTargets(0); + modal.resetDiffNavigation(); + expect(indicator.setText).toHaveBeenLastCalledWith("\u2014"); + }); +}); diff --git a/src/modules/features/ModuleLog.ts b/src/modules/features/ModuleLog.ts index 0755506b..cafaf7e7 100644 --- a/src/modules/features/ModuleLog.ts +++ b/src/modules/features/ModuleLog.ts @@ -1,4 +1,10 @@ -import { computed, reactive, reactiveSource, type ReactiveValue } from "octagonal-wheels/dataobject/reactive"; +import { + computed, + reactive, + reactiveSource, + type ReactiveSource, + type ReactiveValue, +} from "octagonal-wheels/dataobject/reactive"; import { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, @@ -7,6 +13,7 @@ import { type LOG_LEVEL, } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { scheduleTask } from "octagonal-wheels/concurrency/task"; +import { createOwnedResource, type OwnedResource } from "octagonal-wheels/resource"; import { fireAndForget, isDirty, throttle } from "@vrtmrz/livesync-commonlib/compat/common/utils"; import { chunkFetchCounts, @@ -27,7 +34,7 @@ import { AbstractObsidianModule } from "@/modules/AbstractObsidianModule.ts"; import { addIcon, debounce, normalizePath, Notice, stringifyYaml, type WorkspaceLeaf } from "@/deps.ts"; import { LOG_LEVEL_NOTICE, setGlobalLogFunction } from "octagonal-wheels/common/logger"; import { LogPaneView, VIEW_TYPE_LOG } from "./Log/LogPaneView.ts"; -import { serialized } from "octagonal-wheels/concurrency/lock"; +import { withLock } from "octagonal-wheels/concurrency/lock"; import { $msg } from "@/common/translation"; import { P2PLogCollector } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/P2PLogCollector"; import { REMOTE_REQUEST_ACTIVITY_MINIMUM_VISIBLE_MS, getTrackedRequestCount } from "./RemoteActivityStatus.ts"; @@ -46,7 +53,11 @@ import { isValidFilenameInDarwin, isValidFilenameInWidows, } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; -import { MARK_LOG_NETWORK_ERROR, MARK_LOG_SEPARATOR } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; +import { + MARK_LOG_NETWORK_ERROR, + MARK_LOG_SEPARATOR, + type LogFunction, +} from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; import { NetworkWarningStyles } from "@vrtmrz/livesync-commonlib/compat/common/models/setting.const"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { generateReport } from "@/common/reportTool.ts"; @@ -57,20 +68,22 @@ import { // This module cannot be a core module because it depends on the Obsidian UI. -// DI the log again. -const recentLogEntries = reactiveSource([]); -const globalLogFunction = (message: unknown, level?: number, key?: string) => { - const messageX = - message instanceof Error - ? new LiveSyncError("[Error Logged]: " + message.message, { cause: message }) - : typeof message === "string" - ? message - : JSON.stringify(message); - const entry = { message: messageX, level, key } as LogEntry; - recentLogEntries.value = [...recentLogEntries.value, entry]; -}; +function createBufferedLog(entries: ReactiveSource) { + return (message: unknown, level?: number, key?: string) => { + const messageX = + message instanceof Error + ? new LiveSyncError("[Error Logged]: " + message.message, { cause: message }) + : typeof message === "string" + ? message + : JSON.stringify(message); + const entry = { message: messageX, level, key } as LogEntry; + entries.value = [...entries.value, entry]; + }; +} -setGlobalLogFunction(globalLogFunction); +// Legacy Logger calls use this shared buffer; service logs use their owner's buffer. +const recentGlobalLogEntries = reactiveSource([]); +setGlobalLogFunction(createBufferedLog(recentGlobalLogEntries)); // Keep the recent logs in memory for display, but also keep a longer history in logForDump for when the user wants to see more logs. // logForDump is not reactive and is only used for dumping logs when requested, while recentLogs is reactive and is used for displaying logs in the UI. const logForDump = [] as string[]; @@ -113,6 +126,16 @@ function redactLog(log: string) { const showDebugLog = false; export const MARK_DONE = "\u{2009}\u{2009}"; export class ModuleLog extends AbstractObsidianModule { + /** The host releases this connection after the complete asynchronous unload. */ + logResource!: OwnedResource; + private readonly recentLogEntries = reactiveSource([]); + private readonly displayLogEntries = () => { + if (this.recentLogEntries.value.length === 0) return; + const entries = [...this.recentLogEntries.value]; + this.recentLogEntries.value = []; + entries.forEach((entry) => this.__addLog(entry.message, entry.level, entry.key)); + }; + statusBar?: HTMLElement; statusDiv?: HTMLElement; @@ -460,12 +483,7 @@ ${stringifyYaml(info)} return Promise.resolve(true); } private _everyOnloadAfterLoadSettings(): Promise { - recentLogEntries.onChanged((entries) => { - if (entries.value.length === 0) return; - const newEntries = [...entries.value]; - recentLogEntries.value = []; - newEntries.forEach((e) => this.__addLog(e.message, e.level, e.key)); - }); + this.recentLogEntries.onChanged(this.displayLogEntries); eventHub.onEvent(EVENT_FILE_RENAMED, (data) => { void this.setFileStatus(); }); @@ -506,7 +524,7 @@ ${stringifyYaml(info)} writeLogToTheFile(now: Date, vaultName: string, newMessage: string) { fireAndForget(() => - serialized("writeLog", async () => { + withLock("writeLog", async () => { const time = now.toISOString().split("T")[0]; const logDate = `${PREFIXMD_LOGFILE}${time}.md`; const file = await this.core.storageAccess.isExists(normalizePath(logDate)); @@ -597,7 +615,22 @@ ${stringifyYaml(info)} } } override onBindFunction(core: LiveSyncCore, services: typeof core.services): void { - services.API.addLog.setHandler(globalLogFunction); + const log = createBufferedLog(this.recentLogEntries); + const unregisterLog = services.API.addLog.setHandler(log); + const collectGlobalLogs = () => { + const entries = recentGlobalLogEntries.value; + if (entries.length === 0) return; + recentGlobalLogEntries.value = []; + this.recentLogEntries.value = [...this.recentLogEntries.value, ...entries]; + }; + this.logResource = createOwnedResource(log, () => { + unregisterLog(); + recentGlobalLogEntries.offChanged(collectGlobalLogs); + this.recentLogEntries.offChanged(this.displayLogEntries); + this.recentLogEntries.value = []; + }); + recentGlobalLogEntries.onChanged(collectGlobalLogs); + collectGlobalLogs(); services.appLifecycle.onInitialise.addHandler(this._everyOnloadStart.bind(this)); services.appLifecycle.onSettingLoaded.addHandler(this._everyOnloadAfterLoadSettings.bind(this)); services.appLifecycle.onLoaded.addHandler(this._everyOnload.bind(this)); diff --git a/src/modules/features/ModuleLog.unit.spec.ts b/src/modules/features/ModuleLog.unit.spec.ts new file mode 100644 index 00000000..28f48231 --- /dev/null +++ b/src/modules/features/ModuleLog.unit.spec.ts @@ -0,0 +1,228 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/deps.ts", () => ({ + addIcon: vi.fn(), + debounce: (callback: () => void) => callback, + normalizePath: (path: string) => path, + Notice: class {}, + stringifyYaml: vi.fn(), +})); +vi.mock("./Log/LogPaneView.ts", () => ({ LogPaneView: class {}, VIEW_TYPE_LOG: "log" })); +vi.mock("@/common/reportTool.ts", () => ({ generateReport: vi.fn() })); +vi.mock("@/common/utils.ts", () => ({ isValidPath: () => true })); +vi.mock("@/common/translation", () => ({ $msg: (key: string) => key })); +vi.mock("@/common/events.ts", () => ({ + EVENT_FILE_RENAMED: "renamed", + EVENT_LAYOUT_READY: "ready", + EVENT_LEAF_ACTIVE_CHANGED: "active", + EVENT_ON_UNRESOLVED_ERROR: "error", + eventHub: { onEvent: vi.fn() }, +})); +vi.mock("@vrtmrz/livesync-commonlib/compat/replication/trystero/P2PLogCollector", () => ({ + P2PLogCollector: class {}, +})); + +async function createHarness() { + const { ModuleLog } = await import("./ModuleLog.ts"); + const { bindableFunction, allFunction } = + await import("@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils"); + const services = { + API: { + addLog: bindableFunction<(message: unknown, level?: number, key?: string) => void>("log"), + addCommand: vi.fn(), + registerWindow: vi.fn(), + addRibbonIcon: vi.fn(), + registerProtocolHandler: vi.fn(), + }, + context: { events: {}, notices: { hide: vi.fn() } }, + appLifecycle: { + onInitialise: allFunction<() => Promise>(), + onSettingLoaded: allFunction<() => Promise>(), + onLoaded: allFunction<() => Promise>(), + onBeforeUnload: allFunction<() => Promise>(), + }, + }; + const core = { services, settings: {} }; + const module = new ModuleLog({ app: {} } as never, core as never); + const sink = vi.spyOn(module, "__addLog").mockImplementation(() => undefined); + vi.spyOn(module, "observeForLogs").mockImplementation(() => undefined); + vi.spyOn(module, "adjustStatusDivPosition").mockImplementation(() => undefined); + module.onBindFunction(core as never, services as never); + const { Logger } = await import("octagonal-wheels/common/logger"); + return { module, services, sink, Logger }; +} + +describe("ModuleLog service connection", () => { + beforeEach(() => { + vi.resetModules(); + vi.stubGlobal("document", { querySelectorAll: () => [] }); + }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("buffers startup logs and preserves global and service call order", async () => { + const { services, sink, Logger } = await createHarness(); + Logger("before-global"); + services.API.addLog("before-service", 32, "startup"); + Logger("between-global"); + expect(sink).not.toHaveBeenCalled(); + await services.appLifecycle.onSettingLoaded(); + const messages = sink.mock.calls.map(([message]) => message); + expect(messages.filter((message) => typeof message === "string" && /^(before|between)-/.test(message))).toEqual( + ["before-global", "before-service", "between-global"] + ); + expect(sink).toHaveBeenCalledWith("before-service", 32, "startup"); + }); + + it("delivers service logs only to their own module", async () => { + const first = await createHarness(); + const second = await createHarness(); + await first.services.appLifecycle.onSettingLoaded(); + await second.services.appLifecycle.onSettingLoaded(); + first.sink.mockClear(); + second.sink.mockClear(); + + first.services.API.addLog("first", 32, "first-key"); + second.services.API.addLog("second", 64, "second-key"); + expect(first.sink.mock.calls).toEqual([["first", 32, "first-key"]]); + expect(second.sink.mock.calls).toEqual([["second", 64, "second-key"]]); + }); + + it("preserves message conversion, levels, keys, and error causes", async () => { + const { services, sink } = await createHarness(); + await services.appLifecycle.onSettingLoaded(); + sink.mockClear(); + const error = new Error("failed"); + services.API.addLog(error, 128, "failure"); + services.API.addLog({ result: "value" }, 32, "object"); + services.API.addLog("plain"); + const converted = sink.mock.calls[0][0] as Error & { cause: unknown }; + expect(converted.message).toBe("[Error Logged]: failed"); + expect(converted.cause).toBe(error); + expect(sink.mock.calls[0].slice(1)).toEqual([128, "failure"]); + expect(sink.mock.calls.slice(1)).toEqual([ + ['{"result":"value"}', 32, "object"], + ["plain", undefined, undefined], + ]); + }); + + it("detaches both service registration and legacy subscription on disposal", async () => { + const first = await createHarness(); + const second = await createHarness(); + await first.services.appLifecycle.onSettingLoaded(); + await second.services.appLifecycle.onSettingLoaded(); + first.sink.mockClear(); + second.sink.mockClear(); + + await first.module.logResource.dispose(); + await first.module.logResource.dispose(); + expect(() => first.services.API.addLog("retired")).toThrow("not assigned"); + second.services.API.addLog("active", 32, "service"); + first.Logger("legacy-after-retirement", 32, "legacy"); + expect(first.sink).not.toHaveBeenCalled(); + expect(second.sink.mock.calls).toEqual([ + ["active", 32, "service"], + ["legacy-after-retirement", 32, "legacy"], + ]); + await second.module.logResource.dispose(); + }); + + it("leaves a replacement service handler registered when the old owner is disposed", async () => { + const { services, module, sink, Logger } = await createHarness(); + await services.appLifecycle.onSettingLoaded(); + const replacement = vi.fn(); + services.API.addLog.setHandler(replacement, true); + sink.mockClear(); + + await module.logResource.dispose(); + await module.logResource.dispose(); + services.API.addLog("replacement", 32, "key"); + Logger("detached-legacy"); + expect(replacement).toHaveBeenCalledExactlyOnceWith("replacement", 32, "key"); + expect(sink).not.toHaveBeenCalled(); + }); + + it("releases a connection which never reached the settings-loaded phase", async () => { + const { services, module, sink, Logger } = await createHarness(); + services.API.addLog("pending"); + await module.logResource.dispose(); + Logger("after-disposal"); + expect(sink).not.toHaveBeenCalled(); + expect(() => services.API.addLog("retired")).toThrow("not assigned"); + + const replacement = await createHarness(); + await replacement.services.appLifecycle.onSettingLoaded(); + const messages = replacement.sink.mock.calls.map(([message]) => message); + expect(messages).toContain("after-disposal"); + expect(messages).not.toContain("pending"); + await replacement.module.logResource.dispose(); + }); + + it.each(["complete", "fail"] as const)( + "keeps logging through asynchronous database shutdown and releases after it: %s", + async (outcome) => { + const { services, module, sink, Logger } = await createHarness(); + await services.appLifecycle.onSettingLoaded(); + sink.mockClear(); + const { ControlService } = await import("@vrtmrz/livesync-commonlib/compat/services/base/ControlService"); + const { ServiceContext } = await import("@vrtmrz/livesync-commonlib/context"); + const { withOwnedResource } = await import("octagonal-wheels/resource"); + let finish!: () => void; + let fail!: (reason: Error) => void; + let started!: () => void; + const closing = new Promise((resolve) => { + started = resolve; + }); + const closed = new Promise((resolve, reject) => { + finish = resolve; + fail = reject; + }); + const control = new ControlService(new ServiceContext({ translate: (key) => key }), { + appLifecycleService: { + ...services.appLifecycle, + onAppUnload: vi.fn(async () => []), + onUnload: vi.fn(async () => { + services.API.addLog("lifecycle-unload"); + return true; + }), + }, + APIService: services.API, + databaseService: { + localDatabaseDirect: { + onunload: vi.fn(() => services.API.addLog("database-unload")), + close: vi.fn(() => { + started(); + return closed; + }), + }, + }, + } as never); + + const unloading = withOwnedResource(module.logResource, () => control.onUnload()); + await closing; + services.API.addLog("during-close"); + expect(sink.mock.calls.map(([message]) => message)).toEqual([ + "lifecycle-unload", + "database-unload", + "during-close", + ]); + if (outcome === "fail") { + const failure = new Error("database close failed"); + const rejected = expect(unloading).rejects.toBe(failure); + fail(failure); + await rejected; + } else { + finish(); + await unloading; + expect(sink.mock.calls[sink.mock.calls.length - 1][0]).toContain( + "moduleLiveSyncMain.logUnloadingPlugin" + ); + } + expect(() => services.API.addLog("retired")).toThrow("not assigned"); + sink.mockClear(); + Logger("after-unload"); + expect(sink).not.toHaveBeenCalled(); + } + ); +}); diff --git a/src/modules/features/SettingDialogue/LiveSyncSetting.ts b/src/modules/features/SettingDialogue/LiveSyncSetting.ts index 19ff983e..b3b802e4 100644 --- a/src/modules/features/SettingDialogue/LiveSyncSetting.ts +++ b/src/modules/features/SettingDialogue/LiveSyncSetting.ts @@ -14,7 +14,7 @@ import { statusDisplay, type ConfigurationItem, } from "@vrtmrz/livesync-commonlib/compat/common/types"; -import { type ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts"; +import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts"; import { type AllSettingItemKey, getConfig, @@ -24,7 +24,8 @@ import { type AllBooleanItemKey, } from "./settingConstants.ts"; import { $msg } from "@/common/translation"; -import { wrapMemo, type AutoWireOption, type OnUpdateResult } from "./SettingPane.ts"; +import type { AutoWireOption, OnUpdateResult } from "./SettingPane.ts"; +import { onlyWhenChanged } from "octagonal-wheels/function"; import { setButtonDestructiveState } from "./settingComponentStyles.ts"; export class LiveSyncSetting extends Setting { @@ -109,11 +110,15 @@ export class LiveSyncSetting extends Setting { } LiveSyncSetting.env.requestUpdate(); } - autoWireText(key: AllStringItemKey, opt?: AutoWireOption) { + private autoWireStringComponent( + key: AllStringItemKey, + addComponent: (callback: (component: TComponent) => void) => void, + opt?: AutoWireOption + ) { const conf = this.autoWireSetting(key, opt); - this.addText((text) => { + addComponent((text) => { this.autoWiredComponent = text; - const setValue = wrapMemo((value: string) => { + const setValue = onlyWhenChanged((value: string) => { text.setValue(value); }); this.invalidateValue = () => setValue(`${LiveSyncSetting.env.editingSettings[key]}`); @@ -127,26 +132,25 @@ export class LiveSyncSetting extends Setting { } this.autoWireComponent(this.autoWiredComponent, conf, opt); }); + } + autoWireText(key: AllStringItemKey, opt?: AutoWireOption) { + this.autoWireStringComponent( + key, + (callback) => { + this.addText(callback); + }, + opt + ); return this; } autoWireTextArea(key: AllStringItemKey, opt?: AutoWireOption) { - const conf = this.autoWireSetting(key, opt); - this.addTextArea((text) => { - this.autoWiredComponent = text; - const setValue = wrapMemo((value: string) => { - text.setValue(value); - }); - this.invalidateValue = () => setValue(`${LiveSyncSetting.env.editingSettings[key]}`); - this.invalidateValue(); - text.onChange(async (value) => { - await this.commitValue(value); - }); - if (opt?.isPassword) { - text.inputEl.setAttribute("type", "password"); - this.hasPassword = true; - } - this.autoWireComponent(this.autoWiredComponent, conf, opt); - }); + this.autoWireStringComponent( + key, + (callback) => { + this.addTextArea(callback); + }, + opt + ); return this; } autoWireNumeric( @@ -163,7 +167,7 @@ export class LiveSyncSetting extends Setting { text.inputEl.setAttribute("max", `${opt.clampMax}`); } let lastError = false; - const setValue = wrapMemo((value: string) => { + const setValue = onlyWhenChanged((value: string) => { text.setValue(value); }); this.invalidateValue = () => { @@ -209,7 +213,7 @@ export class LiveSyncSetting extends Setting { const conf = this.autoWireSetting(key, opt); this.addToggle((toggle) => { this.autoWiredComponent = toggle; - const setValue = wrapMemo((value: boolean) => { + const setValue = onlyWhenChanged((value: boolean) => { toggle.setValue(opt?.invert ? !value : value); }); this.invalidateValue = () => @@ -228,7 +232,7 @@ export class LiveSyncSetting extends Setting { const conf = this.autoWireSetting(key, opt); this.addDropdown((dropdown) => { this.autoWiredComponent = dropdown; - const setValue = wrapMemo((value: string) => { + const setValue = onlyWhenChanged((value: string) => { dropdown.setValue(value); }); @@ -248,7 +252,7 @@ export class LiveSyncSetting extends Setting { const conf = this.autoWireSetting(key, opt); this.addDropdown((dropdown) => { this.autoWiredComponent = dropdown; - const setValue = wrapMemo((value: string) => { + const setValue = onlyWhenChanged((value: string) => { dropdown.setValue(value); }); diff --git a/src/modules/features/SettingDialogue/LiveSyncSetting.wiring.unit.spec.ts b/src/modules/features/SettingDialogue/LiveSyncSetting.wiring.unit.spec.ts new file mode 100644 index 00000000..5335bc6d --- /dev/null +++ b/src/modules/features/SettingDialogue/LiveSyncSetting.wiring.unit.spec.ts @@ -0,0 +1,227 @@ +import { describe, expect, it, vi } from "vitest"; + +const liveSyncSettingHarness = vi.hoisted(() => ({ + components: [] as unknown[], + settingElements: [] as unknown[], +})); + +vi.mock("@/deps.ts", () => { + class StringComponent { + values: string[] = []; + changeHandler?: (value: string) => Promise; + placeholder?: string; + inputEl = { + attributes: new Map(), + setAttribute(name: string, value: string) { + this.attributes.set(name, value); + }, + toggleClass: (_name: string, _enabled: boolean) => {}, + }; + setValue(value: string) { + this.values.push(value); + } + onChange(handler: (value: string) => Promise) { + this.changeHandler = handler; + return this; + } + setPlaceholder(value: string) { + this.placeholder = value; + return this; + } + constructor() { + liveSyncSettingHarness.components.push(this); + } + } + class TextComponent extends StringComponent {} + class TextAreaComponent extends StringComponent {} + class Setting { + settingEl = { + classes: [] as { name: string; enabled: boolean }[], + toggleClass(name: string, enabled: boolean) { + this.classes.push({ name, enabled }); + }, + }; + constructor(_containerEl: HTMLElement) { + liveSyncSettingHarness.settingElements.push(this.settingEl); + } + setDesc(_description: string | DocumentFragment) { + return this; + } + setName(_name: string | DocumentFragment) { + return this; + } + addText(callback: (component: TextComponent) => void) { + callback(new TextComponent()); + return this; + } + addTextArea(callback: (component: TextAreaComponent) => void) { + callback(new TextAreaComponent()); + return this; + } + setTooltip(_description: string) { + return this; + } + setDisabled(_disabled: boolean) { + return this; + } + } + return { Setting, TextComponent, TextAreaComponent, ButtonComponent: class {} }; +}); +vi.mock("./ObsidianLiveSyncSettingTab.ts", () => ({})); +vi.mock("@/common/translation", () => ({ $msg: (key: string) => key })); +vi.mock("@vrtmrz/livesync-commonlib/compat/common/types", () => ({ + LEVEL_ADVANCED: 1, + LEVEL_POWER_USER: 2, + statusDisplay: () => "", +})); +vi.mock("./settingConstants.ts", () => ({ + getConfig: (key: string) => ({ + name: key, + desc: `description for ${key}`, + placeHolder: `configured placeholder for ${key}`, + }), +})); +vi.mock("./settingComponentStyles.ts", () => ({ setButtonDestructiveState: vi.fn() })); +vi.mock("octagonal-wheels/collection", () => ({ unique: (items: unknown[]) => [...new Set(items)] })); + +import { LiveSyncSetting } from "./LiveSyncSetting.ts"; + +type WireKind = "text" | "textarea"; +type StringComponentHarness = { + values: string[]; + changeHandler?: (value: string) => Promise; + placeholder?: string; + inputEl: { attributes: Map }; +}; +type SettingElementHarness = { classes: { name: string; enabled: boolean }[] }; + +function deferred() { + let resolve!: () => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +function wireStringSetting( + kind: WireKind, + options: { + holdValue?: boolean; + isPassword?: boolean; + placeHolder?: string; + onUpdate?: () => { visibility?: boolean }; + } = {} +) { + const key = kind === "text" ? "syncOnlyRegEx" : "ignoreFiles"; + const events: string[] = []; + const editingSettings: Record = { [key]: 17 }; + const saveSettings = vi.fn(async (_keys: string[]) => {}); + const requestUpdate = vi.fn(() => events.push("update")); + const env = { + settingComponents: [] as unknown[], + editingSettings, + saveSettings, + requestUpdate, + }; + LiveSyncSetting.env = env as never; + + const setting = new LiveSyncSetting({} as HTMLElement); + const onUpdate = vi.fn(options.onUpdate ?? (() => ({ visibility: false }))); + const wireOptions = { ...options, onUpdate }; + if (kind === "text") { + setting.autoWireText(key as never, wireOptions); + } else { + setting.autoWireTextArea(key as never, wireOptions); + } + + return { + key, + events, + editingSettings, + saveSettings, + requestUpdate, + setting, + component: setting.autoWiredComponent as unknown as StringComponentHarness, + settingElement: setting.settingEl as unknown as SettingElementHarness, + onUpdate, + }; +} + +describe("LiveSyncSetting string controls", () => { + it.each(["text", "textarea"] as const)("wires %s through the shared string commit contract", async (kind) => { + const { + key, + events, + editingSettings, + saveSettings, + requestUpdate, + setting, + component, + settingElement, + onUpdate, + } = wireStringSetting(kind, { isPassword: true, placeHolder: "option placeholder" }); + + expect(component.values).toEqual(["17"]); + setting.invalidateValue?.(); + expect(component.values).toEqual(["17"]); + editingSettings[key] = 18; + setting.invalidateValue?.(); + expect(component.values).toEqual(["17", "18"]); + expect(component.inputEl.attributes.get("type")).toBe("password"); + expect(setting.hasPassword).toBe(true); + expect(setting.updateHandlers.size).toBe(1); + expect(onUpdate).toHaveBeenCalledTimes(1); + expect(setting._getComputedStatus()).toEqual({ visibility: false }); + expect(onUpdate).toHaveBeenCalledTimes(2); + expect(settingElement.classes).toContainEqual({ name: "sls-setting-hidden", enabled: true }); + if (kind === "text") { + expect(component.placeholder).toBe("configured placeholder for syncOnlyRegEx"); + } else { + expect(component.placeholder).toBeUndefined(); + } + + const pendingSave = deferred(); + saveSettings.mockImplementationOnce(async (keys) => { + events.push(`save:${keys.join(",")}`); + await pendingSave.promise; + events.push("saved"); + }); + const commit = component.changeHandler!("changed"); + expect(editingSettings[key]).toBe("changed"); + expect(saveSettings).toHaveBeenCalledWith([key]); + expect(events).toEqual([`save:${key}`]); + expect(requestUpdate).not.toHaveBeenCalled(); + pendingSave.resolve(); + await commit; + expect(events).toEqual([`save:${key}`, "saved", "update"]); + + events.length = 0; + await component.changeHandler!("changed"); + expect(saveSettings).toHaveBeenCalledTimes(1); + expect(events).toEqual(["update"]); + + events.length = 0; + const failure = new Error("setting save failed"); + saveSettings.mockImplementationOnce(async (keys) => { + events.push(`save:${keys.join(",")}`); + throw failure; + }); + await expect(component.changeHandler!("failed")).rejects.toBe(failure); + expect(editingSettings[key]).toBe("failed"); + expect(events).toEqual([`save:${key}`]); + expect(requestUpdate).toHaveBeenCalledTimes(2); + }); + + it.each(["text", "textarea"] as const)("keeps %s held values in the editing copy without saving", async (kind) => { + const { key, editingSettings, saveSettings, requestUpdate, component, events } = wireStringSetting(kind, { + holdValue: true, + }); + + await component.changeHandler!("held value"); + + expect(editingSettings[key]).toBe("held value"); + expect(saveSettings).not.toHaveBeenCalled(); + expect(requestUpdate).toHaveBeenCalledOnce(); + expect(events).toEqual(["update"]); + }); +}); diff --git a/src/modules/features/SettingDialogue/PaneSelector.ts b/src/modules/features/SettingDialogue/PaneSelector.ts index 783d6e99..77a98c0a 100644 --- a/src/modules/features/SettingDialogue/PaneSelector.ts +++ b/src/modules/features/SettingDialogue/PaneSelector.ts @@ -1,4 +1,8 @@ -import { LEVEL_ADVANCED, type CustomRegExpSource } from "@vrtmrz/livesync-commonlib/compat/common/types"; +import { + LEVEL_ADVANCED, + type CustomRegExpSource, + type CustomRegExpSourceList, +} from "@vrtmrz/livesync-commonlib/compat/common/types"; import { constructCustomRegExpList, splitCustomRegExpList } from "@vrtmrz/livesync-commonlib/compat/common/utils"; import MultipleRegExpControl from "./MultipleRegExpControl.svelte"; import { LiveSyncSetting as Setting } from "./LiveSyncSetting.ts"; @@ -7,6 +11,35 @@ import type { ObsidianLiveSyncSettingTab } from "./ObsidianLiveSyncSettingTab.ts import type { PageFunctions } from "./SettingPane.ts"; import { visibleOnly } from "./SettingPane.ts"; import { $msg } from "@/common/translation"; + +type RegExpControlOwner = Pick< + ObsidianLiveSyncSettingTab, + "saveAllDirtySettings" | "requestPageRefresh" | "lifetimeComponent" +>; + +function mountMultipleRegExpControl( + owner: RegExpControlOwner, + target: HTMLElement, + source: CustomRegExpSourceList, + delimiter: D, + assign: (serializedPatterns: CustomRegExpSourceList) => void +): void { + const patterns = splitCustomRegExpList(source, delimiter); + const control = mount(MultipleRegExpControl, { + target, + props: { + patterns, + originals: [...patterns], + apply: async (newPatterns: CustomRegExpSource[]) => { + assign(constructCustomRegExpList(newPatterns, delimiter)); + await owner.saveAllDirtySettings(); + owner.requestPageRefresh(); + }, + }, + }); + owner.lifetimeComponent.register(() => void unmount(control)); +} + export function paneSelector(this: ObsidianLiveSyncSettingTab, paneEl: HTMLElement, { addPanel }: PageFunctions): void { void addPanel(paneEl, $msg("Normal Files")).then((paneEl) => { const syncFilesSetting = new Setting(paneEl) @@ -16,37 +49,29 @@ export function paneSelector(this: ObsidianLiveSyncSettingTab, paneEl: HTMLEleme "(RegExp) Limit normal files to matching paths. Leave empty to allow all normal file paths. Hidden File Sync uses its own target patterns." ) ); - const syncFilesControl = mount(MultipleRegExpControl, { - target: syncFilesSetting.controlEl, - props: { - patterns: splitCustomRegExpList(this.editingSettings.syncOnlyRegEx, "|[]|"), - originals: splitCustomRegExpList(this.editingSettings.syncOnlyRegEx, "|[]|"), - apply: async (newPatterns: CustomRegExpSource[]) => { - this.editingSettings.syncOnlyRegEx = constructCustomRegExpList(newPatterns, "|[]|"); - await this.saveAllDirtySettings(); - this.requestPageRefresh(); - }, - }, - }); - this.lifetimeComponent.register(() => void unmount(syncFilesControl)); + mountMultipleRegExpControl( + this, + syncFilesSetting.controlEl, + this.editingSettings.syncOnlyRegEx, + "|[]|", + (value) => { + this.editingSettings.syncOnlyRegEx = value; + } + ); const nonSyncFilesSetting = new Setting(paneEl) .setName($msg("Non-Synchronising files")) .setDesc($msg("(RegExp) Skip changes to matching normal and hidden files, locally and remotely.")); - const nonSyncFilesControl = mount(MultipleRegExpControl, { - target: nonSyncFilesSetting.controlEl, - props: { - patterns: splitCustomRegExpList(this.editingSettings.syncIgnoreRegEx, "|[]|"), - originals: splitCustomRegExpList(this.editingSettings.syncIgnoreRegEx, "|[]|"), - apply: async (newPatterns: CustomRegExpSource[]) => { - this.editingSettings.syncIgnoreRegEx = constructCustomRegExpList(newPatterns, "|[]|"); - await this.saveAllDirtySettings(); - this.requestPageRefresh(); - }, - }, - }); - this.lifetimeComponent.register(() => void unmount(nonSyncFilesControl)); + mountMultipleRegExpControl( + this, + nonSyncFilesSetting.controlEl, + this.editingSettings.syncIgnoreRegEx, + "|[]|", + (value) => { + this.editingSettings.syncIgnoreRegEx = value; + } + ); new Setting(paneEl).autoWireNumeric("syncMaxSizeInMB", { clampMin: 0 }); new Setting(paneEl).autoWireToggle("useIgnoreFiles"); @@ -58,41 +83,30 @@ export function paneSelector(this: ObsidianLiveSyncSettingTab, paneEl: HTMLEleme const targetPatternSetting = new Setting(paneEl) .setName($msg("Target patterns")) .setDesc($msg("Patterns to match files for syncing")); - const patTarget = splitCustomRegExpList(this.editingSettings.syncInternalFilesTargetPatterns, ","); - const targetPatternControl = mount(MultipleRegExpControl, { - target: targetPatternSetting.controlEl, - props: { - patterns: patTarget, - originals: [...patTarget], - apply: async (newPatterns: CustomRegExpSource[]) => { - this.editingSettings.syncInternalFilesTargetPatterns = constructCustomRegExpList(newPatterns, ","); - await this.saveAllDirtySettings(); - this.requestPageRefresh(); - }, - }, - }); - this.lifetimeComponent.register(() => void unmount(targetPatternControl)); + mountMultipleRegExpControl( + this, + targetPatternSetting.controlEl, + this.editingSettings.syncInternalFilesTargetPatterns, + ",", + (value) => { + this.editingSettings.syncInternalFilesTargetPatterns = value; + } + ); const defaultSkipPattern = "\\/node_modules\\/, \\/\\.git\\/, ^\\.git\\/, \\/obsidian-livesync\\/"; const defaultSkipPatternXPlat = defaultSkipPattern + ",\\/workspace$ ,\\/workspace.json$,\\/workspace-mobile.json$"; - const pat = splitCustomRegExpList(this.editingSettings.syncInternalFilesIgnorePatterns, ","); const patSetting = new Setting(paneEl).setName($msg("Ignore patterns")).setDesc(""); - - const ignorePatternControl = mount(MultipleRegExpControl, { - target: patSetting.controlEl, - props: { - patterns: pat, - originals: [...pat], - apply: async (newPatterns: CustomRegExpSource[]) => { - this.editingSettings.syncInternalFilesIgnorePatterns = constructCustomRegExpList(newPatterns, ","); - await this.saveAllDirtySettings(); - this.requestPageRefresh(); - }, - }, - }); - this.lifetimeComponent.register(() => void unmount(ignorePatternControl)); + mountMultipleRegExpControl( + this, + patSetting.controlEl, + this.editingSettings.syncInternalFilesIgnorePatterns, + ",", + (value) => { + this.editingSettings.syncInternalFilesIgnorePatterns = value; + } + ); const addDefaultPatterns = async (patterns: string) => { const oldList = splitCustomRegExpList(this.editingSettings.syncInternalFilesIgnorePatterns, ","); @@ -122,22 +136,14 @@ export function paneSelector(this: ObsidianLiveSyncSettingTab, paneEl: HTMLEleme const overwritePatterns = new Setting(paneEl) .setName($msg("Overwrite patterns")) .setDesc($msg("Patterns to match files for overwriting instead of merging")); - const patTarget2 = splitCustomRegExpList(this.editingSettings.syncInternalFileOverwritePatterns, ","); - const overwritePatternControl = mount(MultipleRegExpControl, { - target: overwritePatterns.controlEl, - props: { - patterns: patTarget2, - originals: [...patTarget2], - apply: async (newPatterns: CustomRegExpSource[]) => { - this.editingSettings.syncInternalFileOverwritePatterns = constructCustomRegExpList( - newPatterns, - "," - ); - await this.saveAllDirtySettings(); - this.requestPageRefresh(); - }, - }, - }); - this.lifetimeComponent.register(() => void unmount(overwritePatternControl)); + mountMultipleRegExpControl( + this, + overwritePatterns.controlEl, + this.editingSettings.syncInternalFileOverwritePatterns, + ",", + (value) => { + this.editingSettings.syncInternalFileOverwritePatterns = value; + } + ); }); } diff --git a/src/modules/features/SettingDialogue/PaneSelector.unit.spec.ts b/src/modules/features/SettingDialogue/PaneSelector.unit.spec.ts new file mode 100644 index 00000000..a53efdf2 --- /dev/null +++ b/src/modules/features/SettingDialogue/PaneSelector.unit.spec.ts @@ -0,0 +1,229 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const paneSelectorHarness = vi.hoisted(() => ({ + mounts: [] as { component: unknown; target: unknown; props: Record; handle: unknown }[], + unmounted: [] as unknown[], + settings: [] as { containerEl: HTMLElement; controlEl: unknown }[], + buttons: [] as { text: string; handler?: () => void | Promise }[], +})); + +vi.mock("@/common/translation", () => ({ $msg: (value: string) => value })); +vi.mock("./MultipleRegExpControl.svelte", () => ({ default: { name: "MultipleRegExpControl" } })); +vi.mock("svelte", () => ({ + mount: vi.fn((component: unknown, options: { target: unknown; props: Record }) => { + const handle = { id: paneSelectorHarness.mounts.length }; + paneSelectorHarness.mounts.push({ component, target: options.target, props: options.props, handle }); + return handle; + }), + unmount: vi.fn((component: unknown) => { + paneSelectorHarness.unmounted.push(component); + }), +})); +vi.mock("./LiveSyncSetting.ts", () => ({ + LiveSyncSetting: class { + controlEl: { id: number }; + constructor(containerEl: HTMLElement) { + this.controlEl = { id: paneSelectorHarness.settings.length }; + paneSelectorHarness.settings.push({ containerEl, controlEl: this.controlEl }); + } + setName(_name: string) { + return this; + } + setDesc(_description: string) { + return this; + } + autoWireNumeric(_key: string, _options: unknown) { + return this; + } + autoWireToggle(_key: string) { + return this; + } + autoWireTextArea(_key: string, _options: unknown) { + return this; + } + addButton( + callback: (button: { + setButtonText: (text: string) => unknown; + onClick: (handler: () => void | Promise) => unknown; + }) => void + ) { + const button = { + text: "", + handler: undefined as (() => void | Promise) | undefined, + setButtonText(text: string) { + this.text = text; + return this; + }, + onClick(handler: () => void | Promise) { + this.handler = handler; + paneSelectorHarness.buttons.push(this); + return this; + }, + }; + callback(button); + return this; + } + }, +})); + +import { paneSelector } from "./PaneSelector.ts"; + +type PatternControlProps = { + patterns: string[]; + originals: string[]; + apply: (patterns: string[]) => Promise; +}; + +function deferred() { + let resolve!: () => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +async function renderSelector() { + const normalPanel = { id: "normal" } as unknown as HTMLElement; + const hiddenPanel = { id: "hidden" } as unknown as HTMLElement; + const addPanel = vi.fn((_parent: HTMLElement, title: string) => + Promise.resolve(title === "Normal Files" ? normalPanel : hiddenPanel) + ); + const disposers: (() => void)[] = []; + const tab = { + editingSettings: { + syncOnlyRegEx: "alpha|[]|beta", + syncIgnoreRegEx: "skip|[]|also-skip", + syncInternalFilesTargetPatterns: "target-a,target-b", + syncInternalFilesIgnorePatterns: "existing,\\/node_modules\\/", + syncInternalFileOverwritePatterns: "overwrite-a,overwrite-b", + }, + saveAllDirtySettings: vi.fn(async () => {}), + requestPageRefresh: vi.fn(), + lifetimeComponent: { register: (dispose: () => void) => disposers.push(dispose) }, + }; + + paneSelector.call(tab as never, {} as HTMLElement, { addPanel } as never); + await vi.waitFor(() => expect(paneSelectorHarness.mounts).toHaveLength(5)); + return { tab, addPanel, disposers }; +} + +function controlProps(index: number): PatternControlProps { + return paneSelectorHarness.mounts[index]!.props as unknown as PatternControlProps; +} + +beforeEach(() => { + paneSelectorHarness.mounts.length = 0; + paneSelectorHarness.unmounted.length = 0; + paneSelectorHarness.settings.length = 0; + paneSelectorHarness.buttons.length = 0; + vi.clearAllMocks(); +}); + +describe("paneSelector multiple-pattern controls", () => { + it("keeps all five bindings, independent originals, delimiter choices, save ordering, and owned cleanup", async () => { + const { tab, addPanel, disposers } = await renderSelector(); + expect(addPanel.mock.calls.map(([, title]) => title)).toEqual(["Normal Files", "Hidden Files"]); + expect(paneSelectorHarness.mounts.map(({ props }) => (props as PatternControlProps).patterns)).toEqual([ + ["alpha", "beta"], + ["skip", "also-skip"], + ["target-a", "target-b"], + ["existing", "\\/node_modules\\/"], + ["overwrite-a", "overwrite-b"], + ]); + + for (let index = 0; index < paneSelectorHarness.mounts.length; index++) { + const { patterns, originals } = controlProps(index); + expect(originals).toEqual(patterns); + expect(originals).not.toBe(patterns); + patterns.push("edited-current-value"); + expect(originals).not.toContain("edited-current-value"); + } + + const cases = [ + { + index: 0, + key: "syncOnlyRegEx", + patterns: ["new-normal-a", "new-normal-b"], + value: "new-normal-a|[]|new-normal-b", + }, + { + index: 1, + key: "syncIgnoreRegEx", + patterns: ["new-skip-a", "new-skip-b"], + value: "new-skip-a|[]|new-skip-b", + }, + { + index: 2, + key: "syncInternalFilesTargetPatterns", + patterns: ["new-target-a", "new-target-b"], + value: "new-target-a,new-target-b", + }, + { + index: 3, + key: "syncInternalFilesIgnorePatterns", + patterns: ["new-ignore-a", "new-ignore-b"], + value: "new-ignore-a,new-ignore-b", + }, + { + index: 4, + key: "syncInternalFileOverwritePatterns", + patterns: ["new-overwrite-a", "new-overwrite-b"], + value: "new-overwrite-a,new-overwrite-b", + }, + ] as const; + const events: string[] = []; + for (let index = 0; index < cases.length; index++) { + const item = cases[index]!; + const pendingSave = deferred(); + events.length = 0; + tab.saveAllDirtySettings.mockImplementationOnce(async () => { + events.push(`save:${item.key}`); + await pendingSave.promise; + events.push("saved"); + }); + tab.requestPageRefresh.mockImplementationOnce(() => { + events.push("refresh"); + }); + + const applying = controlProps(item.index).apply([...item.patterns]); + expect(tab.editingSettings[item.key as keyof typeof tab.editingSettings]).toBe(item.value); + expect(tab.saveAllDirtySettings).toHaveBeenCalledTimes(index + 1); + expect(events).toEqual([`save:${item.key}`]); + expect(tab.requestPageRefresh).toHaveBeenCalledTimes(index); + + pendingSave.resolve(); + await applying; + expect(events).toEqual([`save:${item.key}`, "saved", "refresh"]); + expect(tab.requestPageRefresh).toHaveBeenCalledTimes(index + 1); + } + + const failure = new Error("settings save failed"); + tab.saveAllDirtySettings.mockRejectedValueOnce(failure); + await expect(controlProps(2).apply(["assigned-before-failure"])).rejects.toBe(failure); + expect(tab.editingSettings.syncInternalFilesTargetPatterns).toBe("assigned-before-failure"); + expect(tab.requestPageRefresh).toHaveBeenCalledTimes(cases.length); + + expect(disposers).toHaveLength(5); + disposers.forEach((dispose) => dispose()); + expect(paneSelectorHarness.unmounted).toEqual(paneSelectorHarness.mounts.map(({ handle }) => handle)); + }); + + it("merges default hidden-file ignores with existing patterns and removes duplicates", async () => { + const { tab } = await renderSelector(); + const defaultButton = paneSelectorHarness.buttons.find(({ text }) => text === "Default"); + const crossPlatformButton = paneSelectorHarness.buttons.find(({ text }) => text === "Cross-platform"); + expect(defaultButton?.handler).toBeDefined(); + expect(crossPlatformButton?.handler).toBeDefined(); + + await defaultButton!.handler!(); + expect(tab.editingSettings.syncInternalFilesIgnorePatterns).toBe( + "existing,\\/node_modules\\/, \\/\\.git\\/, ^\\.git\\/, \\/obsidian-livesync\\/" + ); + await crossPlatformButton!.handler!(); + expect(tab.editingSettings.syncInternalFilesIgnorePatterns).toBe( + "existing,\\/node_modules\\/, \\/\\.git\\/, ^\\.git\\/, \\/obsidian-livesync\\/,\\/workspace$ ,\\/workspace.json$,\\/workspace-mobile.json$" + ); + expect(tab.saveAllDirtySettings).toHaveBeenCalledTimes(2); + expect(tab.requestPageRefresh).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/modules/features/SettingDialogue/SettingPane.ts b/src/modules/features/SettingDialogue/SettingPane.ts index 442c2f93..57bd447c 100644 --- a/src/modules/features/SettingDialogue/SettingPane.ts +++ b/src/modules/features/SettingDialogue/SettingPane.ts @@ -97,16 +97,6 @@ export function findAttrFromParent(el: HTMLElement, attr: string): string { return ""; } -export function wrapMemo(func: (arg: T) => void) { - let buf: T | undefined = undefined; - return (arg: T) => { - if (buf !== arg) { - func(arg); - buf = arg; - } - }; -} - /** * Defers pane construction until the owning settings page can confirm that its * render scope is still active. diff --git a/src/modules/features/SettingDialogue/utilFixCouchDBSetting.ts b/src/modules/features/SettingDialogue/utilFixCouchDBSetting.ts index 4a72be8e..246ce622 100644 --- a/src/modules/features/SettingDialogue/utilFixCouchDBSetting.ts +++ b/src/modules/features/SettingDialogue/utilFixCouchDBSetting.ts @@ -13,6 +13,7 @@ import { generateCredentialObject } from "@vrtmrz/livesync-commonlib/compat/repl import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { isUnauthorizedError } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc"; import { normaliseCouchDBConfiguration } from "@/common/couchdbConfiguration"; +import { isCouchDBVersionAtLeast, normaliseResponseHeaderKeys } from "@/modules/features/couchDBDiagnosticValues"; export const checkConfig = async ( checkResultDiv: HTMLDivElement | undefined, @@ -89,19 +90,6 @@ export const checkConfig = async ( addResult($msg("obsidianLiveSyncSettingTab.serverVersion", { info: serverBanner })); const versionMatch = serverBanner.match(/CouchDB(\/([0-9.]+))?/); const versionStr = versionMatch ? versionMatch[2] : "0.0.0"; - const versionParts = `${versionStr}.0.0.0`.split("."); - // Compare version string with the target version. - // version must be a string like "3.2.1" or "3.10.2", and must be two or three parts. - function isGreaterThanOrEqual(version: string) { - const targetParts = version.split("."); - for (let i = 0; i < targetParts.length; i++) { - // compare as number if possible (so 3.10 > 3.2, 3.10.1b > 3.10.1a) - const result = versionParts[i].localeCompare(targetParts[i], undefined, { numeric: true }); - if (result > 0) return true; - if (result < 0) return false; - } - return true; - } // Admin check // for database creation and deletion if (!(editingSettings.couchDB_USER in responseConfig.admins)) { @@ -109,7 +97,7 @@ export const checkConfig = async ( } else { addResult($msg("obsidianLiveSyncSettingTab.okAdminPrivileges")); } - if (isGreaterThanOrEqual("3.2.0")) { + if (isCouchDBVersionAtLeast(versionStr, "3.2.0")) { // HTTP user-authorization check if (responseConfig?.chttpd?.require_valid_user != "true") { isSuccessful = false; @@ -148,7 +136,7 @@ export const checkConfig = async ( } else { addResult($msg("obsidianLiveSyncSettingTab.okWwwAuth")); } - if (isGreaterThanOrEqual("3.2.0")) { + if (isCouchDBVersionAtLeast(versionStr, "3.2.0")) { if (responseConfig?.chttpd?.enable_cors != "true") { isSuccessful = false; addResult($msg("obsidianLiveSyncSettingTab.errEnableCorsChttpd")); @@ -240,12 +228,7 @@ export const checkConfig = async ( undefined, customHeaders ); - const responseHeaders = Object.fromEntries( - Object.entries(rr.headers).map((e) => { - e[0] = `${e[0]}`.toLowerCase(); - return e; - }) - ); + const responseHeaders = normaliseResponseHeaderKeys(rr.headers); addResult($msg("obsidianLiveSyncSettingTab.msgOriginCheck", { org })); if (responseHeaders["access-control-allow-credentials"] != "true") { addResult($msg("obsidianLiveSyncSettingTab.errCorsNotAllowingCredentials")); diff --git a/src/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte b/src/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte index a163ceee..2c640ee0 100644 --- a/src/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte +++ b/src/modules/features/SetupWizard/dialogs/SetupRemoteCouchDB.svelte @@ -314,14 +314,16 @@ {#if processing} {translateMessage("Checking connection... Please wait.")} {:else} + {#if setupMode === "settings"} + + {translateMessage( + "Saving without a successful connection test keeps this profile, but automatic synchronisation may fail until the connection is corrected." + )} + + {/if} checkAndCommit()} /> {#if setupMode === "settings"} - - {translateMessage( - "Saving without a successful connection test keeps this profile, but automatic synchronisation may fail until the connection is corrected." - )} - = | ResultError | ResultMessage | ResultErrorMessage; -/** - * Compares two version strings to determine if the baseVersion is greater than or equal to the version. - * @param baseVersion a.b.c format - * @param version a.b.c format - * @returns true if baseVersion is greater than or equal to version, false otherwise - */ -function isGreaterThanOrEqual(baseVersion: string, version: string) { - const versionParts = `${baseVersion}.0.0.0`.split("."); - const targetParts = version.split("."); - for (let i = 0; i < targetParts.length; i++) { - // compare as number if possible (so 3.10 > 3.2, 3.10.1b > 3.10.1a) - const result = versionParts[i].localeCompare(targetParts[i], undefined, { numeric: true }); - if (result > 0) return true; - if (result < 0) return false; - } - return true; -} - /** * Updates the remote CouchDB setting with the given key and value. * @param setting Connection settings @@ -153,7 +136,7 @@ export const checkConfig = async (editingSettings: ObsidianLiveSyncSettings) => } else { addSuccess($msg("obsidianLiveSyncSettingTab.okAdminPrivileges")); } - if (isGreaterThanOrEqual(versionStr, "3.2.0")) { + if (isCouchDBVersionAtLeast(versionStr, "3.2.0")) { // HTTP user-authorization check if (responseConfig?.chttpd?.require_valid_user != "true") { addError( @@ -189,7 +172,7 @@ export const checkConfig = async (editingSettings: ObsidianLiveSyncSettings) => } else { addSuccess($msg("obsidianLiveSyncSettingTab.okWwwAuth")); } - if (isGreaterThanOrEqual(versionStr, "3.2.0")) { + if (isCouchDBVersionAtLeast(versionStr, "3.2.0")) { if (responseConfig?.chttpd?.enable_cors != "true") { addError( $msg("obsidianLiveSyncSettingTab.errEnableCorsChttpd"), @@ -287,12 +270,7 @@ export const checkConfig = async (editingSettings: ObsidianLiveSyncSettings) => undefined, customHeaders ); - const responseHeaders = Object.fromEntries( - Object.entries(rr.headers).map((e) => { - e[0] = `${e[0]}`.toLowerCase(); - return e; - }) - ); + const responseHeaders = normaliseResponseHeaderKeys(rr.headers); addMessage($msg("obsidianLiveSyncSettingTab.msgOriginCheck", { org })); if (responseHeaders["access-control-allow-credentials"] != "true") { addErrorMessage($msg("obsidianLiveSyncSettingTab.errCorsNotAllowingCredentials")); diff --git a/src/modules/features/StatusBarDisplay.timeout.unit.spec.ts b/src/modules/features/StatusBarDisplay.timeout.unit.spec.ts new file mode 100644 index 00000000..15bda8f1 --- /dev/null +++ b/src/modules/features/StatusBarDisplay.timeout.unit.spec.ts @@ -0,0 +1,142 @@ +import { reactiveSource } from "octagonal-wheels/dataobject/reactive"; +import { createDeferredAction, type TimeoutScheduler } from "octagonal-wheels/concurrency/deferredAction"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + STATUS_COUNTER_INACTIVE_LINGER_MS, + createChunkFetchCounterLabel, + createMinimumVisibleActivityCount, + createPaddedCounterLabel, +} from "./StatusBarDisplay.ts"; + +vi.mock("octagonal-wheels/concurrency/deferredAction", { spy: true }); + +function createdControllers() { + return vi.mocked(createDeferredAction).mock.results.map((result) => { + if (result.type !== "return") throw new Error("Expected a real timeout controller"); + return result.value; + }); +} + +function createScheduler() { + const entries: { callback: () => void; delayMs: number; cancel: ReturnType }[] = []; + const schedule = vi.fn((callback, delayMs) => { + const cancel = vi.fn(); + entries.push({ callback, delayMs, cancel }); + return cancel; + }); + return { schedule, entries }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-24T00:00:00Z")); +}); + +afterEach(() => { + vi.clearAllTimers(); + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +describe("status timeout composition", () => { + it("uses the real shared controller for minimum visibility, resumption, and disposal", () => { + const scheduler = createScheduler(); + const source = reactiveSource(0); + const display = createMinimumVisibleActivityCount(source, 150, scheduler.schedule); + expect(createDeferredAction).toHaveBeenCalledExactlyOnceWith(expect.any(Function), scheduler.schedule); + const controller = createdControllers()[0]; + const schedule = vi.spyOn(controller, "schedule"); + const cancel = vi.spyOn(controller, "cancel"); + const dispose = vi.spyOn(controller, "dispose"); + + source.value = 1; + vi.advanceTimersByTime(50); + source.value = 0; + expect(schedule).toHaveBeenCalledExactlyOnceWith(100); + expect(scheduler.entries[0].delayMs).toBe(100); + source.value = 2; + expect(cancel).toHaveBeenCalledTimes(3); + expect(scheduler.entries[0].cancel).toHaveBeenCalledTimes(1); + scheduler.entries[0].callback(); + expect(display.value).toBe(2); + + source.value = 0; + scheduler.entries[1].callback(); + expect(display.value).toBe(0); + source.value = 3; + source.value = 0; + display.dispose(); + display.dispose(); + scheduler.entries[2].callback(); + expect(display.value).toBe(3); + expect(scheduler.entries[2].cancel).toHaveBeenCalledTimes(1); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it("uses the shared controller for the counter linger and cancels through it when activity resumes", () => { + const scheduler = createScheduler(); + const source = reactiveSource(0); + const display = createPaddedCounterLabel(source, "A", 25, scheduler.schedule); + expect(createDeferredAction).toHaveBeenCalledExactlyOnceWith(expect.any(Function), scheduler.schedule); + const controller = createdControllers()[0]; + const schedule = vi.spyOn(controller, "schedule"); + const cancel = vi.spyOn(controller, "cancel"); + const dispose = vi.spyOn(controller, "dispose"); + source.value = 1; + source.value = 0; + expect(schedule).toHaveBeenCalledExactlyOnceWith(25); + source.value = 2; + expect(cancel).toHaveBeenCalledTimes(3); + expect(scheduler.entries[0].cancel).toHaveBeenCalledTimes(1); + scheduler.entries[0].callback(); + expect(display.value).toBe(" A\u20072"); + expect(display.count.value).toBe(2); + + source.value = 0; + scheduler.entries[1].callback(); + expect(display.value).toBe(""); + expect(display.count.value).toBeUndefined(); + source.value = 3; + source.value = 0; + display.dispose(); + scheduler.entries[2].callback(); + expect(display.value).toBe(" A\u20070"); + expect(display.count.value).toBe(0); + expect(scheduler.entries[2].cancel).toHaveBeenCalledTimes(1); + expect(dispose).toHaveBeenCalledTimes(1); + }); + + it("passes the scheduler to independent chunk-count controllers and disposes both children", () => { + const scheduler = createScheduler(); + const source = reactiveSource({ initial: 2, retrying: 3 }); + const display = createChunkFetchCounterLabel(source, scheduler.schedule); + expect(display.initialCount.value).toBe(2); + expect(display.retryingCount.value).toBe(3); + expect(createDeferredAction).toHaveBeenCalledTimes(2); + expect(vi.mocked(createDeferredAction).mock.calls.map((args) => args[1])).toEqual([ + scheduler.schedule, + scheduler.schedule, + ]); + const controllers = createdControllers(); + const disposers = controllers.map((controller) => vi.spyOn(controller, "dispose")); + source.value = { initial: 0, retrying: 0 }; + expect(scheduler.entries.map((entry) => entry.delayMs)).toEqual([ + STATUS_COUNTER_INACTIVE_LINGER_MS, + STATUS_COUNTER_INACTIVE_LINGER_MS, + ]); + scheduler.entries[0].callback(); + expect(display.value.replace(/\u2007/g, "")).toBe(" 🔁0"); + expect(display.initialCount.value).toBeUndefined(); + expect(display.retryingCount.value).toBe(0); + display.dispose(); + display.dispose(); + expect(scheduler.entries[0].cancel).not.toHaveBeenCalled(); + expect(scheduler.entries[1].cancel).toHaveBeenCalledTimes(1); + for (const dispose of disposers) expect(dispose).toHaveBeenCalledTimes(1); + scheduler.entries[1].callback(); + expect(display.value.replace(/\u2007/g, "")).toBe(" 🔁0"); + expect(display.initialCount.value).toBeUndefined(); + expect(display.retryingCount.value).toBe(0); + }); +}); diff --git a/src/modules/features/StatusBarDisplay.ts b/src/modules/features/StatusBarDisplay.ts index 9c24a972..cc63aca9 100644 --- a/src/modules/features/StatusBarDisplay.ts +++ b/src/modules/features/StatusBarDisplay.ts @@ -1,7 +1,13 @@ import { reactiveSource, type ReactiveSource, type ReactiveValue } from "octagonal-wheels/dataobject/reactive"; +import { createDeferredAction, type TimeoutScheduler } from "octagonal-wheels/concurrency/deferredAction"; import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; +const scheduleStatusTimeout: TimeoutScheduler = (callback, delayMs) => { + const handle = compatGlobal.setTimeout(callback, delayMs); + return () => compatGlobal.clearTimeout(handle); +}; + const STATUS_COUNTER_PADDING = "\u2007".repeat(10); export const STATUS_COUNTER_INACTIVE_LINGER_MS = 3_000; @@ -37,32 +43,32 @@ function asDisposableReactiveValue(value: ReactiveSource, dispose: () => v /** * Mirrors an activity count while keeping each visible period on screen for a * minimum total lifetime. The delay applies only when the source becomes zero. + * + * @param source - The activity count to observe. + * @param minimumVisibleMs - The minimum visible lifetime, clamped to zero. + * @param scheduleTimeout - The scheduler for this display. Defaults to the + * existing compatible host, preserving its timer method receivers. */ export function createMinimumVisibleActivityCount( source: ReactiveValue, - minimumVisibleMs: number + minimumVisibleMs: number, + scheduleTimeout: TimeoutScheduler = scheduleStatusTimeout ): DisposableReactiveValue { const minimumLifetime = Math.max(0, minimumVisibleMs); const displayed = reactiveSource(Math.max(0, source.value)); let visibleSince = displayed.value > 0 ? Date.now() : undefined; - let hideTimer: number | undefined; let disposed = false; - const cancelHide = () => { - if (hideTimer === undefined) return; - compatGlobal.clearTimeout(hideTimer); - hideTimer = undefined; - }; const hideIfIdle = () => { - hideTimer = undefined; if (disposed || Math.max(0, source.value) > 0) return; displayed.value = 0; visibleSince = undefined; }; + const hideTimeout = createDeferredAction(hideIfIdle, scheduleTimeout); const update = () => { if (disposed) return; const nextCount = Math.max(0, source.value); - cancelHide(); + hideTimeout.cancel(); if (nextCount > 0) { if (displayed.value === 0) { visibleSince = Date.now(); @@ -80,7 +86,7 @@ export function createMinimumVisibleActivityCount( if (remaining === 0) { hideIfIdle(); } else { - hideTimer = compatGlobal.setTimeout(hideIfIdle, remaining); + hideTimeout.schedule(remaining); } }; @@ -88,7 +94,7 @@ export function createMinimumVisibleActivityCount( return asDisposableReactiveValue(displayed, () => { if (disposed) return; disposed = true; - cancelHide(); + hideTimeout.dispose(); source.offChanged(update); }); } @@ -96,24 +102,31 @@ export function createMinimumVisibleActivityCount( /** * Formats a counter with a stable width and briefly retains its zero value so * that the completion of queued work remains visible. + * + * @param source - The counter to observe. + * @param mark - The label prefix before the padded count. + * @param inactiveLingerMs - The delay before clearing a zero count, clamped to zero. + * @param scheduleTimeout - The scheduler for this display. Defaults to the + * existing compatible host, preserving its timer method receivers. */ export function createPaddedCounterLabel( source: ReactiveValue, mark: string, - inactiveLingerMs = STATUS_COUNTER_INACTIVE_LINGER_MS + inactiveLingerMs = STATUS_COUNTER_INACTIVE_LINGER_MS, + scheduleTimeout: TimeoutScheduler = scheduleStatusTimeout ): PaddedCounterLabel { const linger = Math.max(0, inactiveLingerMs); const formatted = reactiveSource(""); const displayedCount = reactiveSource(undefined); let maximumLength = 1; - let clearTimer: number | undefined; let disposed = false; - const cancelClear = () => { - if (clearTimer === undefined) return; - compatGlobal.clearTimeout(clearTimer); - clearTimer = undefined; - }; + const clearLabelTimeout = createDeferredAction(() => { + if (disposed) return; + formatted.value = ""; + displayedCount.value = undefined; + maximumLength = 1; + }, scheduleTimeout); const format = (count: number) => { const requiredLength = `${Math.abs(count)}`.length + 1; maximumLength = Math.max(maximumLength, requiredLength); @@ -121,25 +134,19 @@ export function createPaddedCounterLabel( }; const update = () => { if (disposed) return; - cancelClear(); + clearLabelTimeout.cancel(); const count = source.value; formatted.value = format(count); displayedCount.value = count; if (count !== 0) return; - clearTimer = compatGlobal.setTimeout(() => { - clearTimer = undefined; - if (disposed) return; - formatted.value = ""; - displayedCount.value = undefined; - maximumLength = 1; - }, linger); + clearLabelTimeout.schedule(linger); }; source.onChanged(update); const display = asDisposableReactiveValue(formatted, () => { if (disposed) return; disposed = true; - cancelClear(); + clearLabelTimeout.dispose(); source.offChanged(update); }); return { @@ -162,14 +169,28 @@ export function createPaddedCounterLabel( /** * Displays the disjoint initial and retry chunk-fetch counts with the same * padding and inactive linger behaviour as the other status counters. + * + * @param source - The initial and retrying chunk-fetch counts. + * @param scheduleTimeout - The scheduler passed to each independent child display. */ export function createChunkFetchCounterLabel( - source: ReactiveValue<{ initial: number; retrying: number }> + source: ReactiveValue<{ initial: number; retrying: number }>, + scheduleTimeout: TimeoutScheduler = scheduleStatusTimeout ): ChunkFetchCounterLabel { const initialCount = reactiveSource(0); const retryingCount = reactiveSource(0); - const initialLabel = createPaddedCounterLabel(initialCount, "🛄"); - const retryingLabel = createPaddedCounterLabel(retryingCount, "🔁"); + const initialLabel = createPaddedCounterLabel( + initialCount, + "🛄", + STATUS_COUNTER_INACTIVE_LINGER_MS, + scheduleTimeout + ); + const retryingLabel = createPaddedCounterLabel( + retryingCount, + "🔁", + STATUS_COUNTER_INACTIVE_LINGER_MS, + scheduleTimeout + ); const formatted = reactiveSource(`${initialLabel.value}${retryingLabel.value}`); let updatingCounts = false; let disposed = false; diff --git a/src/modules/features/StatusBarDisplay.unit.spec.ts b/src/modules/features/StatusBarDisplay.unit.spec.ts index 5ce8f69e..c704cce4 100644 --- a/src/modules/features/StatusBarDisplay.unit.spec.ts +++ b/src/modules/features/StatusBarDisplay.unit.spec.ts @@ -1,3 +1,4 @@ +import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions"; import { reactive, reactiveSource } from "octagonal-wheels/dataobject/reactive"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -194,3 +195,121 @@ describe("createChunkFetchCounterLabel", () => { expect(withoutPadding()).toBe(" 🔁0"); }); }); + +describe("status timeout ownership", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-10-24T00:00:00Z")); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it("counts the minimum lifetime from construction when activity is already visible", () => { + const source = reactiveSource(2); + const display = createMinimumVisibleActivityCount(source, 150); + vi.advanceTimersByTime(40); + source.value = 0; + + vi.advanceTimersByTime(109); + expect(display.value).toBe(2); + vi.advanceTimersByTime(1); + expect(display.value).toBe(0); + display.dispose(); + }); + + it("hides immediately for a non-positive minimum lifetime and clamps negative activity", () => { + const source = reactiveSource(-2); + const display = createMinimumVisibleActivityCount(source, -10); + expect(display.value).toBe(0); + source.value = 1; + source.value = -2; + + expect(display.value).toBe(0); + expect(vi.getTimerCount()).toBe(0); + display.dispose(); + }); + + it("keeps a zero-duration counter clear asynchronous", () => { + const source = reactiveSource(0); + const display = createPaddedCounterLabel(source, "A", 0); + source.value = 1; + source.value = 0; + + expect(display.value).toBe(" A\u20070"); + vi.advanceTimersByTime(0); + expect(display.value).toBe(""); + display.dispose(); + }); + + it.each(["activity", "label"] as const)( + "retains timer receivers, zero handles, and idempotent disposal for %s", + (kind) => { + const source = reactiveSource(0); + const offChanged = vi.spyOn(source, "offChanged"); + const setTimer = vi.spyOn(compatGlobal as Window, "setTimeout").mockReturnValue(0); + const clearTimer = vi.spyOn(compatGlobal as Window, "clearTimeout").mockImplementation(() => {}); + const display = + kind === "activity" + ? createMinimumVisibleActivityCount(source, 150) + : createPaddedCounterLabel(source, "A", 150); + + source.value = 1; + source.value = 0; + expect(setTimer).toHaveBeenCalledTimes(1); + expect(setTimer.mock.contexts[0]).toBe(compatGlobal); + expect(setTimer.mock.calls[0][1]).toBe(150); + source.value = 2; + expect(clearTimer).toHaveBeenCalledExactlyOnceWith(0); + source.value = 0; + display.dispose(); + display.dispose(); + source.value = 3; + + expect(setTimer).toHaveBeenCalledTimes(2); + expect(clearTimer).toHaveBeenCalledTimes(2); + expect(clearTimer.mock.calls).toEqual([[0], [0]]); + expect(clearTimer.mock.contexts).toEqual([compatGlobal, compatGlobal]); + expect(offChanged).toHaveBeenCalledTimes(1); + } + ); + + it("keeps pending clears independent between display instances", () => { + const firstSource = reactiveSource(0); + const secondSource = reactiveSource(0); + const first = createPaddedCounterLabel(firstSource, "A", 100); + const second = createPaddedCounterLabel(secondSource, "B", 100); + firstSource.value = 1; + firstSource.value = 0; + secondSource.value = 1; + secondSource.value = 0; + expect(vi.getTimerCount()).toBe(2); + + first.dispose(); + expect(vi.getTimerCount()).toBe(1); + vi.advanceTimersByTime(100); + expect(first.value).toBe(" A\u20070"); + expect(second.value).toBe(""); + second.dispose(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("disposes both child timers and its source subscription for chunk counts", () => { + const source = reactiveSource({ initial: 2, retrying: 3 }); + const offChanged = vi.spyOn(source, "offChanged"); + const display = createChunkFetchCounterLabel(source); + source.value = { initial: 0, retrying: 0 }; + const lastLabel = display.value; + expect(vi.getTimerCount()).toBe(2); + + display.dispose(); + display.dispose(); + expect(vi.getTimerCount()).toBe(0); + vi.advanceTimersByTime(STATUS_COUNTER_INACTIVE_LINGER_MS); + source.value = { initial: 4, retrying: 5 }; + expect(display.value).toBe(lastLabel); + expect(offChanged).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/modules/features/couchDBDiagnosticValues.ts b/src/modules/features/couchDBDiagnosticValues.ts new file mode 100644 index 00000000..41208ab5 --- /dev/null +++ b/src/modules/features/couchDBDiagnosticValues.ts @@ -0,0 +1,19 @@ +export function isCouchDBVersionAtLeast(baseVersion: string, targetVersion: string) { + const versionParts = `${baseVersion}.0.0.0`.split("."); + const targetParts = targetVersion.split("."); + for (let i = 0; i < targetParts.length; i++) { + const result = versionParts[i].localeCompare(targetParts[i], undefined, { numeric: true }); + if (result > 0) return true; + if (result < 0) return false; + } + return true; +} + +export function normaliseResponseHeaderKeys(headers: Record) { + return Object.fromEntries( + Object.entries(headers).map((entry) => { + entry[0] = `${entry[0]}`.toLowerCase(); + return entry; + }) + ); +} diff --git a/src/modules/features/couchDBDiagnosticValues.unit.spec.ts b/src/modules/features/couchDBDiagnosticValues.unit.spec.ts new file mode 100644 index 00000000..485a7ba7 --- /dev/null +++ b/src/modules/features/couchDBDiagnosticValues.unit.spec.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { isCouchDBVersionAtLeast, normaliseResponseHeaderKeys } from "./couchDBDiagnosticValues"; + +describe("CouchDB diagnostic values", () => { + describe("version comparison", () => { + it.each([ + ["3.10.2", "3.2.0", true], + ["3.2.0", "3.10.0", false], + ["3.2", "3.2.0", true], + ["3.10.1b", "3.10.1a", true], + ["3.10.1a", "3.10.1b", false], + ["3.x", "3.2", true], + ])("compares %s with %s", (baseVersion, targetVersion, expected) => { + expect(isCouchDBVersionAtLeast(baseVersion, targetVersion)).toBe(expected); + }); + }); + + describe("response header keys", () => { + it("lower-cases keys while retaining last collision values and entry order", () => { + const headers = { Server: "first", server: "last", Zed: "z", Alpha: "a" }; + + const normalised = normaliseResponseHeaderKeys(headers); + + expect(normalised).toEqual({ server: "last", zed: "z", alpha: "a" }); + expect(Object.keys(normalised)).toEqual(["server", "zed", "alpha"]); + expect(headers).toEqual({ Server: "first", server: "last", Zed: "z", Alpha: "a" }); + }); + }); +}); diff --git a/src/modules/services/LiveSyncUI/DialogHost.svelte b/src/modules/services/LiveSyncUI/DialogHost.svelte index 6265c7bf..79862b17 100644 --- a/src/modules/services/LiveSyncUI/DialogHost.svelte +++ b/src/modules/services/LiveSyncUI/DialogHost.svelte @@ -67,7 +67,7 @@ gap: 0.5em; display: flex; flex-direction: column; - padding-bottom: var(--keyboard-height, 0px); + padding-bottom: 0; user-select: text; -webkit-user-select: text; } @@ -76,6 +76,10 @@ margin-left: 10px; } + .dialog-host :global(.button-group > button) { + margin-left: 0; + } + .dialog-host :global(.button-group) { display: flex; flex-direction: column; diff --git a/src/modules/services/ObsidianAPIService.ts b/src/modules/services/ObsidianAPIService.ts index c455d172..ebf4e011 100644 --- a/src/modules/services/ObsidianAPIService.ts +++ b/src/modules/services/ObsidianAPIService.ts @@ -16,6 +16,43 @@ declare module "obsidian" { } } +const cacheReuseDirectives = new Set(["max-age", "max-stale", "min-fresh", "only-if-cached"]); + +function splitHeaderDirectives(value: string): string[] { + const directives: string[] = []; + let start = 0; + let inQuotes = false; + let escaped = false; + + for (let index = 0; index < value.length; index++) { + const character = value[index]; + if (inQuotes && escaped) { + escaped = false; + continue; + } + if (inQuotes && character === "\\") { + escaped = true; + continue; + } + if (character === '"') { + inQuotes = !inQuotes; + } else if (character === "," && !inQuotes) { + const directive = value.slice(start, index).trim(); + if (directive) directives.push(directive); + start = index + 1; + } + } + + const lastDirective = value.slice(start).trim(); + if (lastDirective) directives.push(lastDirective); + return directives; +} + +function getDirectiveName(directive: string): string { + const equalsIndex = directive.indexOf("="); + return (equalsIndex === -1 ? directive : directive.slice(0, equalsIndex)).trim().toLowerCase(); +} + export class ObsidianAPIService extends InjectableAPIService { _customHandler: ObsHttpHandler | undefined; _confirmInstance: Confirm; @@ -182,6 +219,30 @@ export class ObsidianAPIService extends InjectableAPIService key.toLowerCase() === "cache-control" + ); + const cacheControlDirectives = cacheControlKeys + .flatMap((key) => splitHeaderDirectives(transformedHeaders[key])) + .filter((directive) => { + const name = getDirectiveName(directive); + return !cacheReuseDirectives.has(name) && name !== "no-cache" && name !== "no-store"; + }); + const cacheControlKey = cacheControlKeys[0] ?? "cache-control"; + for (const key of cacheControlKeys) delete transformedHeaders[key]; + transformedHeaders[cacheControlKey] = [...cacheControlDirectives, "no-cache", "no-store"].join(", "); + + const pragmaKeys = Object.keys(transformedHeaders).filter((key) => key.toLowerCase() === "pragma"); + const pragmaDirectives = pragmaKeys + .flatMap((key) => splitHeaderDirectives(transformedHeaders[key])) + .filter((directive) => getDirectiveName(directive) !== "no-cache"); + const pragmaKey = pragmaKeys[0] ?? "pragma"; + for (const key of pragmaKeys) delete transformedHeaders[key]; + transformedHeaders[pragmaKey] = [...pragmaDirectives, "no-cache"].join(", "); + } const contentType = transformedHeaders["content-type"] ?? transformedHeaders["Content-Type"] ?? "application/json"; const requestParam: RequestUrlParam = { diff --git a/src/modules/services/ObsidianAPIService.unit.spec.ts b/src/modules/services/ObsidianAPIService.unit.spec.ts index 6523bfaf..10f13081 100644 --- a/src/modules/services/ObsidianAPIService.unit.spec.ts +++ b/src/modules/services/ObsidianAPIService.unit.spec.ts @@ -1,6 +1,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { RequestUrlParam } from "@/deps"; const mocks = vi.hoisted(() => ({ + requestUrl: vi.fn< + (request: RequestUrlParam) => Promise<{ + status: number; + headers: Record; + arrayBuffer: ArrayBuffer; + }> + >(), platform: { isMobile: false, }, @@ -8,12 +16,12 @@ const mocks = vi.hoisted(() => ({ vi.mock("@/deps.ts", () => ({ Platform: mocks.platform, - requestUrl: vi.fn(), + requestUrl: mocks.requestUrl, })); vi.mock("@/deps", () => ({ Platform: mocks.platform, - requestUrl: vi.fn(), + requestUrl: mocks.requestUrl, })); vi.mock("@/modules/essentialObsidian/APILib/ObsHttpHandler", () => ({ @@ -36,6 +44,8 @@ function createService(workspace: Record, isMobile = false): Ob beforeEach(() => { mocks.platform.isMobile = false; vi.clearAllMocks(); + mocks.requestUrl.mockReset(); + mocks.requestUrl.mockResolvedValue({ status: 200, headers: {}, arrayBuffer: new ArrayBuffer(0) }); }); describe("ObsidianAPIService.showWindowOnRight", () => { @@ -65,3 +75,115 @@ describe("ObsidianAPIService.showWindowOnRight", () => { expect(workspace.revealLeaf).toHaveBeenCalledWith(rightLeaf); }); }); + +describe("ObsidianAPIService.nativeFetch cache control", () => { + const url = "https://couchdb.example.test/vault/f%3Ametadata"; + + function sentHeaders(): Headers { + expect(mocks.requestUrl).toHaveBeenCalledOnce(); + return new Headers(mocks.requestUrl.mock.calls[0][0].headers); + } + + it.each(["GET", "HEAD", "get", "head"])("bypasses cached responses for an explicit no-store %s", async (method) => { + const headers = { + Authorization: "Basic fixture", + "Cache-Control": "public, max-age=3600", + "cAcHe-CoNtRoL": "max-stale", + Pragma: "cache", + }; + + await createService({}).nativeFetch(url, { method, cache: "no-store", headers }); + + expect(sentHeaders().get("cache-control")).toBe("public, no-cache, no-store"); + expect(sentHeaders().get("pragma")).toBe("cache, no-cache"); + expect(sentHeaders().get("authorization")).toBe("Basic fixture"); + expect(headers["Cache-Control"]).toBe("public, max-age=3600"); + expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method }); + }); + + it("preserves non-conflicting cache directives when enforcing no-store", async () => { + const headers = { + "Cache-Control": 'max-age=3600, no-transform, custom="a,b"', + "cAcHe-CoNtRoL": 'max-stale=30, min-fresh=5, only-if-cached, custom-two="max-age=1, only-if-cached"', + Pragma: 'legacy="a,b", no-cache', + "X-Custom-Header": "preserved", + }; + const originalHeaders = { ...headers }; + + await createService({}).nativeFetch(url, { cache: "no-store", headers }); + + const sent = mocks.requestUrl.mock.calls[0][0].headers ?? {}; + expect(sent["Cache-Control"]).toBe( + 'no-transform, custom="a,b", custom-two="max-age=1, only-if-cached", no-cache, no-store' + ); + expect(sent["cAcHe-CoNtRoL"]).toBeUndefined(); + expect(sent.Pragma).toBe('legacy="a,b", no-cache'); + expect(sent["X-Custom-Header"]).toBe("preserved"); + expect(headers).toEqual(originalHeaders); + }); + + it("uses the Request cache mode when the options do not override it", async () => { + const request = new Request(url, { + method: "HEAD", + cache: "no-store", + headers: { "X-Fixture": "request" }, + }); + const headers = new Headers({ "X-Other": "options" }); + + await createService({}).nativeFetch(request, { headers }); + + expect(sentHeaders().get("cache-control")).toBe("no-cache, no-store"); + expect(sentHeaders().get("pragma")).toBe("no-cache"); + expect(sentHeaders().get("x-fixture")).toBe("request"); + expect(sentHeaders().get("x-other")).toBe("options"); + expect(headers.has("cache-control")).toBe(false); + expect(request.headers.has("cache-control")).toBe(false); + }); + + it("lets the options request no-store for an otherwise cacheable Request", async () => { + await createService({}).nativeFetch(new Request(url, { cache: "force-cache" }), { cache: "no-store" }); + + expect(sentHeaders().get("cache-control")).toBe("no-cache, no-store"); + }); + + it("lets an explicit default cache mode override a no-store Request", async () => { + await createService({}).nativeFetch(new Request(url, { cache: "no-store" }), { cache: "default" }); + + expect(sentHeaders().has("cache-control")).toBe(false); + expect(sentHeaders().has("pragma")).toBe(false); + }); + + it.each([undefined, "default", "no-cache", "reload", "force-cache"] as const)( + "preserves caller headers for cache mode %s", + async (cache) => { + const headers = { "Cache-Control": "max-age=60", Pragma: "fixture" }; + + await createService({}).nativeFetch(url, { cache, headers }); + + expect(mocks.requestUrl.mock.calls[0][0].headers).toEqual(headers); + } + ); + + it("does not add cache directives to an ordinary request", async () => { + await createService({}).nativeFetch(url); + + expect(sentHeaders().has("cache-control")).toBe(false); + expect(sentHeaders().has("pragma")).toBe(false); + }); + + it.each(["POST", "PUT", "DELETE"])("preserves a no-store %s request", async (method) => { + const headers = { "Cache-Control": "no-transform" }; + + await createService({}).nativeFetch(url, { method, cache: "no-store", headers }); + + expect(mocks.requestUrl.mock.calls[0][0]).toMatchObject({ url, method, headers }); + expect(sentHeaders().has("pragma")).toBe(false); + }); + + it("uses the effective method from the options", async () => { + await createService({}).nativeFetch(new Request(url, { cache: "no-store" }), { method: "POST" }); + + expect(sentHeaders().has("cache-control")).toBe(false); + expect(sentHeaders().has("pragma")).toBe(false); + }); +}); diff --git a/src/modules/services/SvelteDialogSession.ts b/src/modules/services/SvelteDialogSession.ts index 3643aae7..07435b50 100644 --- a/src/modules/services/SvelteDialogSession.ts +++ b/src/modules/services/SvelteDialogSession.ts @@ -34,6 +34,7 @@ export interface SvelteDialogSessionOptions; initialData?: TInitial; renderer?: SvelteDialogRenderer; + signal?: AbortSignal; } const svelteDialogRenderer: SvelteDialogRenderer = { @@ -46,11 +47,7 @@ const svelteDialogRenderer: SvelteDialogRenderer = { }; /** Owns one Svelte dialogue result and mount lifecycle for a concrete host surface. */ -export class SvelteDialogSession< - TResult, - TInitial, - TContext extends ServiceContext = ServiceContext, -> { +export class SvelteDialogSession { private readonly surface: SvelteDialogSurface; private readonly context: TContext; private readonly dependencies: SvelteDialogManagerDependencies; @@ -58,11 +55,13 @@ export class SvelteDialogSession< private readonly component: ComponentHasResult; private readonly initialData?: TInitial; private readonly renderer: SvelteDialogRenderer; + private readonly signal?: AbortSignal; private result?: TResult; private pendingResult?: PromiseWithResolvers; private mountedDialog?: MountedDialog; private unregisterUnload?: () => void; + private unregisterAbort?: () => void; constructor(options: SvelteDialogSessionOptions) { this.surface = options.surface; @@ -72,10 +71,14 @@ export class SvelteDialogSession< this.component = options.component; this.initialData = options.initialData; this.renderer = options.renderer ?? svelteDialogRenderer; + this.signal = options.signal; } onOpen(): void { + if (this.signal?.aborted) throw this.signal.reason ?? new Error("Plug-in unloaded"); this.surface.contentEl.replaceChildren(); + this.unregisterAbort?.(); + this.unregisterAbort = undefined; this.unregisterUnload?.(); this.unregisterUnload = undefined; if (this.pendingResult) { @@ -85,14 +88,23 @@ export class SvelteDialogSession< this.result = undefined; const pendingResult = promiseWithResolvers(); this.pendingResult = pendingResult; - this.unregisterUnload = this.context.events.onceEvent(EVENT_PLUGIN_UNLOADED, () => { + const cancel = (reason: unknown) => { if (this.pendingResult !== pendingResult) { return; } this.pendingResult = undefined; - pendingResult.reject(new Error("Plug-in unloaded")); + pendingResult.reject(reason); this.surface.close(); + }; + this.unregisterUnload = this.context.events.onceEvent(EVENT_PLUGIN_UNLOADED, () => { + cancel(new Error("Plug-in unloaded")); }); + const signal = this.signal; + if (signal) { + const onAbort = () => cancel(signal.reason ?? new Error("Plug-in unloaded")); + signal.addEventListener("abort", onAbort, { once: true }); + this.unregisterAbort = () => signal.removeEventListener("abort", onAbort); + } this.mountedDialog = this.renderer.mount(this.dialogHost, this.surface.contentEl, { onSetupContext: (props: DialogSvelteComponentBaseProps) => { @@ -124,6 +136,8 @@ export class SvelteDialogSession< } onClose(): void { + this.unregisterAbort?.(); + this.unregisterAbort = undefined; this.unregisterUnload?.(); this.unregisterUnload = undefined; this.pendingResult?.resolve(this.result); diff --git a/src/serviceFeatures/basicCommands.ts b/src/serviceFeatures/basicCommands.ts index 2bd622e5..8c6c3095 100644 --- a/src/serviceFeatures/basicCommands.ts +++ b/src/serviceFeatures/basicCommands.ts @@ -1,3 +1,4 @@ +import { createCommandCheck } from "@vrtmrz/obsidian-plugin-kit/command"; import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger"; import { fireAndForget } from "octagonal-wheels/promises"; import type { NecessaryServices } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; @@ -69,14 +70,12 @@ export function useBasicCommandsFeature(host: BasicCommandsHost): void { services.API.addCommand({ id: "livesync-dump", name: $msg("Copy database information for the active file"), - checkCallback: (checking) => { - const file = services.vault.getActiveFilePath(); - if (!file) return false; - if (!checking) { + checkCallback: createCommandCheck( + () => services.vault.getActiveFilePath(), + (file) => { fireAndForget(() => copyFileDatabaseInfo(createFileDatabaseInfoCore(host), file)); } - return true; - }, + ), }); services.API.addCommand({ @@ -115,13 +114,12 @@ export function useBasicCommandsFeature(host: BasicCommandsHost): void { services.API.addCommand({ id: "livesync-scan-files", name: "Scan storage and database again", - checkCallback: (checking) => { - if (!services.setting.currentSettings().useAdvancedMode) return false; - if (!checking) { + checkCallback: createCommandCheck( + () => services.setting.currentSettings().useAdvancedMode, + () => { fireAndForget(() => services.vault.scanVault(true)); } - return true; - }, + ), }); services.API.addCommand({ @@ -135,13 +133,12 @@ export function useBasicCommandsFeature(host: BasicCommandsHost): void { services.API.addCommand({ id: "livesync-abortsync", name: "Abort synchronization immediately", - checkCallback: (checking) => { - if (!services.setting.currentSettings().useAdvancedMode) return false; - if (!checking) { + checkCallback: createCommandCheck( + () => services.setting.currentSettings().useAdvancedMode, + () => { fireAndForget(() => services.replication.stopActiveTransfer()); } - return true; - }, + ), }); return Promise.resolve(true); diff --git a/src/serviceFeatures/basicCommands.unit.spec.ts b/src/serviceFeatures/basicCommands.unit.spec.ts index 356caa2a..0e10adf0 100644 --- a/src/serviceFeatures/basicCommands.unit.spec.ts +++ b/src/serviceFeatures/basicCommands.unit.spec.ts @@ -228,6 +228,49 @@ describe("useBasicCommandsFeature", () => { expect(adapter).not.toBe(fixture.host); }); + it("rechecks availability on execution and captures the current file only once", async () => { + const fixture = createFixture(); + await initialise(fixture); + vi.mocked(copyFileDatabaseInfo).mockClear(); + const dump = fixture.getCommand("livesync-dump"); + expect(dump.checkCallback?.(true)).toBe(true); + fixture.services.vault.getActiveFilePath.mockReturnValue(undefined); + expect(dump.checkCallback?.(false)).toBe(false); + expect(copyFileDatabaseInfo).not.toHaveBeenCalled(); + + fixture.services.vault.getActiveFilePath + .mockReset() + .mockReturnValueOnce("selected.md") + .mockReturnValue("later.md"); + expect(dump.checkCallback?.(false)).toBe(true); + await vi.waitFor(() => expect(copyFileDatabaseInfo).toHaveBeenCalledOnce()); + expect(fixture.services.vault.getActiveFilePath).toHaveBeenCalledOnce(); + expect(vi.mocked(copyFileDatabaseInfo).mock.calls[0][1]).toBe("selected.md"); + + fixture.settings.useAdvancedMode = true; + for (const id of ["livesync-scan-files", "livesync-abortsync"]) { + expect(fixture.getCommand(id).checkCallback?.(true)).toBe(true); + } + fixture.settings.useAdvancedMode = false; + for (const id of ["livesync-scan-files", "livesync-abortsync"]) { + expect(fixture.getCommand(id).checkCallback?.(false)).toBe(false); + } + expect(fixture.services.vault.scanVault).not.toHaveBeenCalled(); + expect(fixture.services.replication.stopActiveTransfer).not.toHaveBeenCalled(); + }); + + it("propagates an availability failure without running the command", async () => { + const fixture = createFixture(); + await initialise(fixture); + vi.mocked(copyFileDatabaseInfo).mockClear(); + const failure = new Error("Vault unavailable"); + fixture.services.vault.getActiveFilePath.mockImplementation(() => { + throw failure; + }); + expect(() => fixture.getCommand("livesync-dump").checkCallback?.(false)).toThrow(failure); + expect(copyFileDatabaseInfo).not.toHaveBeenCalled(); + }); + it("commits pending file events from the batch command", async () => { const fixture = createFixture(); await initialise(fixture); diff --git a/src/serviceFeatures/compatibilityReview.ts b/src/serviceFeatures/compatibilityReview.ts index 7c79f98f..bdc7b90a 100644 --- a/src/serviceFeatures/compatibilityReview.ts +++ b/src/serviceFeatures/compatibilityReview.ts @@ -1,6 +1,9 @@ import { fireAndForget } from "octagonal-wheels/promises"; import { VER } from "@vrtmrz/livesync-commonlib/compat/common/types"; -import type { LiveSyncCore } from "@/main.ts"; +import type { + NecessaryServices, + NecessaryServicesInterfaces, +} from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; import { COMPATIBILITY_PAUSE_SETTING_MESSAGE, DATABASE_COMPATIBILITY_VERSION_KEY, @@ -24,6 +27,16 @@ export interface CompatibilityReviewUi { clearReminder(): void; } +export type CompatibilityReviewDependencies = NecessaryServicesInterfaces< + "setting" | "config" | "vault" | "control", + never +>; + +export type CompatibilityReviewHost = NecessaryServices< + "setting" | "config" | "vault" | "control" | "appLifecycle" | "API", + never +>; + export class CompatibilityReviewController { private pause: CompatibilityPause | undefined; private activeReview: Promise | undefined; @@ -31,7 +44,7 @@ export class CompatibilityReviewController { private disposed = false; constructor( - private readonly core: LiveSyncCore, + private readonly core: CompatibilityReviewDependencies, private readonly ui: CompatibilityReviewUi, private readonly currentVersion: number = VER ) {} @@ -161,7 +174,10 @@ export class CompatibilityReviewController { } } -export function useCompatibilityReview(core: LiveSyncCore, ui: CompatibilityReviewUi): CompatibilityReviewController { +export function useCompatibilityReview( + core: CompatibilityReviewHost, + ui: CompatibilityReviewUi +): CompatibilityReviewController { const controller = new CompatibilityReviewController(core, ui); core.services.appLifecycle.onSettingLoaded.addHandler(() => controller.initialise()); core.services.appLifecycle.onLayoutReady.addHandler(() => { diff --git a/src/serviceFeatures/compatibilityReview.unit.spec.ts b/src/serviceFeatures/compatibilityReview.unit.spec.ts index 44a1b05b..61a508f0 100644 --- a/src/serviceFeatures/compatibilityReview.unit.spec.ts +++ b/src/serviceFeatures/compatibilityReview.unit.spec.ts @@ -1,4 +1,8 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, expectTypeOf, it, vi } from "vitest"; +import type { + NecessaryServices, + NecessaryServicesInterfaces, +} from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; import { COMPATIBILITY_PAUSE_SETTING_MESSAGE, DATABASE_COMPATIBILITY_VERSION_KEY, @@ -101,6 +105,32 @@ function createFixture( } describe("compatibility review controller", () => { + it("accepts operation services without retaining an application-core contract", () => { + type Dependencies = NecessaryServicesInterfaces<"setting" | "config" | "vault" | "control", never>; + type ControllerHost = ConstructorParameters[0]; + + expectTypeOf().toExtend(); + expectTypeOf().toEqualTypeOf<"services" | "serviceModules">(); + expectTypeOf().toEqualTypeOf<"setting" | "config" | "vault" | "control">(); + expectTypeOf().toBeNever(); + }); + + it("composes the review from its operation services and lifecycle and command ports", () => { + type Dependencies = NecessaryServices< + "setting" | "config" | "vault" | "control" | "appLifecycle" | "API", + never + >; + type FeatureHost = Parameters[0]; + + expectTypeOf().toExtend(); + expectTypeOf().toEqualTypeOf<"services" | "serviceModules">(); + expectTypeOf().toEqualTypeOf< + "context" | "setting" | "config" | "vault" | "control" | "appLifecycle" | "API" + >(); + expectTypeOf().toBeNever(); + expectTypeOf>().toEqualTypeOf(); + }); + beforeEach(() => { vi.restoreAllMocks(); }); diff --git a/src/serviceFeatures/conflictResolution/checker.ts b/src/serviceFeatures/conflictResolution/checker.ts index 9882761d..21597d59 100644 --- a/src/serviceFeatures/conflictResolution/checker.ts +++ b/src/serviceFeatures/conflictResolution/checker.ts @@ -1,5 +1,6 @@ import { LOG_LEVEL_NOTICE, + LOG_LEVEL_VERBOSE, type FilePath, type FilePathWithPrefix, type ObsidianLiveSyncSettings, @@ -7,7 +8,7 @@ import { import type { IConflictService, IVaultService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; import type { ReactiveSource } from "octagonal-wheels/dataobject/reactive"; -import { QueueProcessor } from "octagonal-wheels/concurrency/processor"; +import { createConflictCheckQueue } from "@vrtmrz/livesync-commonlib/conflicts"; export type ConflictCheckingSettings = Pick; @@ -17,6 +18,7 @@ export interface ConflictCheckingDependencies { "getOptionalConflictCheckMethod" | "queueCheckFor" | "resolve" | "resolveByNewest" >; readonly conflictProcessQueueCount: ReactiveSource; + readonly cancelActiveResolution: (filename: FilePathWithPrefix) => void; readonly currentSettings: () => ConflictCheckingSettings; readonly vault: Pick; readonly log: LogFunction; @@ -30,25 +32,21 @@ export interface ConflictCheckingHandlers { /** Create conflict-checking handlers while retaining scheduling state privately. */ export function createConflictCheckingHandlers(dependencies: ConflictCheckingDependencies): ConflictCheckingHandlers { - const conflictQueue = new QueueProcessor( - async (filenames: FilePathWithPrefix[]) => { - const filename = filenames[0]; - return await dependencies.conflict.resolve(filename); + const conflictQueue = createConflictCheckQueue({ + async process(filename) { + try { + await dependencies.conflict.resolve(filename); + } catch (error) { + dependencies.log(`Conflict check failed: ${filename}`); + dependencies.log(error, LOG_LEVEL_VERBOSE); + } + }, + // The resolver serialises shared user interactions; independent checks + // and automatic merges can use the remaining execution slots. + concurrency: 10, + onProgress(remaining) { + dependencies.conflictProcessQueueCount.value = remaining; }, - { - suspended: false, - batchSize: 1, - // No need to limit concurrency to `1` here, subsequent process will handle it, - // and some cases do not need to be synchronised (for example, auto-merge). - // Global concurrency is limited by the resolver with the UI. - concurrentLimit: 10, - delay: 0, - keepResultUntilDownstreamConnected: false, - totalRemainingReactiveSource: dependencies.conflictProcessQueueCount, - } - ).replaceEnqueueProcessor((queue, newEntity) => { - const newQueue = [...queue].filter((entry) => entry != newEntity); - return [...newQueue, newEntity]; }); const queueCheckForIfOpen = async (file: FilePathWithPrefix): Promise => { @@ -72,11 +70,15 @@ export function createConflictCheckingHandlers(dependencies: ConflictCheckingDep // The conflict should be resolved by the newer entry. await dependencies.conflict.resolveByNewest(file); } else { + // The next batch waits for active checks to settle. Cancel the + // older same-file resolution now so its dialogue cannot hold the + // replacement behind that wait. + dependencies.cancelActiveResolution(file); conflictQueue.enqueue(file); } }; - const ensureAllProcessed = (): Promise => conflictQueue.waitForAllProcessed(); + const ensureAllProcessed = (): Promise => conflictQueue.waitForIdle(); return { queueCheckForIfOpen, diff --git a/src/serviceFeatures/conflictResolution/conflictResolution.unit.spec.ts b/src/serviceFeatures/conflictResolution/conflictResolution.unit.spec.ts index a3f618d9..30a0d279 100644 --- a/src/serviceFeatures/conflictResolution/conflictResolution.unit.spec.ts +++ b/src/serviceFeatures/conflictResolution/conflictResolution.unit.spec.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, expectTypeOf, it, vi } from "vitest"; import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService"; import { @@ -6,6 +6,7 @@ import { DEFAULT_SETTINGS, LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, + LOG_LEVEL_VERBOSE, MISSING_OR_ERROR, NOT_CONFLICTED, type FilePath, @@ -15,6 +16,12 @@ import { } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { EVENT_CONFLICT_CANCELLED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; import { NO_INTERACTION } from "@vrtmrz/livesync-commonlib/replication"; +import type { IAPIService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService"; +import { + createInstanceLogFunction, + MARK_LOG_SEPARATOR, + type LogFunction, +} from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; import type { ConflictResolutionHost } from "./index"; import { createConflictResolutionOperations, useConflictResolutionFeature } from "./index"; import type { ConflictResolutionOperationsDependencies } from "./operations"; @@ -64,13 +71,15 @@ function createHarness(options: HarnessOptions = {}) { vault: { getActiveFilePath: activeFile }, }; const serviceModules = { databaseFileAccess, fileHandler, storageAccess }; + const log = createInstanceLogFunction("SF:ConflictResolution", services.API as unknown as IAPIService); if (options.compose !== false) { - useConflictResolutionFeature({ services, serviceModules } as unknown as ConflictResolutionHost); + useConflictResolutionFeature({ services, serviceModules, log } as unknown as ConflictResolutionHost); } return { addLog, + log, conflict, context, databaseFileAccess, @@ -151,6 +160,87 @@ function createOperationsHarness() { } describe("conflict resolution serviceFeature", () => { + it("declares a borrowed logger and only the services used by conflict resolution", () => { + expectTypeOf().toEqualTypeOf< + "context" | "appLifecycle" | "conflict" | "database" | "replication" | "setting" | "vault" + >(); + expectTypeOf().toEqualTypeOf(); + }); + + it("uses the injected logger for the checker and resolver without an API service", async () => { + const harness = createHarness({ + compose: false, + settings: { checkConflictOnlyOnOpen: true }, + activeFile: "active.md" as FilePathWithPrefix, + }); + const { API, ...services } = harness.services; + const log = vi.fn(); + useConflictResolutionFeature({ + services, + serviceModules: harness.serviceModules, + log, + } as unknown as ConflictResolutionHost); + harness.databaseFileAccess.fetchEntryMeta.mockResolvedValue(false); + + await harness.conflict.queueCheckForIfOpen("other.md" as FilePathWithPrefix); + await harness.conflict.resolveByNewest("missing.md" as FilePathWithPrefix); + + expect(log).toHaveBeenNthCalledWith( + 1, + "other.md is conflicted, merging process has been postponed.", + LOG_LEVEL_NOTICE + ); + expect(log).toHaveBeenNthCalledWith(2, "Could not get current revision of missing.md"); + expect(API.addLog).not.toHaveBeenCalled(); + }); + + it("keeps separately selected loggers isolated when legacy API services are shared", async () => { + const first = createHarness({ compose: false }); + const second = createHarness({ compose: false }); + const firstLog = vi.fn(); + const secondLog = vi.fn(); + const sharedAPI = first.services.API; + useConflictResolutionFeature({ + services: first.services, + serviceModules: first.serviceModules, + log: firstLog, + } as unknown as ConflictResolutionHost); + useConflictResolutionFeature({ + services: { ...second.services, API: sharedAPI }, + serviceModules: second.serviceModules, + log: secondLog, + } as unknown as ConflictResolutionHost); + first.databaseFileAccess.fetchEntryMeta.mockResolvedValue(false); + second.databaseFileAccess.fetchEntryMeta.mockResolvedValue(false); + + await first.conflict.resolveByNewest("first.md" as FilePathWithPrefix); + await second.conflict.resolveByNewest("second.md" as FilePathWithPrefix); + + expect(firstLog).toHaveBeenCalledExactlyOnceWith("Could not get current revision of first.md"); + expect(secondLog).toHaveBeenCalledExactlyOnceWith("Could not get current revision of second.md"); + expect(sharedAPI.addLog).not.toHaveBeenCalled(); + }); + + it("retains the prefix, level, and key selected by the host logger", async () => { + const harness = createHarness({ compose: false }); + const { API, ...services } = harness.services; + useConflictResolutionFeature({ + services, + serviceModules: harness.serviceModules, + log: harness.log, + } as unknown as ConflictResolutionHost); + + await harness.conflict.resolveByDeletingRevision("same.md" as FilePathWithPrefix, "1-old", "same"); + await harness.conflict.resolveAllConflictedFilesByNewerOnes(); + + expect(API.addLog).toHaveBeenCalledWith( + `[SF:ConflictResolution]${MARK_LOG_SEPARATOR} same.md has been merged automatically`, + LOG_LEVEL_INFO, + "" + ); + expect(API.addLog).toHaveBeenLastCalledWith("Done!", LOG_LEVEL_NOTICE, "resolveAllConflictedFilesByNewerOnes"); + }); + it("keeps resolver operations on narrow collaborators and extension seams", async () => { const harness = createOperationsHarness(); const path = "same.md" as FilePathWithPrefix; @@ -557,6 +647,7 @@ describe("conflict resolution serviceFeature", () => { useConflictResolutionFeature({ services: harness.services, serviceModules: harness.serviceModules, + log: harness.log, } as unknown as ConflictResolutionHost); for (const registration of registrations) { @@ -604,6 +695,86 @@ describe("conflict resolution serviceFeature", () => { expect(harness.tryAutoMerge).toHaveBeenCalledWith(path, true); }); + it("replaces an active same-file dialogue when a later check enters the queue", async () => { + const path = "queued-dialogue-replacement.md" as FilePathWithPrefix; + const harness = createHarness({ settings: { syncAfterMerge: false } }); + harness.tryAutoMerge.mockResolvedValue({ + rightRev: "2-conflict", + leftLeaf: leaf("2-current", "Current\n", 2), + rightLeaf: leaf("2-conflict", "Conflict\n", 3), + }); + let finishFirst: (() => void) | undefined; + const resolveByUserInteraction = vi + .fn() + .mockImplementationOnce( + () => + new Promise((resolve) => { + finishFirst = () => resolve(false); + }) + ) + .mockResolvedValue(false); + harness.conflict.resolveByUserInteraction.addHandler(resolveByUserInteraction); + const subscription = new AbortController(); + harness.context.events.onEvent( + EVENT_CONFLICT_CANCELLED, + (filename) => { + if (filename === path) finishFirst?.(); + }, + { signal: subscription.signal } + ); + + try { + await harness.conflict.queueCheckFor(path); + await vi.waitFor(() => expect(resolveByUserInteraction).toHaveBeenCalledOnce()); + + await harness.conflict.queueCheckFor(path); + + await vi.waitFor(() => expect(resolveByUserInteraction).toHaveBeenCalledTimes(2)); + await expect(harness.conflict.ensureAllProcessed()).resolves.toBe(true); + expect(harness.conflict.conflictProcessQueueCount.value).toBe(0); + } finally { + subscription.abort(); + finishFirst?.(); + await harness.conflict.ensureAllProcessed(); + } + }); + + it("does not open an outdated dialogue when a later check is queued during inspection", async () => { + const path = "queued-during-inspection.md" as FilePathWithPrefix; + const harness = createHarness(); + const manualConflict = { + rightRev: "2-conflict", + leftLeaf: leaf("2-current", "Current\n", 2), + rightLeaf: leaf("2-conflict", "Conflict\n", 3), + }; + let finishInspection!: (result: typeof manualConflict) => void; + harness.tryAutoMerge + .mockImplementationOnce( + () => + new Promise((resolve) => { + finishInspection = resolve; + }) + ) + .mockResolvedValueOnce({ ok: NOT_CONFLICTED }); + const resolveByUserInteraction = vi.fn(async () => false); + harness.conflict.resolveByUserInteraction.addHandler(resolveByUserInteraction); + + try { + await harness.conflict.queueCheckFor(path); + await vi.waitFor(() => expect(harness.tryAutoMerge).toHaveBeenCalledOnce()); + + await harness.conflict.queueCheckFor(path); + finishInspection(manualConflict); + await expect(harness.conflict.ensureAllProcessed()).resolves.toBe(true); + + expect(resolveByUserInteraction).not.toHaveBeenCalled(); + expect(harness.tryAutoMerge).toHaveBeenCalledTimes(2); + } finally { + finishInspection?.(manualConflict); + await harness.conflict.ensureAllProcessed(); + } + }); + it("limits concurrent conflict checks and reports queued and active work", async () => { const paths = Array.from({ length: 11 }, (_, index) => `concurrent-${index}.md` as FilePathWithPrefix); const harness = createHarness(); @@ -724,6 +895,56 @@ describe("conflict resolution serviceFeature", () => { } }); + it("continues checking accepted and later paths after one resolver rejects", async () => { + const harness = createHarness(); + const failure = new Error("Conflict check failed"); + const resolve = vi.fn().mockRejectedValueOnce(failure).mockResolvedValue(undefined); + harness.conflict.resolve.setHandler(resolve, true); + const first = "failed.md" as FilePathWithPrefix; + const second = "accepted.md" as FilePathWithPrefix; + const later = "later.md" as FilePathWithPrefix; + + await Promise.all([harness.conflict.queueCheckFor(first), harness.conflict.queueCheckFor(second)]); + await expect(harness.conflict.ensureAllProcessed()).resolves.toBe(true); + await harness.conflict.queueCheckFor(later); + await expect(harness.conflict.ensureAllProcessed()).resolves.toBe(true); + + expect(resolve.mock.calls.map(([path]) => path)).toEqual([first, second, later]); + expect(harness.addLog).toHaveBeenCalledWith( + `[SF:ConflictResolution]${MARK_LOG_SEPARATOR} Conflict check failed: ${first}`, + LOG_LEVEL_INFO, + "" + ); + expect(harness.addLog).toHaveBeenCalledWith(failure, LOG_LEVEL_VERBOSE, ""); + expect(harness.conflict.conflictProcessQueueCount.value).toBe(0); + }); + + it("drains requeued automatic checks when every resolver slot is occupied", async () => { + const harness = createHarness({ settings: { syncAfterMerge: false } }); + const paths = Array.from({ length: 10 }, (_, index) => `requeue-${index}.md` as FilePathWithPrefix); + const firstChecks = new Set(); + const finishers: Array<() => void> = []; + harness.tryAutoMerge.mockImplementation(async (path: FilePathWithPrefix) => { + if (firstChecks.has(path)) return { ok: NOT_CONFLICTED }; + firstChecks.add(path); + await new Promise((resolve) => { + finishers.push(resolve); + }); + return { ok: AUTO_MERGED }; + }); + + await Promise.all(paths.map((path) => harness.conflict.queueCheckFor(path))); + await vi.waitFor(() => expect(finishers).toHaveLength(10)); + const completion = harness.conflict.ensureAllProcessed(); + for (const finish of finishers) finish(); + + await expect(completion).resolves.toBe(true); + for (const path of paths) { + expect(harness.tryAutoMerge.mock.calls.filter(([checked]) => checked === path)).toHaveLength(2); + } + expect(harness.conflict.conflictProcessQueueCount.value).toBe(0); + }); + it("honours optional conflict handlers before entering the check queue", async () => { const path = "optional.md" as FilePathWithPrefix; const harness = createHarness(); @@ -788,8 +1009,10 @@ describe("conflict resolution serviceFeature", () => { it("resolves an identical pair, emits cancellation, and rechecks after merging", async () => { const path = "independently-created.md" as FilePathWithPrefix; const harness = createHarness({ settings: { syncAfterMerge: false } }); - const cancelled: FilePathWithPrefix[] = []; - harness.context.events.onEvent(EVENT_CONFLICT_CANCELLED, (filename) => cancelled.push(filename)); + const cancelled: Array<{ filename: FilePathWithPrefix; checks: number }> = []; + harness.context.events.onEvent(EVENT_CONFLICT_CANCELLED, (filename) => { + cancelled.push({ filename, checks: harness.tryAutoMerge.mock.calls.length }); + }); const leftLeaf = leaf("1-left", "Same content\n", 1000); const rightLeaf = leaf("1-right", "Same content\n", 2000); harness.tryAutoMerge @@ -806,7 +1029,15 @@ describe("conflict resolution serviceFeature", () => { expect(harness.fileHandler.deleteRevisionFromDB).toHaveBeenCalledWith(path, "1-left"); expect(harness.fileHandler.dbToStorage).toHaveBeenCalledWith(path, path, true); - expect(cancelled).toEqual([path, path]); + // A queued recheck may also cancel its finishing resolver. Keep both + // merge and final-inspection notifications covered without fixing the count. + expect(cancelled).toEqual( + expect.arrayContaining([ + { filename: path, checks: 1 }, + { filename: path, checks: 2 }, + ]) + ); + expect(cancelled.every(({ filename }) => filename === path)).toBe(true); expect(harness.tryAutoMerge).toHaveBeenCalledTimes(2); expect(AUTO_MERGED).toBeDefined(); }); diff --git a/src/serviceFeatures/conflictResolution/index.ts b/src/serviceFeatures/conflictResolution/index.ts index 2d446681..5ca0293b 100644 --- a/src/serviceFeatures/conflictResolution/index.ts +++ b/src/serviceFeatures/conflictResolution/index.ts @@ -1,25 +1,23 @@ -import type { NecessaryServices } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; -import { createInstanceLogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; -import type { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService"; -import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context"; +import { + createServiceFeature, + type NecessaryServices, +} from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; +import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; +import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub"; import { createConflictCheckingHandlers } from "./checker"; import { createConflictResolutionOperations } from "./operations"; -type ConflictResolutionServices = NecessaryServices< - "API" | "appLifecycle" | "conflict" | "database" | "replication" | "setting" | "vault", - "databaseFileAccess" | "fileHandler" | "storageAccess" ->; - -export type ConflictResolutionHost = ConflictResolutionServices & { - readonly services: ConflictResolutionServices["services"] & { - readonly conflict: InjectableConflictService; - }; +export type ConflictResolutionHost = NecessaryServices< + "appLifecycle" | "conflict" | "database" | "replication" | "setting" | "vault", + "databaseFileAccess" | "fileHandler" | "storageAccess", + InjectableServiceHub +> & { + readonly log: LogFunction; }; -/** Compose the host-neutral conflict checker and resolver handlers. */ -export function useConflictResolutionFeature(host: ConflictResolutionHost): void { - const { services, serviceModules } = host; - const log = createInstanceLogFunction("SF:ConflictResolution", services.API); +/** Compose the host-neutral conflict handlers with the host's borrowed logger. */ +export const useConflictResolutionFeature = createServiceFeature((host: ConflictResolutionHost): void => { + const { services, serviceModules, log } = host; const operations = createConflictResolutionOperations({ events: services.context.events, databaseFileAccess: serviceModules.databaseFileAccess, @@ -36,6 +34,7 @@ export function useConflictResolutionFeature(host: ConflictResolutionHost): void const checking = createConflictCheckingHandlers({ conflict: services.conflict, conflictProcessQueueCount: services.conflict.conflictProcessQueueCount, + cancelActiveResolution: operations.cancelActiveResolution, currentSettings: () => services.setting.currentSettings(), vault: services.vault, log, @@ -48,7 +47,7 @@ export function useConflictResolutionFeature(host: ConflictResolutionHost): void services.conflict.resolve.setHandler(operations.resolve); services.conflict.resolveByNewest.setHandler(operations.resolveByNewest); services.conflict.resolveAllConflictedFilesByNewerOnes.setHandler(operations.resolveAllConflictedFilesByNewerOnes); -} +}); export { createConflictCheckingHandlers } from "./checker"; export type { ConflictCheckingDependencies, ConflictCheckingHandlers } from "./checker"; diff --git a/src/serviceFeatures/conflictResolution/operations.ts b/src/serviceFeatures/conflictResolution/operations.ts index 297e214d..afb974f8 100644 --- a/src/serviceFeatures/conflictResolution/operations.ts +++ b/src/serviceFeatures/conflictResolution/operations.ts @@ -27,7 +27,7 @@ import type { LiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context"; import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess"; import { EVENT_CONFLICT_CANCELLED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; import { NO_INTERACTION } from "@vrtmrz/livesync-commonlib/replication"; -import { isLockAcquired, serialized } from "octagonal-wheels/concurrency/lock"; +import { isLockAcquired, withLock } from "octagonal-wheels/concurrency/lock"; import diff_match_patch from "diff-match-patch"; import { stripAllPrefixes, isPlainText } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path"; @@ -54,6 +54,7 @@ export interface ConflictResolutionOperationsDependencies { } export interface ConflictResolutionOperations { + readonly cancelActiveResolution: (filename: FilePathWithPrefix) => void; readonly resolveByDeletingRevision: ( path: FilePathWithPrefix, deleteRevision: string, @@ -72,6 +73,13 @@ export function createConflictResolutionOperations( const latestResolveRequestByFilename = new Map(); let nextResolveRequestId = 0; + const cancelActiveResolution = (filename: FilePathWithPrefix): void => { + latestResolveRequestByFilename.delete(filename); + if (isLockAcquired(`conflict-resolve:${filename}`)) { + dependencies.events.emitEvent(EVENT_CONFLICT_CANCELLED, filename); + } + }; + const resolveByDeletingRevision = async ( path: FilePathWithPrefix, deleteRevision: string, @@ -184,7 +192,7 @@ export function createConflictResolutionOperations( // per-file lock. Dialogues for other paths remain untouched. dependencies.events.emitEvent(EVENT_CONFLICT_CANCELLED, filename); } - return await serialized(serialisationKey, async () => { + return await withLock(serialisationKey, async () => { if (latestResolveRequestByFilename.get(filename) !== requestId) { return; } @@ -299,6 +307,7 @@ export function createConflictResolutionOperations( }; return { + cancelActiveResolution, resolveByDeletingRevision, checkConflictAndPerformAutoMerge, resolve, diff --git a/src/serviceFeatures/interactiveConflictResolution/operations.ts b/src/serviceFeatures/interactiveConflictResolution/operations.ts index 4c583e31..3196b126 100644 --- a/src/serviceFeatures/interactiveConflictResolution/operations.ts +++ b/src/serviceFeatures/interactiveConflictResolution/operations.ts @@ -22,7 +22,7 @@ import type { import type { LiveSyncLocalDB } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB"; import type { LiveSyncEventHub } from "@vrtmrz/livesync-commonlib/context"; import { fireAndForget } from "octagonal-wheels/promises"; -import { serialized } from "octagonal-wheels/concurrency/lock"; +import { withLock } from "octagonal-wheels/concurrency/lock"; import { NO_INTERACTION } from "@vrtmrz/livesync-commonlib/replication"; import { displayRev } from "@vrtmrz/livesync-commonlib/compat/common/utils"; import { EVENT_CONFLICT_CANCELLED, EVENT_ON_UNRESOLVED_ERROR } from "@/common/events.ts"; @@ -160,7 +160,7 @@ export function createInteractiveConflictResolutionOperations( // UI for resolving different files should proceed one-by-one. A newer // request for the active file replaces its dialogue instead of waiting // behind a comparison which is already stale. - return await serialized(dialogueSerialisationKey, async () => { + return await withLock(dialogueSerialisationKey, async () => { if (disposed || latestRequestByFilename.get(filename) !== requestId) { return false; } diff --git a/src/serviceFeatures/onLayoutReady/enablei18n.ts b/src/serviceFeatures/onLayoutReady/enablei18n.ts index 5a83eafb..74f367eb 100644 --- a/src/serviceFeatures/onLayoutReady/enablei18n.ts +++ b/src/serviceFeatures/onLayoutReady/enablei18n.ts @@ -1,9 +1,12 @@ import { getLanguage, Notice, requireApiVersion } from "@/deps"; +import type { NecessaryServices } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; import { createServiceFeature } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; import { SUPPORTED_I18N_LANGS, type I18N_LANGS } from "@/common/rosetta"; import { $msg, __onMissingTranslation, setLang } from "@/common/translation"; import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger"; +type EnableI18nHost = NecessaryServices<"setting" | "API" | "appLifecycle", never>; + function tryGetLanguage(onError: (error: unknown) => void) { if (requireApiVersion("1.8.7")) { try { @@ -48,7 +51,8 @@ class ObsidianLanguageAppliedNotice { } } -export const enableI18nFeature = createServiceFeature(async ({ services: { setting, API, appLifecycle } }) => { +export const enableI18nFeature = createServiceFeature(async (host: EnableI18nHost) => { + const { setting, API, appLifecycle } = host.services; // Clear missing translation handler to avoid unnecessary warnings. __onMissingTranslation(() => {}); let isChanged = false; diff --git a/src/serviceFeatures/onLayoutReady/enablei18n.unit.spec.ts b/src/serviceFeatures/onLayoutReady/enablei18n.unit.spec.ts index 8f88eb62..960c414d 100644 --- a/src/serviceFeatures/onLayoutReady/enablei18n.unit.spec.ts +++ b/src/serviceFeatures/onLayoutReady/enablei18n.unit.spec.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, expectTypeOf, it, vi } from "vitest"; const noticeState = vi.hoisted(() => ({ instances: [] as Array<{ hide: ReturnType; duration: number }>, @@ -63,6 +63,14 @@ describe("automatic display language", () => { vi.restoreAllMocks(); }); + it("declares only the services it needs and returns a boolean promise", () => { + type FeatureHost = Parameters[0]; + + expectTypeOf().toEqualTypeOf<"context" | "setting" | "API" | "appLifecycle">(); + expectTypeOf().toBeNever(); + expectTypeOf>().toEqualTypeOf>(); + }); + it("lets start-up continue and opens translation details only from a persistent Notice", async () => { const settings = { displayLanguage: "" }; const applyPartial = vi.fn(async (partial: Partial) => Object.assign(settings, partial)); diff --git a/src/serviceFeatures/replication/ReplicateResultProcessor.counting.unit.spec.ts b/src/serviceFeatures/replication/ReplicateResultProcessor.counting.unit.spec.ts new file mode 100644 index 00000000..2a76766c --- /dev/null +++ b/src/serviceFeatures/replication/ReplicateResultProcessor.counting.unit.spec.ts @@ -0,0 +1,46 @@ +import { reactiveSource } from "octagonal-wheels/dataobject/reactive"; +import { promiseWithResolvers } from "octagonal-wheels/promises"; +import { describe, expect, it } from "vitest"; +import { ReplicateResultProcessor } from "./ReplicateResultProcessor"; + +describe("ReplicateResultProcessor.withCounting", () => { + it("counts overlapping callbacks until each settles and preserves results and errors", async () => { + const originalCount = 7; + const count = reactiveSource(originalCount); + const processor = new ReplicateResultProcessor({} as never); + const firstResult = { task: "first" }; + const firstResultGate = promiseWithResolvers(); + const secondResultGate = promiseWithResolvers(); + const firstStarted = promiseWithResolvers(); + const secondStarted = promiseWithResolvers(); + const failure = new Error("second callback failed"); + let firstCountAtStart = 0; + let secondCountAtStart = 0; + + const first = processor.withCounting(async () => { + firstCountAtStart = count.value; + firstStarted.resolve(undefined); + return firstResultGate.promise; + }, count); + await firstStarted.promise; + expect(firstCountAtStart).toBe(originalCount + 1); + expect(count.value).toBe(originalCount + 1); + + const second = processor.withCounting(async () => { + secondCountAtStart = count.value; + secondStarted.resolve(undefined); + return secondResultGate.promise; + }, count); + await secondStarted.promise; + expect(secondCountAtStart).toBe(originalCount + 2); + expect(count.value).toBe(originalCount + 2); + + firstResultGate.resolve(firstResult); + await expect(first).resolves.toBe(firstResult); + expect(count.value).toBe(originalCount + 1); + + secondResultGate.reject(failure); + await expect(second).rejects.toBe(failure); + expect(count.value).toBe(originalCount); + }); +}); diff --git a/src/serviceFeatures/replication/ReplicateResultProcessor.ts b/src/serviceFeatures/replication/ReplicateResultProcessor.ts index 8c481064..a45d3658 100644 --- a/src/serviceFeatures/replication/ReplicateResultProcessor.ts +++ b/src/serviceFeatures/replication/ReplicateResultProcessor.ts @@ -25,7 +25,8 @@ import { } from "@vrtmrz/livesync-commonlib/compat/common/logger"; import { fireAndForget, isAnyNote, throttle } from "@vrtmrz/livesync-commonlib/compat/common/utils"; import { Semaphore } from "octagonal-wheels/concurrency/semaphore_v2"; -import { serialized } from "octagonal-wheels/concurrency/lock"; +import { withCounting } from "octagonal-wheels/concurrency/counting"; +import { withLock } from "octagonal-wheels/concurrency/lock"; import type { ReactiveSource } from "octagonal-wheels/dataobject/reactive_v2"; import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; import { isNotFoundError } from "@vrtmrz/livesync-commonlib/compat/common/utils.doc"; @@ -223,12 +224,7 @@ export class ReplicateResultProcessor { * @returns result of the procedure */ async withCounting(proc: () => Promise, countValue: ReactiveSource) { - countValue.value++; - try { - return await proc(); - } finally { - countValue.value--; - } + return withCounting(countValue, proc); } /** @@ -538,7 +534,7 @@ export class ReplicateResultProcessor { private _applyToDatabase(doc_: PouchDB.Core.ExistingDocument) { const dbDoc = doc_ as LoadedEntry; // It has no `data` const path = this.getPath(dbDoc); - return serialized(`replication-process:${dbDoc._id}`, async () => { + return withLock(`replication-process:${dbDoc._id}`, async () => { const docNote = `${path} (${shortenId(dbDoc._id)}, ${shortenRev(dbDoc._rev)})`; const isRequired = await this.checkIsChangeRequiredForDatabaseProcessing(dbDoc); if (!isRequired) { diff --git a/src/serviceFeatures/replication/centralCompatibilityRecovery.ts b/src/serviceFeatures/replication/centralCompatibilityRecovery.ts index 19148427..b3753568 100644 --- a/src/serviceFeatures/replication/centralCompatibilityRecovery.ts +++ b/src/serviceFeatures/replication/centralCompatibilityRecovery.ts @@ -1,48 +1,22 @@ -import { - VERSIONING_DOCID, - type EntryDoc, - type ObsidianLiveSyncSettings, -} from "@vrtmrz/livesync-commonlib/compat/common/types"; +import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { assessTweakCompatibility } from "@vrtmrz/livesync-commonlib/settings"; import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, Logger } from "octagonal-wheels/common/logger"; -import { skipIfDuplicated } from "octagonal-wheels/concurrency/lock"; -import { balanceChunkPurgedDBs, purgeUnreferencedChunks } from "@vrtmrz/livesync-commonlib/compat/pouchdb/chunks"; -import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; import { CENTRAL_COMPATIBILITY_REJECTION_REASONS, - REPLICATION_PROGRESS_PRESENTATIONS, - assessRemoteFeatureDocument, - describeRemoteFeatureRejection, type ReplicatorInstance, type ReplicationFailureRequest, } from "@vrtmrz/livesync-commonlib/replication"; import { $msg } from "@/common/translation"; -import { usesLegacyIndexedDBAdapter } from "@/common/compatibilitySettings"; import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore"; -import type PouchDB from "pouchdb-core"; - -async function canInterpretCleanupDatabase(db: PouchDB.Database): Promise { - try { - const assessment = assessRemoteFeatureDocument(await db.get(VERSIONING_DOCID)); - if (assessment.status === "supported" || assessment.status === "older-generation") return true; - Logger(`Database cleanup cancelled: ${describeRemoteFeatureRejection(assessment)}`, LOG_LEVEL_NOTICE); - } catch (error) { - Logger("Database cleanup cancelled: feature compatibility could not be checked.", LOG_LEVEL_NOTICE); - Logger(error, LOG_LEVEL_INFO); - } - return false; -} type CentralCompatibilityRecoveryServices = Pick< LiveSyncBaseCore["services"], - "API" | "appLifecycle" | "replicator" | "setting" | "tweakValue" + "appLifecycle" | "replicator" | "setting" | "tweakValue" >; /** Collaborators for applying a compatibility decision to its failed publication. */ interface CentralCompatibilityRecoveryContext { readonly confirm: LiveSyncBaseCore["confirm"]; - /** Obtain the database only when recovery runs, after initialisation or reset. */ - readonly getLocalDatabase: () => LiveSyncBaseCore["localDatabase"]; readonly rebuilder: LiveSyncBaseCore["rebuilder"]; readonly services: CentralCompatibilityRecoveryServices; } @@ -72,89 +46,11 @@ function canMarkRemoteResolved(replicator: ReplicatorInstance): replicator is Re * replacement; the failure result is never re-read from the current instance. */ export function createCentralCompatibilityRecovery(context: CentralCompatibilityRecoveryContext) { - async function reconcileCleanedRemote( - showProgress: boolean, - setting: ObsidianLiveSyncSettings, - expectedContext: ReplicationFailureRequest["context"] - ) { - Logger("The remote database has been cleaned.", showProgress ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO); - await skipIfDuplicated("cleanup", async () => { - if (!(await canInterpretCleanupDatabase(context.getLocalDatabase().localDatabase))) return; - const count = await purgeUnreferencedChunks(context.getLocalDatabase().localDatabase, true); - const message = `The remote database has been cleaned up. -To synchronize, this device must be also cleaned up. ${count} chunk(s) will be erased from this device. -However, If there are many chunks to be deleted, maybe fetching again is faster. -We will lose the history of this device if we fetch the remote database again. -Even if you choose to clean up, you will see this option again if you exit Obsidian and then synchronise again.`; - const CHOICE_FETCH = "Fetch again"; - const CHOICE_CLEAN = "Cleanup"; - const CHOICE_DISMISS = "Dismiss"; - const selected = await context.confirm.confirmWithMessage( - "Cleaned", - message, - [CHOICE_FETCH, CHOICE_CLEAN, CHOICE_DISMISS], - CHOICE_DISMISS, - 30 - ); - if (selected == CHOICE_FETCH) { - await context.rebuilder.$performRebuildDB("localOnly"); - } - if (selected != CHOICE_CLEAN) return; - - await context.services.replicator.runBoundedRemoteActivity( - () => - context.services.replicator.runWithActiveReplicatorContext(async (activeContext) => { - if (activeContext !== expectedContext) return; - const replicator = activeContext.replicator; - if (!(replicator instanceof LiveSyncCouchDBReplicator)) return; - const localDatabase = context.getLocalDatabase(); - const remoteDatabase = await replicator.connectRemoteCouchDBWithSetting( - setting, - context.services.API.isMobile(), - true - ); - if (typeof remoteDatabase == "string") { - Logger(remoteDatabase, LOG_LEVEL_NOTICE); - return false; - } - - try { - await purgeUnreferencedChunks(localDatabase.localDatabase, false); - localDatabase.clearCaches(); - const replicated = await context.services.replicator.runFiniteReplicationActivity( - () => replicator.openOneShotReplication(setting, showProgress, false, "sync", true), - { label: "replication" } - ); - if (replicated) { - await balanceChunkPurgedDBs(localDatabase.localDatabase, remoteDatabase.db); - await purgeUnreferencedChunks(localDatabase.localDatabase, false); - localDatabase.clearCaches(); - await replicator.markRemoteResolved(setting); - Logger( - "The local database has been cleaned up.", - showProgress ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO - ); - } else { - Logger( - "Replication has been cancelled. Please try it again.", - showProgress ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO - ); - } - } finally { - await remoteDatabase.close(); - } - }), - { label: "database-cleanup" } - ); - }); - } - async function handleReplicationFailure(request: ReplicationFailureRequest): Promise { - const { context: failedContext, interaction, outcome, progressPresentation, setting } = request; - const showProgress = progressPresentation === REPLICATION_PROGRESS_PRESENTATIONS.NOTICE; + const { context: failedContext, interaction, outcome, setting } = request; if (interaction.kind === "forbidden") { // Automatic requests may report the failure, but must not enter - // tweak, lock, fetch, unlock, or cleanup dialogues. + // tweak, lock, fetch, or unlock dialogues. Logger("Replication failed on an unattended path.", LOG_LEVEL_INFO); return false; } @@ -198,14 +94,6 @@ Even if you choose to clean up, you will see this option again if you exit Obsid ) { return false; } - if ( - recovery.reason === CENTRAL_COMPATIBILITY_REJECTION_REASONS.NODE_CLEANED && - usesLegacyIndexedDBAdapter(setting) - ) { - await reconcileCleanedRemote(showProgress, setting, failedContext); - return false; - } - const message = $msg("Replicator.Dialogue.Locked.Message"); const CHOICE_FETCH = $msg("Replicator.Dialogue.Locked.Action.Fetch"); const CHOICE_DISMISS = $msg("Replicator.Dialogue.Locked.Action.Dismiss"); @@ -240,5 +128,5 @@ Even if you choose to clean up, you will see this option again if you exit Obsid return false; } - return Object.freeze({ handleReplicationFailure, reconcileCleanedRemote }); + return Object.freeze({ handleReplicationFailure }); } diff --git a/src/serviceFeatures/replication/centralCompatibilityRecovery.unit.spec.ts b/src/serviceFeatures/replication/centralCompatibilityRecovery.unit.spec.ts index b4a0de5a..4cb8f587 100644 --- a/src/serviceFeatures/replication/centralCompatibilityRecovery.unit.spec.ts +++ b/src/serviceFeatures/replication/centralCompatibilityRecovery.unit.spec.ts @@ -16,55 +16,86 @@ const chunkMocks = vi.hoisted(() => ({ })); vi.mock("@vrtmrz/livesync-commonlib/compat/pouchdb/chunks", () => chunkMocks); -vi.mock("@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator", () => ({ - LiveSyncCouchDBReplicator: class {}, -})); -import { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator"; import { createCentralCompatibilityRecovery } from "./centralCompatibilityRecovery"; describe("central compatibility recovery", () => { - it("does not count chunks for cleanup when local feature requirements are unknown", async () => { - chunkMocks.purgeUnreferencedChunks.mockClear(); - const confirmWithMessage = vi.fn(async () => "Dismiss"); - const recovery = createCentralCompatibilityRecovery({ - confirm: { confirmWithMessage }, - getLocalDatabase: () => ({ - localDatabase: { - get: vi.fn(async (id: string) => ({ - _id: id, - type: "versioninfo", - version: 13, - used_features: ["future-format-v7"], - })), - }, - }), - services: { replicator: {} }, - } as never); + describe.each([ + { name: "unknown local features", version: 13, used_features: ["future-format-v7"] }, + { name: "a legacy local version", version: 11 }, + { name: "the current local version", version: 12 }, + ])("compatibility: cleaned IndexedDB remote with $name", (localVersion) => { + it.each(["fetch", "unlock", "dismiss"] as const)( + "uses standard recovery for an explicit %s choice without legacy cleanup", + async (choice) => { + chunkMocks.purgeUnreferencedChunks.mockClear(); + chunkMocks.balanceChunkPurgedDBs.mockClear(); + const markRemoteResolved = vi.fn(async (_setting: unknown) => undefined); + const failedContext = { + provider: {}, + replicator: { markRemoteResolved }, + configurationIdentity: "cleaned-remote-test", + }; + const setting = { useIndexedDBAdapter: true } as ObsidianLiveSyncSettings; + const scheduleFetch = vi.fn(async () => undefined); + const scheduleRestart = vi.fn(); + const getLocalDatabase = vi.fn(() => ({ + localDatabase: { + get: vi.fn(async () => ({ _id: VERSIONING_DOCID, type: "versioninfo", ...localVersion })), + }, + })); + const confirmWithMessage = vi.fn(async () => "Dismiss"); + const askSelectStringDialogue = vi.fn( + async (_message: string, choices: string[]) => + choices[choice === "fetch" ? 0 : choice === "unlock" ? 1 : 2] + ); + const recovery = createCentralCompatibilityRecovery({ + confirm: { askSelectStringDialogue, confirmWithMessage }, + getLocalDatabase, + rebuilder: { scheduleFetch }, + services: { + appLifecycle: { scheduleRestart }, + replicator: { + runWithActiveReplicatorContext: async (task: (context: unknown) => unknown) => + await task(failedContext), + }, + }, + } as never); - await recovery.reconcileCleanedRemote(true, {} as ObsidianLiveSyncSettings, {} as never); + const result = await recovery.handleReplicationFailure({ + context: failedContext, + setting, + outcome: replicationFailed(new Error("cleaned"), { + reason: CENTRAL_COMPATIBILITY_REJECTION_REASONS.NODE_CLEANED, + }), + progressPresentation: REPLICATION_PROGRESS_PRESENTATIONS.NOTICE, + interaction: USER_INITIATED_REPLICATION_AUTHORITY, + } as never); - expect(chunkMocks.purgeUnreferencedChunks).not.toHaveBeenCalled(); - expect(confirmWithMessage).not.toHaveBeenCalled(); - }); - - it("allows cleanup counting for a legacy local version document", async () => { - chunkMocks.purgeUnreferencedChunks.mockClear(); - const confirmWithMessage = vi.fn(async () => "Dismiss"); - const recovery = createCentralCompatibilityRecovery({ - confirm: { confirmWithMessage }, - getLocalDatabase: () => ({ - localDatabase: { - get: vi.fn(async (id: string) => ({ _id: id, type: "versioninfo", version: 11 })), - }, - }), - services: { replicator: {} }, - } as never); - - await recovery.reconcileCleanedRemote(true, {} as ObsidianLiveSyncSettings, {} as never); - - expect(chunkMocks.purgeUnreferencedChunks).toHaveBeenCalledWith(expect.anything(), true); - expect(confirmWithMessage).toHaveBeenCalledOnce(); + expect(result).toBe(false); + expect(askSelectStringDialogue).toHaveBeenCalledOnce(); + const choices = askSelectStringDialogue.mock.calls[0][1]; + expect(choices).toHaveLength(3); + expect(askSelectStringDialogue).toHaveBeenCalledWith(expect.any(String), choices, { + title: expect.any(String), + defaultAction: choices[2], + timeout: 60, + }); + expect(confirmWithMessage).not.toHaveBeenCalled(); + expect(getLocalDatabase).not.toHaveBeenCalled(); + expect(chunkMocks.purgeUnreferencedChunks).not.toHaveBeenCalled(); + expect(chunkMocks.balanceChunkPurgedDBs).not.toHaveBeenCalled(); + expect(scheduleFetch).toHaveBeenCalledTimes(choice === "fetch" ? 1 : 0); + expect(scheduleRestart).toHaveBeenCalledTimes(choice === "fetch" ? 1 : 0); + expect(markRemoteResolved).toHaveBeenCalledTimes(choice === "unlock" ? 1 : 0); + if (choice === "fetch") { + expect(scheduleFetch.mock.invocationCallOrder[0]).toBeLessThan( + scheduleRestart.mock.invocationCallOrder[0] + ); + } + if (choice === "unlock") expect(markRemoteResolved).toHaveBeenCalledWith(setting); + } + ); }); it("passes the failed attempt's exact tweak assessment to mismatch resolution", async () => { @@ -139,7 +170,6 @@ describe("central compatibility recovery", () => { try { const recovery = createCentralCompatibilityRecovery({ confirm: {}, - getLocalDatabase: () => ({}), rebuilder: {}, services: { appLifecycle: {}, @@ -187,7 +217,6 @@ describe("central compatibility recovery", () => { }); const recovery = createCentralCompatibilityRecovery({ confirm: {}, - getLocalDatabase: () => ({}), rebuilder: {}, services: { appLifecycle: {}, @@ -255,7 +284,6 @@ describe("central compatibility recovery", () => { ); const recovery = createCentralCompatibilityRecovery({ confirm: {}, - getLocalDatabase: () => ({}), rebuilder: {}, services: { appLifecycle: {}, @@ -287,7 +315,10 @@ describe("central compatibility recovery", () => { expect(setPreferredRemoteTweakSettings.mock.calls[0][0]).not.toBe(effectiveSetting); }); - it("does not apply an unlock selected for a replaced failed publication", async () => { + it.each([ + CENTRAL_COMPATIBILITY_REJECTION_REASONS.NODE_LOCKED, + CENTRAL_COMPATIBILITY_REJECTION_REASONS.NODE_CLEANED, + ])("does not apply an unlock for a replaced failed publication: %s", async (reason) => { const failedMarkResolved = vi.fn(async () => undefined); const replacementMarkResolved = vi.fn(async () => undefined); const failedContext = { provider: {}, replicator: { markRemoteResolved: failedMarkResolved } }; @@ -299,7 +330,6 @@ describe("central compatibility recovery", () => { confirm: { askSelectStringDialogue: vi.fn(async (_message: string, choices: string[]) => choices[1]), }, - getLocalDatabase: () => ({}), rebuilder: {}, services: { appLifecycle: { scheduleRestart: vi.fn() }, @@ -311,9 +341,9 @@ describe("central compatibility recovery", () => { await recovery.handleReplicationFailure({ context: failedContext, - setting: {}, + setting: { useIndexedDBAdapter: true }, outcome: replicationFailed(new Error("locked"), { - reason: CENTRAL_COMPATIBILITY_REJECTION_REASONS.NODE_LOCKED, + reason, }), progressPresentation: REPLICATION_PROGRESS_PRESENTATIONS.NOTICE, interaction: USER_INITIATED_REPLICATION_AUTHORITY, @@ -324,69 +354,55 @@ describe("central compatibility recovery", () => { expect(replacementMarkResolved).not.toHaveBeenCalled(); }); - it("keeps cleaned-remote replication and balancing inside the shared activity boundary", async () => { - const activityFinished = vi.fn(); - const runBoundedRemoteActivity = vi.fn(async (task: () => unknown) => { - try { - return await task(); - } finally { - activityFinished(); - } - }); - const runFiniteReplicationActivity = vi.fn(async (task: () => unknown) => await task()); - const openOneShotReplication = vi.fn(async () => true); - const remoteDatabase = { - close: vi.fn(async () => undefined), + it.each([ + NO_INTERACTION, + { + kind: "permitted" as const, + permissions: { ...USER_INITIATED_REPLICATION_AUTHORITY.permissions, failureRecovery: false }, + }, + ])("does not recover a cleaned remote without recovery authority: %j", async (interaction) => { + const askSelectStringDialogue = vi.fn(); + const confirmWithMessage = vi.fn(); + const getLocalDatabase = vi.fn(); + const scheduleFetch = vi.fn(); + const scheduleRestart = vi.fn(); + const markRemoteResolved = vi.fn(); + const failedContext = { + provider: {}, + replicator: { markRemoteResolved }, + configurationIdentity: "cleaned-remote-test", }; - const close = vi.fn(async () => undefined); - const activeReplicator = Object.assign(new LiveSyncCouchDBReplicator({} as never), { - connectRemoteCouchDBWithSetting: vi.fn(async () => ({ db: remoteDatabase, close })), - openOneShotReplication, - markRemoteResolved: vi.fn(async () => undefined), - }); - const expectedContext = { provider: {}, replicator: activeReplicator }; const runWithActiveReplicatorContext = vi.fn(async (task: (context: unknown) => unknown) => - task(expectedContext) + task(failedContext) ); - const localDatabase = { - localDatabase: { - get: vi.fn(async () => ({ _id: VERSIONING_DOCID, type: "versioninfo", version: 12 })), - }, - clearCaches: vi.fn(), - }; - const getLocalDatabase = vi.fn(() => localDatabase); const recovery = createCentralCompatibilityRecovery({ - confirm: { confirmWithMessage: vi.fn(async () => "Cleanup") }, + confirm: { askSelectStringDialogue, confirmWithMessage }, getLocalDatabase, - rebuilder: {}, + rebuilder: { scheduleFetch }, services: { - appLifecycle: {}, - API: { isMobile: vi.fn(() => false) }, - replicator: { - runBoundedRemoteActivity, - runFiniteReplicationActivity, - runWithActiveReplicatorContext, - }, - tweakValue: {}, + appLifecycle: { scheduleRestart }, + replicator: { runWithActiveReplicatorContext }, }, } as never); - await recovery.reconcileCleanedRemote(true, {} as ObsidianLiveSyncSettings, expectedContext as never); + await expect( + recovery.handleReplicationFailure({ + context: failedContext, + setting: { useIndexedDBAdapter: true }, + outcome: replicationFailed(new Error("cleaned"), { + reason: CENTRAL_COMPATIBILITY_REJECTION_REASONS.NODE_CLEANED, + }), + progressPresentation: REPLICATION_PROGRESS_PRESENTATIONS.NOTICE, + interaction, + } as never) + ).resolves.toBe(false); - expect(runBoundedRemoteActivity).toHaveBeenCalledWith(expect.any(Function), { - label: "database-cleanup", - }); - expect(runFiniteReplicationActivity).toHaveBeenCalledWith(expect.any(Function), { - label: "replication", - }); - expect(runWithActiveReplicatorContext).toHaveBeenCalledOnce(); - expect(openOneShotReplication).toHaveBeenCalledOnce(); - expect(openOneShotReplication.mock.invocationCallOrder[0]).toBeLessThan( - activityFinished.mock.invocationCallOrder[0] - ); - expect(chunkMocks.balanceChunkPurgedDBs).toHaveBeenCalledOnce(); - expect(getLocalDatabase).toHaveBeenCalled(); - expect(close).toHaveBeenCalledOnce(); - expect(close.mock.invocationCallOrder[0]).toBeLessThan(activityFinished.mock.invocationCallOrder[0]); + expect(askSelectStringDialogue).not.toHaveBeenCalled(); + expect(confirmWithMessage).not.toHaveBeenCalled(); + expect(getLocalDatabase).not.toHaveBeenCalled(); + expect(scheduleFetch).not.toHaveBeenCalled(); + expect(scheduleRestart).not.toHaveBeenCalled(); + expect(runWithActiveReplicatorContext).not.toHaveBeenCalled(); + expect(markRemoteResolved).not.toHaveBeenCalled(); }); }); diff --git a/src/serviceFeatures/replication/index.ts b/src/serviceFeatures/replication/index.ts index 473c7bb7..884822fc 100644 --- a/src/serviceFeatures/replication/index.ts +++ b/src/serviceFeatures/replication/index.ts @@ -83,10 +83,8 @@ export function useReplicationFeature core.localDatabase, rebuilder: core.rebuilder, services: { - API: services.API, appLifecycle: services.appLifecycle, replicator: services.replicator, setting: services.setting, diff --git a/src/serviceFeatures/setupObsidian/setupUri.ts b/src/serviceFeatures/setupObsidian/setupUri.ts index 3295c5d3..96b33378 100644 --- a/src/serviceFeatures/setupObsidian/setupUri.ts +++ b/src/serviceFeatures/setupObsidian/setupUri.ts @@ -1,3 +1,4 @@ +import { createCommandCheck } from "@vrtmrz/obsidian-plugin-kit/command"; import { LOG_LEVEL_NOTICE, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types"; import type { LogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; import { createInstanceLogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils"; @@ -112,33 +113,40 @@ export function useSetupURIFeature(host: NecessaryServices<"API" | "UI" | "setti host.services.API.addCommand({ id: "livesync-copysetupuri", name: "Copy settings as a new setup URI", - checkCallback: (checking) => { - if (!host.services.setting.currentSettings().isConfigured) return false; - if (!checking) fireAndForget(copySetupURI(host, log)); - return true; - }, + checkCallback: createCommandCheck( + () => host.services.setting.currentSettings().isConfigured, + () => { + fireAndForget(copySetupURI(host, log)); + } + ), }); host.services.API.addCommand({ id: "livesync-copysetupuri-short", name: "Copy settings as a new setup URI (With customization sync)", - checkCallback: (checking) => { - const settings = host.services.setting.currentSettings(); - if (!settings.isConfigured || !settings.usePluginSync) return false; - if (!checking) fireAndForget(copySetupURI(host, log, false)); - return true; - }, + checkCallback: createCommandCheck( + () => { + const settings = host.services.setting.currentSettings(); + return settings.isConfigured && settings.usePluginSync; + }, + () => { + fireAndForget(copySetupURI(host, log, false)); + } + ), }); host.services.API.addCommand({ id: "livesync-copysetupurifull", name: "Copy settings as a new setup URI (Full)", - checkCallback: (checking) => { - const settings = host.services.setting.currentSettings(); - if (!settings.isConfigured || !settings.useAdvancedMode) return false; - if (!checking) fireAndForget(copySetupURIFull(host, log)); - return true; - }, + checkCallback: createCommandCheck( + () => { + const settings = host.services.setting.currentSettings(); + return settings.isConfigured && settings.useAdvancedMode; + }, + () => { + fireAndForget(copySetupURIFull(host, log)); + } + ), }); host.services.context.events.onEvent(EVENT_REQUEST_COPY_SETUP_URI, () => diff --git a/src/serviceFeatures/setupObsidian/setupUri.unit.spec.ts b/src/serviceFeatures/setupObsidian/setupUri.unit.spec.ts index 8a23d50a..9f632f67 100644 --- a/src/serviceFeatures/setupObsidian/setupUri.unit.spec.ts +++ b/src/serviceFeatures/setupObsidian/setupUri.unit.spec.ts @@ -221,7 +221,7 @@ describe("setupObsidian/setupUri", () => { }, UI: { confirm: { - askString: vi.fn(() => "pass"), + askString: vi.fn(async () => false), }, promptCopyToClipboard: vi.fn(() => true), }, @@ -269,7 +269,7 @@ describe("setupObsidian/setupUri", () => { }, UI: { confirm: { - askString: vi.fn(() => "pass"), + askString: vi.fn(async () => false), }, promptCopyToClipboard: vi.fn(() => true), }, @@ -292,5 +292,12 @@ describe("setupObsidian/setupUri", () => { settings.useAdvancedMode = true; expect(command("livesync-copysetupuri-short").checkCallback?.(true)).toBe(true); expect(command("livesync-copysetupurifull").checkCallback?.(true)).toBe(true); + expect(host.services.UI.confirm.askString).not.toHaveBeenCalled(); + settings.isConfigured = false; + for (const registered of commands) expect(registered.checkCallback?.(false)).toBe(false); + expect(host.services.UI.confirm.askString).not.toHaveBeenCalled(); + settings.isConfigured = true; + for (const registered of commands) expect(registered.checkCallback?.(false)).toBe(true); + expect(host.services.UI.confirm.askString).toHaveBeenCalledTimes(3); }); }); diff --git a/src/serviceFeatures/useDatabaseDiagnostics.ts b/src/serviceFeatures/useDatabaseDiagnostics.ts index 2fa974c8..85e582b9 100644 --- a/src/serviceFeatures/useDatabaseDiagnostics.ts +++ b/src/serviceFeatures/useDatabaseDiagnostics.ts @@ -1,3 +1,4 @@ +import { createCommandCheck } from "@vrtmrz/obsidian-plugin-kit/command"; import { createServiceFeature, type NecessaryServices, @@ -46,22 +47,23 @@ export const useDatabaseDiagnosticsFeature = createServiceFeature((host: Databas id: "analyse-database", name: "Analyse Database Usage (advanced)", icon: "database-search", - checkCallback: (checking) => { - if (!services.setting.currentSettings().useAdvancedMode || !services.database.isDatabaseReady()) - return false; - if (!checking) void open("usage"); - return true; - }, + checkCallback: createCommandCheck( + () => services.setting.currentSettings().useAdvancedMode && services.database.isDatabaseReady(), + () => { + void open("usage"); + } + ), }); services.API.addCommand({ id: "inspect-database-integrity", name: $msg("Inspect database integrity"), icon: "database-search", - checkCallback: (checking) => { - if (!services.database.isDatabaseReady()) return false; - if (!checking) void open("integrity"); - return true; - }, + checkCallback: createCommandCheck( + () => services.database.isDatabaseReady(), + () => { + void open("integrity"); + } + ), }); return Promise.resolve(true); }); diff --git a/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts b/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts index 8372b5ef..df948e2a 100644 --- a/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts +++ b/src/serviceFeatures/useDatabaseDiagnostics.unit.spec.ts @@ -82,6 +82,26 @@ describe("database diagnostics serviceFeature", () => { expect(data.openDialog).not.toHaveBeenCalled(); }); + it.each(["analyse-database", "inspect-database-integrity"])( + "rechecks database availability before executing %s", + async (id) => { + const data = fixture(); + data.settings.useAdvancedMode = true; + for (const initialise of data.initialise) await initialise(); + const command = data.commands.find((candidate) => candidate.id === id)!; + expect(command.checkCallback(true)).toBe(true); + expect(data.openDialog).not.toHaveBeenCalled(); + data.setReady(false); + expect(command.checkCallback(false)).toBe(false); + expect(data.openDialog).not.toHaveBeenCalled(); + data.setReady(true); + expect(command.checkCallback(false)).toBe(true); + expect(data.openDialog).toHaveBeenCalledOnce(); + expect(data.openDialog.mock.calls[0][1].kind).toBe(id === "analyse-database" ? "usage" : "integrity"); + data.close(); + } + ); + it("uses the existing dialogue manager and performs no analysis until the run choice is submitted", async () => { vi.mocked(runDatabaseDiagnostic).mockClear(); const data = fixture(); diff --git a/src/serviceFeatures/useJoinedDevices.ts b/src/serviceFeatures/useJoinedDevices.ts index 53b7dbc9..6baa9529 100644 --- a/src/serviceFeatures/useJoinedDevices.ts +++ b/src/serviceFeatures/useJoinedDevices.ts @@ -9,6 +9,7 @@ import { import { REMOTE_RESOURCE_KINDS, type JoinedDevicesSnapshot } from "@vrtmrz/livesync-commonlib/replication"; import type { ObsidianNoticeGroups } from "@/modules/services/ObsidianNoticeGroups"; import JoinedDevicesDialog from "@/modules/services/LiveSyncUI/dialogues/JoinedDevicesDialog.svelte"; +import { withOwnedRemoteResource } from "@/common/ownedRemoteResource"; import { getCouchDBReplicatorConfigurationIdentity, getObjectStorageReplicatorConfigurationIdentity, @@ -119,9 +120,9 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi settings ); if (!resource) throw new Error("Joined devices are unavailable"); - try { + return withOwnedRemoteResource(resource, async (ownedResource) => { assertCurrent(); - const snapshot = await resource.read(); + const snapshot = await ownedResource.read(); assertCurrent(); notify(snapshot, id, identity!); return joinedDevicesView(snapshot, { @@ -130,9 +131,7 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi pluginVersion: services.API.getPluginVersion(), appVersion: services.API.getAppVersion(), }); - } finally { - await resource.dispose(); - } + }); }, forget: async (id) => { if (id === (await ownId())) throw new Error("This device cannot be forgotten"); @@ -141,14 +140,12 @@ export function useJoinedDevicesFeature(host: JoinedDevicesHost, notices: Obsidi settings ); if (!resource) throw new Error("Joined devices are unavailable"); - try { + return withOwnedRemoteResource(resource, async (ownedResource) => { assertCurrent(); - const result = await resource.forget(id); + const result = await ownedResource.forget(id); assertCurrent(); return result; - } finally { - await resource.dispose(); - } + }); }, cancel: () => controller.abort(), }; diff --git a/src/serviceFeatures/useJoinedDevices.unit.spec.ts b/src/serviceFeatures/useJoinedDevices.unit.spec.ts index 188afe5a..0c5670c2 100644 --- a/src/serviceFeatures/useJoinedDevices.unit.spec.ts +++ b/src/serviceFeatures/useJoinedDevices.unit.spec.ts @@ -110,6 +110,67 @@ describe("joined devices service feature", () => { expect(await pending).toEqual({ status: "handled", value: "closed" }); await expect(options.refresh()).rejects.toThrow(); }); + it("propagates a failed refresh after releasing its resource", async () => { + const data = fixture(); + const failure = new Error("joined devices read failed"); + data.read.mockRejectedValueOnce(failure); + const { options, pending } = await open(data); + + await expect(options.refresh()).rejects.toBe(failure); + expect(data.dispose).toHaveBeenCalledOnce(); + data.close(); + await pending; + }); + it.each(["refresh", "forget"] as const)( + "waits for resource disposal after %s", + async (operationName) => { + const data = fixture(); + let finishDisposal!: () => void; + data.dispose.mockImplementationOnce( + () => + new Promise((resolve) => { + finishDisposal = resolve; + }) + ); + const { options, pending } = await open(data); + const operation = operationName === "refresh" ? options.refresh() : options.forget("other"); + let settled = false; + void operation.then( + () => { + settled = true; + }, + () => { + settled = true; + } + ); + + await vi.waitFor(() => expect(data.dispose).toHaveBeenCalledOnce()); + expect(settled).toBe(false); + finishDisposal(); + await expect(operation).resolves.toBeDefined(); + expect(settled).toBe(true); + data.close(); + await pending; + } + ); + it.each(["refresh", "forget"] as const)( + "uses disposal failure precedence when %s also fails", + async (operationName) => { + const data = fixture(); + const operationFailure = new Error("joined devices operation failed"); + const disposalFailure = new Error("joined devices disposal failed"); + if (operationName === "refresh") data.read.mockRejectedValueOnce(operationFailure); + else data.forget.mockRejectedValueOnce(operationFailure); + data.dispose.mockRejectedValueOnce(disposalFailure); + const { options, pending } = await open(data); + const operation = operationName === "refresh" ? options.refresh() : options.forget("other"); + + await expect(operation).rejects.toBe(disposalFailure); + expect(data.dispose).toHaveBeenCalledOnce(); + data.close(); + await pending; + } + ); it("prevents self removal and fences an old dialogue after a remote switch", async () => { const data = fixture(); const { options, pending } = await open(data); diff --git a/src/serviceFeatures/useP2PReplicatorUI.ts b/src/serviceFeatures/useP2PReplicatorUI.ts index afa1b112..09b60fb4 100644 --- a/src/serviceFeatures/useP2PReplicatorUI.ts +++ b/src/serviceFeatures/useP2PReplicatorUI.ts @@ -1,6 +1,6 @@ import { eventHub, EVENT_REQUEST_OPEN_P2P } from "@/common/events"; import type { NecessaryServices } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; -import { type UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/UseP2PReplicatorResult"; +import type { P2PServiceViews } from "@vrtmrz/livesync-commonlib/p2p"; import { P2PServerStatusPaneView, VIEW_TYPE_P2P_SERVER_STATUS, @@ -47,22 +47,9 @@ export function hasP2PConfiguration(settings: Partial) */ export function useP2PReplicatorUI( - host: NecessaryServices< - | "API" - | "appLifecycle" - | "setting" - | "vault" - | "database" - | "databaseEvents" - | "keyValueDB" - | "replication" - | "config" - | "UI" - | "replicator", - never - >, + host: NecessaryServices<"API" | "appLifecycle" | "setting" | "replicator", never>, core: LiveSyncCore, - replicator: UseP2PReplicatorResult, + replicator: P2PServiceViews, openInteractiveReplication: OpenInteractiveP2PReplication ) { const api = host.services.API as { diff --git a/src/serviceFeatures/useP2PReplicatorUI.unit.spec.ts b/src/serviceFeatures/useP2PReplicatorUI.unit.spec.ts index 87e4cf4f..650c864d 100644 --- a/src/serviceFeatures/useP2PReplicatorUI.unit.spec.ts +++ b/src/serviceFeatures/useP2PReplicatorUI.unit.spec.ts @@ -1,4 +1,6 @@ -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import type { NecessaryServices } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule"; +import type { P2PServiceViews } from "@vrtmrz/livesync-commonlib/p2p"; import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; import { eventHub, EVENT_REQUEST_OPEN_P2P } from "@/common/events"; @@ -16,6 +18,19 @@ import { useP2PReplicatorUI } from "./useP2PReplicatorUI"; const noopOpenInteractiveReplication = () => Promise.resolve(false); describe("useP2PReplicatorUI commands", () => { + it("accepts only the UI services and focused P2P views", () => { + type Dependencies = NecessaryServices<"API" | "appLifecycle" | "setting" | "replicator", never>; + type Arguments = Parameters; + + expectTypeOf().toExtend(); + expectTypeOf().toEqualTypeOf< + "context" | "API" | "appLifecycle" | "setting" | "replicator" + >(); + expectTypeOf().toBeNever(); + expectTypeOf().toExtend(); + expectTypeOf().toEqualTypeOf(); + }); + it("waits for settings to load before deciding whether to show the P2P ribbon", async () => { let initialise: (() => Promise) | undefined; let settingLoaded: (() => Promise) | undefined; diff --git a/terser.config.mjs b/terser.config.mjs index e15bc67a..feae72e9 100644 --- a/terser.config.mjs +++ b/terser.config.mjs @@ -60,7 +60,12 @@ const terserOption = { hoist_vars: true, // toplevel: "vars", }, - mangle: false, + // Keep property keys, reflective names, and eval-visible bindings stable. + mangle: { + properties: false, + toplevel: false, + eval: false, + }, ecma: 2018, // specify one of: 5, 2015, 2016, etc. enclose: false, // or specify true, or "args:values" diff --git a/test/README.md b/test/README.md index 9387f5e4..1541424c 100644 --- a/test/README.md +++ b/test/README.md @@ -32,7 +32,7 @@ NODE_OPTIONS=--max-old-space-size=3072 npm run test:unit -- --maxWorkers=1 | --- | --- | | Multiple-device file conflicts and stale-file protection | [CouchDB procedure below](#multiple-device-conflict-regression-tests) | | Obsidian startup, file watching, persistence, and UI | [Real Obsidian E2E](e2e-obsidian/README.md) | -| CLI subprocesses, filesystem workflows, and P2P | [CLI Deno tests](../src/apps/cli/testdeno/test_dev_deno.md) and [test authoring](../src/apps/cli/testdeno/CONTRIBUTING_TESTS.md) | +| CLI subprocesses, filesystem workflows, and P2P | [CLI Node.js tests](../test/e2e-cli/test_dev_node.md) and [test authoring](../test/e2e-cli/CONTRIBUTING_TESTS.md) | | WebApp, WebPeer, and browser interoperability | [Browser application tests](browser-apps/README.md) | ## Community Review checks and CI confirmation @@ -58,8 +58,8 @@ Run the native CLI subprocess suite after building the CLI: ```bash NODE_OPTIONS=--max-old-space-size=3072 npm run build --workspace self-hosted-livesync-cli -cd src/apps/cli/testdeno -deno test -A --no-check test-mirror.ts +cd test/e2e-cli +node --test --test-concurrency=1 test-mirror.ts ``` The expected result is seven passing steps. These cover storage-only and database-only files, database deletion, an ordinary local edit, incoming database content, an omitted Vault path, and local content with unknown provenance. @@ -75,7 +75,64 @@ NODE_OPTIONS=--max-old-space-size=3072 npm run build:docker --workspace self-hos npm run test:e2e:docker:mirror --workspace self-hosted-livesync-cli ``` -The expected result is `PASS=7 FAIL=0`. These mirror suites use temporary local databases and need no CouchDB service. The complete `test:e2e:docker:all` command also runs the other Docker CLI suites and manages a disposable CouchDB fixture; use it to check the full Docker CI gate. Keep the ordinary-edit and unknown-provenance scenarios aligned between the Deno and Bash suites. +The expected result is `PASS=7 FAIL=0`. These mirror suites use temporary local databases and need no CouchDB service. The complete `test:e2e:docker:all` command also runs the other Docker CLI suites and manages a disposable CouchDB fixture; use it to check the full Docker CI gate. Keep the ordinary-edit and unknown-provenance scenarios aligned between the Node.js and Bash suites. + +## CLI Fetch database continuity + +The standard CLI CI suite and direct task runner execute +`test:fetch-database-continuity` against a dedicated RustFS container. Each +scenario creates its own bucket. The test binds a uniquely named container to a +dynamic loopback port, then stops only that container. Redacted evidence is +written under the operating system's temporary directory and is not uploaded as +an artefact. + +For a local run, build the CLI and run +`node test/e2e-cli/run-task.ts test:fetch-database-continuity` with Node.js 26 +and Docker available. The managed container uses `accessKey` and `secretKey` +from the CLI `.test.env` file when present, and defaults to `minioadmin` for +both. No endpoint, bucket, or evidence-directory variables are needed. Set +`LIVESYNC_ISSUE1304_EVIDENCE_DIR` to choose another evidence directory outside +the worktree. + +To use an external S3-compatible fixture without Docker, set all four +`LIVESYNC_ISSUE1304_MINIO_ENDPOINT`, `LIVESYNC_ISSUE1304_MINIO_ACCESS_KEY`, +`LIVESYNC_ISSUE1304_MINIO_SECRET_KEY`, and `LIVESYNC_ISSUE1304_MINIO_BUCKET`. +The bucket must be fresh and empty for every run. Partial external fixture +configuration is rejected. This test writes remote data; never point it at a +production bucket. + +The test seeds an encrypted, path-obfuscated file with a second-generation +revision, imports one Setup URI on two clients, fetches on A without +`--write-settings`, and restarts A while B remains online. It checks the saved +database suffix, file bytes, original revision, and absence of a duplicate root. +Managed runs execute the explicit `--write-settings` control automatically in +a separate scenario bucket. For an external fixture, set +`LIVESYNC_ISSUE1304_POSITIVE_CONTROL=1` and +`LIVESYNC_ISSUE1304_POSITIVE_CONTROL_BUCKET` to a second fresh bucket to run +that control. + +For historical reproduction against the unmodified implementation, set +`LIVESYNC_ISSUE1304_VERIFY_LEGACY_FAILURE=1`. This preserves the original +same-content conflict-resolution sequence and its file-preservation assertion; +the affected implementation fails because A's file disappears. Leave this +variable unset when validating the correction. The test saves redacted process +logs, file hashes, and database evidence. It does not cover Obsidian runtime or +CouchDB behaviour. + +## Object Storage Chunk recovery regression tests + +After building the CLI, run the focused recovery test with Node.js 26 or later: + +```bash +NODE_OPTIONS=--max-old-space-size=3072 npm run build --workspace self-hosted-livesync-cli +NODE_OPTIONS=--max-old-space-size=3072 MINIO_ENDPOINT=http://127.0.0.1:9000 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_BUCKET_NAME=livesync-test npm run test:e2e:task --workspace self-hosted-livesync-cli -- test:journal-chunk-recovery +``` + +The test manages the disposable `minio-test` RustFS container on ports 9000 and 9001. Run it separately from other tests using that container or those ports. It creates a distinct bucket and temporary Vaults for each case and removes its test container afterwards. + +Both the unencrypted and encrypted cases rebuild from a complete file containing raw unit separators (`U+001F`), Fetch into another Vault, and compare the exact bytes. A fresh CLI process reads the persisted content. The test then damages the receiving Vault file and verifies that Fetch retains that newer local file and preserves both contents as independent conflict branches. Each revision is read explicitly so the assertions do not depend on which branch PouchDB selects as the winner. After moving the damaged file to a separate backup, another Fetch into the empty Vault restores the remote contents. It retains `--write-settings` to exercise explicit settings writeback; database selection is now saved automatically, and the continuity test below covers omission of that option. This test is included in the standard CLI CI task list. + +This establishes the CLI and shared Rebuild/Fetch behaviour. It does not exercise Obsidian's Hidden File Sync or Customisation Sync initialisation, nor inject a damaged Chunk under an existing content-addressed ID. ## Multiple-device conflict regression tests diff --git a/test/apps/browser/BrowserConfirm.unit.spec.ts b/test/apps/browser/BrowserConfirm.unit.spec.ts new file mode 100644 index 00000000..d279eefd --- /dev/null +++ b/test/apps/browser/BrowserConfirm.unit.spec.ts @@ -0,0 +1,73 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; +import { EVENT_PLUGIN_UNLOADED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; + +const kit = vi.hoisted(() => ({ + uiOptions: [] as { document: Document; signal: AbortSignal }[], + notices: [] as { document: Document; dispose: ReturnType; show: ReturnType }[], +})); +vi.mock("@vrtmrz/browser-ui-kit", () => ({ + createBrowserUi(options: { document: Document; signal: AbortSignal }) { + kit.uiOptions.push(options); + return { promptText: vi.fn(async () => "selected") }; + }, + BrowserUiNotifications: class { + dispose = vi.fn(); + show = vi.fn(); + constructor(options: { document: Document }) { + kit.notices.push({ document: options.document, dispose: this.dispose, show: this.show }); + } + }, +})); +import { BrowserConfirm } from "@/apps/browser/BrowserConfirm"; +import { createBrowserUiScope } from "@/apps/browser/BrowserUiScope"; + +beforeEach(() => { + kit.uiOptions.length = 0; + kit.notices.length = 0; +}); + +describe("BrowserConfirm UI owner", () => { + it("passes the same document and lifetime to existing Kit capabilities", async () => { + const context = createServiceContext(); + const anchor = { textContent: "", click: vi.fn() }; + const document = { createElement: vi.fn(() => anchor) } as unknown as Document; + const ui = createBrowserUiScope(context, { document }); + const confirm = new BrowserConfirm(context, ui); + expect(kit.uiOptions[0]).toMatchObject({ document, signal: ui.signal }); + expect(kit.notices[0].document).toBe(document); + await expect(confirm.askString("Title", "Value", "Placeholder")).resolves.toBe("selected"); + confirm.askInPopup("key", "Go {HERE}", (link) => { + link.textContent = "Open"; + }); + expect(document.createElement).toHaveBeenCalledWith("a"); + expect(kit.notices[0].show).toHaveBeenCalledWith( + "key", + expect.objectContaining({ action: expect.objectContaining({ label: "Open" }) }) + ); + context.events.emitEvent(EVENT_PLUGIN_UNLOADED); + expect(ui.signal.aborted).toBe(true); + expect(kit.notices[0].dispose).toHaveBeenCalledOnce(); + }); + it("disposes only the notifications belonging to the unloaded context", () => { + const first = createServiceContext(); + const second = createServiceContext(); + new BrowserConfirm(first, createBrowserUiScope(first, { document: {} as Document })); + new BrowserConfirm(second, createBrowserUiScope(second, { document: {} as Document })); + first.events.emitEvent(EVENT_PLUGIN_UNLOADED); + first.events.emitEvent(EVENT_PLUGIN_UNLOADED); + expect(kit.notices[0].dispose).toHaveBeenCalledOnce(); + expect(kit.notices[1].dispose).not.toHaveBeenCalled(); + second.events.emitEvent(EVENT_PLUGIN_UNLOADED); + }); + it("disposes notifications immediately for a pre-aborted owner", () => { + const context = createServiceContext(); + const parent = new AbortController(); + parent.abort(); + new BrowserConfirm(context, createBrowserUiScope(context, { signal: parent.signal })); + expect(kit.uiOptions[0].signal.aborted).toBe(true); + expect(kit.notices[0].dispose).toHaveBeenCalledOnce(); + context.events.emitEvent(EVENT_PLUGIN_UNLOADED); + expect(kit.notices[0].dispose).toHaveBeenCalledOnce(); + }); +}); diff --git a/test/apps/browser/BrowserMenu.unit.spec.ts b/test/apps/browser/BrowserMenu.unit.spec.ts new file mode 100644 index 00000000..b0ef0fb6 --- /dev/null +++ b/test/apps/browser/BrowserMenu.unit.spec.ts @@ -0,0 +1,286 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const browserMenuHarness = vi.hoisted(() => { + const createDocument = () => { + const created: { tag: string; removed: boolean; remove: ReturnType }[] = []; + const appended: unknown[] = []; + const document = { + createElement: vi.fn((tag: string) => { + const element: { tag: string; removed: boolean; remove: ReturnType } = { + tag, + removed: false, + remove: vi.fn(() => { + element.removed = true; + }), + }; + created.push(element); + return element; + }), + body: { + appendChild: vi.fn((element: unknown) => { + appended.push(element); + return element; + }), + }, + }; + return { document, created, appended }; + }; + + return { + documents: { + active: createDocument(), + explicit: createDocument(), + }, + mounts: [] as { + component: unknown; + target: unknown; + props: Record; + handle: { id: number }; + }[], + unmounts: [] as unknown[], + }; +}); + +vi.mock("@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions", () => ({ + _activeDocument: browserMenuHarness.documents.active.document, +})); +vi.mock("svelte", () => ({ + mount: vi.fn((component: unknown, options: { target: unknown; props: Record }) => { + const handle = { id: browserMenuHarness.mounts.length }; + browserMenuHarness.mounts.push({ component, target: options.target, props: options.props, handle }); + return handle; + }), + unmount: vi.fn((handle: unknown) => { + browserMenuHarness.unmounts.push(handle); + return Promise.resolve(); + }), +})); + +import { Menu, MenuItem, MenuSeparator } from "@/apps/browser/BrowserMenu"; + +type BrowserMenuOptionsFixture = { document?: Document; signal?: AbortSignal }; + +function menuWithOptions(options: BrowserMenuOptionsFixture): Menu { + const MenuWithOptions = Menu as unknown as new (options: BrowserMenuOptionsFixture) => Menu; + return new MenuWithOptions(options); +} + +function explicitDocument(): Document { + return browserMenuHarness.documents.explicit.document as unknown as Document; +} + +function closeMenu(index: number): () => void { + const close = browserMenuHarness.mounts[index]!.props.closeMenu; + if (typeof close !== "function") throw new Error("Menu mount did not receive a close callback"); + return close as () => void; +} + +async function settlesAfterMicrotask(promise: Promise): Promise { + let settled = false; + void promise.then(() => { + settled = true; + }); + await Promise.resolve(); + return settled; +} + +beforeEach(() => { + browserMenuHarness.documents.active.created.length = 0; + browserMenuHarness.documents.active.appended.length = 0; + browserMenuHarness.documents.explicit.created.length = 0; + browserMenuHarness.documents.explicit.appended.length = 0; + browserMenuHarness.mounts.length = 0; + browserMenuHarness.unmounts.length = 0; + vi.clearAllMocks(); +}); + +afterEach(() => { + for (const mount of browserMenuHarness.mounts) { + const close = mount.props.closeMenu; + if (typeof close === "function") (close as () => void)(); + } +}); + +describe("browser Menu lifecycle", () => { + it("preserves item, separator, chained construction, and positioned display APIs", async () => { + const menu = new Menu(); + const handler = vi.fn(async () => {}); + + expect( + menu.addItem((item) => { + item.setTitle("Open").setIcon("folder").onClick(handler); + }) + ).toBe(menu); + expect(menu.addSeparator()).toBe(menu); + expect(menu.items[0]).toBeInstanceOf(MenuItem); + expect(menu.items[0]).toMatchObject({ type: "item", title: "Open", icon: "folder", handler }); + expect(menu.items[1]).toBeInstanceOf(MenuSeparator); + + const displayed = menu.showAtPosition({ x: 12, y: 34 }); + expect(browserMenuHarness.documents.active.document.createElement).toHaveBeenCalledWith("div"); + expect(browserMenuHarness.documents.active.appended).toEqual([browserMenuHarness.mounts[0]!.target]); + expect(browserMenuHarness.mounts[0]!.props).toMatchObject({ items: menu.items, x: 12, y: 34 }); + expect(browserMenuHarness.mounts[0]!.target).toBe(browserMenuHarness.documents.active.created[0]); + closeMenu(0)(); + await expect(displayed).resolves.toBeUndefined(); + }); + + it("keeps the returned close promise pending until a menu close action", async () => { + const menu = new Menu(); + const displayed = menu.showAtPosition({ x: 1, y: 2 }); + + expect(await settlesAfterMicrotask(displayed)).toBe(false); + closeMenu(0)(); + await expect(displayed).resolves.toBeUndefined(); + }); + + it("resolves the active display promise when hidden without a menu action", async () => { + const menu = new Menu(); + const displayed = menu.showAtPosition({ x: 1, y: 2 }); + + menu.hide(); + + await expect(displayed).resolves.toBeUndefined(); + }); + + it("closes a replaced display while keeping the replacement active", async () => { + const menu = new Menu(); + const first = menu.showAtPosition({ x: 1, y: 2 }); + const staleClose = closeMenu(0); + const firstElement = browserMenuHarness.mounts[0]!.target as { removed: boolean }; + const second = menu.showAtPosition({ x: 3, y: 4 }); + const secondElement = browserMenuHarness.mounts[1]!.target as { removed: boolean }; + + await expect(first).resolves.toBeUndefined(); + expect(firstElement.removed).toBe(true); + expect(secondElement.removed).toBe(false); + expect(menu.waitingForClose?.promise).toBe(second); + expect(await settlesAfterMicrotask(second)).toBe(false); + + try { + staleClose(); + expect(menu.waitingForClose?.promise).toBe(second); + expect(secondElement.removed).toBe(false); + expect(await settlesAfterMicrotask(second)).toBe(false); + } finally { + closeMenu(1)(); + } + await expect(second).resolves.toBeUndefined(); + }); + + it("mounts a menu in its explicitly supplied document", async () => { + const menu = menuWithOptions({ document: explicitDocument() }); + + const displayed = menu.showAtPosition({ x: 5, y: 6 }); + + expect(browserMenuHarness.documents.explicit.document.createElement).toHaveBeenCalledWith("div"); + expect(browserMenuHarness.documents.explicit.appended).toEqual([browserMenuHarness.mounts[0]!.target]); + expect(browserMenuHarness.mounts[0]!.target).toBe(browserMenuHarness.documents.explicit.created[0]); + expect(browserMenuHarness.documents.active.document.createElement).not.toHaveBeenCalled(); + closeMenu(0)(); + await expect(displayed).resolves.toBeUndefined(); + }); + + it("aborting one menu closes only that menu", async () => { + const firstOwner = new AbortController(); + const secondOwner = new AbortController(); + const firstMenu = menuWithOptions({ document: explicitDocument(), signal: firstOwner.signal }); + const secondMenu = menuWithOptions({ document: explicitDocument(), signal: secondOwner.signal }); + const first = firstMenu.showAtPosition({ x: 1, y: 2 }); + const second = secondMenu.showAtPosition({ x: 3, y: 4 }); + let firstSettled = false; + void first.then(() => { + firstSettled = true; + }); + firstOwner.abort(); + + await Promise.resolve(); + expect(firstSettled).toBe(true); + expect(secondMenu.waitingForClose?.promise).toBe(second); + expect(await settlesAfterMicrotask(second)).toBe(false); + closeMenu(1)(); + await expect(second).resolves.toBeUndefined(); + }); + + it("does not mount for an already aborted owner", async () => { + const owner = new AbortController(); + owner.abort(); + const menu = menuWithOptions({ document: explicitDocument(), signal: owner.signal }); + + const displayed = menu.showAtPosition({ x: 1, y: 2 }); + + expect(browserMenuHarness.mounts).toHaveLength(0); + expect(await settlesAfterMicrotask(displayed)).toBe(true); + }); + + it("does not mount again after its owner aborts", async () => { + const owner = new AbortController(); + const menu = menuWithOptions({ document: explicitDocument(), signal: owner.signal }); + const first = menu.showAtPosition({ x: 1, y: 2 }); + owner.abort(); + + expect(await settlesAfterMicrotask(first)).toBe(true); + const reopened = menu.showAtPosition({ x: 3, y: 4 }); + expect(await settlesAfterMicrotask(reopened)).toBe(true); + expect(browserMenuHarness.mounts).toHaveLength(1); + }); + + it.each(["hide", "dismissal", "replacement", "abort"] as const)( + "unmounts each Svelte component exactly once for %s", + async (reason) => { + const owner = new AbortController(); + const menu = menuWithOptions({ document: explicitDocument(), signal: owner.signal }); + const first = menu.showAtPosition({ x: 1, y: 2 }); + let second: Promise | undefined; + + if (reason === "hide") { + menu.hide(); + } else if (reason === "dismissal") { + closeMenu(0)(); + } else if (reason === "replacement") { + second = menu.showAtPosition({ x: 3, y: 4 }); + closeMenu(1)(); + } else { + owner.abort(); + } + + expect(await settlesAfterMicrotask(first)).toBe(true); + if (second) expect(await settlesAfterMicrotask(second)).toBe(true); + for (const mount of browserMenuHarness.mounts) { + expect(browserMenuHarness.unmounts.filter((handle) => handle === mount.handle)).toHaveLength(1); + } + } + ); + + it("removes the menu element before resolving the close promise", async () => { + const menu = new Menu(); + const displayed = menu.showAtPosition({ x: 1, y: 2 }); + const element = browserMenuHarness.mounts[0]!.target as { removed: boolean }; + let removedWhenResolved = false; + void displayed.then(() => { + removedWhenResolved = element.removed; + }); + + menu.hide(); + + await displayed; + expect(removedWhenResolved).toBe(true); + }); + + it("removes its abort listener when closed so a later abort does not unmount it again", async () => { + const owner = new AbortController(); + const addListener = vi.spyOn(owner.signal, "addEventListener"); + const removeListener = vi.spyOn(owner.signal, "removeEventListener"); + const menu = menuWithOptions({ document: explicitDocument(), signal: owner.signal }); + const displayed = menu.showAtPosition({ x: 1, y: 2 }); + const handle = browserMenuHarness.mounts[0]!.handle; + + menu.hide(); + await expect(displayed).resolves.toBeUndefined(); + owner.abort(); + + expect(addListener.mock.calls.map(([type]) => type)).toContain("abort"); + expect(removeListener.mock.calls.map(([type]) => type)).toContain("abort"); + expect(browserMenuHarness.unmounts.filter((unmounted) => unmounted === handle)).toHaveLength(1); + }); +}); diff --git a/test/apps/browser/BrowserSvelteDialogManager.unit.spec.ts b/test/apps/browser/BrowserSvelteDialogManager.unit.spec.ts new file mode 100644 index 00000000..71e8ebad --- /dev/null +++ b/test/apps/browser/BrowserSvelteDialogManager.unit.spec.ts @@ -0,0 +1,199 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; +import { EVENT_PLUGIN_UNLOADED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; +import type { + ComponentHasResult, + DialogHostProps, + SvelteDialogManagerDependencies, +} from "@vrtmrz/livesync-commonlib/compat/services/implements/base/SvelteDialog"; + +const rendering = vi.hoisted(() => ({ + mount: vi.fn(), + unmount: vi.fn(async (_component: unknown) => {}), + legacy: { document: undefined as Document | undefined }, +})); +vi.mock("svelte", async (importOriginal) => ({ + ...(await importOriginal()), + mount: rendering.mount, + unmount: rendering.unmount, +})); +vi.mock("@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions", () => ({ + get _activeDocument() { + return rendering.legacy.document; + }, +})); +vi.mock("@/modules/services/LiveSyncUI/DialogHost.svelte", () => ({ default: () => {} })); + +import { + BrowserModal, + BrowserSvelteDialogManager, + LiveSyncBrowserDialog, +} from "@/apps/browser/BrowserSvelteDialogManager"; +import { createBrowserUiScope } from "@/apps/browser/BrowserUiScope"; + +class TestElement { + children: TestElement[] = []; + parentElement: TestElement | null = null; + hidden = false; + className = ""; + textContent = ""; + constructor( + readonly ownerDocument: TestDocument, + readonly tagName: string + ) {} + appendChild(child: TestElement) { + child.remove(); + child.parentElement = this; + this.children.push(child); + return child; + } + remove() { + if (this.parentElement) + this.parentElement.children = this.parentElement.children.filter((child) => child !== this); + this.parentElement = null; + } + replaceChildren() { + for (const child of [...this.children]) child.remove(); + } +} +class TestDocument { + body = new TestElement(this, "body"); + createElement(tag: string) { + return new TestElement(this, tag); + } +} +const asDocument = (document: TestDocument) => document as unknown as Document; +const component = (() => {}) as unknown as ComponentHasResult; +type Context = ReturnType; +type UiOptions = { document?: Document; signal?: AbortSignal }; +function createManager(options?: UiOptions) { + const context = createServiceContext(); + return { + context, + manager: new BrowserSvelteDialogManager( + context, + {} as SvelteDialogManagerDependencies, + options ? createBrowserUiScope(context, options) : undefined + ), + }; +} +function lastMount() { + const call = rendering.mount.mock.calls[rendering.mount.mock.calls.length - 1]; + return { target: call[1].target as TestElement, props: call[1].props as DialogHostProps }; +} + +beforeEach(() => { + vi.clearAllMocks(); + rendering.legacy.document = asDocument(new TestDocument()); + rendering.mount.mockImplementation(() => ({})); +}); +afterEach(() => { + rendering.legacy.document = undefined; +}); + +describe("Browser Svelte dialogue ownership", () => { + it("releases every per-dialogue context subscription on legacy direct close", async () => { + const subscriptions: ReturnType[] = []; + const context = { + events: { + onceEvent: vi.fn(() => { + const unregister = vi.fn(); + subscriptions.push(unregister); + return unregister; + }), + }, + } as unknown as Context; + const dialog = new LiveSyncBrowserDialog(context, {} as SvelteDialogManagerDependencies, component); + dialog.open(); + const result = dialog.waitForClose(); + dialog.close(); + await expect(result).resolves.toBeUndefined(); + expect(subscriptions.length).toBeGreaterThan(0); + for (const unsubscribe of subscriptions) expect(unsubscribe).toHaveBeenCalledOnce(); + }); + + it("preserves the legacy document, normal result, and component cleanup", async () => { + const { manager } = createManager(); + const result = manager.openSvelteDialog(component, "initial"); + const { target, props } = lastMount(); + expect(target.ownerDocument).toBe(rendering.legacy.document); + expect(props.getInitialData?.()).toBe("initial"); + props.setResult("selected"); + props.closeDialog(); + await expect(result).resolves.toBe("selected"); + expect((rendering.legacy.document as unknown as TestDocument).body.children).toHaveLength(0); + expect(rendering.unmount).toHaveBeenCalledOnce(); + }); + it("preserves rejection when the owning context unloads", async () => { + const { context, manager } = createManager(); + const result = manager.openSvelteDialog(component); + const rejected = expect(result).rejects.toThrow("Plug-in unloaded"); + context.events.emitEvent(EVENT_PLUGIN_UNLOADED); + await rejected; + expect(rendering.unmount).toHaveBeenCalledOnce(); + expect((rendering.legacy.document as unknown as TestDocument).body.children).toHaveLength(0); + }); + it("uses its supplied document even when the legacy document changes before opening", async () => { + const targetDocument = new TestDocument(); + const { manager } = createManager({ document: asDocument(targetDocument) }); + rendering.legacy.document = asDocument(new TestDocument()); + const result = manager.openSvelteDialog(component); + const { target, props } = lastMount(); + expect(target.ownerDocument).toBe(targetDocument); + expect(targetDocument.body.children).toHaveLength(1); + expect((rendering.legacy.document as unknown as TestDocument).body.children).toHaveLength(0); + props.closeDialog(); + await expect(result).resolves.toBeUndefined(); + }); + it("aborts one owner without closing another owner's dialogue", async () => { + const firstOwner = new AbortController(); + const firstDocument = new TestDocument(); + const secondDocument = new TestDocument(); + const first = createManager({ document: asDocument(firstDocument), signal: firstOwner.signal }); + const second = createManager({ document: asDocument(secondDocument) }); + const firstResult = first.manager.openSvelteDialog(component); + let observedFirst: unknown = "still pending"; + void firstResult.then( + () => { + observedFirst = "resolved"; + }, + (error) => { + observedFirst = error; + } + ); + const secondResult = second.manager.openSvelteDialog(component); + const secondProps = lastMount().props; + const reason = new Error("fixture owner ended"); + firstOwner.abort(reason); + await vi.waitFor(() => expect(observedFirst).toBe(reason), { timeout: 1000, interval: 20 }); + expect(firstDocument.body.children).toHaveLength(0); + expect(secondDocument.body.children).toHaveLength(1); + secondProps.setResult("second"); + secondProps.closeDialog(); + await expect(secondResult).resolves.toBe("second"); + }); + it("does not render for an already aborted owner or reopen after unload", async () => { + const owner = new AbortController(); + const reason = new Error("fixture owner ended"); + owner.abort(reason); + const aborted = createManager({ signal: owner.signal }); + const rejected = aborted.manager.openSvelteDialog(component).catch((error) => error); + expect(rendering.mount).not.toHaveBeenCalled(); + await expect(rejected).resolves.toBe(reason); + const unloaded = createManager(); + unloaded.context.events.emitEvent(EVENT_PLUGIN_UNLOADED); + const afterUnload = unloaded.manager.openSvelteDialog(component).catch((error) => error); + expect(rendering.mount).not.toHaveBeenCalled(); + await expect(afterUnload).resolves.toBeInstanceOf(Error); + }); + it("keeps a native modal in the document which created its elements", () => { + const document = new TestDocument(); + const modal = new BrowserModal(asDocument(document)); + rendering.legacy.document = asDocument(new TestDocument()); + modal.open(); + expect(document.body.children).toEqual([modal.baseEl]); + expect((modal.contentEl as unknown as TestElement).ownerDocument).toBe(document); + modal.close(); + expect(document.body.children).toHaveLength(0); + }); +}); diff --git a/test/apps/browser/BrowserUiScope.unit.spec.ts b/test/apps/browser/BrowserUiScope.unit.spec.ts new file mode 100644 index 00000000..abf32433 --- /dev/null +++ b/test/apps/browser/BrowserUiScope.unit.spec.ts @@ -0,0 +1,56 @@ +import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; +import { EVENT_PLUGIN_UNLOADED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; +import { describe, expect, it, vi } from "vitest"; +import { createBrowserUiScope } from "@/apps/browser/BrowserUiScope"; + +describe("browser UI scope", () => { + it("owns its document and cancellation independently from another application", () => { + const firstContext = createServiceContext(); + const secondContext = createServiceContext(); + const firstDocument = {} as Document; + const secondDocument = {} as Document; + const first = createBrowserUiScope(firstContext, { document: firstDocument }); + const second = createBrowserUiScope(secondContext, { document: secondDocument }); + const abort = vi.fn(); + first.signal.addEventListener("abort", abort); + firstContext.events.emitEvent(EVENT_PLUGIN_UNLOADED); + firstContext.events.emitEvent(EVENT_PLUGIN_UNLOADED); + expect(first.document).toBe(firstDocument); + expect(second.document).toBe(secondDocument); + expect(first.signal.aborted).toBe(true); + expect(first.signal.reason).toEqual(new Error("Plug-in unloaded")); + expect(second.signal.aborted).toBe(false); + expect(abort).toHaveBeenCalledOnce(); + secondContext.events.emitEvent(EVENT_PLUGIN_UNLOADED); + }); + it("forwards parent cancellation and unregisters the context subscription", () => { + const context = createServiceContext(); + const parent = new AbortController(); + const unregister = vi.fn(); + const once = vi.spyOn(context.events, "onceEvent").mockReturnValue(unregister); + const scope = createBrowserUiScope(context, { signal: parent.signal }); + const reason = new Error("fixture host ended"); + parent.abort(reason); + expect(scope.signal.aborted).toBe(true); + expect(scope.signal.reason).toBe(reason); + expect(unregister).toHaveBeenCalledOnce(); + once.mockRestore(); + }); + it("removes the parent listener on unload and does not subscribe an already aborted parent", () => { + const context = createServiceContext(); + const parent = new AbortController(); + const remove = vi.spyOn(parent.signal, "removeEventListener"); + const scope = createBrowserUiScope(context, { signal: parent.signal }); + context.events.emitEvent(EVENT_PLUGIN_UNLOADED); + expect(remove).toHaveBeenCalledWith("abort", expect.any(Function)); + const reason = scope.signal.reason; + parent.abort(new Error("later")); + expect(scope.signal.reason).toBe(reason); + const closedContext = createServiceContext(); + const subscribe = vi.spyOn(closedContext.events, "onceEvent"); + const closed = createBrowserUiScope(closedContext, { signal: parent.signal }); + expect(closed.signal.aborted).toBe(true); + expect(closed.signal.reason).toBe(parent.signal.reason); + expect(subscribe).not.toHaveBeenCalled(); + }); +}); diff --git a/test/apps/browser/SvelteDialogSession.unit.spec.ts b/test/apps/browser/SvelteDialogSession.unit.spec.ts index f87fe25f..08b46313 100644 --- a/test/apps/browser/SvelteDialogSession.unit.spec.ts +++ b/test/apps/browser/SvelteDialogSession.unit.spec.ts @@ -86,3 +86,64 @@ describe("SvelteDialogSession", () => { expect(close).toHaveBeenCalledOnce(); }); }); + +describe("SvelteDialogSession owner signal", () => { + it("rejects and releases the mounted dialogue when its owner aborts", async () => { + const context = createServiceContext(); + const controller = new AbortController(); + const lifecycle = { signal: controller.signal }; + const unmount = vi.fn().mockResolvedValue(undefined); + let session: SvelteDialogSession; + const close = vi.fn(() => session.onClose()); + session = new SvelteDialogSession({ + ...lifecycle, + surface: { + contentEl: { replaceChildren: vi.fn() } as unknown as HTMLElement, + close, + setTitle: vi.fn(), + }, + context, + dependencies: {} as SvelteDialogManagerDependencies, + dialogHost: (() => {}) as unknown as Component, + component: (() => {}) as unknown as ComponentHasResult, + renderer: { mount: () => ({}) as never, unmount }, + }); + session.onOpen(); + let observed: unknown = "pending"; + void session.waitForClose().catch((error) => { + observed = error; + }); + const reason = new Error("fixture owner ended"); + controller.abort(reason); + await vi.waitFor(() => expect(observed).toBe(reason), { timeout: 1000, interval: 20 }); + expect(close).toHaveBeenCalledOnce(); + expect(unmount).toHaveBeenCalledOnce(); + context.events.emitEvent(EVENT_PLUGIN_UNLOADED); + expect(close).toHaveBeenCalledOnce(); + }); + + it("detaches an owner signal when normal close has already settled the result", async () => { + const context = createServiceContext(); + const controller = new AbortController(); + const remove = vi.spyOn(controller.signal, "removeEventListener"); + const lifecycle = { signal: controller.signal }; + const close = vi.fn(); + const session = new SvelteDialogSession({ + ...lifecycle, + surface: { contentEl: { replaceChildren: vi.fn() } as unknown as HTMLElement, close, setTitle: vi.fn() }, + context, + dependencies: {} as SvelteDialogManagerDependencies, + dialogHost: (() => {}) as unknown as Component, + component: (() => {}) as unknown as ComponentHasResult, + renderer: { mount: () => ({}) as never, unmount: vi.fn().mockResolvedValue(undefined) }, + }); + session.onOpen(); + const result = session.waitForClose(); + session.onClose(); + await expect(result).resolves.toBeUndefined(); + expect(remove).toHaveBeenCalledWith("abort", expect.any(Function)); + controller.abort(); + context.events.emitEvent(EVENT_PLUGIN_UNLOADED); + expect(close).not.toHaveBeenCalled(); + }); +}); diff --git a/test/bench-network/Dockerfile.runner b/test/bench-network/Dockerfile.runner index 180141dd..12dbb8f0 100644 --- a/test/bench-network/Dockerfile.runner +++ b/test/bench-network/Dockerfile.runner @@ -1,14 +1,11 @@ # syntax=docker/dockerfile:1 -FROM node:24-slim +FROM node:26-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl unzip python3 make g++ iproute2 libjpeg-turbo-progs time \ && rm -rf /var/lib/apt/lists/* -ENV DENO_INSTALL=/usr/local -RUN curl -fsSL https://deno.land/install.sh | sh - WORKDIR /workspace COPY package.json package-lock.json ./ @@ -20,22 +17,10 @@ RUN npm ci COPY . . RUN NODE_OPTIONS=--max-old-space-size=3072 LIVESYNC_CLI_TEST_SUPPORT=1 npm run build -w self-hosted-livesync-cli -WORKDIR /workspace/src/apps/cli/testdeno - -RUN deno cache --lock=deno.lock \ - bench-network-cases.ts \ - bench-latency-sweep.ts \ - bench-p2p-split-node.ts \ - bench-p2p.ts \ - bench-couchdb.ts \ - bench-compression.ts \ - test-p2p-sync.ts \ - test-p2p-automatic-host.ts \ - test-p2p-replicator-replacement.ts \ - test-p2p-relay-disconnect.ts +WORKDIR /workspace/test/e2e-cli COPY test/bench-network/run-bench.sh /usr/local/bin/run-livesync-bench -COPY src/apps/cli/testdeno/run-cli-e2e.sh /usr/local/bin/run-livesync-cli-e2e +COPY test/e2e-cli/run-cli-e2e.sh /usr/local/bin/run-livesync-cli-e2e RUN chmod +x /usr/local/bin/run-livesync-bench /usr/local/bin/run-livesync-cli-e2e CMD ["run-livesync-bench"] diff --git a/test/bench-network/compose.yml b/test/bench-network/compose.yml index 2c2b2407..a4211c62 100644 --- a/test/bench-network/compose.yml +++ b/test/bench-network/compose.yml @@ -74,10 +74,10 @@ services: BENCH_COMMAND: ${BENCH_COMMAND:-cases} BENCH_CASES: ${BENCH_CASES:-couchdb-baseline,p2p-direct-local} BENCH_REPEAT_COUNT: ${BENCH_REPEAT_COUNT:-1} - BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results - BENCH_COMPRESSION_RESULT_ROOT: /workspace/src/apps/cli/testdeno/bench-results + BENCH_CASES_ROOT: /workspace/test/e2e-cli/bench-results + BENCH_COMPRESSION_RESULT_ROOT: /workspace/test/e2e-cli/bench-results BENCH_COMPRESSION_REPEAT_COUNT: ${BENCH_COMPRESSION_REPEAT_COUNT:-1} - BENCH_SWEEP_ROOT: /workspace/src/apps/cli/testdeno/bench-results + BENCH_SWEEP_ROOT: /workspace/test/e2e-cli/bench-results BENCH_SWEEP_RTT_MS: ${BENCH_SWEEP_RTT_MS:-20,50,100,150,300} BENCH_SWEEP_INCLUDE_P2P: ${BENCH_SWEEP_INCLUDE_P2P:-true} BENCH_COUCHDB_MANAGED: "false" @@ -110,7 +110,7 @@ services: TURN_SERVERS: ${TURN_SERVERS:-none} LIVESYNC_P2P_PEERS_RETRY: ${LIVESYNC_P2P_PEERS_RETRY:-1} volumes: - - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results + - ./bench-results:/workspace/test/e2e-cli/bench-results couchdb-shim: build: @@ -156,8 +156,8 @@ services: BENCH_COMMAND: ${BENCH_COMMAND:-cases} BENCH_CASES: ${BENCH_CASES:-couchdb-netem-home-wifi} BENCH_REPEAT_COUNT: ${BENCH_REPEAT_COUNT:-1} - BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results - BENCH_SWEEP_ROOT: /workspace/src/apps/cli/testdeno/bench-results + BENCH_CASES_ROOT: /workspace/test/e2e-cli/bench-results + BENCH_SWEEP_ROOT: /workspace/test/e2e-cli/bench-results BENCH_COUCHDB_MANAGED: "false" BENCH_COUCHDB_BACKEND_URI: http://couchdb-shim:5984 BENCH_SHIM_COUCHDB_URI: http://couchdb-shim:5984 @@ -173,7 +173,7 @@ services: BENCH_SYNC_TIMEOUT: ${BENCH_SYNC_TIMEOUT:-300} BENCH_LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0} volumes: - - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results + - ./bench-results:/workspace/test/e2e-cli/bench-results p2p-signalling-shim: build: @@ -219,7 +219,7 @@ services: BENCH_COMMAND: ${BENCH_COMMAND:-cases} BENCH_CASES: ${BENCH_CASES:-p2p-signalling-netem-home-wifi} BENCH_REPEAT_COUNT: ${BENCH_REPEAT_COUNT:-1} - BENCH_CASES_ROOT: /workspace/src/apps/cli/testdeno/bench-results + BENCH_CASES_ROOT: /workspace/test/e2e-cli/bench-results BENCH_SIGNAL_SHIM_RELAY: ws://p2p-signalling-shim:7777/ BENCH_MD_FILE_COUNT: ${BENCH_MD_FILE_COUNT:-20} BENCH_MD_MIN_SIZE_BYTES: ${BENCH_MD_MIN_SIZE_BYTES:-512} @@ -232,7 +232,7 @@ services: LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: ${LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS:-60000} BENCH_LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0} volumes: - - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results + - ./bench-results:/workspace/test/e2e-cli/bench-results p2p-split-host: build: @@ -250,7 +250,7 @@ services: BENCH_P2P_SPLIT_ROLE: host BENCH_SPLIT_RUN_ID: ${BENCH_SPLIT_RUN_ID:-bench-split-run} BENCH_SPLIT_WORK_ROOT: /p2p-work - BENCH_SPLIT_RESULT_ROOT: /workspace/src/apps/cli/testdeno/bench-results + BENCH_SPLIT_RESULT_ROOT: /workspace/test/e2e-cli/bench-results BENCH_RELAY: ws://nostr-relay:7777/ BENCH_APP_ID: ${BENCH_APP_ID:-self-hosted-livesync-cli-benchmark} BENCH_ROOM_ID: ${BENCH_ROOM_ID:-bench-split-room} @@ -276,7 +276,7 @@ services: LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0} volumes: - p2p-split-work:/p2p-work - - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results + - ./bench-results:/workspace/test/e2e-cli/bench-results p2p-split-client: build: @@ -296,7 +296,7 @@ services: BENCH_P2P_SPLIT_ROLE: client BENCH_SPLIT_RUN_ID: ${BENCH_SPLIT_RUN_ID:-bench-split-run} BENCH_SPLIT_WORK_ROOT: /p2p-work - BENCH_SPLIT_RESULT_ROOT: /workspace/src/apps/cli/testdeno/bench-results + BENCH_SPLIT_RESULT_ROOT: /workspace/test/e2e-cli/bench-results BENCH_RELAY: ws://nostr-relay:7777/ BENCH_APP_ID: ${BENCH_APP_ID:-self-hosted-livesync-cli-benchmark} BENCH_ROOM_ID: ${BENCH_ROOM_ID:-bench-split-room} @@ -317,7 +317,7 @@ services: LIVESYNC_TEST_TEE: ${BENCH_LIVESYNC_TEST_TEE:-0} volumes: - p2p-split-work:/p2p-work - - ./bench-results:/workspace/src/apps/cli/testdeno/bench-results + - ./bench-results:/workspace/test/e2e-cli/bench-results netem-smoke: build: diff --git a/test/bench-network/run-bench.sh b/test/bench-network/run-bench.sh index 5cb158f0..8727aece 100644 --- a/test/bench-network/run-bench.sh +++ b/test/bench-network/run-bench.sh @@ -3,16 +3,16 @@ set -eu case "${BENCH_COMMAND:-cases}" in cases) - exec deno task bench:cases + exec node run-task.ts bench:cases ;; latency-sweep) - exec deno task bench:latency-sweep + exec node run-task.ts bench:latency-sweep ;; compression) - exec deno task bench:compression + exec node run-task.ts bench:compression ;; p2p-split-node) - exec deno task bench:p2p-split-node + exec node run-task.ts bench:p2p-split-node ;; *) echo "Unknown BENCH_COMMAND: ${BENCH_COMMAND}" >&2 diff --git a/test/browser-apps/ui-lifecycle/browser-ui-lifecycle.test.ts b/test/browser-apps/ui-lifecycle/browser-ui-lifecycle.test.ts new file mode 100644 index 00000000..1376500d --- /dev/null +++ b/test/browser-apps/ui-lifecycle/browser-ui-lifecycle.test.ts @@ -0,0 +1,202 @@ +import { assertEquals } from "@std/assert"; +import { resolve } from "@std/path"; +import { chromium } from "playwright"; + +import { observePageFailures, startStaticServer, waitFor } from "../helpers/browser.ts"; + +type OwnerName = "owner-a" | "owner-b" | "owner-c" | "owner-d"; +type DialogOutcome = + | { readonly kind: "resolved"; readonly value: unknown } + | { readonly kind: "rejected"; readonly message: string }; + +interface BrowserUiLifecycleFixture { + readonly ready: boolean; + startDialog(name: OwnerName, initialData?: string): void; + startMenu(name: OwnerName, title: string): void; + clickSavedBackdrop(name: OwnerName): void; + hideMenu(name: OwnerName): void; + unload(name: OwnerName): void; + abortParent(name: OwnerName): void; + state(name: OwnerName): { + readonly dialogs: readonly DialogOutcome[]; + readonly menus: readonly string[]; + readonly hasSavedBackdrop: boolean; + }; +} + +declare global { + interface Window { + browserUiLifecycle?: BrowserUiLifecycleFixture; + } +} + +const fixtureDist = resolve(import.meta.dirname!, "../../../_testdata/browser-ui-lifecycle"); + +Deno.test({ + name: "Browser UI: dialogue and menu lifecycles remain owned by their document and plug-in scope", + sanitizeOps: false, + sanitizeResources: false, + async fn() { + const server = await startStaticServer(fixtureDist); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + const assertNoPageFailures = observePageFailures(page); + await page.goto(server.baseUrl); + await page.waitForFunction(() => window.browserUiLifecycle?.ready === true); + + const startDialog = async (name: OwnerName, initialData?: string) => + await page.evaluate( + ({ name, initialData }) => window.browserUiLifecycle!.startDialog(name, initialData), + { name, initialData } + ); + const startMenu = async (name: OwnerName, title: string) => + await page.evaluate(({ name, title }) => window.browserUiLifecycle!.startMenu(name, title), { + name, + title, + }); + const state = async (name: OwnerName) => + await page.evaluate((ownerName) => window.browserUiLifecycle!.state(ownerName), name); + + const ownerA = page.frameLocator("#owner-a"); + const ownerB = page.frameLocator("#owner-b"); + const ownerC = page.frameLocator("#owner-c"); + const ownerD = page.frameLocator("#owner-d"); + + await startDialog("owner-a", "A-seed"); + await startDialog("owner-b", "B-seed"); + const initial = await page.evaluate(() => ({ + first: window.browserUiLifecycle!.state("owner-a").dialogs, + second: window.browserUiLifecycle!.state("owner-b").dialogs, + })); + assertEquals(initial, { first: [], second: [] }); + assertNoPageFailures(); + await ownerA.getByTestId("lifecycle-dialog").waitFor(); + await ownerB.getByTestId("lifecycle-dialog").waitFor(); + assertEquals(await page.locator(".dialog-host").count(), 0); + assertEquals(await ownerA.locator(".dialog-host").count(), 1); + assertEquals(await ownerB.locator(".dialog-host").count(), 1); + + await ownerA.getByTestId("submit-dialog").click(); + await waitFor(async () => (await state("owner-a")).dialogs.length === 1, "Owner A dialogue did not finish"); + assertEquals((await state("owner-a")).dialogs, [{ kind: "resolved", value: "result:A-seed" }]); + assertEquals(await ownerA.locator(".dialog-host").count(), 0); + assertEquals(await ownerB.locator(".dialog-host").count(), 1); + + await ownerB.getByTestId("close-dialog").click(); + await waitFor(async () => (await state("owner-b")).dialogs.length === 1, "Owner B dialogue did not finish"); + assertEquals((await state("owner-b")).dialogs, [{ kind: "resolved", value: "undefined" }]); + + await startMenu("owner-a", "first menu"); + await ownerA.getByText("first menu", { exact: true }).waitFor(); + assertEquals(await page.locator(".background").count(), 0); + assertEquals(await ownerA.locator(".background").count(), 1); + assertEquals(await ownerB.locator(".background").count(), 0); + + await startMenu("owner-a", "replacement menu"); + await ownerA.getByText("replacement menu", { exact: true }).waitFor(); + await waitFor( + async () => (await state("owner-a")).menus.length === 1, + "Replacing the menu did not close its first promise" + ); + assertEquals((await state("owner-a")).menus, ["resolved"]); + assertEquals((await state("owner-a")).hasSavedBackdrop, true); + + await page.evaluate(() => window.browserUiLifecycle!.clickSavedBackdrop("owner-a")); + assertEquals(await ownerA.locator(".background").count(), 1); + assertEquals((await state("owner-a")).menus, ["resolved"]); + await page.evaluate(() => window.browserUiLifecycle!.hideMenu("owner-a")); + await waitFor( + async () => (await state("owner-a")).menus.length === 2, + "Hiding the replacement menu did not finish" + ); + assertEquals((await state("owner-a")).menus, ["resolved", "resolved"]); + assertEquals(await ownerA.locator(".background").count(), 0); + + await startDialog("owner-a", "unload-A"); + await startMenu("owner-a", "A unload menu"); + await startDialog("owner-b", "still-B"); + await startMenu("owner-b", "B remains open"); + await ownerA.getByTestId("lifecycle-dialog").waitFor(); + await ownerA.getByText("A unload menu", { exact: true }).waitFor(); + await ownerB.getByTestId("lifecycle-dialog").waitFor(); + await ownerB.getByText("B remains open", { exact: true }).waitFor(); + + await page.evaluate(() => window.browserUiLifecycle!.unload("owner-a")); + await waitFor( + async () => + (await state("owner-a")).dialogs.length === 2 && (await state("owner-a")).menus.length === 3, + "Owner A unload did not settle its UI operations" + ); + assertEquals((await state("owner-a")).dialogs[1], { kind: "rejected", message: "Plug-in unloaded" }); + assertEquals(await ownerA.locator(".dialog-host").count(), 0); + assertEquals(await ownerA.locator(".background").count(), 0); + assertEquals(await ownerB.locator(".dialog-host").count(), 1); + assertEquals(await ownerB.locator(".background").count(), 1); + + await startDialog("owner-a", "after-unload"); + await startMenu("owner-a", "after unload menu"); + await waitFor( + async () => + (await state("owner-a")).dialogs.length === 3 && (await state("owner-a")).menus.length === 4, + "Post-unload UI operations did not settle" + ); + assertEquals((await state("owner-a")).dialogs[2], { kind: "rejected", message: "Plug-in unloaded" }); + assertEquals((await state("owner-a")).menus[3], "resolved"); + assertEquals(await ownerA.locator(".dialog-host").count(), 0); + assertEquals(await ownerA.locator(".background").count(), 0); + + await startDialog("owner-c", "parent-abort"); + await startMenu("owner-c", "parent abort menu"); + await ownerC.getByTestId("lifecycle-dialog").waitFor(); + await ownerC.getByText("parent abort menu", { exact: true }).waitFor(); + await page.evaluate(() => window.browserUiLifecycle!.abortParent("owner-c")); + await waitFor( + async () => + (await state("owner-c")).dialogs.length === 1 && (await state("owner-c")).menus.length === 1, + "Parent abort did not settle its UI operations" + ); + assertEquals((await state("owner-c")).dialogs[0], { kind: "rejected", message: "Fixture UI owner ended" }); + assertEquals((await state("owner-c")).menus[0], "resolved"); + assertEquals(await ownerC.locator(".dialog-host").count(), 0); + assertEquals(await ownerC.locator(".background").count(), 0); + assertEquals(await ownerB.locator(".dialog-host").count(), 1); + assertEquals(await ownerB.locator(".background").count(), 1); + + await startDialog("owner-c", "after-parent-abort"); + await startMenu("owner-c", "after parent abort menu"); + await waitFor( + async () => + (await state("owner-c")).dialogs.length === 2 && (await state("owner-c")).menus.length === 2, + "Post-abort UI operations did not settle" + ); + assertEquals((await state("owner-c")).dialogs[1], { kind: "rejected", message: "Fixture UI owner ended" }); + assertEquals((await state("owner-c")).menus[1], "resolved"); + assertEquals(await ownerC.locator(".dialog-host").count(), 0); + assertEquals(await ownerC.locator(".background").count(), 0); + + await startDialog("owner-d", "pre-aborted"); + await startMenu("owner-d", "pre-aborted menu"); + await waitFor( + async () => + (await state("owner-d")).dialogs.length === 1 && (await state("owner-d")).menus.length === 1, + "Pre-aborted UI operations did not settle" + ); + assertEquals((await state("owner-d")).dialogs[0], { kind: "rejected", message: "Fixture UI owner ended" }); + assertEquals((await state("owner-d")).menus[0], "resolved"); + assertEquals(await ownerD.locator(".dialog-host").count(), 0); + assertEquals(await ownerD.locator(".background").count(), 0); + + await page.evaluate(() => window.browserUiLifecycle!.hideMenu("owner-b")); + await waitFor(async () => (await state("owner-b")).menus.length === 1, "Owner B menu did not close"); + await ownerB.getByTestId("close-dialog").click(); + await waitFor(async () => (await state("owner-b")).dialogs.length === 2, "Owner B dialogue did not close"); + assertEquals(await ownerB.locator(".background").count(), 0); + assertEquals(await ownerB.locator(".dialog-host").count(), 0); + assertNoPageFailures(); + } finally { + await browser.close(); + await server.close(); + } + }, +}); diff --git a/test/browser-apps/ui-lifecycle/index.html b/test/browser-apps/ui-lifecycle/index.html new file mode 100644 index 00000000..bbee4dc7 --- /dev/null +++ b/test/browser-apps/ui-lifecycle/index.html @@ -0,0 +1,15 @@ + + + + + + Browser UI lifecycle fixture + + + + + + + + + diff --git a/test/browser-apps/ui-lifecycle/src/TestLifecycleDialog.svelte b/test/browser-apps/ui-lifecycle/src/TestLifecycleDialog.svelte new file mode 100644 index 00000000..22858aaf --- /dev/null +++ b/test/browser-apps/ui-lifecycle/src/TestLifecycleDialog.svelte @@ -0,0 +1,16 @@ + + +
+ + +
diff --git a/test/browser-apps/ui-lifecycle/src/main.ts b/test/browser-apps/ui-lifecycle/src/main.ts new file mode 100644 index 00000000..1f8c3d49 --- /dev/null +++ b/test/browser-apps/ui-lifecycle/src/main.ts @@ -0,0 +1,154 @@ +import { createServiceContext } from "@vrtmrz/livesync-commonlib/context"; +import { EVENT_PLUGIN_UNLOADED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents"; +import { MemoryValueStore } from "@vrtmrz/value-store"; + +import { Menu } from "@/apps/browser/BrowserMenu"; +import { createLiveSyncBrowserServiceHub } from "@/apps/browser/createLiveSyncBrowserServiceHub"; +import TestLifecycleDialog from "./TestLifecycleDialog.svelte"; + +type OwnerName = "owner-a" | "owner-b" | "owner-c" | "owner-d"; +type DialogOutcome = + | { readonly kind: "resolved"; readonly value: unknown } + | { readonly kind: "rejected"; readonly message: string }; + +interface Owner { + readonly context: ReturnType; + readonly document: Document; + readonly hub: ReturnType; + readonly menu: Menu; + readonly parentAbort?: AbortController; + readonly dialogOutcomes: DialogOutcome[]; + readonly menuOutcomes: string[]; + savedBackdrop?: HTMLElement; +} + +interface BrowserUiLifecycleFixture { + readonly ready: boolean; + startDialog(name: OwnerName, initialData?: string): void; + startMenu(name: OwnerName, title: string): void; + clickSavedBackdrop(name: OwnerName): void; + hideMenu(name: OwnerName): void; + unload(name: OwnerName): void; + abortParent(name: OwnerName): void; + state(name: OwnerName): { + readonly dialogs: readonly DialogOutcome[]; + readonly menus: readonly string[]; + readonly hasSavedBackdrop: boolean; + }; +} + +declare global { + interface Window { + browserUiLifecycle?: BrowserUiLifecycleFixture; + } +} + +const owners = new Map(); + +function ownerDocument(name: OwnerName): Document { + const frame = document.getElementById(name); + if (!(frame instanceof HTMLIFrameElement) || !frame.contentDocument) { + throw new Error(`The ${name} frame is not ready`); + } + return frame.contentDocument; +} + +function createOwner(name: OwnerName, options: { parentAbort?: AbortController; preAborted?: boolean } = {}): Owner { + const frameDocument = ownerDocument(name); + const parentAbort = options.parentAbort; + if (options.preAborted) { + parentAbort?.abort(new Error("Fixture UI owner ended")); + } + const context = createServiceContext(); + const hub = createLiveSyncBrowserServiceHub({ + context, + deviceValues: new MemoryValueStore(), + ui: { + document: frameDocument, + ...(parentAbort ? { signal: parentAbort.signal } : {}), + }, + }); + return { + context, + document: frameDocument, + hub, + menu: new Menu({ document: frameDocument, signal: hub.browserUi.signal }), + ...(parentAbort ? { parentAbort } : {}), + dialogOutcomes: [], + menuOutcomes: [], + }; +} + +const ownerCAbort = new AbortController(); +const ownerDAbort = new AbortController(); +ownerDAbort.abort(new Error("Fixture UI owner ended")); + +owners.set("owner-a", createOwner("owner-a")); +owners.set("owner-b", createOwner("owner-b")); +owners.set("owner-c", createOwner("owner-c", { parentAbort: ownerCAbort })); +owners.set("owner-d", createOwner("owner-d", { parentAbort: ownerDAbort, preAborted: true })); + +const fixture: BrowserUiLifecycleFixture = { + ready: true, + startDialog(name, initialData) { + const owner = owners.get(name)!; + try { + void owner.hub.UI.dialogManager.open(TestLifecycleDialog, initialData).then( + (value) => + owner.dialogOutcomes.push({ kind: "resolved", value: value === undefined ? "undefined" : value }), + (error: unknown) => + owner.dialogOutcomes.push({ + kind: "rejected", + message: error instanceof Error ? error.message : String(error), + }) + ); + } catch (error) { + owner.dialogOutcomes.push({ + kind: "rejected", + message: error instanceof Error ? error.message : String(error), + }); + } + }, + startMenu(name, title) { + const owner = owners.get(name)!; + owner.savedBackdrop = owner.document.querySelector(".background") ?? undefined; + try { + void owner.menu + .addItem((item) => item.setTitle(title)) + .showAtPosition({ x: 12, y: 12 }) + .then( + () => owner.menuOutcomes.push("resolved"), + (error: unknown) => + owner.menuOutcomes.push(`rejected:${error instanceof Error ? error.message : String(error)}`) + ); + } catch (error) { + owner.menuOutcomes.push(`rejected:${error instanceof Error ? error.message : String(error)}`); + } + }, + clickSavedBackdrop(name) { + const owner = owners.get(name)!; + const window = owner.document.defaultView; + if (owner.savedBackdrop && window) { + owner.savedBackdrop.dispatchEvent(new window.MouseEvent("click", { bubbles: true })); + } + }, + hideMenu(name) { + owners.get(name)!.menu.hide(); + }, + unload(name) { + owners.get(name)!.context.events.emitEvent(EVENT_PLUGIN_UNLOADED); + }, + abortParent(name) { + owners.get(name)!.parentAbort?.abort(new Error("Fixture UI owner ended")); + }, + state(name) { + const owner = owners.get(name)!; + return { + dialogs: [...owner.dialogOutcomes], + menus: [...owner.menuOutcomes], + hasSavedBackdrop: owner.savedBackdrop !== undefined, + }; + }, +}; + +window.browserUiLifecycle = fixture; diff --git a/test/browser-apps/ui-lifecycle/tsconfig.json b/test/browser-apps/ui-lifecycle/tsconfig.json new file mode 100644 index 00000000..e83a2d8e --- /dev/null +++ b/test/browser-apps/ui-lifecycle/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "target": "ES2020", + "lib": ["ES2020", "DOM", "DOM.Iterable"] + }, + "include": ["src/**/*.ts", "src/**/*.svelte"], + "exclude": [] +} diff --git a/test/browser-apps/ui-lifecycle/vite.config.ts b/test/browser-apps/ui-lifecycle/vite.config.ts new file mode 100644 index 00000000..5c10c5b4 --- /dev/null +++ b/test/browser-apps/ui-lifecycle/vite.config.ts @@ -0,0 +1,25 @@ +import { fileURLToPath } from "node:url"; +import { dirname, resolve } from "node:path"; +import { svelte } from "@sveltejs/vite-plugin-svelte"; +import { defineConfig } from "vite"; + +const fixtureDirectory = dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = resolve(fixtureDirectory, "../../.."); + +export default defineConfig({ + root: fixtureDirectory, + plugins: [svelte()], + resolve: { + alias: { + "@": resolve(repositoryRoot, "src"), + }, + }, + base: "./", + build: { + outDir: resolve(repositoryRoot, "_testdata/browser-ui-lifecycle"), + emptyOutDir: true, + rollupOptions: { + input: resolve(fixtureDirectory, "index.html"), + }, + }, +}); diff --git a/test/build/terserConfig.unit.spec.ts b/test/build/terserConfig.unit.spec.ts new file mode 100644 index 00000000..83672bbf --- /dev/null +++ b/test/build/terserConfig.unit.spec.ts @@ -0,0 +1,58 @@ +import { runInNewContext } from "node:vm"; +import { describe, expect, it } from "vitest"; +import { minify } from "terser"; +import { terserOption } from "../../terser.config.mjs"; + +const fixture = ` + const host = require("obsidian"); + class ModuleLog { + constructor(configuration) { this.savedSettings = configuration; } + read(longParameterName) { + const longIntermediateValue = host.prefix + longParameterName; + return { savedSettings: this.savedSettings, message: longIntermediateValue }; + } + } + function renderMessage(inputValue) { return new ModuleLog({ persistedField: "unchanged" }).read(inputValue); } + function readWithEval(longEvalParameter) { return eval("longEvalParameter"); } + function failWithName() { throw new Error("fixture failure"); } + module.exports = { ModuleLog, renderMessage, readWithEval, failWithName }; +`; + +describe("production minifier", () => { + it("shortens local identifiers while preserving host properties, module names, and diagnostics", async () => { + const result = await minify(fixture, { ...terserOption, sourceMap: false }); + expect(result.code).toBeDefined(); + const sandbox = { + module: { + exports: {} as { + ModuleLog: { name: string }; + renderMessage(input: string): unknown; + readWithEval(input: string): string; + failWithName(): never; + }, + }, + require: (name: string) => { + expect(name).toBe("obsidian"); + return { prefix: "host:" }; + }, + }; + runInNewContext(result.code!, sandbox); + const api = sandbox.module.exports; + expect(api.ModuleLog.name).toBe("ModuleLog"); + expect(api.renderMessage.name).toBe("renderMessage"); + expect(api.renderMessage("sample")).toEqual({ + savedSettings: { persistedField: "unchanged" }, + message: "host:sample", + }); + expect(api.readWithEval("evaluated")).toBe("evaluated"); + let failure: Error | undefined; + try { + api.failWithName(); + } catch (error) { + failure = error as Error; + } + expect(failure?.stack).toContain("failWithName"); + expect(result.code).not.toContain("longParameterName"); + expect(result.code).not.toContain("longIntermediateValue"); + }); +}); diff --git a/src/apps/cli/testdeno/.test.env b/test/e2e-cli/.test.env similarity index 100% rename from src/apps/cli/testdeno/.test.env rename to test/e2e-cli/.test.env diff --git a/test/e2e-cli/CONTRIBUTING_TESTS.md b/test/e2e-cli/CONTRIBUTING_TESTS.md new file mode 100644 index 00000000..4219b281 --- /dev/null +++ b/test/e2e-cli/CONTRIBUTING_TESTS.md @@ -0,0 +1,70 @@ +# Writing CLI Tests with Node.js + +Add CLI end-to-end tests under `test/e2e-cli/`. This suite uses Node.js 26 native TypeScript support and the built-in `node:test` runner so that local filesystem and subprocess scenarios can run on Windows, macOS, and Linux. + +## Principles + +- Keep one scenario per file when practical. +- Reuse helpers from `helpers/` rather than duplicating process, Docker, or settings logic. +- Prefer deterministic data over random inputs unless randomness is explicitly required. +- Ensure every test can clean up automatically. +- Keep assertions actionable with clear failure messages. +- Keep test concurrency at one. Tests share the built CLI and may start local services. +- Use TypeScript syntax which Node can strip without compilation; `test/e2e-cli/tsconfig.json` checks this boundary. + +## Directory structure + +```text +test/e2e-cli/ + helpers/ + backgroundCli.ts + cli.ts + docker.ts + env.ts + p2p.ts + settings.ts + temp.ts + test-*.ts + run-task.ts + tasks.json + ci-tasks.json +``` + +## Test file naming and assertions + +Use `test-.ts`, aligned with existing CLI test names where practical. Import the built-in test and assertion APIs: `test` and `describe` from `node:test`, and assertions from `node:assert/strict`. For example: + +```ts +import assert from "node:assert/strict"; +import { test } from "node:test"; + +test("feature: behaviour", async (t) => { + await t.test("preserves the expected result", () => { + assert.equal(actualValue, expectedValue); + }); +}); +``` + +## Core helper usage + +- `runCli(...)` and `runCliOrFail(...)` execute the CLI and return or validate its result. +- `runCliWithInputOrFail(input, ...)` supports `put` and other stdin-driven commands. +- `TempDir` and `await using` provide automatic temporary-directory cleanup. +- Settings helpers create baseline files and apply CouchDB or P2P configuration. +- Docker helpers start, initialise, and stop local CouchDB, RustFS, and relay services when a test needs them. +- `startCliInBackground(...)` manages long-running host commands such as `p2p-host`. + +Prefer `try`/`finally` when a test needs explicit cleanup in addition to registered test hooks. For network-sensitive commands, use `LIVESYNC_CLI_RETRY` during debugging. Keep Docker container reuse disabled by default unless debugging. + +## Task registration + +Add each supported task name to `tasks.json`. A task record uses one of these forms: + +- `{ "test": ["test-file.ts"] }`, optionally with a `filter` regular expression for named subtests. +- `{ "script": "script.ts" }` for a TypeScript script. +- `{ "tasks": ["task-name"] }` for an ordered task group. +- `{ "command": "...", "args": [...] }` only for the retained legacy benchmark shell scripts. + +Add a task name to `ci-tasks.json` only when it belongs in the standard service-backed CI suite. That file is authoritative for both `run-ci-suite.ts` and the automatic CI matrix. Keep the local `test:local` group service-free so it remains suitable for Windows, macOS, and Linux smoke runs. + +Use `npm run test:e2e:task --workspace self-hosted-livesync-cli -- ` from the repository root to run one task after building the CLI. The standard suite and canonical P2P suite use `npm run test:e2e:cli` and `npm run test:e2e:cli:p2p`, respectively. P2P network scenarios use Compose because host networking and WebRTC candidate selection are not reproducible across environments; individual host-direct P2P tasks are for explicit diagnostics. diff --git a/src/apps/cli/testdeno/bench-compression.ts b/test/e2e-cli/bench-compression.ts similarity index 96% rename from src/apps/cli/testdeno/bench-compression.ts rename to test/e2e-cli/bench-compression.ts index dafef1d5..2db4ba00 100644 --- a/src/apps/cli/testdeno/bench-compression.ts +++ b/test/e2e-cli/bench-compression.ts @@ -1,4 +1,6 @@ -import { join } from "@std/path"; +import { REPOSITORY_ROOT } from "./helpers/paths.ts"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts"; import { assertFilesEqual } from "./helpers/cli.ts"; @@ -82,12 +84,12 @@ const DATASET_KINDS: DatasetKind[] = ["md", "jpg", "png", "json", "ts", "gz", "b const COMPRESSED_MARKER = "\u000eLZ\u001d"; function readEnvString(name: string, fallback: string): string { - const value = Deno.env.get(name)?.trim(); + const value = process.env[name]?.trim(); return value ? value : fallback; } function readEnvPositiveInteger(name: string, fallback: number): number { - const raw = Deno.env.get(name)?.trim(); + const raw = process.env[name]?.trim(); if (!raw) return fallback; const value = Number(raw); if (!Number.isInteger(value) || value <= 0) { @@ -97,7 +99,7 @@ function readEnvPositiveInteger(name: string, fallback: number): number { } function readEnvPositiveNumber(name: string, fallback: number): number { - const raw = Deno.env.get(name)?.trim(); + const raw = process.env[name]?.trim(); if (!raw) return fallback; const value = Number(raw); if (!Number.isFinite(value) || value <= 0) { @@ -107,7 +109,7 @@ function readEnvPositiveNumber(name: string, fallback: number): number { } function readEnvBoolean(name: string, fallback: boolean): boolean { - const raw = Deno.env.get(name)?.trim(); + const raw = process.env[name]?.trim(); if (!raw) return fallback; return /^(1|true|yes|on)$/i.test(raw); } @@ -429,14 +431,14 @@ async function main(): Promise { const passphrase = readEnvString("BENCH_PASSPHRASE", "compression-benchmark-passphrase"); const resultRoot = readEnvString("BENCH_COMPRESSION_RESULT_ROOT", "bench-results"); const resultPath = - Deno.env.get("BENCH_COMPRESSION_RESULT_JSON")?.trim() || + process.env.BENCH_COMPRESSION_RESULT_JSON?.trim() || join(resultRoot, `compression-${new Date().toISOString().replaceAll(":", "-")}.json`); const createdDatabases = new Set(); const results: RunResult[] = []; let managedStarted = false; - await Deno.mkdir(resultRoot, { recursive: true }); - const proxy = startCouchdbProxy({ backendUri, proxyUri, requestedRttMs }); + await mkdir(resultRoot, { recursive: true }); + const proxy = await startCouchdbProxy({ backendUri, proxyUri, requestedRttMs }); try { for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) { @@ -457,8 +459,8 @@ async function main(): Promise { const vaultB = workDir.join("vault-b"); const settingsA = workDir.join("settings-a.json"); const settingsB = workDir.join("settings-b.json"); - await Deno.mkdir(vaultA, { recursive: true }); - await Deno.mkdir(vaultB, { recursive: true }); + await mkdir(vaultA, { recursive: true }); + await mkdir(vaultB, { recursive: true }); await initSettingsFile(settingsA); await initSettingsFile(settingsB); await Promise.all( @@ -502,7 +504,7 @@ async function main(): Promise { databaseName, dataset, }); - const settings = JSON.parse(await Deno.readTextFile(settingsA)) as Record; + const settings = JSON.parse(await readFile(settingsA, "utf8")) as Record; const effectiveSettings = Object.fromEntries( [ "encrypt", @@ -553,8 +555,9 @@ async function main(): Promise { generatedAt: new Date().toISOString(), commonlibVersion: ( JSON.parse( - await Deno.readTextFile( - join(import.meta.dirname!, "../../../../node_modules/@vrtmrz/livesync-commonlib/package.json") + await readFile( + join(REPOSITORY_ROOT, "node_modules/@vrtmrz/livesync-commonlib/package.json"), + "utf8" ) ) as { version: string } ).version, @@ -581,7 +584,7 @@ async function main(): Promise { summary: summariseResults(results), runs: results, }; - await Deno.writeTextFile(resultPath, JSON.stringify(output, null, 2)); + await writeFile(resultPath, JSON.stringify(output, null, 2)); console.log(JSON.stringify(output, null, 2)); console.error(`[Compression benchmark] wrote ${resultPath}`); } finally { @@ -598,6 +601,6 @@ async function main(): Promise { if (import.meta.main) { main().catch((error) => { console.error("[Compression benchmark fatal]", error); - Deno.exit(1); + process.exitCode = 1; }); } diff --git a/src/apps/cli/testdeno/bench-couchdb.ts b/test/e2e-cli/bench-couchdb.ts similarity index 83% rename from src/apps/cli/testdeno/bench-couchdb.ts rename to test/e2e-cli/bench-couchdb.ts index 1bfe616c..b0d0fe2c 100644 --- a/src/apps/cli/testdeno/bench-couchdb.ts +++ b/test/e2e-cli/bench-couchdb.ts @@ -1,3 +1,5 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; import { TempDir } from "./helpers/temp.ts"; import { applyRemoteSyncSettings, @@ -48,12 +50,12 @@ type BenchmarkConfig = { }; function readEnvString(name: string, fallback: string): string { - const value = Deno.env.get(name)?.trim(); + const value = process.env[name]?.trim(); return value && value.length > 0 ? value : fallback; } function readEnvNumber(name: string, fallback: number): number { - const raw = Deno.env.get(name); + const raw = process.env[name]; if (raw === undefined || raw.trim() === "") { return fallback; } @@ -66,7 +68,7 @@ function readEnvNumber(name: string, fallback: number): number { } function readEnvBool(name: string, fallback: boolean): boolean { - const raw = Deno.env.get(name); + const raw = process.env[name]; if (raw === undefined || raw.trim() === "") { return fallback; } @@ -74,7 +76,7 @@ function readEnvBool(name: string, fallback: boolean): boolean { } function readEnvStringArray(name: string, fallback: string[]): string[] { - const raw = Deno.env.get(name)?.trim(); + const raw = process.env[name]?.trim(); if (!raw) { return fallback; } @@ -171,7 +173,7 @@ function buildConfig(): BenchmarkConfig { "This benchmark result is scoped to the configured dataset, remote store, and network model.", ]), verificationMode: parseBenchmarkVerificationMode( - Deno.env.get("BENCH_VERIFY_MODE"), + process.env.BENCH_VERIFY_MODE, ), repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)), repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)), @@ -179,7 +181,7 @@ function buildConfig(): BenchmarkConfig { } function readOptionalResultPath(): string | undefined { - const raw = Deno.env.get("BENCH_RESULT_JSON")?.trim(); + const raw = process.env.BENCH_RESULT_JSON?.trim(); if (!raw) { return undefined; } @@ -201,52 +203,51 @@ export type CouchdbProxyCounters = { responseBodyBytes: number; }; -export function startCouchdbProxy( +export async function startCouchdbProxy( options: { backendUri: string; proxyUri: string; requestedRttMs: number; delay?: (milliseconds: number) => Promise; }, -): CouchdbProxyHandle { +): Promise { const backend = new URL(options.backendUri); const proxy = new URL(options.proxyUri); const halfDelayMs = options.requestedRttMs / 2; const delay = options.delay ?? ((milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds))); - const controller = new AbortController(); const counters: CouchdbProxyCounters = { requestCount: 0, requestBodyBytes: 0, responseBodyBytes: 0, }; - const listener = Deno.serve( - { - hostname: proxy.hostname, - port: Number(proxy.port), - signal: controller.signal, - onError(error) { - console.error(`[Proxy] ${String(error)}`); - return new Response("proxy error", { status: 502 }); - }, - }, - async (request) => { + const server = createServer(async (request, response) => { + try { await delay(halfDelayMs); - const targetUrl = new URL(request.url); + const targetUrl = new URL(request.url ?? "/", proxy); targetUrl.protocol = backend.protocol; targetUrl.host = backend.host; - const headers = new Headers(request.headers); - headers.delete("host"); - headers.delete("content-length"); - - let requestBody: ArrayBuffer | undefined; + const headers = new Headers(); + for (const [name, value] of Object.entries(request.headers)) { + if (value === undefined || name === "host" || name === "content-length") continue; + if (Array.isArray(value)) { + for (const item of value) headers.append(name, item); + } else { + headers.append(name, value); + } + } + let requestBody: Buffer | undefined; if (request.method !== "GET" && request.method !== "HEAD") { try { - requestBody = await request.arrayBuffer(); + const chunks: Buffer[] = []; + for await (const chunk of request as AsyncIterable) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + requestBody = Buffer.concat(chunks); } catch { requestBody = undefined; } @@ -257,7 +258,7 @@ export function startCouchdbProxy( const upstream = await fetch(targetUrl, { method: request.method, headers, - body: requestBody, + body: requestBody === undefined ? undefined : Uint8Array.from(requestBody), redirect: "manual", }); @@ -267,13 +268,33 @@ export function startCouchdbProxy( counters.responseBodyBytes += responseBody.byteLength; await delay(halfDelayMs); - return new Response(responseBody, { - status: upstream.status, - statusText: upstream.statusText, - headers: responseHeaders, - }); - }, - ); + response.statusCode = upstream.status; + response.statusMessage = upstream.statusText; + responseHeaders.forEach((value, name) => response.setHeader(name, value)); + response.end(Buffer.from(responseBody)); + } catch (error) { + console.error(`[Proxy] ${String(error)}`); + if (!response.headersSent) { + response.statusCode = 502; + response.end("proxy error"); + } else { + response.destroy(); + } + } + }); + await new Promise((resolve, reject) => { + const onListening = () => { + server.off("error", onError); + resolve(); + }; + const onError = (error: Error) => { + server.off("listening", onListening); + reject(error); + }; + server.once("error", onError); + server.once("listening", onListening); + server.listen(Number(proxy.port), proxy.hostname); + }); return { applied: true, @@ -287,8 +308,9 @@ export function startCouchdbProxy( }, snapshotCounters: () => ({ ...counters }), stop: async () => { - controller.abort(); - await listener.finished.catch(() => {}); + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); }, }; } @@ -302,8 +324,8 @@ async function main(): Promise { const vaultB = workDir.join("vault-b"); const settingsA = workDir.join("settings-a.json"); const settingsB = workDir.join("settings-b.json"); - await Deno.mkdir(vaultA, { recursive: true }); - await Deno.mkdir(vaultB, { recursive: true }); + await mkdir(vaultA, { recursive: true }); + await mkdir(vaultB, { recursive: true }); await initSettingsFile(settingsA); await initSettingsFile(settingsB); @@ -327,7 +349,7 @@ async function main(): Promise { ); } - const proxy = startCouchdbProxy({ + const proxy = await startCouchdbProxy({ backendUri: config.couchdbBackendUri, proxyUri: config.couchdbProxyUri, requestedRttMs: config.requestedRttMs, @@ -448,7 +470,7 @@ async function main(): Promise { }; if (resultPath) { - await Deno.writeTextFile( + await writeFile( resultPath, JSON.stringify(result, null, 2), ); @@ -477,6 +499,6 @@ async function main(): Promise { if (import.meta.main) { main().catch((error) => { console.error(`[Fatal Error]`, error); - Deno.exit(1); + process.exitCode = 1; }); } diff --git a/src/apps/cli/testdeno/bench-latency-sweep.ts b/test/e2e-cli/bench-latency-sweep.ts similarity index 85% rename from src/apps/cli/testdeno/bench-latency-sweep.ts rename to test/e2e-cli/bench-latency-sweep.ts index 3624b1d5..93489ada 100644 --- a/src/apps/cli/testdeno/bench-latency-sweep.ts +++ b/test/e2e-cli/bench-latency-sweep.ts @@ -1,3 +1,8 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runProcess } from "./helpers/process.ts"; + type SweepResult = { name: string; runner: "p2p" | "couchdb"; @@ -8,7 +13,7 @@ type SweepResult = { }; function readEnvString(name: string, fallback: string): string { - const value = Deno.env.get(name)?.trim(); + const value = process.env[name]?.trim(); return value && value.length > 0 ? value : fallback; } @@ -68,9 +73,8 @@ async function runBenchmark(options: { const suffix = options.repeatCount > 1 ? `-r${String(options.repeatIndex).padStart(2, "0")}` : ""; - const resultPath = `${options.outputDir}/${options.name}${suffix}.json`; + const resultPath = join(options.outputDir, `${options.name}${suffix}.json`); const env = { - ...Deno.env.toObject(), ...options.env, BENCH_RESULT_JSON: resultPath, BENCH_REPEAT_INDEX: String(options.repeatIndex), @@ -78,29 +82,28 @@ async function runBenchmark(options: { }; console.log(`[latency-sweep] running ${options.name}`); - const child = new Deno.Command("deno", { - args: ["task", options.taskName], + const child = await runProcess(process.execPath, { + args: [fileURLToPath(new URL("./run-task.ts", import.meta.url)), options.taskName], cwd: import.meta.dirname, env, stdin: "null", stdout: "inherit", stderr: "inherit", - }).spawn(); - const status = await child.status; - if (status.code !== 0) { - throw new Error(`benchmark failed: ${options.name} (exit ${status.code})`); + }); + if (!child.success) { + throw new Error(`benchmark failed: ${options.name} (exit ${child.code})`); } - return JSON.parse(await Deno.readTextFile(resultPath)) as Record; + return JSON.parse(await readFile(resultPath, "utf8")) as Record; } async function main(): Promise { - const outRoot = readEnvString("BENCH_SWEEP_ROOT", `${import.meta.dirname}/bench-results`); - const outputDir = `${outRoot}/latency-sweep-${timestamp()}`; + const outRoot = readEnvString("BENCH_SWEEP_ROOT", join(import.meta.dirname, "bench-results")); + const outputDir = join(outRoot, `latency-sweep-${timestamp()}`); const rtts = parseRttList(readEnvString("BENCH_SWEEP_RTT_MS", "20,50,100,150,300")); const repeatCount = readEnvInteger("BENCH_REPEAT_COUNT", 1); const base = buildBaseEnv(); - await Deno.mkdir(outputDir, { recursive: true }); + await mkdir(outputDir, { recursive: true }); const results: SweepResult[] = []; if (readEnvString("BENCH_SWEEP_INCLUDE_P2P", "true") !== "false") { @@ -162,7 +165,7 @@ async function main(): Promise { repeatCount, results, }; - await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2)); + await writeFile(join(outputDir, "summary.json"), JSON.stringify(summary, null, 2)); console.log(JSON.stringify(summary, null, 2)); console.log(`[latency-sweep] result directory: ${outputDir}`); } @@ -170,6 +173,6 @@ async function main(): Promise { if (import.meta.main) { main().catch((error) => { console.error("[Fatal Error]", error); - Deno.exit(1); + process.exitCode = 1; }); } diff --git a/src/apps/cli/testdeno/bench-network-cases.ts b/test/e2e-cli/bench-network-cases.ts similarity index 94% rename from src/apps/cli/testdeno/bench-network-cases.ts rename to test/e2e-cli/bench-network-cases.ts index 5795800b..0bad3bf2 100644 --- a/src/apps/cli/testdeno/bench-network-cases.ts +++ b/test/e2e-cli/bench-network-cases.ts @@ -1,3 +1,8 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runProcess } from "./helpers/process.ts"; + export type BenchmarkCase = { name: string; runner: "p2p" | "couchdb"; @@ -10,7 +15,7 @@ export type BenchmarkCase = { }; function readEnvString(name: string, fallback: string): string { - const value = Deno.env.get(name)?.trim(); + const value = process.env[name]?.trim(); return value && value.length > 0 ? value : fallback; } @@ -324,10 +329,9 @@ async function runCase( const suffix = repeatCount > 1 ? `-r${String(repeatIndex).padStart(2, "0")}` : ""; - const resultPath = `${outputDir}/${testCase.name}${suffix}.json`; + const resultPath = join(outputDir, `${testCase.name}${suffix}.json`); const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb"; const env = { - ...Deno.env.toObject(), ...testCase.env, BENCH_RESULT_JSON: resultPath, BENCH_REPEAT_INDEX: String(repeatIndex), @@ -340,8 +344,8 @@ async function runCase( console.log( `[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`, ); - const command = new Deno.Command("deno", { - args: ["task", taskName], + const child = await runProcess(process.execPath, { + args: [fileURLToPath(new URL("./run-task.ts", import.meta.url)), taskName], cwd: import.meta.dirname, env, stdin: "null", @@ -349,13 +353,11 @@ async function runCase( stderr: "inherit", }); - const child = command.spawn(); - const status = await child.status; - if (status.code !== 0) { - throw new Error(`case failed: ${testCase.name} (exit ${status.code})`); + if (!child.success) { + throw new Error(`case failed: ${testCase.name} (exit ${child.code})`); } - const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record< + const result = JSON.parse(await readFile(resultPath, "utf8")) as Record< string, unknown >; @@ -394,16 +396,16 @@ function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] { async function main(): Promise { const outRoot = readEnvString( "BENCH_CASES_ROOT", - `${import.meta.dirname}/bench-results`, + join(import.meta.dirname, "bench-results"), ); - const outputDir = `${outRoot}/cases-${timestamp()}`; - await Deno.mkdir(outputDir, { recursive: true }); + const outputDir = join(outRoot, `cases-${timestamp()}`); + await mkdir(outputDir, { recursive: true }); const allCases = buildCases(); const cases = selectCases(allCases); const repeatCount = readEnvInteger("BENCH_REPEAT_COUNT", 1); - await Deno.writeTextFile( - `${outputDir}/case-manifest.json`, + await writeFile( + join(outputDir, "case-manifest.json"), JSON.stringify( { generatedAt: new Date().toISOString(), @@ -431,8 +433,8 @@ async function main(): Promise { repeatCount, results, }; - await Deno.writeTextFile( - `${outputDir}/summary.json`, + await writeFile( + join(outputDir, "summary.json"), JSON.stringify(summary, null, 2), ); console.log(JSON.stringify(summary, null, 2)); @@ -442,6 +444,6 @@ async function main(): Promise { if (import.meta.main) { main().catch((error) => { console.error("[Fatal Error]", error); - Deno.exit(1); + process.exitCode = 1; }); } diff --git a/src/apps/cli/testdeno/bench-p2p-split-node.ts b/test/e2e-cli/bench-p2p-split-node.ts similarity index 88% rename from src/apps/cli/testdeno/bench-p2p-split-node.ts rename to test/e2e-cli/bench-p2p-split-node.ts index 14bc9456..928a72bc 100644 --- a/src/apps/cli/testdeno/bench-p2p-split-node.ts +++ b/test/e2e-cli/bench-p2p-split-node.ts @@ -1,9 +1,11 @@ -import { join } from "@std/path"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts"; import { discoverPeer } from "./helpers/p2p.ts"; import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; +import { runProcess } from "./helpers/process.ts"; type Role = "host" | "client"; @@ -53,12 +55,12 @@ function errorToRecord(error: unknown): Record { } function readEnvString(name: string, fallback: string): string { - const value = Deno.env.get(name)?.trim(); + const value = process.env[name]?.trim(); return value && value.length > 0 ? value : fallback; } function readEnvNumber(name: string, fallback: number): number { - const raw = Deno.env.get(name); + const raw = process.env[name]; if (raw === undefined || raw.trim() === "") { return fallback; } @@ -74,12 +76,12 @@ function nowMs(): number { } async function commandOutput(command: string, args: string[]): Promise { - const output = await new Deno.Command(command, { + const output = await runProcess(command, { args, stdin: "null", stdout: "piped", stderr: "piped", - }).output(); + }); const stdout = new TextDecoder().decode(output.stdout); const stderr = new TextDecoder().decode(output.stderr); if (!output.success) { @@ -118,7 +120,7 @@ async function applyNetemIfRequested(): Promise { } await commandOk("ip", ["link", "set", "dev", iface, "mtu", String(mtu)]); - await new Deno.Command("tc", { args: ["qdisc", "del", "dev", iface, "root"] }).output(); + await runProcess("tc", { args: ["qdisc", "del", "dev", iface, "root"] }); await commandOk("tc", [ "qdisc", "add", @@ -144,8 +146,8 @@ async function waitForFile(path: string, timeoutMs: number): Promise { const started = Date.now(); while (Date.now() - started < timeoutMs) { try { - const stat = await Deno.stat(path); - if (stat.isFile) { + const fileStat = await stat(path); + if (fileStat.isFile()) { return; } } catch { @@ -157,7 +159,7 @@ async function waitForFile(path: string, timeoutMs: number): Promise { } async function readJsonFile(path: string): Promise { - return JSON.parse(await Deno.readTextFile(path)) as T; + return JSON.parse(await readFile(path, "utf8")) as T; } function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] { @@ -172,7 +174,7 @@ function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] { async function readLatestP2PConnectionStats(path: string): Promise { try { - const lines = (await Deno.readTextFile(path)) + const lines = (await readFile(path, "utf8")) .split(/\r?\n/) .map((line) => line.trim()) .filter((line) => line.length > 0); @@ -188,7 +190,7 @@ function buildCommonConfig() { return { runId, workRoot: join(baseWorkRoot, runId), - resultRoot: readEnvString("BENCH_SPLIT_RESULT_ROOT", "/workspace/src/apps/cli/testdeno/bench-results"), + resultRoot: readEnvString("BENCH_SPLIT_RESULT_ROOT", "/workspace/test/e2e-cli/bench-results"), relay: readEnvString("BENCH_RELAY", "ws://nostr-relay:7777/"), appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"), roomId: readEnvString("BENCH_ROOM_ID", "bench-split-room"), @@ -229,12 +231,12 @@ async function prepareP2PSettings( async function runHost(): Promise { const config = buildCommonConfig(); const netem = await applyNetemIfRequested(); - await Deno.mkdir(config.workRoot, { recursive: true }); - await Deno.mkdir(config.resultRoot, { recursive: true }); + await mkdir(config.workRoot, { recursive: true }); + await mkdir(config.resultRoot, { recursive: true }); const hostVault = join(config.workRoot, "vault-host"); const hostSettings = join(config.workRoot, "settings-host.json"); - await Deno.mkdir(hostVault, { recursive: true }); + await mkdir(hostVault, { recursive: true }); await prepareP2PSettings(hostSettings, "p2p-split-host", config); const seedFiles = await createDeterministicDataset({ @@ -247,7 +249,7 @@ async function runHost(): Promise { binCount: config.binFileCount, binSizeBytes: config.binSizeBytes, }); - await Deno.writeTextFile( + await writeFile( join(config.workRoot, "sample-files.json"), JSON.stringify(pickSampleFiles(seedFiles.entries), null, 2) ); @@ -264,12 +266,12 @@ async function runHost(): Promise { mirrorElapsedMs, netem, }; - await Deno.writeTextFile(join(config.workRoot, "host-ready.json"), JSON.stringify(hostReady, null, 2)); + await writeFile(join(config.workRoot, "host-ready.json"), JSON.stringify(hostReady, null, 2)); const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host"); try { await host.waitUntilContains("P2P host is running", 20_000); - await Deno.writeTextFile( + await writeFile( join(config.workRoot, "p2p-host-ready.json"), JSON.stringify({ generatedAt: new Date().toISOString() }) ); @@ -282,23 +284,23 @@ async function runHost(): Promise { async function runClient(): Promise { const config = buildCommonConfig(); const netem = await applyNetemIfRequested(); - await Deno.mkdir(config.resultRoot, { recursive: true }); + await mkdir(config.resultRoot, { recursive: true }); await waitForFile(join(config.workRoot, "host-ready.json"), config.nodeTimeoutMs); await waitForFile(join(config.workRoot, "p2p-host-ready.json"), config.nodeTimeoutMs); const clientVault = join(config.workRoot, "vault-client"); const clientSettings = join(config.workRoot, "settings-client.json"); const statsPath = join(config.workRoot, "p2p-connection-stats.jsonl"); - await Deno.mkdir(clientVault, { recursive: true }); + await mkdir(clientVault, { recursive: true }); await prepareP2PSettings(clientSettings, "p2p-split-client", config); const hostReady = await readJsonFile(join(config.workRoot, "host-ready.json")); const timestamp = new Date().toISOString().replace(/[-:]/g, "").slice(0, 15); const outputDir = join(config.resultRoot, `p2p-split-${config.profile}-${timestamp}`); - await Deno.mkdir(outputDir, { recursive: true }); + await mkdir(outputDir, { recursive: true }); - const previousStatsPath = Deno.env.get("LIVESYNC_P2P_STATS_JSONL"); - Deno.env.set("LIVESYNC_P2P_STATS_JSONL", statsPath); + const previousStatsPath = process.env.LIVESYNC_P2P_STATS_JSONL; + process.env.LIVESYNC_P2P_STATS_JSONL = statsPath; let stage = "peer-discovery"; let peerDiscoveryCommandElapsedMs: number | undefined; let syncElapsedMs: number | undefined; @@ -338,10 +340,10 @@ async function runClient(): Promise { caseName: "p2p-split-compose", mode: "p2p-split-compose-benchmark", runId: config.runId, - simulationTier: Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "2" : "1", + simulationTier: process.env.BENCH_NETEM_ENABLED === "1" ? "2" : "1", networkProfile: config.profile, networkModel: - Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "split-compose-egress-netem" : "split-compose-no-netem", + process.env.BENCH_NETEM_ENABLED === "1" ? "split-compose-egress-netem" : "split-compose-no-netem", relay: config.relay, turnServers: config.turnServers, turnEnabled: config.turnServers.trim().length > 0, @@ -360,8 +362,8 @@ async function runClient(): Promise { throughputBytesPerSec: Number((hostReady.totalBytes / (syncElapsedMs / 1000)).toFixed(2)), throughputMiBPerSec: Number((hostReady.totalBytes / (syncElapsedMs / 1000) / 1024 / 1024).toFixed(4)), }; - await Deno.writeTextFile(join(outputDir, "summary.json"), JSON.stringify(result, null, 2)); - await Deno.writeTextFile( + await writeFile(join(outputDir, "summary.json"), JSON.stringify(result, null, 2)); + await writeFile( join(config.workRoot, "client-done.json"), JSON.stringify({ generatedAt: new Date().toISOString(), outputDir, ok: true }) ); @@ -374,10 +376,10 @@ async function runClient(): Promise { caseName: "p2p-split-compose", mode: "p2p-split-compose-benchmark", runId: config.runId, - simulationTier: Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "2" : "1", + simulationTier: process.env.BENCH_NETEM_ENABLED === "1" ? "2" : "1", networkProfile: config.profile, networkModel: - Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "split-compose-egress-netem" : "split-compose-no-netem", + process.env.BENCH_NETEM_ENABLED === "1" ? "split-compose-egress-netem" : "split-compose-no-netem", relay: config.relay, turnServers: config.turnServers, turnEnabled: config.turnServers.trim().length > 0, @@ -398,8 +400,8 @@ async function runClient(): Promise { ...errorToRecord(error), }, }; - await Deno.writeTextFile(join(outputDir, "summary.json"), JSON.stringify(result, null, 2)); - await Deno.writeTextFile( + await writeFile(join(outputDir, "summary.json"), JSON.stringify(result, null, 2)); + await writeFile( join(config.workRoot, "client-done.json"), JSON.stringify({ generatedAt: new Date().toISOString(), outputDir, ok: false }) ); @@ -407,9 +409,9 @@ async function runClient(): Promise { throw error; } finally { if (previousStatsPath === undefined) { - Deno.env.delete("LIVESYNC_P2P_STATS_JSONL"); + delete process.env.LIVESYNC_P2P_STATS_JSONL; } else { - Deno.env.set("LIVESYNC_P2P_STATS_JSONL", previousStatsPath); + process.env.LIVESYNC_P2P_STATS_JSONL = previousStatsPath; } } } @@ -430,6 +432,6 @@ async function main(): Promise { if (import.meta.main) { main().catch((error) => { console.error("[Fatal Error]", error); - Deno.exit(1); + process.exitCode = 1; }); } diff --git a/src/apps/cli/testdeno/bench-p2p.ts b/test/e2e-cli/bench-p2p.ts similarity index 95% rename from src/apps/cli/testdeno/bench-p2p.ts rename to test/e2e-cli/bench-p2p.ts index 05512694..fd6eed8c 100644 --- a/src/apps/cli/testdeno/bench-p2p.ts +++ b/test/e2e-cli/bench-p2p.ts @@ -1,3 +1,4 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { TempDir } from "./helpers/temp.ts"; import { applyP2pSettings, @@ -82,12 +83,12 @@ type P2PConnectionStats = { }; function readEnvString(name: string, fallback: string): string { - const value = Deno.env.get(name)?.trim(); + const value = process.env[name]?.trim(); return value && value.length > 0 ? value : fallback; } function readEnvNumber(name: string, fallback: number): number { - const raw = Deno.env.get(name); + const raw = process.env[name]; if (raw === undefined || raw.trim() === "") { return fallback; } @@ -100,7 +101,7 @@ function readEnvNumber(name: string, fallback: number): number { } function readEnvStringArray(name: string, fallback: string[]): string[] { - const raw = Deno.env.get(name)?.trim(); + const raw = process.env[name]?.trim(); if (!raw) { return fallback; } @@ -187,7 +188,7 @@ function buildConfig(): BenchmarkConfig { "This benchmark result is scoped to the configured dataset, network model, and selected ICE path.", ]), verificationMode: parseBenchmarkVerificationMode( - Deno.env.get("BENCH_VERIFY_MODE"), + process.env.BENCH_VERIFY_MODE, ), repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)), repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)), @@ -195,7 +196,7 @@ function buildConfig(): BenchmarkConfig { } function readOptionalResultPath(): string | undefined { - const raw = Deno.env.get("BENCH_RESULT_JSON")?.trim(); + const raw = process.env.BENCH_RESULT_JSON?.trim(); if (!raw) { return undefined; } @@ -206,7 +207,7 @@ async function readLatestP2PConnectionStats( statsPath: string, ): Promise { try { - const text = await Deno.readTextFile(statsPath); + const text = await readFile(statsPath, "utf8"); const lines = text .split(/\r?\n/) .map((line) => line.trim()) @@ -233,13 +234,13 @@ async function main(): Promise { const hostSettings = workDir.join("settings-host.json"); const clientSettings = workDir.join("settings-client.json"); const p2pStatsPath = workDir.join("p2p-connection-stats.jsonl"); - const previousStatsPath = Deno.env.get("LIVESYNC_P2P_STATS_JSONL"); - Deno.env.set("LIVESYNC_P2P_STATS_JSONL", p2pStatsPath); + const previousStatsPath = process.env.LIVESYNC_P2P_STATS_JSONL; + process.env.LIVESYNC_P2P_STATS_JSONL = p2pStatsPath; try { await Promise.all([ - Deno.mkdir(hostVault, { recursive: true }), - Deno.mkdir(clientVault, { recursive: true }), + mkdir(hostVault, { recursive: true }), + mkdir(clientVault, { recursive: true }), initSettingsFile(hostSettings), initSettingsFile(clientSettings), ]); @@ -407,7 +408,7 @@ async function main(): Promise { }; if (resultPath) { - await Deno.writeTextFile( + await writeFile( resultPath, JSON.stringify(result, null, 2), ); @@ -428,9 +429,9 @@ async function main(): Promise { } } finally { if (previousStatsPath === undefined) { - Deno.env.delete("LIVESYNC_P2P_STATS_JSONL"); + delete process.env.LIVESYNC_P2P_STATS_JSONL; } else { - Deno.env.set("LIVESYNC_P2P_STATS_JSONL", previousStatsPath); + process.env.LIVESYNC_P2P_STATS_JSONL = previousStatsPath; } await stopCoturnIfStarted(coturnStarted); await stopLocalRelayIfStarted(relayStarted); @@ -440,6 +441,6 @@ async function main(): Promise { if (import.meta.main) { main().catch((error) => { console.error(`[Fatal Error]`, error); - Deno.exit(1); + process.exitCode = 1; }); } diff --git a/src/apps/cli/testdeno/bench-run-item1.sh b/test/e2e-cli/bench-run-item1.sh similarity index 87% rename from src/apps/cli/testdeno/bench-run-item1.sh rename to test/e2e-cli/bench-run-item1.sh index 53d2998f..95881ffb 100644 --- a/src/apps/cli/testdeno/bench-run-item1.sh +++ b/test/e2e-cli/bench-run-item1.sh @@ -13,13 +13,13 @@ echo "[bench-wrapper] output directory: ${OUT_DIR}" echo "[bench-wrapper] running p2p benchmark" ( cd "${SCRIPT_DIR}" - BENCH_RESULT_JSON="${OUT_DIR}/p2p.json" deno task bench:p2p + BENCH_RESULT_JSON="${OUT_DIR}/p2p.json" node run-task.ts bench:p2p ) echo "[bench-wrapper] running couchdb benchmark with RTT ${BENCH_COUCHDB_RTT_MS:-default} ms (emulating HTTP network latency)" ( cd "${SCRIPT_DIR}" - BENCH_RESULT_JSON="${OUT_DIR}/couchdb.json" deno task bench:couchdb + BENCH_RESULT_JSON="${OUT_DIR}/couchdb.json" node run-task.ts bench:couchdb ) cat > "${OUT_DIR}/README.txt" < }; +const reportPath = + process.env.LIVESYNC_CLI_COVERAGE_REPORT ?? join(REPOSITORY_ROOT, "coverage", "cli-e2e", "coverage-final.json"); +const sourceRoot = (join(REPOSITORY_ROOT, "src") + sep).replaceAll("\\", "/"); +const report = JSON.parse(await readFile(reportPath, "utf8")) as Record; +function coverageFor(suffix: string): FileCoverage { + const found = Object.entries(report).find(([path]) => path.replaceAll("\\", "/").endsWith(suffix)); + assert.ok(found, `Missing coverage for ${suffix}`); + return found[1]; +} + +test("CLI E2E coverage retains executed application functions after source-map remapping", () => { + assert.ok( + Object.keys(report).every((path) => path.replaceAll("\\", "/").startsWith(sourceRoot)), + "The report must contain only source files from this repository" + ); + const counts = Object.values(coverageFor("/src/apps/cli/main.ts").f); + assert.ok(counts.length > 0, "The report must retain application function ranges, rather than default line counts"); + assert.ok( + counts.some((count) => count > 0), + "The CLI application must have executed" + ); + assert.ok( + Object.keys(report).every((path) => !path.replaceAll("\\", "/").includes("/node_modules/")), + "The application report must exclude bundled dependencies" + ); +}); + +test( + "local CLI coverage retains unexecuted P2P functions", + { + skip: (process.env.LIVESYNC_CLI_COVERAGE_TASK ?? "test:local") !== "test:local", + }, + () => { + const counts = Object.values(coverageFor("/src/apps/cli/commands/p2p.ts").f); + assert.ok(counts.length > 0, "The report must retain P2P function ranges"); + assert.ok( + counts.some((count) => count === 0), + "The local suite must not report every P2P function as executed" + ); + } +); diff --git a/src/apps/cli/testdeno/helpers/backgroundCli.ts b/test/e2e-cli/helpers/backgroundCli.ts similarity index 84% rename from src/apps/cli/testdeno/helpers/backgroundCli.ts rename to test/e2e-cli/helpers/backgroundCli.ts index ef81feab..e4c74f89 100644 --- a/src/apps/cli/testdeno/helpers/backgroundCli.ts +++ b/test/e2e-cli/helpers/backgroundCli.ts @@ -1,9 +1,11 @@ +import { writeSync } from "node:fs"; +import { spawnProcess, type RunningProcess } from "./process.ts"; import { CLI_DIR, TEE_ENABLED, formatTeeCommand, createLineTeeWriter } from "./cli.ts"; -import { join } from "@std/path"; +import { join } from "node:path"; const CLI_DIST = join(CLI_DIR, "dist", "index.cjs"); -const VERBOSE_ENABLED = Deno.env.get("LIVESYNC_CLI_VERBOSE") === "1"; -const DEBUG_ENABLED = Deno.env.get("LIVESYNC_CLI_DEBUG") === "1"; +const VERBOSE_ENABLED = process.env.LIVESYNC_CLI_VERBOSE === "1"; +const DEBUG_ENABLED = process.env.LIVESYNC_CLI_DEBUG === "1"; function decorateArgs(args: string[]): string[] { return DEBUG_ENABLED ? ["-d", ...args] : VERBOSE_ENABLED ? ["-v", ...args] : args; @@ -38,24 +40,26 @@ export class BackgroundCliProcess { #stdoutDone: Promise; #stderrDone: Promise; - constructor( - readonly child: Deno.ChildProcess, - readonly args: string[] - ) { + readonly child: RunningProcess; + readonly args: string[]; + + constructor(child: RunningProcess, args: string[]) { + this.child = child; + this.args = args; const cliArgs = decorateArgs(args); this.#stdoutDone = pump( child.stdout, (text) => { this.#stdout += text; }, - TEE_ENABLED ? createLineTeeWriter(child.pid, "stdout", (chunk) => Deno.stdout.writeSync(chunk)) : null + TEE_ENABLED ? createLineTeeWriter(child.pid, "stdout", (chunk) => writeSync(1, chunk)) : null ); this.#stderrDone = pump( child.stderr, (text) => { this.#stderr += text; }, - TEE_ENABLED ? createLineTeeWriter(child.pid, "stderr", (chunk) => Deno.stderr.writeSync(chunk)) : null + TEE_ENABLED ? createLineTeeWriter(child.pid, "stderr", (chunk) => writeSync(2, chunk)) : null ); } @@ -102,16 +106,17 @@ export class BackgroundCliProcess { export function startCliInBackground(...args: string[]): BackgroundCliProcess { const cliArgs = decorateArgs(args); - const child = new Deno.Command("node", { + const child = spawnProcess(process.execPath, { args: [CLI_DIST, ...cliArgs], cwd: CLI_DIR, stdin: "null", stdout: "piped", stderr: "piped", - }).spawn(); + }); if (TEE_ENABLED) { - Deno.stdout.writeSync( + writeSync( + 1, new TextEncoder().encode(`[CLI tee pid=${child.pid}] process(bg): ${formatTeeCommand(cliArgs)}\n`) ); } diff --git a/src/apps/cli/testdeno/helpers/benchmarkVerification.ts b/test/e2e-cli/helpers/benchmarkVerification.ts similarity index 80% rename from src/apps/cli/testdeno/helpers/benchmarkVerification.ts rename to test/e2e-cli/helpers/benchmarkVerification.ts index ace22ed3..c29692ae 100644 --- a/src/apps/cli/testdeno/helpers/benchmarkVerification.ts +++ b/test/e2e-cli/helpers/benchmarkVerification.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; import type { DatasetEntry } from "./dataset.ts"; export type BenchmarkVerificationMode = "all" | "sample"; @@ -9,16 +11,8 @@ export type BenchmarkVerificationResult = { datasetDigestSha256: string; }; -function toHex(bytes: ArrayBuffer): string { - return [...new Uint8Array(bytes)] - .map((value) => value.toString(16).padStart(2, "0")) - .join(""); -} - -async function sha256(bytes: Uint8Array): Promise { - const input = new ArrayBuffer(bytes.byteLength); - new Uint8Array(input).set(bytes); - return toHex(await crypto.subtle.digest("SHA-256", input)); +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); } export function parseBenchmarkVerificationMode( @@ -53,12 +47,12 @@ export async function computeDatasetDigestSha256( ): Promise { const manifest: string[] = []; for (const entry of entries) { - const contentDigest = await sha256(await Deno.readFile(entry.absolutePath)); + const contentDigest = sha256(await readFile(entry.absolutePath)); manifest.push( `${entry.kind}\t${entry.relativePath}\t${entry.size}\t${contentDigest}`, ); } - return await sha256(new TextEncoder().encode(manifest.join("\n"))); + return sha256(new TextEncoder().encode(manifest.join("\n"))); } export async function verifyBenchmarkDataset( diff --git a/src/apps/cli/testdeno/helpers/cli.ts b/test/e2e-cli/helpers/cli.ts similarity index 84% rename from src/apps/cli/testdeno/helpers/cli.ts rename to test/e2e-cli/helpers/cli.ts index 6318f553..ad6f59f1 100644 --- a/src/apps/cli/testdeno/helpers/cli.ts +++ b/test/e2e-cli/helpers/cli.ts @@ -1,13 +1,10 @@ -import { join } from "@std/path"; +import { writeSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { spawnProcess } from "./process.ts"; +import { createHash } from "node:crypto"; -// --------------------------------------------------------------------------- -// Path resolution -// --------------------------------------------------------------------------- -// This file lives at: src/apps/cli/testdeno/helpers/cli.ts -// CLI root (src/apps/cli/) is two levels up. -// import.meta.dirname is available in Deno 1.40+ as an OS-native path string. -export const CLI_DIR: string = join(import.meta.dirname!, "..", ".."); -export const CLI_DIST = join(CLI_DIR, "dist", "index.cjs"); +import { CLI_DIR, CLI_DIST } from "./paths.ts"; +export { CLI_DIR, CLI_DIST } from "./paths.ts"; // --------------------------------------------------------------------------- // Result type @@ -20,9 +17,9 @@ export interface CliResult { code: number; } -export const TEE_ENABLED = Deno.env.get("LIVESYNC_TEST_TEE") === "1"; -const VERBOSE_ENABLED = Deno.env.get("LIVESYNC_CLI_VERBOSE") === "1"; -const DEBUG_ENABLED = Deno.env.get("LIVESYNC_CLI_DEBUG") === "1"; +export const TEE_ENABLED = process.env.LIVESYNC_TEST_TEE === "1"; +const VERBOSE_ENABLED = process.env.LIVESYNC_CLI_VERBOSE === "1"; +const DEBUG_ENABLED = process.env.LIVESYNC_CLI_DEBUG === "1"; function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -40,7 +37,7 @@ function concatChunks(chunks: Uint8Array[]): Uint8Array { } export function formatTeeCommand(args: string[]): string { - return ["node", CLI_DIST, ...args].map((part) => JSON.stringify(part)).join(" "); + return [process.execPath, CLI_DIST, ...args].map((part) => JSON.stringify(part)).join(" "); } export function createLineTeeWriter( @@ -114,27 +111,25 @@ async function collectStream( async function runNodeCommand(args: string[], stdinData?: Uint8Array): Promise { const cliArgs = DEBUG_ENABLED ? ["-d", ...args] : VERBOSE_ENABLED ? ["-v", ...args] : args; - const child = new Deno.Command("node", { + const child = spawnProcess(process.execPath, { args: [CLI_DIST, ...cliArgs], cwd: CLI_DIR, stdin: stdinData ? "piped" : "null", stdout: "piped", stderr: "piped", - }).spawn(); + }); if (TEE_ENABLED) { - Deno.stdout.writeSync( - new TextEncoder().encode(`[CLI tee pid=${child.pid}] process: ${formatTeeCommand(cliArgs)}\n`) - ); + writeSync(1, new TextEncoder().encode(`[CLI tee pid=${child.pid}] process: ${formatTeeCommand(cliArgs)}\n`)); } const stdoutPromise = collectStream( child.stdout, - TEE_ENABLED ? createLineTeeWriter(child.pid, "stdout", (chunk) => Deno.stdout.writeSync(chunk)) : null + TEE_ENABLED ? createLineTeeWriter(child.pid, "stdout", (chunk) => writeSync(1, chunk)) : null ); const stderrPromise = collectStream( child.stderr, - TEE_ENABLED ? createLineTeeWriter(child.pid, "stderr", (chunk) => Deno.stderr.writeSync(chunk)) : null + TEE_ENABLED ? createLineTeeWriter(child.pid, "stderr", (chunk) => writeSync(2, chunk)) : null ); if (stdinData) { @@ -172,7 +167,7 @@ function isTransientNetworkError(message: string): boolean { * do. Does NOT throw on non-zero exit — check `.code` yourself. */ export async function runCli(...args: string[]): Promise { - const retries = Number(Deno.env.get("LIVESYNC_CLI_RETRY") ?? "0"); + const retries = Number(process.env.LIVESYNC_CLI_RETRY ?? "0"); for (let attempt = 0; ; attempt++) { const result = await runNodeCommand(args); if (result.code === 0) return result; @@ -204,7 +199,7 @@ export async function runCliOrFail(...args: string[]): Promise { export async function runCliWithInput(input: string | Uint8Array, ...args: string[]): Promise { const data = typeof input === "string" ? new TextEncoder().encode(input) : input; - const retries = Number(Deno.env.get("LIVESYNC_CLI_RETRY") ?? "0"); + const retries = Number(process.env.LIVESYNC_CLI_RETRY ?? "0"); for (let attempt = 0; ; attempt++) { const result = await runNodeCommand(args, data); if (result.code === 0) return result; @@ -258,12 +253,9 @@ export function assertNotContains(haystack: string, needle: string, message: str } export async function assertFilesEqual(expectedPath: string, actualPath: string, message: string): Promise { - const [expected, actual] = await Promise.all([Deno.readFile(expectedPath), Deno.readFile(actualPath)]); + const [expected, actual] = await Promise.all([readFile(expectedPath), readFile(actualPath)]); if (expected.length !== actual.length || expected.some((b, i) => b !== actual[i])) { - const hex = async (d: Uint8Array) => { - const h = await crypto.subtle.digest("SHA-256", d); - return [...new Uint8Array(h)].map((b) => b.toString(16).padStart(2, "0")).join(""); - }; + const hex = (data: Uint8Array) => createHash("sha256").update(data).digest("hex"); throw new Error( `[FAIL] ${message}\nexpected SHA-256: ${await hex(expected)}\nactual SHA-256: ${await hex(actual)}` ); @@ -275,7 +267,7 @@ export async function assertFilesEqual(expectedPath: string, actualPath: string, // --------------------------------------------------------------------------- export async function readJsonFile>(filePath: string): Promise { - return JSON.parse(await Deno.readTextFile(filePath)) as T; + return JSON.parse(await readFile(filePath, "utf8")) as T; } export function jsonStringField(jsonText: string, field: string): string { diff --git a/src/apps/cli/testdeno/helpers/compressionDataset.ts b/test/e2e-cli/helpers/compressionDataset.ts similarity index 83% rename from src/apps/cli/testdeno/helpers/compressionDataset.ts rename to test/e2e-cli/helpers/compressionDataset.ts index 308836d8..2785ce8d 100644 --- a/src/apps/cli/testdeno/helpers/compressionDataset.ts +++ b/test/e2e-cli/helpers/compressionDataset.ts @@ -1,5 +1,8 @@ -import { join } from "@std/path"; +import { copyFile, mkdtemp, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { runProcess } from "./process.ts"; import type { DatasetEntry, DatasetKind } from "./dataset.ts"; +import { REPOSITORY_ROOT } from "./paths.ts"; export type CompressionDatasetEntry = DatasetEntry & { source: string; @@ -18,8 +21,6 @@ export type JpegEncoder = (inputPpm: string, outputJpeg: string) => Promise { } export async function encodeJpegWithCjpeg(inputPpm: string, outputJpeg: string): Promise { - const command = new Deno.Command("cjpeg", { - args: ["-quality", "85", "-optimize", "-outfile", outputJpeg, inputPpm], - stdin: "null", - stdout: "piped", - stderr: "piped", - }); - let result: Deno.CommandOutput; + let result: Awaited>; try { - result = await command.output(); + result = await runProcess("cjpeg", { + args: ["-quality", "85", "-optimize", "-outfile", outputJpeg, inputPpm], + stdin: "null", + stdout: "piped", + stderr: "piped", + }); } catch (error) { - if (error instanceof Deno.errors.NotFound) { + if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") { throw new Error( "cjpeg is required for the compression benchmark. Use the Compose runner or install libjpeg tools." ); @@ -95,7 +95,7 @@ async function writeRandomBinary(path: string, size: number, seed: string): Prom for (let index = 0; index < bytes.length; index++) { bytes[index] = nextRandom() & 0xff; } - await Deno.writeFile(path, bytes); + await writeFile(path, bytes); } export async function createCompressionBenchmarkDataset(options: { @@ -114,7 +114,7 @@ export async function createCompressionBenchmarkDataset(options: { let jpegGenerator = ""; for (const kind of ALL_KINDS) { - await Deno.mkdir(join(datasetRoot, kind), { recursive: true }); + await mkdir(join(datasetRoot, kind), { recursive: true }); } const addFile = async (kind: DatasetKind, absolutePath: string, source: string) => { @@ -122,13 +122,13 @@ export async function createCompressionBenchmarkDataset(options: { .slice(options.rootDir.length + 1) .split("\\") .join("/"); - const size = (await Deno.stat(absolutePath)).size; + const size = (await stat(absolutePath)).size; entries.push({ kind, relativePath, absolutePath, size, source }); }; const copyRepositoryFile = async (kind: DatasetKind, sourcePath: string, targetName: string) => { const destination = join(datasetRoot, kind, targetName); - await Deno.copyFile(join(repositoryRoot, sourcePath), destination); + await copyFile(join(repositoryRoot, sourcePath), destination); await addFile(kind, destination, sourcePath); }; @@ -150,9 +150,9 @@ export async function createCompressionBenchmarkDataset(options: { "ReplicateResultProcessor.ts" ); - const markdownBytes = await Deno.readFile(join(repositoryRoot, "docs/settings.md")); + const markdownBytes = await readFile(join(repositoryRoot, "docs/settings.md")); const gzipPath = join(datasetRoot, "gz", "settings.md.gz"); - await Deno.writeFile(gzipPath, await gzip(markdownBytes)); + await writeFile(gzipPath, await gzip(markdownBytes)); await addFile("gz", gzipPath, "generated gzip of docs/settings.md"); const randomPath = join(datasetRoot, "bin", "deterministic-random.bin"); @@ -163,13 +163,14 @@ export async function createCompressionBenchmarkDataset(options: { ["smooth-gradient.jpg", false], ["textured-gradient.jpg", true], ] as const) { - const ppmPath = await Deno.makeTempFile({ dir: options.rootDir, prefix: "compression-jpeg-", suffix: ".ppm" }); + const temporaryDirectory = await mkdtemp(join(options.rootDir, "compression-jpeg-")); + const ppmPath = join(temporaryDirectory, "input.ppm"); const jpegPath = join(datasetRoot, "jpg", name); try { - await Deno.writeFile(ppmPath, createSyntheticPpm(640, 480, `${seed}-${name}`, textured)); + await writeFile(ppmPath, createSyntheticPpm(640, 480, `${seed}-${name}`, textured)); jpegGenerator = await jpegEncoder(ppmPath, jpegPath); } finally { - await Deno.remove(ppmPath).catch(() => {}); + await rm(temporaryDirectory, { recursive: true, force: true }).catch(() => {}); } await addFile("jpg", jpegPath, `${jpegGenerator}; ${textured ? "textured" : "smooth"}`); } diff --git a/src/apps/cli/testdeno/helpers/dataset.ts b/test/e2e-cli/helpers/dataset.ts similarity index 85% rename from src/apps/cli/testdeno/helpers/dataset.ts rename to test/e2e-cli/helpers/dataset.ts index 78af167a..8b52800e 100644 --- a/src/apps/cli/testdeno/helpers/dataset.ts +++ b/test/e2e-cli/helpers/dataset.ts @@ -1,3 +1,6 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + export type DeterministicDatasetConfig = { rootDir: string; datasetDirName: string; @@ -65,10 +68,6 @@ function createTextBytes(size: number, fileIndex: number, seed: string): Uint8Ar return out; } -function toPath(rootDir: string, relativePath: string): string { - return `${rootDir}/${relativePath}`; -} - export async function createDeterministicDataset(config: DeterministicDatasetConfig): Promise { if (config.mdCount < 0 || config.binCount < 0) { throw new Error("mdCount and binCount must be non-negative"); @@ -80,11 +79,11 @@ export async function createDeterministicDataset(config: DeterministicDatasetCon throw new Error("mdMinSizeBytes must be <= mdMaxSizeBytes"); } - const datasetRoot = toPath(config.rootDir, config.datasetDirName); - const mdDir = `${datasetRoot}/md`; - const binDir = `${datasetRoot}/bin`; - await Deno.mkdir(mdDir, { recursive: true }); - await Deno.mkdir(binDir, { recursive: true }); + const datasetRoot = join(config.rootDir, config.datasetDirName); + const mdDir = join(datasetRoot, "md"); + const binDir = join(datasetRoot, "bin"); + await mkdir(mdDir, { recursive: true }); + await mkdir(binDir, { recursive: true }); const nextRandom = createXorshift32(fnv1a32(config.seed)); const mdRange = config.mdMaxSizeBytes - config.mdMinSizeBytes + 1; @@ -93,21 +92,21 @@ export async function createDeterministicDataset(config: DeterministicDatasetCon for (let index = 0; index < config.mdCount; index++) { const size = config.mdMinSizeBytes + (nextRandom() % mdRange); const relativePath = `${config.datasetDirName}/md/file-${String(index).padStart(4, "0")}.md`; - const absolutePath = toPath(config.rootDir, relativePath); + const absolutePath = join(config.rootDir, ...relativePath.split("/")); const body = createTextBytes(size, index, config.seed); - await Deno.writeFile(absolutePath, body); + await writeFile(absolutePath, body); entries.push({ kind: "md", relativePath, absolutePath, size }); } for (let index = 0; index < config.binCount; index++) { const size = config.binSizeBytes; const relativePath = `${config.datasetDirName}/bin/file-${String(index).padStart(4, "0")}.bin`; - const absolutePath = toPath(config.rootDir, relativePath); + const absolutePath = join(config.rootDir, ...relativePath.split("/")); const body = new Uint8Array(size); for (let i = 0; i < size; i++) { body[i] = nextRandom() & 0xff; } - await Deno.writeFile(absolutePath, body); + await writeFile(absolutePath, body); entries.push({ kind: "bin", relativePath, absolutePath, size }); } diff --git a/src/apps/cli/testdeno/helpers/docker.ts b/test/e2e-cli/helpers/docker.ts similarity index 88% rename from src/apps/cli/testdeno/helpers/docker.ts rename to test/e2e-cli/helpers/docker.ts index 822f0211..961ba957 100644 --- a/src/apps/cli/testdeno/helpers/docker.ts +++ b/test/e2e-cli/helpers/docker.ts @@ -1,3 +1,6 @@ +import { randomUUID } from "node:crypto"; +import { writeSync } from "node:fs"; +import { spawnProcess } from "./process.ts"; /** * Docker service management for tests. * @@ -13,12 +16,12 @@ type DockerInvoker = { }; let dockerInvokerPromise: Promise | null = null; -const DOCKER_TEE = Deno.env.get("LIVESYNC_DOCKER_TEE") === "1" || Deno.env.get("LIVESYNC_TEST_TEE") === "1"; +const DOCKER_TEE = process.env.LIVESYNC_DOCKER_TEE === "1" || process.env.LIVESYNC_TEST_TEE === "1"; const trackedContainers = new Set(); -const CLEANUP_SIGNALS: Deno.Signal[] = ["SIGINT", "SIGTERM"]; +const CLEANUP_SIGNALS: NodeJS.Signals[] = ["SIGINT", "SIGTERM"]; let signalCleanupHandlersInstalled = false; let signalCleanupInProgress = false; -const signalCleanupHandlers = new Map void>(); +const signalCleanupHandlers = new Map void>(); // --------------------------------------------------------------------------- // Low-level docker wrapper @@ -64,14 +67,14 @@ async function collectStream( async function runCommand(bin: string, args: string[]): Promise<{ code: number; stdout: string; stderr: string }> { try { - const child = new Deno.Command(bin, { + const child = spawnProcess(bin, { args, stdin: "null", stdout: "piped", stderr: "piped", - }).spawn(); - const stdoutPromise = collectStream(child.stdout, DOCKER_TEE ? (chunk) => Deno.stdout.writeSync(chunk) : null); - const stderrPromise = collectStream(child.stderr, DOCKER_TEE ? (chunk) => Deno.stderr.writeSync(chunk) : null); + }); + const stdoutPromise = collectStream(child.stdout, DOCKER_TEE ? (chunk) => writeSync(1, chunk) : null); + const stderrPromise = collectStream(child.stderr, DOCKER_TEE ? (chunk) => writeSync(2, chunk) : null); const [status, stdout, stderr] = await Promise.all([child.status, stdoutPromise, stderrPromise]); const dec = new TextDecoder(); const result = { @@ -81,7 +84,7 @@ async function runCommand(bin: string, args: string[]): Promise<{ code: number; }; return result; } catch (err) { - if (err instanceof Deno.errors.NotFound) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { return { code: 127, stdout: "", @@ -93,7 +96,7 @@ async function runCommand(bin: string, args: string[]): Promise<{ code: number; } async function resolveDockerInvoker(): Promise { - const custom = Deno.env.get("LIVESYNC_DOCKER_COMMAND")?.trim(); + const custom = process.env.LIVESYNC_DOCKER_COMMAND?.trim(); if (custom) { const parsed = parseCommand(custom); const runner: DockerInvoker = { @@ -110,8 +113,8 @@ async function resolveDockerInvoker(): Promise { return runner; } - const mode = (Deno.env.get("LIVESYNC_DOCKER_MODE") ?? "auto").toLowerCase(); - const onWindows = Deno.build.os === "windows"; + const mode = (process.env.LIVESYNC_DOCKER_MODE ?? "auto").toLowerCase(); + const onWindows = process.platform === "win32"; const native: DockerInvoker = { bin: "docker", prefix: [], label: "docker" }; const wsl: DockerInvoker = { bin: "wsl", prefix: [], label: "wsl docker" }; @@ -204,13 +207,13 @@ async function cleanupTrackedContainers(reason: string): Promise { } } -async function handleSignalCleanup(signal: Deno.Signal): Promise { +async function handleSignalCleanup(signal: NodeJS.Signals): Promise { if (signalCleanupInProgress) return; signalCleanupInProgress = true; try { await cleanupTrackedContainers(`signal ${signal}`); } finally { - Deno.exit(signal === "SIGINT" ? 130 : 143); + process.exit(signal === "SIGINT" ? 130 : 143); } } @@ -222,7 +225,7 @@ function ensureSignalCleanupHandlers(): void { void handleSignalCleanup(signal); }; try { - Deno.addSignalListener(signal, listener); + process.on(signal, listener); signalCleanupHandlers.set(signal, listener); } catch { // Unsupported signal on this platform. @@ -234,7 +237,7 @@ function removeSignalCleanupHandlers(): void { if (!signalCleanupHandlersInstalled) return; for (const [signal, listener] of signalCleanupHandlers) { try { - Deno.removeSignalListener(signal, listener); + process.off(signal, listener); } catch { // Ignore if already removed or unsupported. } @@ -575,6 +578,59 @@ export async function startMinio( await waitForMinioBucket(minioEndpoint, accessKey, secretKey, bucket); } +export type IsolatedMinioFixture = { + endpoint: string; + accessKey: string; + secretKey: string; + stop: () => Promise; +}; + +/** Start an S3 fixture owned by this caller without touching the shared test container. */ +export async function startIsolatedMinio(accessKey: string, secretKey: string): Promise { + const container = `livesync-cli-continuity-${randomUUID()}`; + console.log(`[INFO] starting isolated S3 test container: ${container}`); + let tracked = false; + const stop = async () => { + await stopAndRemoveContainer(container); + if (tracked) untrackContainer(container); + }; + + try { + await dockerOrFail( + "run", + "-d", + "--name", + container, + "-p", + "127.0.0.1::9000", + "-e", + `RUSTFS_ACCESS_KEY=${accessKey}`, + "-e", + `RUSTFS_SECRET_KEY=${secretKey}`, + "-e", + "RUSTFS_CONSOLE_ENABLE=true", + "-e", + "RUSTFS_CORS_ALLOWED_ORIGINS=*", + S3_IMAGE, + "/data" + ); + trackContainer(container); + tracked = true; + + const mapping = await dockerOrFail("port", container, "9000/tcp"); + const port = Number(mapping.trim().match(/:(\d+)$/)?.[1]); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error(`Could not determine the isolated S3 fixture port from: ${mapping.trim()}`); + } + const endpoint = `http://127.0.0.1:${port}`; + await fetchRetry(endpoint, { method: "GET" }, 30, 1000, [403, 404]); + return { endpoint, accessKey, secretKey, stop }; + } catch (error) { + await stop(); + throw error; + } +} + // --------------------------------------------------------------------------- // P2P relay (strfry) // --------------------------------------------------------------------------- diff --git a/src/apps/cli/testdeno/helpers/env.ts b/test/e2e-cli/helpers/env.ts similarity index 90% rename from src/apps/cli/testdeno/helpers/env.ts rename to test/e2e-cli/helpers/env.ts index 4187d399..4d890c20 100644 --- a/src/apps/cli/testdeno/helpers/env.ts +++ b/test/e2e-cli/helpers/env.ts @@ -1,3 +1,4 @@ +import { readFile } from "node:fs/promises"; /** * Load a .env-style file (KEY=value per line) into a plain object. * Equivalent to `source $TEST_ENV_FILE; set -a` in bash. @@ -10,7 +11,7 @@ * # comment lines are ignored */ export async function loadEnvFile(filePath: string): Promise> { - const text = await Deno.readTextFile(filePath); + const text = await readFile(filePath, "utf8"); const result: Record = {}; for (const line of text.split("\n")) { const trimmed = line.trim(); diff --git a/src/apps/cli/testdeno/helpers/measuredCli.ts b/test/e2e-cli/helpers/measuredCli.ts similarity index 85% rename from src/apps/cli/testdeno/helpers/measuredCli.ts rename to test/e2e-cli/helpers/measuredCli.ts index 52e9d98f..bcf292d6 100644 --- a/src/apps/cli/testdeno/helpers/measuredCli.ts +++ b/test/e2e-cli/helpers/measuredCli.ts @@ -1,3 +1,4 @@ +import { runProcess } from "./process.ts"; import { CLI_DIR, CLI_DIST } from "./cli.ts"; export type CliProcessMeasurement = { @@ -13,17 +14,17 @@ const MARKER = "__LIVESYNC_GNU_TIME__"; export async function runMeasuredCliOrFail(...args: string[]): Promise { const started = performance.now(); - let output: Deno.CommandOutput; + let output: Awaited>; try { - output = await new Deno.Command("/usr/bin/time", { - args: ["-f", `${MARKER}%U\t%S\t%M`, "node", CLI_DIST, ...args], + output = await runProcess("/usr/bin/time", { + args: ["-f", `${MARKER}%U\t%S\t%M`, process.execPath, CLI_DIST, ...args], cwd: CLI_DIR, stdin: "null", stdout: "piped", stderr: "piped", - }).output(); + }); } catch (error) { - if (error instanceof Deno.errors.NotFound) { + if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") { throw new Error( "GNU /usr/bin/time is required for the compression benchmark. Use the Compose runner or install GNU time." ); diff --git a/test/e2e-cli/helpers/net.ts b/test/e2e-cli/helpers/net.ts new file mode 100644 index 00000000..52520187 --- /dev/null +++ b/test/e2e-cli/helpers/net.ts @@ -0,0 +1,59 @@ +import { createConnection, createServer } from "node:net"; + +type WaitForPortOptions = { timeoutMs?: number; intervalMs?: number; connectTimeoutMs?: number }; + +async function connectWithTimeout(hostname: string, port: number, timeoutMs: number): Promise { + await new Promise((resolve, reject) => { + const socket = createConnection({ host: hostname, port }); + socket.setTimeout(timeoutMs, () => socket.destroy(new Error(`connect timeout after ${timeoutMs}ms`))); + socket.once("error", reject); + socket.once("connect", () => { + socket.destroy(); + resolve(); + }); + }); +} + +export async function waitForPort(hostname: string, port: number, options: WaitForPortOptions = {}): Promise { + const timeoutMs = options.timeoutMs ?? 15000; + const intervalMs = options.intervalMs ?? 250; + const started = Date.now(); + let lastError: unknown; + while (Date.now() - started < timeoutMs) { + try { + await connectWithTimeout(hostname, port, options.connectTimeoutMs ?? 1000); + return; + } catch (error) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + } + throw new Error( + `Port ${hostname}:${port} did not become ready within ${timeoutMs}ms` + + (lastError ? ` (last error: ${String(lastError)})` : "") + ); +} + +export async function getFreePort(hostname = "127.0.0.1"): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, hostname, resolve); + }); + const address = server.address(); + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + if (!address || typeof address === "string") throw new Error("The listener did not bind a TCP port"); + return address.port; +} + +export async function getOptimalLoopbackIp(): Promise { + for (const hostname of ["127.0.0.1", "::1"]) { + try { + await getFreePort(hostname); + return hostname; + } catch { + /* Try the other loopback family. */ + } + } + return "127.0.0.1"; +} diff --git a/src/apps/cli/testdeno/helpers/p2p.ts b/test/e2e-cli/helpers/p2p.ts similarity index 90% rename from src/apps/cli/testdeno/helpers/p2p.ts rename to test/e2e-cli/helpers/p2p.ts index 6be7ebcf..af730686 100644 --- a/src/apps/cli/testdeno/helpers/p2p.ts +++ b/test/e2e-cli/helpers/p2p.ts @@ -1,3 +1,4 @@ + import { runCli } from "./cli.ts"; import { isLocalP2pRelay, startP2pRelay, stopP2pRelay, startCoturn, stopCoturn } from "./docker.ts"; import { waitForPort } from "./net.ts"; @@ -35,7 +36,7 @@ export async function discoverPeer( timeoutSeconds: number, targetPeer?: string ): Promise { - const retries = Math.max(0, Number(Deno.env.get("LIVESYNC_P2P_PEERS_RETRY") ?? "3")); + const retries = Math.max(0, Number(process.env.LIVESYNC_P2P_PEERS_RETRY ?? "3")); let lastCombined = ""; for (let attempt = 0; attempt <= retries; attempt++) { @@ -82,9 +83,9 @@ export async function maybeStartLocalRelay(relay: string): Promise { } const endpoint = parseRelayEndpoint(relay); await waitForPort(endpoint.hostname, endpoint.port, { - timeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000"), - intervalMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_INTERVAL_MS") ?? "250"), - connectTimeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_CONNECT_TIMEOUT_MS") ?? "1000"), + timeoutMs: Number(process.env.LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS ?? "15000"), + intervalMs: Number(process.env.LIVESYNC_P2P_RELAY_READY_INTERVAL_MS ?? "250"), + connectTimeoutMs: Number(process.env.LIVESYNC_P2P_RELAY_CONNECT_TIMEOUT_MS ?? "1000"), }); // Docker proxy accepts TCP connections instantly before the container's internal process is fully ready. // Wait an additional few seconds to ensure strfry is actually accepting WebSockets. diff --git a/test/e2e-cli/helpers/paths.ts b/test/e2e-cli/helpers/paths.ts new file mode 100644 index 00000000..0cb38d28 --- /dev/null +++ b/test/e2e-cli/helpers/paths.ts @@ -0,0 +1,5 @@ +import { join, resolve } from "node:path"; + +export const REPOSITORY_ROOT = resolve(import.meta.dirname, "../../.."); +export const CLI_DIR = join(REPOSITORY_ROOT, "src", "apps", "cli"); +export const CLI_DIST = join(CLI_DIR, "dist", "index.cjs"); diff --git a/test/e2e-cli/helpers/process.test.ts b/test/e2e-cli/helpers/process.test.ts new file mode 100644 index 00000000..43636fa0 --- /dev/null +++ b/test/e2e-cli/helpers/process.test.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { runProcess, spawnProcess } from "./process.ts"; + +test("subprocesses preserve arguments, environment overrides, and both output streams", async () => { + const argument = "a path with spaces/日本語.txt"; + const result = await runProcess(process.execPath, { + args: [ + "--eval", + "process.stdout.write(JSON.stringify({arg:process.argv[1],value:process.env.LIVESYNC_PROCESS_TEST,inherited:process.env.PATH||process.env.Path}));process.stderr.write('diagnostic');process.exitCode=7", + argument, + ], + env: { LIVESYNC_PROCESS_TEST: "override" }, + }); + assert.equal(result.code, 7); + assert.equal(result.success, false); + const output = JSON.parse(new TextDecoder().decode(result.stdout)); + assert.equal(output.arg, argument); + assert.equal(output.value, "override"); + assert.ok(output.inherited); + assert.equal(new TextDecoder().decode(result.stderr), "diagnostic"); +}); + +test("piped stdin closes and large stdout and stderr drain before completion", async () => { + const child = spawnProcess(process.execPath, { + args: [ + "--eval", + "let data='';process.stdin.setEncoding('utf8');process.stdin.on('data',v=>data+=v);process.stdin.on('end',()=>{process.stdout.write(data.repeat(100000));process.stderr.write('e'.repeat(100000));})", + ], + stdin: "piped", + }); + const stdout = new Response(child.stdout).text(); + const stderr = new Response(child.stderr).text(); + const writer = child.stdin.getWriter(); + await writer.write(new TextEncoder().encode("abc")); + await writer.close(); + const [status, out, err] = await Promise.all([child.status, stdout, stderr]); + assert.equal(status.success, true); + assert.equal(out, "abc".repeat(100000)); + assert.equal(err, "e".repeat(100000)); +}); + +test("a missing executable rejects with its OS error", async () => { + await assert.rejects(runProcess("livesync-missing-test-executable-847281"), { code: "ENOENT" }); +}); diff --git a/test/e2e-cli/helpers/process.ts b/test/e2e-cli/helpers/process.ts new file mode 100644 index 00000000..bc20dd57 --- /dev/null +++ b/test/e2e-cli/helpers/process.ts @@ -0,0 +1,76 @@ +import { spawn } from "node:child_process"; +import { constants } from "node:os"; +import { Readable, Writable } from "node:stream"; + +export type ProcessStatus = { code: number; success: boolean; signal: NodeJS.Signals | null }; +export type ProcessOutput = ProcessStatus & { stdout: Uint8Array; stderr: Uint8Array }; +export type ProcessOptions = { + args?: string[]; + cwd?: string | URL; + env?: NodeJS.ProcessEnv; + stdin?: "null" | "piped" | "inherit"; + stdout?: "piped" | "inherit"; + stderr?: "piped" | "inherit"; +}; +export type RunningProcess = { + readonly pid: number; + readonly stdin: WritableStream; + readonly stdout: ReadableStream; + readonly stderr: ReadableStream; + readonly status: Promise; + kill(signal?: NodeJS.Signals): void; +}; + +export function spawnProcess(command: string, options: ProcessOptions = {}): RunningProcess { + const child = spawn(command, options.args ?? [], { + cwd: options.cwd, + env: { ...process.env, ...options.env }, + stdio: [ + options.stdin === "inherit" ? "inherit" : options.stdin === "piped" ? "pipe" : "ignore", + options.stdout === "inherit" ? "inherit" : "pipe", + options.stderr === "inherit" ? "inherit" : "pipe", + ], + windowsHide: true, + }); + const status = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => + resolve({ + code: code ?? (signal ? 128 + (constants.signals[signal] ?? 0) : 1), + success: code === 0 && signal === null, + signal, + }) + ); + }); + return { + pid: child.pid ?? 0, + get stdin() { + if (!child.stdin) throw new Error("The child process stdin is not piped"); + return Writable.toWeb(child.stdin) as WritableStream; + }, + get stdout() { + if (!child.stdout) throw new Error("The child process stdout is not piped"); + return Readable.toWeb(child.stdout) as ReadableStream; + }, + get stderr() { + if (!child.stderr) throw new Error("The child process stderr is not piped"); + return Readable.toWeb(child.stderr) as ReadableStream; + }, + status, + kill(signal = "SIGTERM") { + child.kill(signal); + }, + }; +} + +export async function runProcess(command: string, options: ProcessOptions = {}): Promise { + const child = spawnProcess(command, options); + const collect = async (stream: ReadableStream): Promise => + new Uint8Array(await new Response(stream).arrayBuffer()); + const [status, stdout, stderr] = await Promise.all([ + child.status, + options.stdout === "inherit" ? new Uint8Array() : collect(child.stdout), + options.stderr === "inherit" ? new Uint8Array() : collect(child.stderr), + ]); + return { ...status, stdout, stderr }; +} diff --git a/src/apps/cli/testdeno/helpers/settings.ts b/test/e2e-cli/helpers/settings.ts similarity index 77% rename from src/apps/cli/testdeno/helpers/settings.ts rename to test/e2e-cli/helpers/settings.ts index bb999b13..3a3e5226 100644 --- a/src/apps/cli/testdeno/helpers/settings.ts +++ b/test/e2e-cli/helpers/settings.ts @@ -1,3 +1,5 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { runProcess } from "./process.ts"; import { CLI_DIR, runCliOrFail } from "./cli.ts"; // --------------------------------------------------------------------------- @@ -41,49 +43,27 @@ export async function generateSetupUriFromSettings( "})();", ].join("\n"); - const scriptPath = await Deno.makeTempFile({ - dir: CLI_DIR, - prefix: "livesync-setup-uri-", - suffix: ".mts", + const { code, stdout, stderr } = await runProcess(process.execPath, { + args: ["--input-type=module", "--eval", script], + cwd: CLI_DIR, + env: { SETTINGS_FILE: settingsFile, SETUP_PASSPHRASE: setupPassphrase }, }); - await Deno.writeTextFile(scriptPath, script); - - try { - const cmd = new Deno.Command("npx", { - args: ["tsx", scriptPath], - cwd: CLI_DIR, - env: { - SETTINGS_FILE: settingsFile, - SETUP_PASSPHRASE: setupPassphrase, - }, - stdin: "null", - stdout: "piped", - stderr: "piped", - }); - - const { code, stdout, stderr } = await cmd.output(); - const dec = new TextDecoder(); - if (code !== 0) { - throw new Error( - `Failed to generate setup URI (code ${code})\nstdout: ${dec.decode(stdout)}\nstderr: ${dec.decode(stderr)}` - ); - } - - const uri = dec.decode(stdout).trim(); - if (!uri) { - throw new Error("Failed to generate setup URI: output is empty"); - } - return uri; - } finally { - await Deno.remove(scriptPath).catch(() => {}); + const dec = new TextDecoder(); + if (code !== 0) { + throw new Error( + `Failed to generate setup URI (code ${code})\nstdout: ${dec.decode(stdout)}\nstderr: ${dec.decode(stderr)}` + ); } + const uri = dec.decode(stdout).trim(); + if (!uri) throw new Error("Failed to generate setup URI: output is empty"); + return uri; } /** Set isConfigured=true in a settings file (required for mirror / scan). */ export async function markSettingsConfigured(settingsFile: string): Promise { - const data = JSON.parse(await Deno.readTextFile(settingsFile)); + const data = JSON.parse(await readFile(settingsFile, "utf8")); data.isConfigured = true; - await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2)); + await writeFile(settingsFile, JSON.stringify(data, null, 2)); } // --------------------------------------------------------------------------- @@ -102,7 +82,7 @@ export async function applyCouchdbSettings( couchdbDbname: string, liveSync = false ): Promise { - const data = JSON.parse(await Deno.readTextFile(settingsFile)); + const data = JSON.parse(await readFile(settingsFile, "utf8")); data.couchDB_URI = couchdbUri; data.couchDB_USER = couchdbUser; data.couchDB_PASSWORD = couchdbPassword; @@ -114,7 +94,7 @@ export async function applyCouchdbSettings( data.usePluginSync = false; } data.isConfigured = true; - await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2)); + await writeFile(settingsFile, JSON.stringify(data, null, 2)); } export async function applyRemoteSyncSettings( @@ -135,7 +115,7 @@ export async function applyRemoteSyncSettings( usePathObfuscation?: boolean; } ): Promise { - const data = JSON.parse(await Deno.readTextFile(settingsFile)); + const data = JSON.parse(await readFile(settingsFile, "utf8")); if (options.remoteType === "COUCHDB") { data.remoteType = ""; @@ -166,7 +146,7 @@ export async function applyRemoteSyncSettings( data.usePathObfuscation = options.usePathObfuscation; } data.isConfigured = true; - await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2)); + await writeFile(settingsFile, JSON.stringify(data, null, 2)); } // --------------------------------------------------------------------------- @@ -186,7 +166,7 @@ export async function applyP2pSettings( autoAccept = "~.*", turnServers = "turn:127.0.0.1:3478" ): Promise { - const data = JSON.parse(await Deno.readTextFile(settingsFile)); + const data = JSON.parse(await readFile(settingsFile, "utf8")); data.P2P_Enabled = true; data.P2P_AutoStart = false; data.P2P_AutoBroadcast = false; @@ -201,11 +181,11 @@ export async function applyP2pSettings( data.P2P_turnCredential = "testpass"; data.P2P_IsHeadless = true; data.isConfigured = true; - await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2)); + await writeFile(settingsFile, JSON.stringify(data, null, 2)); } export async function applyP2pTestTweaks(settingsFile: string, deviceName: string, passphrase: string): Promise { - const data = JSON.parse(await Deno.readTextFile(settingsFile)); + const data = JSON.parse(await readFile(settingsFile, "utf8")); data.remoteType = "ONLY_P2P"; data.encrypt = true; data.passphrase = passphrase; @@ -215,5 +195,5 @@ export async function applyP2pTestTweaks(settingsFile: string, deviceName: strin data.usePluginSyncV2 = true; data.P2P_DevicePeerName = deviceName; data.isConfigured = true; - await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2)); + await writeFile(settingsFile, JSON.stringify(data, null, 2)); } diff --git a/test/e2e-cli/helpers/temp.ts b/test/e2e-cli/helpers/temp.ts new file mode 100644 index 00000000..b558e8d9 --- /dev/null +++ b/test/e2e-cli/helpers/temp.ts @@ -0,0 +1,20 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** A temporary directory which is removed when its `await using` scope ends. */ +export class TempDir implements AsyncDisposable { + readonly path: string; + private constructor(path: string) { + this.path = path; + } + static async create(prefix = "livesync-node-test"): Promise { + return new TempDir(await mkdtemp(join(tmpdir(), `${prefix}.`))); + } + join(...parts: string[]): string { + return join(this.path, ...parts); + } + async [Symbol.asyncDispose](): Promise { + await rm(this.path, { recursive: true, force: true }).catch(() => {}); + } +} diff --git a/src/apps/cli/testdeno/relay-disconnect-probe.mjs b/test/e2e-cli/relay-disconnect-probe.mjs similarity index 100% rename from src/apps/cli/testdeno/relay-disconnect-probe.mjs rename to test/e2e-cli/relay-disconnect-probe.mjs diff --git a/src/apps/cli/testdeno/run-ci-suite.ts b/test/e2e-cli/run-ci-suite.ts similarity index 65% rename from src/apps/cli/testdeno/run-ci-suite.ts rename to test/e2e-cli/run-ci-suite.ts index 0eb1edfc..d82fc023 100644 --- a/src/apps/cli/testdeno/run-ci-suite.ts +++ b/test/e2e-cli/run-ci-suite.ts @@ -1,18 +1,20 @@ +import { spawnProcess } from "./helpers/process.ts"; +import { fileURLToPath } from "node:url"; import TASKS from "./ci-tasks.json" with { type: "json" }; for (const [index, task] of TASKS.entries()) { console.log(`\n[CLI E2E ${index + 1}/${TASKS.length}] ${task}`); - const child = new Deno.Command(Deno.execPath(), { - args: ["task", task], + const child = spawnProcess(process.execPath, { + args: [fileURLToPath(new URL("./run-task.ts", import.meta.url)), task], cwd: import.meta.dirname, stdin: "inherit", stdout: "inherit", stderr: "inherit", - }).spawn(); + }); const status = await child.status; if (!status.success) { console.error(`[CLI E2E] ${task} failed with exit code ${status.code}.`); - Deno.exit(status.code); + process.exit(status.code); } } diff --git a/src/apps/cli/testdeno/run-cli-e2e.sh b/test/e2e-cli/run-cli-e2e.sh similarity index 94% rename from src/apps/cli/testdeno/run-cli-e2e.sh rename to test/e2e-cli/run-cli-e2e.sh index 164a4e4a..3358547f 100644 --- a/src/apps/cli/testdeno/run-cli-e2e.sh +++ b/test/e2e-cli/run-cli-e2e.sh @@ -5,7 +5,7 @@ TASK="${CLI_E2E_TASK:-test:p2p:ci}" case "$TASK" in test:p2p-host|test:p2p-peers|test:p2p-sync|test:p2p-automatic-host|test:p2p-replacement|test:p2p-relay-disconnect|test:p2p:ci|test:p2p-three-nodes|test:p2p-upload-download) - exec deno task "$TASK" + exec node run-task.ts "$TASK" ;; *) echo "Unknown CLI_E2E_TASK: $TASK" >&2 diff --git a/test/e2e-cli/run-compose-p2p.ts b/test/e2e-cli/run-compose-p2p.ts new file mode 100644 index 00000000..0da9d824 --- /dev/null +++ b/test/e2e-cli/run-compose-p2p.ts @@ -0,0 +1,53 @@ +import { spawnProcess, type ProcessStatus } from "./helpers/process.ts"; +import { REPOSITORY_ROOT } from "./helpers/paths.ts"; +import { realpath } from "node:fs/promises"; +const repositoryRoot = await realpath(REPOSITORY_ROOT); +const composeArgs = ["compose", "-f", "test/bench-network/compose.yml"]; +const p2pEnvironment = { + CLI_E2E_TASK: process.env.CLI_E2E_TASK ?? "test:p2p:ci", + RELAY: process.env.RELAY ?? "ws://nostr-relay:7777/", + PEERS_TIMEOUT: process.env.PEERS_TIMEOUT ?? "20", + SYNC_TIMEOUT: process.env.SYNC_TIMEOUT ?? "60", + LIVESYNC_USE_COTURN: process.env.LIVESYNC_USE_COTURN ?? "0", + TURN_SERVERS: process.env.TURN_SERVERS ?? "none", + LIVESYNC_P2P_PEERS_RETRY: process.env.LIVESYNC_P2P_PEERS_RETRY ?? "1", + LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: process.env.LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS ?? "60000", + BENCH_LIVESYNC_TEST_TEE: process.env.BENCH_LIVESYNC_TEST_TEE ?? "0", + LIVESYNC_CLI_DEBUG: process.env.LIVESYNC_CLI_DEBUG ?? "0", + LIVESYNC_CLI_VERBOSE: process.env.LIVESYNC_CLI_VERBOSE ?? "0", +}; + +async function runDocker(args: string[], env?: Record): Promise { + return await spawnProcess("docker", { + args, + cwd: repositoryRoot, + env, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + }).status; +} + +let testStatus: ProcessStatus | undefined; +try { + testStatus = await runDocker( + [...composeArgs, "run", "--build", "--rm", "bench-runner", "run-livesync-cli-e2e"], + p2pEnvironment + ); +} finally { + const cleanupStatus = await runDocker([...composeArgs, "down", "-v", "--remove-orphans"]); + if (!cleanupStatus.success) { + console.error(`[CLI E2E] Compose cleanup failed with exit code ${cleanupStatus.code}.`); + if (testStatus?.success) { + process.exit(cleanupStatus.code); + } + } +} + +if (!testStatus?.success) { + const code = testStatus?.code ?? 1; + console.error(`[CLI E2E] Compose P2P suite failed with exit code ${code}.`); + process.exit(code); +} + +console.log("\n[CLI E2E] Compose P2P suite passed."); diff --git a/test/e2e-cli/run-coverage.ts b/test/e2e-cli/run-coverage.ts new file mode 100644 index 00000000..c0ab4480 --- /dev/null +++ b/test/e2e-cli/run-coverage.ts @@ -0,0 +1,61 @@ +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnProcess } from "./helpers/process.ts"; +import { CLI_DIR, REPOSITORY_ROOT } from "./helpers/paths.ts"; + +const require = createRequire(import.meta.url); +const reportDirectory = join(REPOSITORY_ROOT, "coverage", "cli-e2e"); +const task = process.argv[2] ?? "test:local"; + +async function run(args: string[], cwd: string, env: NodeJS.ProcessEnv = {}): Promise { + const result = await spawnProcess(process.execPath, { + args, + cwd, + env, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + }).status; + if (!result.success) throw new Error(`Coverage command failed with exit code ${result.code}`); +} + +if (task === "test:p2p:compose" || task === "test:e2e:all") { + throw new Error("CLI coverage collects host processes; select a host-direct task such as test:local or test:ci"); +} +const viteCli = join(dirname(require.resolve("vite/package.json")), "bin", "vite.js"); +await run([viteCli, "build"], CLI_DIR, { LIVESYNC_CLI_COVERAGE: "1" }); +const c8Cli = require.resolve("c8/bin/c8.js"); +await run( + [ + c8Cli, + "--all", + "--src=src/apps/cli", + // Match native paths and Node's file URL source-map cache. + "--include=**/src/**/*.ts", + "--exclude-after-remap", + "--exclude=**/test-support/**", + "--exclude=**/test/**", + "--exclude=**/*.unit.spec.ts", + "--exclude=**/*.test.ts", + "--reporter=text", + "--reporter=json", + "--reporter=html", + `--reports-dir=${reportDirectory}`, + `--temp-directory=${join(reportDirectory, "raw")}`, + process.execPath, + fileURLToPath(new URL("./run-task.ts", import.meta.url)), + task, + ...process.argv.slice(3), + ], + REPOSITORY_ROOT +); +await run( + ["--test", "--test-concurrency=1", fileURLToPath(new URL("./coverage-report.test.ts", import.meta.url))], + REPOSITORY_ROOT, + { + LIVESYNC_CLI_COVERAGE_REPORT: join(reportDirectory, "coverage-final.json"), + LIVESYNC_CLI_COVERAGE_TASK: task, + } +); +console.log(`CLI E2E coverage: ${reportDirectory}`); diff --git a/test/e2e-cli/run-task.ts b/test/e2e-cli/run-task.ts new file mode 100644 index 00000000..b92845ae --- /dev/null +++ b/test/e2e-cli/run-task.ts @@ -0,0 +1,59 @@ +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { join } from "node:path"; +import { CLI_DIR } from "./helpers/paths.ts"; +import { spawnProcess } from "./helpers/process.ts"; + +type Task = { test?: string[]; filter?: string; script?: string; tasks?: string[]; command?: string; args?: string[] }; +const taskDirectory = import.meta.dirname; + +export async function runTask(name: string, extraArgs: string[] = []): Promise { + if (Number(process.versions.node.split(".")[0]) < 26) throw new Error("CLI E2E tests require Node.js 26 or newer"); + const tasks = JSON.parse(await readFile(new URL("./tasks.json", import.meta.url), "utf8")) as Record; + const task = tasks[name]; + if (!task) throw new Error(`Unknown CLI task '${name}'. Available tasks: ${Object.keys(tasks).join(", ")}`); + if (task.tasks) { + for (const child of task.tasks) { + const code = await runTask(child, extraArgs); + if (code !== 0) return code; + } + return 0; + } + const envFile = join(CLI_DIR, ".test.env"); + const envArgs = existsSync(envFile) ? [`--env-file=${envFile}`] : []; + let command = process.execPath; + let args: string[]; + if (task.test) { + args = [ + ...envArgs, + "--test", + "--test-concurrency=1", + ...(task.filter ? [`--test-name-pattern=${task.filter}`] : []), + ...extraArgs, + ...task.test.map((file) => fileURLToPath(new URL(file, import.meta.url))), + ]; + } else if (task.script) { + args = [...envArgs, fileURLToPath(new URL(task.script, import.meta.url)), ...extraArgs]; + } else if (task.command) { + command = task.command; + args = [...(task.args ?? []), ...extraArgs]; + } else throw new Error(`CLI task '${name}' has no executable entry point`); + const child = spawnProcess(command, { + args, + cwd: taskDirectory, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + }); + return (await child.status).code; +} + +if (import.meta.main) { + try { + process.exitCode = await runTask(process.argv[2] ?? "test", process.argv.slice(3)); + } catch (error) { + console.error(error); + process.exitCode = 1; + } +} diff --git a/test/e2e-cli/tasks.json b/test/e2e-cli/tasks.json new file mode 100644 index 00000000..ee62a00e --- /dev/null +++ b/test/e2e-cli/tasks.json @@ -0,0 +1,209 @@ +{ + "test:helpers": { + "test": [ + "helpers/process.test.ts", + "test-ci-task-contract.ts" + ] + }, + "test": { + "tasks": [ + "test:ci" + ] + }, + "test:ci": { + "script": "run-ci-suite.ts" + }, + "test:p2p:compose": { + "script": "run-compose-p2p.ts" + }, + "test:local": { + "tasks": [ + "test:setup-put-cat", + "test:mirror", + "test:daemon" + ] + }, + "test:daemon": { + "test": [ + "test-daemon.ts" + ] + }, + "test:daemon-startup": { + "test": [ + "test-daemon-startup.ts" + ] + }, + "test:decoupled-vault": { + "test": [ + "test-decoupled-vault.ts" + ] + }, + "test:remote-commands": { + "test": [ + "test-remote-commands.ts" + ] + }, + "test:settings-writeback": { + "test": [ + "test-settings-writeback.ts" + ] + }, + "test:fetch-database-continuity": { + "test": [ + "test-fetch-database-continuity.ts" + ] + }, + "test:remote-administration-exit-codes": { + "test": [ + "test-remote-administration-exit-codes.ts" + ] + }, + "test:push-pull": { + "test": [ + "test-push-pull.ts" + ] + }, + "test:setup-put-cat": { + "test": [ + "test-setup-put-cat.ts" + ] + }, + "test:mirror": { + "test": [ + "test-mirror.ts" + ] + }, + "test:sync-two-local": { + "test": [ + "test-sync-two-local-databases.ts" + ] + }, + "test:sync-locked-remote": { + "test": [ + "test-sync-locked-remote.ts" + ] + }, + "test:p2p-host": { + "test": [ + "test-p2p-host.ts" + ] + }, + "test:p2p-peers": { + "test": [ + "test-p2p-peers-local-relay.ts" + ] + }, + "test:p2p-sync": { + "test": [ + "test-p2p-sync.ts" + ] + }, + "test:p2p-automatic-host": { + "test": [ + "test-p2p-automatic-host.ts" + ] + }, + "test:p2p-replacement": { + "test": [ + "test-p2p-replicator-replacement.ts" + ] + }, + "test:p2p-relay-disconnect": { + "test": [ + "test-p2p-relay-disconnect.ts" + ] + }, + "test:p2p:ci": { + "test": [ + "test-p2p-sync.ts", + "test-p2p-automatic-host.ts", + "test-p2p-replicator-replacement.ts", + "test-p2p-relay-disconnect.ts" + ] + }, + "test:p2p-three-nodes": { + "test": [ + "test-p2p-three-nodes-conflict.ts" + ] + }, + "test:p2p-upload-download": { + "test": [ + "test-p2p-upload-download-repro.ts" + ] + }, + "test:benchmark-contract": { + "test": [ + "test-benchmark-contract.ts" + ] + }, + "bench:p2p": { + "script": "bench-p2p.ts" + }, + "bench:couchdb": { + "script": "bench-couchdb.ts" + }, + "bench:compression": { + "script": "bench-compression.ts" + }, + "bench:cases": { + "script": "bench-network-cases.ts" + }, + "bench:latency-sweep": { + "script": "bench-latency-sweep.ts" + }, + "bench:p2p-split-node": { + "script": "bench-p2p-split-node.ts" + }, + "bench:item1": { + "command": "bash", + "args": [ + "./bench-run-item1.sh" + ] + }, + "bench:item1:full": { + "command": "bash", + "args": [ + "-lc", + "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh" + ] + }, + "test:e2e-couchdb": { + "test": [ + "test-e2e-two-vaults-couchdb.ts" + ] + }, + "test:journal-chunk-recovery": { + "test": [ + "test-journal-chunk-recovery.ts" + ] + }, + "test:e2e-matrix": { + "test": [ + "test-e2e-two-vaults-matrix.ts" + ] + }, + "test:e2e-matrix:couchdb-enc0": { + "test": [ + "test-e2e-two-vaults-matrix.ts" + ], + "filter": "e2e matrix: COUCHDB-enc0" + }, + "test:e2e-matrix:couchdb-enc1": { + "test": [ + "test-e2e-two-vaults-matrix.ts" + ], + "filter": "e2e matrix: COUCHDB-enc1" + }, + "test:e2e-matrix:minio-enc0": { + "test": [ + "test-e2e-two-vaults-matrix.ts" + ], + "filter": "e2e matrix: MINIO-enc0" + }, + "test:e2e-matrix:minio-enc1": { + "test": [ + "test-e2e-two-vaults-matrix.ts" + ], + "filter": "e2e matrix: MINIO-enc1" + } +} diff --git a/src/apps/cli/testdeno/test-benchmark-contract.ts b/test/e2e-cli/test-benchmark-contract.ts similarity index 69% rename from src/apps/cli/testdeno/test-benchmark-contract.ts rename to test/e2e-cli/test-benchmark-contract.ts index 5a201fec..40c0600d 100644 --- a/src/apps/cli/testdeno/test-benchmark-contract.ts +++ b/test/e2e-cli/test-benchmark-contract.ts @@ -1,4 +1,9 @@ -import { assert, assertEquals, assertStringIncludes } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { join } from "node:path"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { type BenchmarkCase, buildCases } from "./bench-network-cases.ts"; import { startCouchdbProxy } from "./bench-couchdb.ts"; import { @@ -7,14 +12,10 @@ import { } from "./helpers/benchmarkVerification.ts"; import type { DatasetEntry } from "./helpers/dataset.ts"; import { createCompressionBenchmarkDataset } from "./helpers/compressionDataset.ts"; +import { getFreePort } from "./helpers/net.ts"; -function getFreePort(): number { - const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 }); - try { - return (listener.addr as Deno.NetAddr).port; - } finally { - listener.close(); - } +function assertStringIncludes(actual: string, expected: string, message?: string): void { + assert.ok(actual.includes(expected), message ?? `Expected ${JSON.stringify(actual)} to include ${JSON.stringify(expected)}`); } function getCase(cases: BenchmarkCase[], name: string): BenchmarkCase { @@ -42,7 +43,7 @@ function parsedLimitations(testCase: BenchmarkCase): string[] { return parsed; } -Deno.test("benchmark cases record scope and limitations for paper use", () => { +test("benchmark cases record scope and limitations for paper use", () => { const cases = buildCases(); assert(cases.length > 0); @@ -67,31 +68,34 @@ Deno.test("benchmark cases record scope and limitations for paper use", () => { testCase.limitations.length > 0, `${testCase.name} must list limitations`, ); - assertEquals( + assert.deepStrictEqual( testCase.env.BENCH_MEASUREMENT_SCOPE, testCase.measurementScope, ); - assertEquals(parsedLimitations(testCase), testCase.limitations); - assertEquals( + assert.deepStrictEqual(parsedLimitations(testCase), testCase.limitations); + assert.deepStrictEqual( testCase.env.BENCH_VERIFY_MODE, "all", `${testCase.name} must verify the complete dataset`, ); } }); -Deno.test("CouchDB latency proxy applies half the requested RTT in each direction", async () => { - const backendPort = getFreePort(); - const proxyPort = getFreePort(); +test("CouchDB latency proxy applies half the requested RTT in each direction", async () => { + const backend = createServer((_request, response) => response.end("ok")); + const backendPort = await new Promise((resolve, reject) => { + backend.once("error", reject); + backend.listen(0, "127.0.0.1", () => { + const address = backend.address(); + if (!address || typeof address === "string") { + reject(new Error("Could not read the benchmark backend address")); + return; + } + resolve(address.port); + }); + }); + const proxyPort = await getFreePort(); const delays: number[] = []; - const backend = Deno.serve( - { - hostname: "127.0.0.1", - port: backendPort, - onListen() {}, - }, - () => new Response("ok"), - ); - const proxy = startCouchdbProxy({ + const proxy = await startCouchdbProxy({ backendUri: `http://127.0.0.1:${backendPort}`, proxyUri: `http://127.0.0.1:${proxyPort}`, requestedRttMs: 20, @@ -103,10 +107,10 @@ Deno.test("CouchDB latency proxy applies half the requested RTT in each directio try { const response = await fetch(`http://127.0.0.1:${proxyPort}/probe`); - assertEquals(await response.text(), "ok"); - assertEquals(proxy.directionalDelayMs, 10); - assertEquals(delays, [10, 10]); - assertEquals(proxy.snapshotCounters(), { + assert.deepStrictEqual(await response.text(), "ok"); + assert.deepStrictEqual(proxy.directionalDelayMs, 10); + assert.deepStrictEqual(delays, [10, 10]); + assert.deepStrictEqual(proxy.snapshotCounters(), { requestCount: 1, requestBodyBytes: 0, responseBodyBytes: 2, @@ -116,34 +120,34 @@ Deno.test("CouchDB latency proxy applies half the requested RTT in each directio method: "POST", body: "abc", }); - assertEquals(await posted.text(), "ok"); - assertEquals(proxy.snapshotCounters(), { + assert.deepStrictEqual(await posted.text(), "ok"); + assert.deepStrictEqual(proxy.snapshotCounters(), { requestCount: 1, requestBodyBytes: 3, responseBodyBytes: 2, }); } finally { await proxy.stop(); - await backend.shutdown(); + await new Promise((resolve, reject) => { + backend.close((error) => error ? reject(error) : resolve()); + }); } - const halfMillisecondProxy = startCouchdbProxy({ + const halfMillisecondProxy = await startCouchdbProxy({ backendUri: "http://127.0.0.1:1", - proxyUri: `http://127.0.0.1:${getFreePort()}`, + proxyUri: `http://127.0.0.1:${await getFreePort()}`, requestedRttMs: 1, delay: () => Promise.resolve(), }); try { - assertEquals(halfMillisecondProxy.directionalDelayMs, 0.5); + assert.deepStrictEqual(halfMillisecondProxy.directionalDelayMs, 0.5); } finally { await halfMillisecondProxy.stop(); } }); -Deno.test("compression benchmark dataset covers representative file kinds deterministically", async () => { - const fixtureRoot = await Deno.makeTempDir({ - prefix: "livesync-compression-contract-", - }); +test("compression benchmark dataset covers representative file kinds deterministically", async () => { + const fixtureRoot = await mkdtemp(join(tmpdir(), "livesync-compression-contract-")); const repositoryRoot = `${fixtureRoot}/repository`; const vaultA = `${fixtureRoot}/vault-a`; const vaultB = `${fixtureRoot}/vault-b`; @@ -161,21 +165,21 @@ Deno.test("compression benchmark dataset covers representative file kinds determ try { for (const [index, relativePath] of repositoryFiles.entries()) { const absolutePath = `${repositoryRoot}/${relativePath}`; - await Deno.mkdir( + await mkdir( absolutePath.slice(0, absolutePath.lastIndexOf("/")), { recursive: true }, ); - await Deno.writeFile( + await writeFile( absolutePath, new TextEncoder().encode( `fixture-${index}-${relativePath}\n`.repeat(20), ), ); } - await Deno.mkdir(vaultA, { recursive: true }); - await Deno.mkdir(vaultB, { recursive: true }); + await mkdir(vaultA, { recursive: true }); + await mkdir(vaultB, { recursive: true }); const jpegEncoder = async (_input: string, output: string) => { - await Deno.writeFile( + await writeFile( output, new Uint8Array([ 0xff, @@ -205,7 +209,7 @@ Deno.test("compression benchmark dataset covers representative file kinds determ jpegEncoder, }); - assertEquals(first.filesByKind, { + assert.deepStrictEqual(first.filesByKind, { md: 3, jpg: 2, png: 2, @@ -214,9 +218,9 @@ Deno.test("compression benchmark dataset covers representative file kinds determ gz: 1, bin: 1, }); - assertEquals(first.totalFiles, 13); - assertEquals(first.jpegGenerator, "contract JPEG encoder"); - assertEquals( + assert.deepStrictEqual(first.totalFiles, 13); + assert.deepStrictEqual(first.jpegGenerator, "contract JPEG encoder"); + assert.deepStrictEqual( first.entries.map((entry) => [ entry.kind, entry.relativePath, @@ -230,11 +234,11 @@ Deno.test("compression benchmark dataset covers representative file kinds determ ); assert(first.entries.every((entry) => entry.size > 0)); } finally { - await Deno.remove(fixtureRoot, { recursive: true }).catch(() => {}); + await rm(fixtureRoot, { recursive: true }).catch(() => {}); } }); -Deno.test("benchmark verification mode selects either all files or a labelled sample", () => { +test("benchmark verification mode selects either all files or a labelled sample", () => { const entries: DatasetEntry[] = [ { kind: "md", relativePath: "a.md", absolutePath: "/a", size: 1 }, { kind: "md", relativePath: "b.md", absolutePath: "/b", size: 1 }, @@ -243,15 +247,15 @@ Deno.test("benchmark verification mode selects either all files or a labelled sa { kind: "bin", relativePath: "e.bin", absolutePath: "/e", size: 1 }, ]; - assertEquals(parseBenchmarkVerificationMode("ALL"), "all"); - assertEquals(selectVerificationEntries(entries, "all").length, entries.length); + assert.deepStrictEqual(parseBenchmarkVerificationMode("ALL"), "all"); + assert.deepStrictEqual(selectVerificationEntries(entries, "all").length, entries.length); const sample = selectVerificationEntries(entries, "sample"); assert(sample.length > 0 && sample.length < entries.length); assert(sample.some((entry) => entry.kind === "md")); assert(sample.some((entry) => entry.kind === "bin")); }); -Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", () => { +test("P2P signalling-shim cases do not claim to shape the note-data path", () => { const cases = buildCases(); for ( const name of [ @@ -260,10 +264,10 @@ Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", ] ) { const testCase = getCase(cases, name); - assertEquals(testCase.runner, "p2p"); - assertEquals(testCase.env.BENCH_TURN_SERVERS, ""); - assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2"); - assertEquals( + assert.deepStrictEqual(testCase.runner, "p2p"); + assert.deepStrictEqual(testCase.env.BENCH_TURN_SERVERS, ""); + assert.deepStrictEqual(testCase.env.BENCH_SIMULATION_TIER, "2"); + assert.deepStrictEqual( testCase.env.BENCH_NETWORK_MODEL, "compose-netem-signalling-shim", ); @@ -285,12 +289,12 @@ Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", } }); -Deno.test("placeholder and TURN cases are clearly non-evidence for broad P2P performance", () => { +test("placeholder and TURN cases are clearly non-evidence for broad P2P performance", () => { const cases = buildCases(); const smartphone = getCase(cases, "p2p-smartphone-vpn-direct"); - assertEquals(smartphone.env.BENCH_SIMULATION_TIER, "unmeasured"); - assertEquals(smartphone.env.BENCH_NETWORK_MODEL, "local-runner-no-netem"); + assert.deepStrictEqual(smartphone.env.BENCH_SIMULATION_TIER, "unmeasured"); + assert.deepStrictEqual(smartphone.env.BENCH_NETWORK_MODEL, "local-runner-no-netem"); assert( smartphone.limitations.some((limitation) => limitation.includes("must not be reported as smartphone") @@ -310,15 +314,15 @@ Deno.test("placeholder and TURN cases are clearly non-evidence for broad P2P per ); }); -Deno.test("CouchDB netem cases are marked as remote-store baselines", () => { +test("CouchDB netem cases are marked as remote-store baselines", () => { const cases = buildCases(); for ( const name of ["couchdb-netem-home-wifi", "couchdb-netem-tethering-vpn"] ) { const testCase = getCase(cases, name); - assertEquals(testCase.runner, "couchdb"); - assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2"); - assertEquals( + assert.deepStrictEqual(testCase.runner, "couchdb"); + assert.deepStrictEqual(testCase.env.BENCH_SIMULATION_TIER, "2"); + assert.deepStrictEqual( testCase.env.BENCH_NETWORK_MODEL, "compose-netem-tcp-shim", ); diff --git a/test/e2e-cli/test-ci-task-contract.ts b/test/e2e-cli/test-ci-task-contract.ts new file mode 100644 index 00000000..22a94565 --- /dev/null +++ b/test/e2e-cli/test-ci-task-contract.ts @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { test } from "node:test"; + +test("fetch database continuity is exercised by standard CLI CI", async () => { + const ciTasks = JSON.parse(await readFile(new URL("./ci-tasks.json", import.meta.url), "utf8")) as string[]; + const tasks = JSON.parse(await readFile(new URL("./tasks.json", import.meta.url), "utf8")) as Record< + string, + { test?: string[] } + >; + const workflow = await readFile(new URL("../../.github/workflows/cli-node-tests.yml", import.meta.url), "utf8"); + + assert(ciTasks.includes("test:helpers"), "the standard CI suite must run the task registration contract"); + assert(tasks["test:helpers"]?.test?.includes("test-ci-task-contract.ts")); + assert(ciTasks.includes("test:fetch-database-continuity"), "the continuity task must be in the standard CI suite"); + assert.deepEqual(tasks["test:fetch-database-continuity"]?.test, ["test-fetch-database-continuity.ts"]); + assert.match(workflow, /test\/e2e-cli\/ci-tasks\.json/, "the automatic CI matrix must read the authoritative task list"); +}); diff --git a/src/apps/cli/testdeno/test-daemon-startup.ts b/test/e2e-cli/test-daemon-startup.ts similarity index 78% rename from src/apps/cli/testdeno/test-daemon-startup.ts rename to test/e2e-cli/test-daemon-startup.ts index 8810faa1..e811e6ba 100644 --- a/src/apps/cli/testdeno/test-daemon-startup.ts +++ b/test/e2e-cli/test-daemon-startup.ts @@ -1,5 +1,7 @@ -import { assertEquals } from "@std/assert"; -import { join } from "@std/path"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, readFile, stat, unlink, utimes, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; import { runCliOrFail, runCliWithInputOrFail } from "./helpers/cli.ts"; import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts"; @@ -8,7 +10,7 @@ import { startCouchdb, stopCouchdb } from "./helpers/docker.ts"; function envOrDefault(keys: string[], fallback: string): string { for (const key of keys) { - const value = Deno.env.get(key)?.trim(); + const value = process.env[key]?.trim(); if (value) return value; } return fallback; @@ -23,10 +25,10 @@ async function waitForText(filePath: string, expected: string, timeoutMs = 45_00 let actual = ""; while (Date.now() < deadline) { try { - actual = await Deno.readTextFile(filePath); + actual = await readFile(filePath, "utf8"); if (actual === expected) return; } catch (error) { - if (!(error instanceof Deno.errors.NotFound)) throw error; + if ((error as { code?: string }).code !== "ENOENT") throw error; } await waitForTick(); } @@ -39,9 +41,9 @@ async function waitForMissing(filePath: string, timeoutMs = 45_000): Promise {}); } -Deno.test("daemon: startup scan uploads, reconciles, and reflects CouchDB files", async () => { +test("daemon: startup scan uploads, reconciles, and reflects CouchDB files", async () => { await using workDir = await TempDir.create("livesync-cli-daemon-startup"); const couchdbUri = envOrDefault(["COUCHDB_URI", "hostname"], "http://127.0.0.1:5989").replace(/\/$/, ""); @@ -74,12 +76,12 @@ Deno.test("daemon: startup scan uploads, reconciles, and reflects CouchDB files" const settingsC = workDir.join("settings-c.json"); await Promise.all([ - Deno.mkdir(databaseA, { recursive: true }), - Deno.mkdir(databaseB, { recursive: true }), - Deno.mkdir(databaseC, { recursive: true }), - Deno.mkdir(vaultA, { recursive: true }), - Deno.mkdir(vaultB, { recursive: true }), - Deno.mkdir(vaultC, { recursive: true }), + mkdir(databaseA, { recursive: true }), + mkdir(databaseB, { recursive: true }), + mkdir(databaseC, { recursive: true }), + mkdir(vaultA, { recursive: true }), + mkdir(vaultB, { recursive: true }), + mkdir(vaultC, { recursive: true }), ]); const startupPath = "notes/present-before-start.md"; @@ -91,11 +93,11 @@ Deno.test("daemon: startup scan uploads, reconciles, and reflects CouchDB files" const deleteFileB = join(vaultB, deletePath); const remoteOnlyFileB = join(vaultB, remoteOnlyPath); - await Deno.mkdir(join(vaultA, "notes"), { recursive: true }); - await Deno.writeTextFile(startupFileA, "created before daemon startup\n"); + await mkdir(join(vaultA, "notes"), { recursive: true }); + await writeFile(startupFileA, "created before daemon startup\n"); const initialTime = new Date(Date.now() - 10_000); - await Deno.utime(startupFileA, initialTime, initialTime); - await Deno.writeTextFile(deleteFileA, "delete this after the first run\n"); + await utimes(startupFileA, initialTime, initialTime); + await writeFile(deleteFileA, "delete this after the first run\n"); let daemonA: BackgroundCliProcess | undefined; let daemonB: BackgroundCliProcess | undefined; @@ -118,10 +120,10 @@ Deno.test("daemon: startup scan uploads, reconciles, and reflects CouchDB files" await waitForText(deleteFileB, "delete this after the first run\n"); // Changes made while A is stopped must be found by its next startup scan. - assertEquals(await daemonA.stop(), 0, daemonA.combined); + assert.deepStrictEqual(await daemonA.stop(), 0, daemonA.combined); daemonA = undefined; - await Deno.writeTextFile(startupFileA, "edited while daemon was stopped\n"); - await Deno.remove(deleteFileA); + await writeFile(startupFileA, "edited while daemon was stopped\n"); + await unlink(deleteFileA); daemonA = startCliInBackground(databaseA, "--vault", vaultA, "--settings", settingsA, "daemon"); await daemonA.waitUntilContains("[Daemon] Initial replication complete", 45_000); @@ -142,8 +144,8 @@ Deno.test("daemon: startup scan uploads, reconciles, and reflects CouchDB files" ); await runCliOrFail(databaseC, "--vault", vaultC, "--settings", settingsC, "sync"); await waitForText(remoteOnlyFileB, "created in a different local database\n"); - assertEquals(await Deno.readTextFile(startupFileB), "edited while daemon was stopped\n"); - assertEquals((await Deno.stat(remoteOnlyFileB)).isFile, true); + assert.deepStrictEqual(await readFile(startupFileB, "utf8"), "edited while daemon was stopped\n"); + assert.deepStrictEqual(((await stat(remoteOnlyFileB))).isFile(), true); } finally { await stopDaemon(daemonB); await stopDaemon(daemonA); diff --git a/src/apps/cli/testdeno/test-daemon.ts b/test/e2e-cli/test-daemon.ts similarity index 68% rename from src/apps/cli/testdeno/test-daemon.ts rename to test/e2e-cli/test-daemon.ts index d5f16ffa..618a4341 100644 --- a/src/apps/cli/testdeno/test-daemon.ts +++ b/test/e2e-cli/test-daemon.ts @@ -1,5 +1,5 @@ /** - * Deno port of test-daemon-linux.sh + * Node.js port of test-daemon-linux.sh * * Tests daemon-related ignore rules behaviour. * @@ -13,33 +13,36 @@ * 3. import: .gitignore directive → patterns from .gitignore are merged * * Run: - * deno test -A test-daemon.ts + * node --test test-daemon.ts */ -import { join } from "@std/path"; -import { assertEquals } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; + +import { join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; import { runCliOrFail, runCli, assertContains, assertNotContains } from "./helpers/cli.ts"; import { initSettingsFile, markSettingsConfigured } from "./helpers/settings.ts"; -Deno.test("daemon: ignore rules behaviour", async (t) => { +test("daemon: ignore rules behaviour", async (t) => { // ------------------------------------------------------------------------- // Case 1: .livesync/ignore with *.tmp → ignored file not synced to database // ------------------------------------------------------------------------- - await t.step("case 1: .livesync/ignore *.tmp prevents sync", async () => { + await t.test("case 1: .livesync/ignore *.tmp prevents sync", async () => { await using workDir = await TempDir.create("livesync-cli-daemon-c1"); const settingsFile = workDir.join("data.json"); const vaultDir = workDir.join("vault"); - await Deno.mkdir(join(vaultDir, ".livesync"), { recursive: true }); - await Deno.mkdir(join(vaultDir, "notes"), { recursive: true }); + await mkdir(join(vaultDir, ".livesync"), { recursive: true }); + await mkdir(join(vaultDir, "notes"), { recursive: true }); await initSettingsFile(settingsFile); await markSettingsConfigured(settingsFile); - await Deno.writeTextFile(join(vaultDir, ".livesync", "ignore"), "*.tmp\n"); - await Deno.writeTextFile(join(vaultDir, "notes", "normal.md"), "normal content\n"); - await Deno.writeTextFile(join(vaultDir, "notes", "scratch.tmp"), "tmp content\n"); + await writeFile(join(vaultDir, ".livesync", "ignore"), "*.tmp\n"); + await writeFile(join(vaultDir, "notes", "normal.md"), "normal content\n"); + await writeFile(join(vaultDir, "notes", "scratch.tmp"), "tmp content\n"); console.log("[INFO] Running mirror for Case 1..."); await runCliOrFail(vaultDir, "--settings", settingsFile, "mirror"); @@ -47,8 +50,8 @@ Deno.test("daemon: ignore rules behaviour", async (t) => { // The normal file should be in the database. const resultNormal = workDir.join("case1-normal.txt"); await runCliOrFail(vaultDir, "--settings", settingsFile, "pull", "notes/normal.md", resultNormal); - const normalContent = await Deno.readTextFile(resultNormal); - assertEquals(normalContent, "normal content\n", "normal.md content mismatch after mirror"); + const normalContent = await readFile(resultNormal, "utf8"); + assert.deepStrictEqual(normalContent, "normal content\n", "normal.md content mismatch after mirror"); // The .tmp file should NOT be in the database. const dbList = await runCliOrFail(vaultDir, "--settings", settingsFile, "ls"); @@ -60,50 +63,50 @@ Deno.test("daemon: ignore rules behaviour", async (t) => { // ------------------------------------------------------------------------- // Case 2: .livesync/ignore absent → no error, and normal synchronisation continues // ------------------------------------------------------------------------- - await t.step("case 2: .livesync/ignore absent does not cause failure", async () => { + await t.test("case 2: .livesync/ignore absent does not cause failure", async () => { await using workDir = await TempDir.create("livesync-cli-daemon-c2"); const settingsFile = workDir.join("data2.json"); const vaultDir = workDir.join("vault2"); - await Deno.mkdir(join(vaultDir, "notes"), { recursive: true }); + await mkdir(join(vaultDir, "notes"), { recursive: true }); await initSettingsFile(settingsFile); await markSettingsConfigured(settingsFile); // No .livesync directory at all. - await Deno.writeTextFile(join(vaultDir, "notes", "hello.md"), "hello\n"); + await writeFile(join(vaultDir, "notes", "hello.md"), "hello\n"); console.log("[INFO] Running mirror for Case 2..."); const result = await runCli(vaultDir, "--settings", settingsFile, "mirror"); - assertEquals(result.code, 0, "mirror exited non-zero when .livesync/ignore is absent"); + assert.deepStrictEqual(result.code, 0, "mirror exited non-zero when .livesync/ignore is absent"); // The normal file should have been synced. const resultHello = workDir.join("case2-hello.txt"); await runCliOrFail(vaultDir, "--settings", settingsFile, "pull", "notes/hello.md", resultHello); - const helloContent = await Deno.readTextFile(resultHello); - assertEquals(helloContent, "hello\n", "file content mismatch when .livesync/ignore is absent"); + const helloContent = await readFile(resultHello, "utf8"); + assert.deepStrictEqual(helloContent, "hello\n", "file content mismatch when .livesync/ignore is absent"); console.log("[PASS] Case 2 verified successfully"); }); // ------------------------------------------------------------------------- // Case 3: import: .gitignore merges patterns // ------------------------------------------------------------------------- - await t.step("case 3: import: .gitignore directive merges patterns", async () => { + await t.test("case 3: import: .gitignore directive merges patterns", async () => { await using workDir = await TempDir.create("livesync-cli-daemon-c3"); const settingsFile = workDir.join("data3.json"); const vaultDir = workDir.join("vault3"); - await Deno.mkdir(join(vaultDir, ".livesync"), { recursive: true }); - await Deno.mkdir(join(vaultDir, "notes"), { recursive: true }); + await mkdir(join(vaultDir, ".livesync"), { recursive: true }); + await mkdir(join(vaultDir, "notes"), { recursive: true }); await initSettingsFile(settingsFile); await markSettingsConfigured(settingsFile); - await Deno.writeTextFile(join(vaultDir, ".livesync", "ignore"), "import: .gitignore\n"); - await Deno.writeTextFile(join(vaultDir, ".gitignore"), "# gitignore comment\n*.log\nbuild/\n"); + await writeFile(join(vaultDir, ".livesync", "ignore"), "import: .gitignore\n"); + await writeFile(join(vaultDir, ".gitignore"), "# gitignore comment\n*.log\nbuild/\n"); - await Deno.writeTextFile(join(vaultDir, "notes", "regular.md"), "regular note\n"); - await Deno.writeTextFile(join(vaultDir, "notes", "debug.log"), "log content\n"); + await writeFile(join(vaultDir, "notes", "regular.md"), "regular note\n"); + await writeFile(join(vaultDir, "notes", "debug.log"), "log content\n"); console.log("[INFO] Running mirror for Case 3..."); await runCliOrFail(vaultDir, "--settings", settingsFile, "mirror"); diff --git a/src/apps/cli/testdeno/test-decoupled-vault.ts b/test/e2e-cli/test-decoupled-vault.ts similarity index 63% rename from src/apps/cli/testdeno/test-decoupled-vault.ts rename to test/e2e-cli/test-decoupled-vault.ts index f1a017f5..c4f4114e 100644 --- a/src/apps/cli/testdeno/test-decoupled-vault.ts +++ b/test/e2e-cli/test-decoupled-vault.ts @@ -1,39 +1,42 @@ /** - * Deno port of test-decoupled-vault-linux.sh + * Node.js port of test-decoupled-vault-linux.sh * * Tests push, pull, and mirror command behaviour when the vault directory is * decoupled (separated) from the database directory. * * Run: - * deno test -A test-decoupled-vault.ts + * node --test test-decoupled-vault.ts */ -import { join } from "@std/path"; -import { assertEquals } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, readFile, rm, unlink, writeFile } from "node:fs/promises"; + +import { join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; import { runCliOrFail } from "./helpers/cli.ts"; import { applyCouchdbSettings, initSettingsFile, markSettingsConfigured } from "./helpers/settings.ts"; import { startCouchdb, stopCouchdb } from "./helpers/docker.ts"; -const REMOTE_PATH = Deno.env.get("REMOTE_PATH") ?? "test/push-pull-decoupled.txt"; +const REMOTE_PATH = process.env["REMOTE_PATH"] ?? "test/push-pull-decoupled.txt"; -Deno.test("decoupled database and vault", async () => { +test("decoupled database and vault", async () => { await using workDir = await TempDir.create("livesync-cli-decoupled"); const settingsFile = workDir.join("data.json"); const vaultDir = workDir.join("vault"); const dbDir = workDir.join("db"); - await Deno.mkdir(join(vaultDir, "test"), { recursive: true }); - await Deno.mkdir(dbDir, { recursive: true }); + await mkdir(join(vaultDir, "test"), { recursive: true }); + await mkdir(dbDir, { recursive: true }); - const uri = Deno.env.get("COUCHDB_URI") ?? "http://127.0.0.1:5989/"; - const user = Deno.env.get("COUCHDB_USER") ?? "admin"; - const password = Deno.env.get("COUCHDB_PASSWORD") ?? "testpassword"; - const dbname = Deno.env.get("COUCHDB_DBNAME") ?? `decoupled-${Date.now()}`; + const uri = process.env["COUCHDB_URI"] ?? "http://127.0.0.1:5989/"; + const user = process.env["COUCHDB_USER"] ?? "admin"; + const password = process.env["COUCHDB_PASSWORD"] ?? "testpassword"; + const dbname = process.env["COUCHDB_DBNAME"] ?? `decoupled-${Date.now()}`; - const shouldStartDocker = Deno.env.get("LIVESYNC_START_DOCKER") !== "0"; - const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1"; + const shouldStartDocker = process.env["LIVESYNC_START_DOCKER"] !== "0"; + const keepDocker = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1"; if (shouldStartDocker) { await startCouchdb(uri, user, password, dbname); @@ -53,7 +56,7 @@ Deno.test("decoupled database and vault", async () => { const srcFile = workDir.join("push-source.txt"); const pulledFile = workDir.join("pull-result.txt"); const content = `push-pull-decoupled-test ${new Date().toISOString()}\n`; - await Deno.writeTextFile(srcFile, content); + await writeFile(srcFile, content); // 1. Test push command with decoupled vault directory console.log(`[INFO] push with decoupled vault -> ${REMOTE_PATH}`); @@ -63,21 +66,21 @@ Deno.test("decoupled database and vault", async () => { console.log(`[INFO] pull with decoupled vault <- ${REMOTE_PATH}`); await runCliOrFail(dbDir, "--vault", vaultDir, "--settings", settingsFile, "pull", REMOTE_PATH, pulledFile); - const pulled = await Deno.readTextFile(pulledFile); - assertEquals(pulled, content, "push/pull roundtrip with decoupled vault content mismatch"); + const pulled = await readFile(pulledFile, "utf8"); + assert.deepStrictEqual(pulled, content, "push/pull roundtrip with decoupled vault content mismatch"); console.log("[PASS] push/pull roundtrip with decoupled vault matched"); // 3. Clean up pulled file and vault test directory to verify mirror - await Deno.remove(pulledFile).catch(() => {}); - await Deno.remove(join(vaultDir, "test"), { recursive: true }).catch(() => {}); + await unlink(pulledFile).catch(() => {}); + await rm(join(vaultDir, "test"), { recursive: true }).catch(() => {}); // 4. Test mirror command with decoupled vault directory console.log("[INFO] mirror with decoupled vault"); await runCliOrFail(dbDir, "--vault", vaultDir, "--settings", settingsFile, "mirror"); const restoredFile = join(vaultDir, REMOTE_PATH); - const restored = await Deno.readTextFile(restoredFile); - assertEquals(restored, content, "mirror with decoupled vault content mismatch"); + const restored = await readFile(restoredFile, "utf8"); + assert.deepStrictEqual(restored, content, "mirror with decoupled vault content mismatch"); console.log("[PASS] mirror with decoupled vault matched"); } finally { if (shouldStartDocker && !keepDocker) { diff --git a/src/apps/cli/testdeno/test-e2e-two-vaults-couchdb.ts b/test/e2e-cli/test-e2e-two-vaults-couchdb.ts similarity index 95% rename from src/apps/cli/testdeno/test-e2e-two-vaults-couchdb.ts rename to test/e2e-cli/test-e2e-two-vaults-couchdb.ts index c112b702..ba3e9206 100644 --- a/src/apps/cli/testdeno/test-e2e-two-vaults-couchdb.ts +++ b/test/e2e-cli/test-e2e-two-vaults-couchdb.ts @@ -1,4 +1,6 @@ -import { assert } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, writeFile } from "node:fs/promises"; import { TempDir } from "./helpers/temp.ts"; import { runCli, @@ -14,7 +16,7 @@ type RemoteType = "COUCHDB" | "MINIO"; function requireEnv(...keys: string[]): string { for (const key of keys) { - const value = Deno.env.get(key)?.trim(); + const value = process.env[key]?.trim(); if (value) return value; } throw new Error(`Required env var is missing: ${keys.join(" or ")}`); @@ -49,10 +51,10 @@ export async function runScenario(remoteType: RemoteType, encrypt: boolean): Pro const pullDst = workDir.join("pull-destination.txt"); const pushBinarySrc = workDir.join("push-source.bin"); const pullBinaryDst = workDir.join("pull-destination.bin"); - await Deno.mkdir(vaultA, { recursive: true }); - await Deno.mkdir(vaultB, { recursive: true }); + await mkdir(vaultA, { recursive: true }); + await mkdir(vaultB, { recursive: true }); - const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1"; + const keepDocker = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1"; if (remoteType === "COUCHDB") { await startCouchdb(couchdbUri, couchdbUser, couchdbPassword, dbname); } else { @@ -168,7 +170,7 @@ export async function runScenario(remoteType: RemoteType, encrypt: boolean): Pro ).trimEnd(); assert(secondVisible === `second-sync-round-${dbSuffix}`); - await Deno.writeTextFile(pushSrc, `pushed-content-${dbSuffix}\n`); + await writeFile(pushSrc, `pushed-content-${dbSuffix}\n`); await runCliOrFail(vaultA, "--settings", settingsA, "push", pushSrc, targetPush); await runCliWithInputOrFail(`put-content-${dbSuffix}\n`, vaultA, "--settings", settingsA, "put", targetPut); await syncBoth(); @@ -181,7 +183,7 @@ export async function runScenario(remoteType: RemoteType, encrypt: boolean): Pro const binary = new Uint8Array(4096); binary.fill(0x61); - await Deno.writeFile(pushBinarySrc, binary); + await writeFile(pushBinarySrc, binary); await runCliOrFail(vaultA, "--settings", settingsA, "push", pushBinarySrc, targetPushBinary); await syncBoth(); await runCliOrFail(vaultB, "--settings", settingsB, "pull", targetPushBinary, pullBinaryDst); @@ -294,10 +296,10 @@ export async function runScenario(remoteType: RemoteType, encrypt: boolean): Pro } } -Deno.test("e2e: two vaults over CouchDB without encryption", async () => { +test("e2e: two vaults over CouchDB without encryption", async () => { await runScenario("COUCHDB", false); }); -Deno.test("e2e: two vaults over CouchDB with encryption", async () => { +test("e2e: two vaults over CouchDB with encryption", async () => { await runScenario("COUCHDB", true); }); diff --git a/src/apps/cli/testdeno/test-e2e-two-vaults-matrix.ts b/test/e2e-cli/test-e2e-two-vaults-matrix.ts similarity index 87% rename from src/apps/cli/testdeno/test-e2e-two-vaults-matrix.ts rename to test/e2e-cli/test-e2e-two-vaults-matrix.ts index 757d6b57..eb99a1c0 100644 --- a/src/apps/cli/testdeno/test-e2e-two-vaults-matrix.ts +++ b/test/e2e-cli/test-e2e-two-vaults-matrix.ts @@ -1,3 +1,4 @@ +import { test } from "node:test"; import { runScenario } from "./test-e2e-two-vaults-couchdb.ts"; type MatrixCase = { @@ -14,7 +15,7 @@ const matrixCases: MatrixCase[] = [ ]; for (const tc of matrixCases) { - Deno.test(`e2e matrix: ${tc.label}`, async () => { + test(`e2e matrix: ${tc.label}`, async () => { await runScenario(tc.remoteType, tc.encrypt); }); } diff --git a/test/e2e-cli/test-fetch-database-continuity.ts b/test/e2e-cli/test-fetch-database-continuity.ts new file mode 100644 index 00000000..8f38548f --- /dev/null +++ b/test/e2e-cli/test-fetch-database-continuity.ts @@ -0,0 +1,637 @@ +import assert from "node:assert/strict"; +import { createHash, randomUUID } from "node:crypto"; +import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { after, before, test } from "node:test"; +import { CreateBucketCommand, S3Client } from "@aws-sdk/client-s3"; +import { CLI_DIR, CLI_DIST } from "./helpers/cli.ts"; +import { startIsolatedMinio, type IsolatedMinioFixture } from "./helpers/docker.ts"; +import { applyRemoteSyncSettings, generateSetupUriFromSettings, initSettingsFile } from "./helpers/settings.ts"; +import { TempDir } from "./helpers/temp.ts"; +import { spawnProcess, type RunningProcess } from "./helpers/process.ts"; + +type RemoteSettings = { + endpoint: string; + accessKey: string; + secretKey: string; + bucket: string; + evidenceDir: string; +}; + +type CliResult = { code: number; stdout: string; stderr: string; combined: string }; +type ScenarioMode = "reproduction" | "write-settings-control"; + +const SETUP_PASSPHRASE = "issue-1304-setup-passphrase"; +const RESOLUTION_MARKER = "Resolving [same]: Conflicted revision has been deleted"; +const INITIAL_REPLICATION_COMPLETE = "[Daemon] Initial replication complete"; +const INITIAL_REPLICATION_FAILED = "[Daemon] Initial replication failed"; +const VERIFY_LEGACY_FAILURE = process.env["LIVESYNC_ISSUE1304_VERIFY_LEGACY_FAILURE"] === "1"; +const EXPLICIT_FIXTURE_VARIABLES = [ + "LIVESYNC_ISSUE1304_MINIO_ENDPOINT", + "LIVESYNC_ISSUE1304_MINIO_ACCESS_KEY", + "LIVESYNC_ISSUE1304_MINIO_SECRET_KEY", + "LIVESYNC_ISSUE1304_MINIO_BUCKET", +]; +const EXPLICIT_FIXTURE = EXPLICIT_FIXTURE_VARIABLES.some((name) => process.env[name]?.trim()); + +if (EXPLICIT_FIXTURE && EXPLICIT_FIXTURE_VARIABLES.some((name) => !process.env[name]?.trim())) { + throw new Error(`External fixture mode requires all of: ${EXPLICIT_FIXTURE_VARIABLES.join(", ")}`); +} + +let managedFixture: IsolatedMinioFixture | undefined; +if (!EXPLICIT_FIXTURE) { + before(async () => { + managedFixture = await startIsolatedMinio( + process.env["accessKey"]?.trim() ?? "minioadmin", + process.env["secretKey"]?.trim() ?? "minioadmin" + ); + }); + after(async () => { + await managedFixture?.stop(); + }); +} + +function requiredEnv(...names: string[]): string { + for (const name of names) { + const value = process.env[name]?.trim(); + if (value) return value; + } + throw new Error(`Required environment variable is missing: ${names.join(" or ")}`); +} + +function evidenceDirectory(): string { + return resolve( + process.env["LIVESYNC_ISSUE1304_EVIDENCE_DIR"]?.trim() || join(tmpdir(), "livesync-cli-issue1304-evidence") + ); +} + +function remoteSettings(): RemoteSettings { + return { + endpoint: requiredEnv("LIVESYNC_ISSUE1304_MINIO_ENDPOINT").replace(/\/$/, ""), + accessKey: requiredEnv("LIVESYNC_ISSUE1304_MINIO_ACCESS_KEY"), + secretKey: requiredEnv("LIVESYNC_ISSUE1304_MINIO_SECRET_KEY"), + bucket: requiredEnv("LIVESYNC_ISSUE1304_MINIO_BUCKET"), + evidenceDir: evidenceDirectory(), + }; +} + +async function createManagedBucket(remote: Pick, mode: ScenarioMode): Promise { + const shortMode = mode === "reproduction" ? "repro" : "control"; + const bucket = `livesync-ci-1304-${shortMode}-${randomUUID().replaceAll("-", "")}`; + const client = new S3Client({ + endpoint: remote.endpoint, + region: "us-east-1", + forcePathStyle: true, + credentials: { accessKeyId: remote.accessKey, secretAccessKey: remote.secretKey }, + }); + try { + await client.send(new CreateBucketCommand({ Bucket: bucket })); + } finally { + client.destroy(); + } + return bucket; +} + +async function scenarioRemoteSettings(mode: ScenarioMode): Promise { + if (EXPLICIT_FIXTURE) { + const remote = remoteSettings(); + if (mode === "write-settings-control") remote.bucket = requiredEnv("LIVESYNC_ISSUE1304_POSITIVE_CONTROL_BUCKET"); + return remote; + } + + if (!managedFixture) throw new Error("The managed S3 fixture was not started"); + const remote = { + endpoint: managedFixture.endpoint, + accessKey: managedFixture.accessKey, + secretKey: managedFixture.secretKey, + }; + return { + ...remote, + bucket: await createManagedBucket(remote, mode), + evidenceDir: evidenceDirectory(), + }; +} + +function redactText(value: string, secrets: readonly string[]): string { + let redacted = value; + for (const secret of secrets) { + if (!secret) continue; + redacted = redacted.split(secret).join("[redacted]"); + const encoded = encodeURIComponent(secret); + if (encoded !== secret) redacted = redacted.split(encoded).join("[redacted]"); + } + return redacted; +} + +async function streamText(stream: ReadableStream, onText: (text: string) => void): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (value) onText(decoder.decode(value, { stream: true })); + } + const tail = decoder.decode(); + if (tail) onText(tail); + } finally { + reader.releaseLock(); + } +} + +class CapturedCliProcess { + readonly child: RunningProcess; + #stdout = ""; + #stderr = ""; + #secrets = [ + process.env["LIVESYNC_ISSUE1304_MINIO_ENDPOINT"] ?? "", + process.env["LIVESYNC_ISSUE1304_MINIO_ACCESS_KEY"] ?? "", + process.env["LIVESYNC_ISSUE1304_MINIO_SECRET_KEY"] ?? "", + SETUP_PASSPHRASE, + "issue-1304-content-passphrase", + ]; + #streamsDone: Promise; + + constructor(args: string[]) { + this.child = spawnProcess(process.execPath, { + args: [CLI_DIST, ...args], + cwd: CLI_DIR, + stdin: "null", + stdout: "piped", + stderr: "piped", + }); + this.#streamsDone = Promise.all([ + streamText(this.child.stdout, (text) => (this.#stdout += text)), + streamText(this.child.stderr, (text) => (this.#stderr += text)), + ]).then(() => undefined); + } + + get combined(): string { + return this.#stdout + this.#stderr; + } + + async waitUntil(predicate: (output: string) => boolean, description: string, timeoutMs = 60_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate(this.combined)) return; + const tick = Math.min(100, deadline - Date.now()); + const event = await Promise.race([ + this.child.status.then((status) => ({ kind: "exit" as const, status })), + new Promise<{ kind: "tick" }>((resolveTick) => setTimeout(() => resolveTick({ kind: "tick" }), tick)), + ]); + if (event.kind === "exit") { + await this.#streamsDone; + if (predicate(this.combined)) return; + throw new Error( + `${description}; process exited with ${event.status.code}\n${redactText(this.combined, this.#secrets).slice(-5000)}` + ); + } + } + throw new Error(`Timed out waiting for ${description}\n${redactText(this.combined, this.#secrets).slice(-5000)}`); + } + + async stop(): Promise { + try { + this.child.kill("SIGTERM"); + } catch { + // A completed process needs no further signal. + } + const status = await this.child.status; + await this.#streamsDone; + return status.code; + } +} + +async function runCliQuietly(args: string[], input?: string): Promise { + const child = spawnProcess(process.execPath, { + args: [CLI_DIST, ...args], + cwd: CLI_DIR, + stdin: input === undefined ? "null" : "piped", + stdout: "piped", + stderr: "piped", + }); + let stdout = ""; + let stderr = ""; + const streamsDone = Promise.all([ + streamText(child.stdout, (text) => (stdout += text)), + streamText(child.stderr, (text) => (stderr += text)), + ]); + if (input !== undefined) { + const writer = child.stdin.getWriter(); + await writer.write(new TextEncoder().encode(input)); + await writer.close(); + } + const [status] = await Promise.all([child.status, streamsDone]); + return { code: status.code, stdout, stderr, combined: stdout + stderr }; +} + +function countOccurrences(value: string, needle: string): number { + return value.split(needle).length - 1; +} + +function safeCommand( + label: string, + result: CliResult, + commandLog: Array<{ label: string; code: number; output: string }>, + secrets: readonly string[] +): void { + const output = redactText(result.combined, secrets); + commandLog.push({ label, code: result.code, output }); + assert.equal(result.code, 0, `${label} failed with exit code ${result.code}\n${output.slice(-5000)}`); +} + +async function fileState(filePath: string): Promise<{ exists: boolean; bytes?: number; sha256?: string }> { + try { + const contents = await readFile(filePath); + return { + exists: true, + bytes: contents.byteLength, + sha256: createHash("sha256").update(contents).digest("hex"), + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return { exists: false }; + throw error; + } +} + +async function collectDatabaseTree(root: string, relative = ""): Promise> { + const result: Array<{ path: string; bytes: number; sha256: string }> = []; + let entries; + try { + entries = await readdir(join(root, relative), { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return result; + throw error; + } + entries.sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + if (!relative && entry.name === ".livesync") continue; + const child = relative ? join(relative, entry.name) : entry.name; + const absolute = join(root, child); + if (entry.isDirectory()) { + result.push(...(await collectDatabaseTree(root, child))); + } else if (entry.isFile()) { + const contents = await readFile(absolute); + result.push({ + path: child, + bytes: contents.byteLength, + sha256: createHash("sha256").update(contents).digest("hex"), + }); + } + } + return result; +} + +async function copyDatabaseTree(source: string, destination: string): Promise { + await mkdir(destination, { recursive: true }); + let entries; + try { + entries = await readdir(source, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + for (const entry of entries) { + if (entry.name === ".livesync") continue; + await cp(join(source, entry.name), join(destination, entry.name), { recursive: true, errorOnExist: false }); + } +} + +async function sanitisedRuntimeState(root: string, secrets: readonly string[]): Promise> { + const runtimeRoot = join(root, ".livesync", "runtime"); + const files: Record = {}; + let entries; + try { + entries = await readdir(runtimeRoot, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return files; + throw error; + } + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith(".json")) continue; + const text = await readFile(join(runtimeRoot, entry.name), "utf8"); + try { + files[entry.name] = JSON.stringify(JSON.parse(text), (_key, value: unknown) => { + return typeof value === "string" ? redactText(value, secrets) : value; + }, 2); + } catch { + files[entry.name] = redactText(text, secrets); + } + } + return files; +} + +async function settingsSummary(settingsPath: string): Promise | null> { + try { + const settings = JSON.parse(await readFile(settingsPath, "utf8")) as Record; + return { + additionalSuffixOfDatabaseName: settings.additionalSuffixOfDatabaseName ?? null, + remoteType: settings.remoteType ?? null, + liveSync: settings.liveSync ?? null, + syncOnStart: settings.syncOnStart ?? null, + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} + +async function runScenario(mode: ScenarioMode, remote: RemoteSettings): Promise { + const evidenceRoot = remote.evidenceDir; + const evidenceName = `${mode}-${Date.now()}-${process.pid}`; + const evidenceDir = join(evidenceRoot, evidenceName); + const workDir = await TempDir.create(`livesync-cli-issue1304-${mode}`); + const scenarioId = `${Date.now()}-${process.pid}`; + const suffix = `issue1304-${scenarioId}`; + const targetPath = `notes/issue1304-${scenarioId}/continuity.md`; + const initialContent = `issue-1304 initial ${scenarioId}\n`; + const finalContent = `issue-1304 final ${scenarioId}\n`; + const finalBytes = new TextEncoder().encode(finalContent); + const secrets = [remote.endpoint, remote.accessKey, remote.secretKey, SETUP_PASSPHRASE, "issue-1304-content-passphrase"]; + const commandLog: Array<{ label: string; code: number; output: string }> = []; + const processes: Record = {}; + const settingsTimeline: Record = {}; + const postResolutionInfo: Record | null> = {}; + let observedFinalFileA: { exists: boolean; bytes?: number; sha256?: string } | undefined; + let observedFinalContentA: string | undefined; + let setupUri: string | undefined; + let failure: unknown; + + const databaseSeed = workDir.join("database-seed"); + const databaseA = workDir.join("database-a"); + const databaseB = workDir.join("database-b"); + const vaultSeed = workDir.join("vault-seed"); + const vaultA = workDir.join("vault-a"); + const vaultB = workDir.join("vault-b"); + const settingsSeed = workDir.join("settings-seed.json"); + const settingsA = workDir.join("settings-a.json"); + const settingsB = workDir.join("settings-b.json"); + const finalFileA = join(vaultA, targetPath); + const finalFileB = join(vaultB, targetPath); + + try { + await Promise.all([ + mkdir(databaseSeed, { recursive: true }), + mkdir(databaseA, { recursive: true }), + mkdir(databaseB, { recursive: true }), + mkdir(vaultSeed, { recursive: true }), + mkdir(vaultA, { recursive: true }), + mkdir(vaultB, { recursive: true }), + ]); + await initSettingsFile(settingsSeed); + await applyRemoteSyncSettings(settingsSeed, { + remoteType: "MINIO", + minioBucket: remote.bucket, + minioEndpoint: remote.endpoint, + minioAccessKey: remote.accessKey, + minioSecretKey: remote.secretKey, + encrypt: true, + passphrase: "issue-1304-content-passphrase", + usePathObfuscation: true, + }); + const seedSettings = JSON.parse(await readFile(settingsSeed, "utf8")) as Record; + seedSettings.additionalSuffixOfDatabaseName = suffix; + await writeFile(settingsSeed, JSON.stringify(seedSettings, null, 2)); + settingsTimeline.setupSuffix = suffix; + setupUri = await generateSetupUriFromSettings(settingsSeed, SETUP_PASSPHRASE, true); + secrets.push(setupUri); + + const seedRun = async (label: string, args: string[], input?: string) => { + const result = await runCliQuietly(args, input); + safeCommand(label, result, commandLog, secrets); + return result; + }; + + await seedRun("seed-put-initial", [databaseSeed, "--vault", vaultSeed, "--settings", settingsSeed, "put", targetPath], initialContent); + await seedRun("seed-sync-initial", [databaseSeed, "--vault", vaultSeed, "--settings", settingsSeed, "sync"]); + await seedRun("seed-put-final", [databaseSeed, "--vault", vaultSeed, "--settings", settingsSeed, "put", targetPath], finalContent); + await seedRun("seed-sync-final", [databaseSeed, "--vault", vaultSeed, "--settings", settingsSeed, "sync"]); + const seedInfoResult = await seedRun("seed-info", [databaseSeed, "--vault", vaultSeed, "--settings", settingsSeed, "info", targetPath]); + const seedInfo = JSON.parse(seedInfoResult.stdout) as { revision?: string; revisions?: string[] }; + const seedRevisionGeneration = Number(seedInfo.revision?.split("-")[0] ?? 0); + assert( + seedRevisionGeneration >= 2 && (seedInfo.revisions?.length ?? 0) >= 1, + `Expected a remote seed revision with generation >= 2; got ${JSON.stringify({ revision: seedInfo.revision, revisions: seedInfo.revisions })}` + ); + settingsTimeline.seedRevision = seedInfo.revision; + + await initSettingsFile(settingsA); + await initSettingsFile(settingsB); + const setupA = await runCliQuietly( + [databaseA, "--vault", vaultA, "--settings", settingsA, "setup", setupUri], + `${SETUP_PASSPHRASE}\n` + ); + safeCommand("setup-client-a", setupA, commandLog, secrets); + const setupB = await runCliQuietly( + [databaseB, "--vault", vaultB, "--settings", settingsB, "setup", setupUri], + `${SETUP_PASSPHRASE}\n` + ); + safeCommand("setup-client-b", setupB, commandLog, secrets); + + const bFetchArgs = [databaseB, "--vault", vaultB, "--settings", settingsB, "--force", "fetch", "--write-settings"]; + const bFetch = await runCliQuietly(bFetchArgs); + safeCommand("fetch-client-b", bFetch, commandLog, secrets); + assert.equal(await readFile(finalFileB, "utf8"), finalContent, "Client B did not fetch the final file bytes"); + settingsTimeline.clientBSuffixAfterFetch = (await settingsSummary(settingsB))?.additionalSuffixOfDatabaseName; + assert.equal(settingsTimeline.clientBSuffixAfterFetch, "headless-app"); + + // Client B remains live so it can resolve the equal-content root after A + // scans its old database and uploads a new root. + processes.clientB = new CapturedCliProcess([ + "-v", + databaseB, + "--vault", + vaultB, + "--settings", + settingsB, + "--interval", + "1", + "daemon", + ]); + await processes.clientB.waitUntil((output) => output.includes(INITIAL_REPLICATION_COMPLETE), "Client B initial replication"); + + settingsTimeline.clientASuffixBeforeFetch = (await settingsSummary(settingsA))?.additionalSuffixOfDatabaseName; + assert.equal(settingsTimeline.clientASuffixBeforeFetch, suffix); + const aFetchArgs = [databaseA, "--vault", vaultA, "--settings", settingsA, "--force", "fetch"]; + if (mode === "write-settings-control") aFetchArgs.push("--write-settings"); + const aFetch = await runCliQuietly(aFetchArgs); + safeCommand("fetch-client-a", aFetch, commandLog, secrets); + assert.equal(await readFile(finalFileA, "utf8"), finalContent, "Client A did not fetch the final file bytes"); + settingsTimeline.clientASuffixAfterFetch = (await settingsSummary(settingsA))?.additionalSuffixOfDatabaseName; + assert.equal( + settingsTimeline.clientASuffixAfterFetch, + mode === "reproduction" && VERIFY_LEGACY_FAILURE ? suffix : "headless-app", + "Client A settings suffix did not match the selected fetch mode" + ); + + // This startup selects A's persisted suffix. In the reproduction mode it + // scans the old, empty database while the fetched bytes are in the vault. + processes.clientAStartup = new CapturedCliProcess([ + "-v", + databaseA, + "--vault", + vaultA, + "--settings", + settingsA, + "daemon", + ]); + await processes.clientAStartup.waitUntil( + (output) => output.includes(INITIAL_REPLICATION_COMPLETE) || output.includes(INITIAL_REPLICATION_FAILED), + "Client A startup replication result", + 45_000 + ); + await processes.clientAStartup.stop(); + + const markResolved = await runCliQuietly([databaseA, "--vault", vaultA, "--settings", settingsA, "mark-resolved"]); + safeCommand("mark-resolved-client-a", markResolved, commandLog, secrets); + + if (mode === "reproduction" && VERIFY_LEGACY_FAILURE) { + processes.clientA = new CapturedCliProcess([ + "-v", + databaseA, + "--vault", + vaultA, + "--settings", + settingsA, + "--interval", + "1", + "daemon", + ]); + await processes.clientA.waitUntil((output) => output.includes(INITIAL_REPLICATION_COMPLETE), "Client A interval daemon initial replication"); + + const processingMarker = `Processing ${targetPath}`; + const aProcessingBeforeResolution = countOccurrences(processes.clientA.combined, processingMarker); + await processes.clientB.waitUntil((output) => output.includes(RESOLUTION_MARKER), "Client B automatic same-content resolution"); + await processes.clientA.waitUntil( + (output) => countOccurrences(output, processingMarker) > aProcessingBeforeResolution, + "Client A to reflect the post-resolution revision", + 45_000 + ); + } else { + processes.clientA = new CapturedCliProcess([ + "-v", + databaseA, + "--vault", + vaultA, + "--settings", + settingsA, + "--interval", + "1", + "daemon", + ]); + await processes.clientA.waitUntil((output) => output.includes(INITIAL_REPLICATION_COMPLETE), "Client A control daemon initial replication"); + await new Promise((resolveDelay) => setTimeout(resolveDelay, 2_500)); + assert.equal(await readFile(finalFileA, "utf8"), finalContent, "Client A control lost the fetched file"); + } + } catch (error) { + failure = error; + } + + for (const processCapture of Object.values(processes)) { + if (!processCapture) continue; + try { + await processCapture.stop(); + } catch (error) { + if (failure === undefined) failure = error; + } + } + + try { + await mkdir(evidenceDir, { recursive: true }); + for (const [name, database, vault, settingsPath] of [ + ["clientA", databaseA, vaultA, settingsA], + ["clientB", databaseB, vaultB, settingsB], + ] as const) { + const result = await runCliQuietly([database, "--vault", vault, "--settings", settingsPath, "info", targetPath]); + const output = redactText(result.combined, secrets); + commandLog.push({ label: `post-resolution-info-${name}`, code: result.code, output }); + if (result.code === 0) { + try { + postResolutionInfo[name] = JSON.parse(result.stdout) as Record; + } catch { + postResolutionInfo[name] = null; + } + } else { + postResolutionInfo[name] = null; + } + } + observedFinalFileA = await fileState(finalFileA); + if (observedFinalFileA.exists) observedFinalContentA = await readFile(finalFileA, "utf8"); + const databaseTrees: Record> = {}; + const runtimeState: Record> = {}; + const databasePaths = { seed: databaseSeed, clientA: databaseA, clientB: databaseB }; + for (const [name, path] of Object.entries(databasePaths)) { + databaseTrees[name] = await collectDatabaseTree(path); + await copyDatabaseTree(path, join(evidenceDir, `database-${name}`)); + runtimeState[name] = await sanitisedRuntimeState(path, secrets); + if (Object.keys(runtimeState[name]).length > 0) { + const runtimeDir = join(evidenceDir, `runtime-${name}`); + await mkdir(runtimeDir, { recursive: true }); + for (const [fileName, contents] of Object.entries(runtimeState[name])) { + await writeFile(join(runtimeDir, fileName), contents); + } + } + } + const logs = Object.fromEntries( + Object.entries(processes).map(([name, processCapture]) => [ + name, + processCapture ? redactText(processCapture.combined, secrets) : "", + ]) + ); + const fileStates = { + clientA: await fileState(finalFileA), + clientB: await fileState(finalFileB), + }; + const evidence = { + scenario: mode, + targetPath, + expectedContentBytes: finalBytes.byteLength, + expectedContentSha256: createHash("sha256").update(finalBytes).digest("hex"), + settingsTimeline, + settings: { + seed: await settingsSummary(settingsSeed), + clientA: await settingsSummary(settingsA), + clientB: await settingsSummary(settingsB), + }, + fileStates, + postResolutionInfo, + databaseTrees, + runtimeState, + commands: commandLog, + processLogs: logs, + remote: { endpoint: "[configured]", bucket: remote.bucket }, + failure: failure instanceof Error ? redactText(failure.stack ?? failure.message, secrets) : String(failure ?? ""), + }; + await writeFile(join(evidenceDir, "evidence.json"), JSON.stringify(evidence, null, 2)); + } catch (error) { + if (failure === undefined) failure = error; + else if (failure instanceof Error) { + failure = new Error(`${failure.message}\nEvidence capture failed: ${error instanceof Error ? error.message : String(error)}`); + } + } + + await workDir[Symbol.asyncDispose](); + if (failure !== undefined) throw failure; + assert.equal(observedFinalFileA?.exists, true, "Client A file is absent after the resolved journal was applied"); + assert.equal(observedFinalContentA, finalContent, "Client A lost the fetched file after remote conflict resolution"); + if (!VERIFY_LEGACY_FAILURE) { + const infoA = postResolutionInfo.clientA; + const infoB = postResolutionInfo.clientB; + assert(infoA && infoB, "Could not inspect both local databases after the remote resolution"); + assert.equal(infoA.conflicts, "N/A", "Client A still has a duplicate root revision after the resolution"); + assert.equal(infoA.revision, infoB.revision, "Client A did not apply the revision selected by Client B after the resolved journal"); + assert.equal(infoA.revision, settingsTimeline.seedRevision, "Daemon startup replaced the fetched revision ancestry"); + assert.equal(processes.clientB?.combined.includes(RESOLUTION_MARKER), false, "Client A created an unnecessary competing root"); + } +} + +test("CLI fetch: files remain after same-content duplicate root resolution", async () => { + await runScenario("reproduction", await scenarioRemoteSettings("reproduction")); +}); + +if (!EXPLICIT_FIXTURE || process.env["LIVESYNC_ISSUE1304_POSITIVE_CONTROL"] === "1") { + test("CLI fetch: --write-settings keeps the selected database across daemon startup", async () => { + await runScenario("write-settings-control", await scenarioRemoteSettings("write-settings-control")); + }); +} diff --git a/test/e2e-cli/test-journal-chunk-recovery.ts b/test/e2e-cli/test-journal-chunk-recovery.ts new file mode 100644 index 00000000..345fbc88 --- /dev/null +++ b/test/e2e-cli/test-journal-chunk-recovery.ts @@ -0,0 +1,212 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { TempDir } from "./helpers/temp.ts"; +import { assertFilesEqual, runCliOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; +import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts"; +import { startMinio, stopMinio } from "./helpers/docker.ts"; + +function requireEnv(...keys: string[]): string { + for (const key of keys) { + const value = process.env[key]?.trim(); + if (value) return value; + } + throw new Error(`Required env var is missing: ${keys.join(" or ")}`); +} + +function createJournalRecoveryBucketName(baseBucket: string, suffix: string): string { + const bucketSuffix = `journal-recovery-${suffix}`; + const maxBaseLength = 63 - bucketSuffix.length - 1; + const truncatedBase = baseBucket.slice(0, maxBaseLength).replace(/-+$/, ""); + return truncatedBase ? `${truncatedBase}-${bucketSuffix}` : bucketSuffix; +} + +function createJournalFixture(): string { + return [ + "Journal chunk recovery fixture: UTF-8, newlines, and backslashes.\n", + ...Array.from( + { length: 128 }, + (_, index) => + `record ${String(index).padStart(3, "0")}: prefix\\path café 日本語 \u001f middle-${index} \u001f suffix-${index} remains after the separator.\n` + ), + "Final suffix after every control separator remains present.\n", + ].join(""); +} + +async function runScenario(encrypt: boolean): Promise { + const suffix = `${encrypt ? "enc1" : "enc0"}-${Date.now()}-${Math.floor(Math.random() * 100000)}`; + const minioEndpoint = requireEnv("MINIO_ENDPOINT", "minioEndpoint").replace(/\/$/, ""); + const minioAccessKey = requireEnv("MINIO_ACCESS_KEY", "accessKey"); + const minioSecretKey = requireEnv("MINIO_SECRET_KEY", "secretKey"); + const minioBucket = createJournalRecoveryBucketName(requireEnv("MINIO_BUCKET_NAME", "bucketName"), suffix); + const passphrase = "e2e-passphrase"; + const fileContent = createJournalFixture(); + const expectedBytes = Buffer.from(fileContent, "utf8"); + const targetPath = "e2e/journal-chunk-recovery.md"; + const keepDocker = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1"; + + await using workDir = await TempDir.create(`livesync-cli-journal-recovery-${suffix}`); + const sourceDatabase = workDir.join("source-db"); + const receiverDatabase = workDir.join("receiver-db"); + const sourceVault = workDir.join("source-vault"); + const receiverVault = workDir.join("receiver-vault"); + const sourceSettings = workDir.join("source-settings.json"); + const receiverSettings = workDir.join("receiver-settings.json"); + const sourceFile = join(sourceVault, targetPath); + const receiverFile = join(receiverVault, targetPath); + await Promise.all([ + mkdir(sourceDatabase, { recursive: true }), + mkdir(receiverDatabase, { recursive: true }), + mkdir(sourceVault, { recursive: true }), + mkdir(receiverVault, { recursive: true }), + ]); + await mkdir(dirname(sourceFile), { recursive: true }); + await writeFile(sourceFile, expectedBytes); + + try { + await startMinio(minioEndpoint, minioAccessKey, minioSecretKey, minioBucket); + await initSettingsFile(sourceSettings); + await initSettingsFile(receiverSettings); + await applyRemoteSyncSettings(sourceSettings, { + remoteType: "MINIO", + minioBucket, + minioEndpoint, + minioAccessKey, + minioSecretKey, + encrypt, + passphrase, + enableCompression: false, + }); + await applyRemoteSyncSettings(receiverSettings, { + remoteType: "MINIO", + minioBucket, + minioEndpoint, + minioAccessKey, + minioSecretKey, + encrypt, + passphrase, + enableCompression: false, + }); + + await runCliOrFail( + sourceDatabase, + "--settings", + sourceSettings, + "--vault", + sourceVault, + "rebuild", + "--force", + "--write-settings" + ); + await runCliOrFail( + receiverDatabase, + "--settings", + receiverSettings, + "--vault", + receiverVault, + "fetch", + "--force", + "--write-settings" + ); + + await assertFilesEqual(sourceFile, receiverFile, "Fetched Vault file does not match the rebuild source"); + const fetchedContent = sanitiseCatStdout( + await runCliOrFail(receiverDatabase, "--settings", receiverSettings, "cat", targetPath) + ); + assert.equal(fetchedContent, fileContent, "A fresh CLI process should read the exact fetched file content"); + + const damagedContent = "Damaged receiver Vault copy; the remote still has the complete source.\n"; + await writeFile(receiverFile, damagedContent); + await runCliOrFail( + receiverDatabase, + "--settings", + receiverSettings, + "--vault", + receiverVault, + "fetch", + "--force", + "--write-settings" + ); + + assert.equal( + await readFile(receiverFile, "utf8"), + damagedContent, + "Fetch does not forcibly overwrite a newer local file" + ); + const info: unknown = JSON.parse( + await runCliOrFail(receiverDatabase, "--settings", receiverSettings, "info", targetPath) + ); + assert.ok(info && typeof info === "object" && "revision" in info && "conflicts" in info); + assert.ok(typeof info.revision === "string" && typeof info.conflicts === "string"); + assert.match(info.revision, /^1-[\da-f]+$/, "The fetched file should retain an independent root"); + assert.match(info.conflicts, /^1-[\da-f]+$/, "The differing local file should be preserved as a conflict"); + assert.notEqual(info.revision, info.conflicts); + const retainedContents: string[] = []; + for (const revision of [info.revision, info.conflicts]) { + retainedContents.push( + sanitiseCatStdout( + await runCliOrFail( + receiverDatabase, + "--settings", + receiverSettings, + "cat-rev", + targetPath, + revision + ) + ) + ); + } + assert.deepEqual( + retainedContents.sort(), + [fileContent, damagedContent].sort(), + "Fetch should retain both the intact remote and differing local contents without inferring their ancestry" + ); + await rename(receiverFile, workDir.join("damaged-vault-backup.md")); + await runCliOrFail( + receiverDatabase, + "--settings", + receiverSettings, + "--vault", + receiverVault, + "fetch", + "--force", + "--write-settings" + ); + await assertFilesEqual(sourceFile, receiverFile, "Fetching into an empty Vault should restore the remote file"); + assert.equal(await readFile(workDir.join("damaged-vault-backup.md"), "utf8"), damagedContent); + const restoredContent = sanitiseCatStdout( + await runCliOrFail(receiverDatabase, "--settings", receiverSettings, "cat", targetPath) + ); + assert.equal(restoredContent, fileContent, "A repeated fetch should retain the remote file content"); + assert.deepEqual( + await readFile(sourceFile), + expectedBytes, + "The complete source Vault fixture should remain intact" + ); + } finally { + if (!keepDocker) { + await stopMinio().catch(() => {}); + } + } +} + +test("journal recovery bucket name construction stays within S3 limits", () => { + const ciBucket = createJournalRecoveryBucketName("livesync-test-bucket-ci", "enc0-1793522628122-62377"); + assert.ok(ciBucket.length <= 63, "The CI fixture must stay within the S3 bucket-name length limit"); + assert.match(ciBucket, /^[a-z0-9][a-z0-9-]*[a-z0-9]$/); + + const longBase = "a".repeat(63); + const unencryptedBucket = createJournalRecoveryBucketName(longBase, "enc0-1793522628122-62377"); + const encryptedBucket = createJournalRecoveryBucketName(longBase, "enc1-1793522628122-62377"); + assert.ok(unencryptedBucket.length <= 63); + assert.ok(encryptedBucket.length <= 63); + assert.match(unencryptedBucket, /^[a-z0-9][a-z0-9-]*[a-z0-9]$/); + assert.match(encryptedBucket, /^[a-z0-9][a-z0-9-]*[a-z0-9]$/); + assert.notEqual(unencryptedBucket, encryptedBucket, "Encryption scenarios should have distinct buckets"); +}); + +test("journal chunk recovery through MINIO", async (t) => { + await t.test("without encryption", async () => runScenario(false)); + await t.test("with encryption", async () => runScenario(true)); +}); diff --git a/src/apps/cli/testdeno/test-mirror.ts b/test/e2e-cli/test-mirror.ts similarity index 76% rename from src/apps/cli/testdeno/test-mirror.ts rename to test/e2e-cli/test-mirror.ts index 1b2bcc22..1dd19622 100644 --- a/src/apps/cli/testdeno/test-mirror.ts +++ b/test/e2e-cli/test-mirror.ts @@ -1,5 +1,5 @@ /** - * Deno port of test-mirror-linux.sh + * Node.js port of test-mirror-linux.sh * * Tests the `mirror` command — bidirectional synchronisation between a local * storage directory (vault) and an in-process database. @@ -16,15 +16,18 @@ * No external services are required. * * Run: - * deno test -A test-mirror.ts + * node --test test-mirror.ts */ -import { assert, assertEquals } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { copyFile, mkdir, readFile, stat, utimes, writeFile } from "node:fs/promises"; + import { TempDir } from "./helpers/temp.ts"; import { runCliOrFail, runCliWithInputOrFail } from "./helpers/cli.ts"; import { initSettingsFile, markSettingsConfigured } from "./helpers/settings.ts"; -Deno.test("mirror: storage <-> DB synchronisation", async (t) => { +test("mirror: storage <-> DB synchronisation", async (t) => { await using workDir = await TempDir.create("livesync-cli-mirror"); // ------------------------------------------------------------------- @@ -33,20 +36,20 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { const settingsFile = workDir.join("data.json"); const vaultDir = workDir.join("vault"); const dbDir = workDir.join("db"); - await Deno.mkdir(workDir.join("vault", "test"), { recursive: true }); - await Deno.mkdir(dbDir, { recursive: true }); + await mkdir(workDir.join("vault", "test"), { recursive: true }); + await mkdir(dbDir, { recursive: true }); await initSettingsFile(settingsFile); // isConfigured=true is required for canProceedScan in the mirror command. await markSettingsConfigured(settingsFile); - const data = JSON.parse(await Deno.readTextFile(settingsFile)); + const data = JSON.parse(await readFile(settingsFile, "utf8")); data.writeDocumentsIfConflicted = true; - await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2)); + await writeFile(settingsFile, JSON.stringify(data, null, 2)); // Copy settings to the DB directory (separated-path mode) const dbSettings = workDir.join("db", "settings.json"); - await Deno.copyFile(settingsFile, dbSettings); + await copyFile(settingsFile, dbSettings); /** Run mirror in separated-path mode: DB dir ≠ vault dir. */ const runMirror = () => runCliOrFail(dbDir, "--settings", dbSettings, "mirror", vaultDir); @@ -61,17 +64,17 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { // ------------------------------------------------------------------- // Case 1: storage-only -> DB (UPDATE DATABASE) // ------------------------------------------------------------------- - await t.step("case 1: storage-only file is synced into DB", async () => { + await t.test("case 1: storage-only file is synced into DB", async () => { const storageFile = workDir.join("vault", "test", "storage-only.md"); - await Deno.writeTextFile(storageFile, "storage-only content\n"); + await writeFile(storageFile, "storage-only content\n"); await runMirror(); const resultFile = workDir.join("case1-pull.txt"); await dbRun("pull", "test/storage-only.md", resultFile); - const storageContent = await Deno.readTextFile(storageFile); - const pulledContent = await Deno.readTextFile(resultFile); + const storageContent = await readFile(storageFile, "utf8"); + const pulledContent = await readFile(resultFile, "utf8"); assert( storageContent === pulledContent, `storage-only file NOT synced into DB\nexpected: ${storageContent}\ngot: ${pulledContent}` @@ -82,14 +85,14 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { // ------------------------------------------------------------------- // Case 2: DB-only -> storage (UPDATE STORAGE) // ------------------------------------------------------------------- - await t.step("case 2: DB-only file is restored to storage", async () => { + await t.test("case 2: DB-only file is restored to storage", async () => { await dbRun( "push", // write inline via push (pipe not needed — push takes a file path) // create a temp file with content and push it await (async () => { const tmp = workDir.join("db-only-src.txt"); - await Deno.writeTextFile(tmp, "db-only content\n"); + await writeFile(tmp, "db-only content\n"); return tmp; })(), "test/db-only.md" @@ -101,7 +104,7 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { await runMirror(); assert(await exists(storagePath), "DB-only file NOT restored to storage after mirror"); - const content = await Deno.readTextFile(storagePath); + const content = await readFile(storagePath, "utf8"); assert(content === "db-only content\n", `DB-only file restored but content mismatch: '${content}'`); console.log("[PASS] case 2: DB-only file was restored to storage"); }); @@ -109,9 +112,9 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { // ------------------------------------------------------------------- // Case 3: DB-deleted -> storage untouched // ------------------------------------------------------------------- - await t.step("case 3: DB-deleted entry is NOT restored to storage", async () => { + await t.test("case 3: DB-deleted entry is NOT restored to storage", async () => { const deletedSrc = workDir.join("deleted-src.txt"); - await Deno.writeTextFile(deletedSrc, "to-be-deleted\n"); + await writeFile(deletedSrc, "to-be-deleted\n"); await dbRun("push", deletedSrc, "test/deleted.md"); await dbRun("rm", "test/deleted.md"); @@ -125,29 +128,29 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { // ------------------------------------------------------------------- // Case 4: storage newer -> DB updated (SYNC: STORAGE -> DB) // ------------------------------------------------------------------- - await t.step("case 4: storage newer than DB -> DB is updated", async () => { + await t.test("case 4: storage newer than DB -> DB is updated", async () => { // Seed DB with old content (mtime ~ now) const seedFile = workDir.join("case4-seed.txt"); - await Deno.writeTextFile(seedFile, "old content\n"); + await writeFile(seedFile, "old content\n"); await dbRun("push", seedFile, "test/sync-storage-newer.md"); // Reflect the shared base into the actual Vault before editing it. await runMirror(); - assertEquals(await Deno.readTextFile(workDir.join("vault", "test", "sync-storage-newer.md")), "old content\n"); + assert.deepStrictEqual(await readFile(workDir.join("vault", "test", "sync-storage-newer.md"), "utf8"), "old content\n"); // Write new content to storage with a timestamp 1 hour in the future const storageFile = workDir.join("vault", "test", "sync-storage-newer.md"); - await Deno.writeTextFile(storageFile, "new content\n"); - await Deno.utime(storageFile, new Date(), new Date(Date.now() + 3600_000)); + await writeFile(storageFile, "new content\n"); + await utimes(storageFile, new Date(), new Date(Date.now() + 3600_000)); await runMirror(); const resultFile = workDir.join("case4-pull.txt"); const info = JSON.parse(await dbRun("info", "test/sync-storage-newer.md")); - assertEquals(info.conflicts, "N/A", "An ordinary local edit must not create a conflict"); + assert.deepStrictEqual(info.conflicts, "N/A", "An ordinary local edit must not create a conflict"); await dbRun("pull", "test/sync-storage-newer.md", resultFile); - const storageContent = await Deno.readTextFile(storageFile); - const pulledContent = await Deno.readTextFile(resultFile); + const storageContent = await readFile(storageFile, "utf8"); + const pulledContent = await readFile(resultFile, "utf8"); assert( storageContent === pulledContent, `DB NOT updated to match newer storage file\nexpected: ${storageContent}\ngot: ${pulledContent}` @@ -158,20 +161,20 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { // ------------------------------------------------------------------- // Case 5: DB newer -> storage updated (SYNC: DB -> STORAGE) // ------------------------------------------------------------------- - await t.step("case 5: DB newer than storage -> storage is updated", async () => { + await t.test("case 5: DB newer than storage -> storage is updated", async () => { // Write old content to storage with a timestamp 1 hour in the past const storageFile = workDir.join("vault", "test", "sync-db-newer.md"); - await Deno.writeTextFile(storageFile, "old storage content\n"); - await Deno.utime(storageFile, new Date(), new Date(Date.now() - 3600_000)); + await writeFile(storageFile, "old storage content\n"); + await utimes(storageFile, new Date(), new Date(Date.now() - 3600_000)); // Write new content to DB only (mtime ~ now, newer than the storage file) const dbNewFile = workDir.join("case5-db-new.txt"); - await Deno.writeTextFile(dbNewFile, "new db content\n"); + await writeFile(dbNewFile, "new db content\n"); await dbRun("push", dbNewFile, "test/sync-db-newer.md"); await runMirror(); - const content = await Deno.readTextFile(storageFile); + const content = await readFile(storageFile, "utf8"); assert(content === "new db content\n", `storage NOT updated to match newer DB entry (got: '${content}')`); console.log("[PASS] case 5: storage updated to match newer DB entry"); }); @@ -179,15 +182,15 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { // ------------------------------------------------------------------- // Case 6: compatibility mode (vault path = DB path) // ------------------------------------------------------------------- - await t.step("case 6: compatibility mode (omitted vault-path)", async () => { + await t.test("case 6: compatibility mode (omitted vault-path)", async () => { const compatFile = workDir.join("vault", "compat.md"); - await Deno.writeTextFile(compatFile, "compat-content\n"); + await writeFile(compatFile, "compat-content\n"); await runMirrorCompat(); const resultFile = workDir.join("case6-pull.txt"); await compatRun("pull", "compat.md", resultFile); - const pulled = await Deno.readTextFile(resultFile); + const pulled = await readFile(resultFile, "utf8"); assert(pulled === "compat-content\n", `Compatibility mode failed to sync file into DB (got: '${pulled}')`); console.log("[PASS] case 6: compatibility mode works"); }); @@ -196,13 +199,13 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { // Case 7: unknown local origin must preserve both contents regardless of mtime. // This deliberately uses put: push would record a file provenance entry. // ------------------------------------------------------------------- - await t.step("case 7: unknown local content is preserved, deduplicated, and resolved", async () => { + await t.test("case 7: unknown local content is preserved, deduplicated, and resolved", async () => { const path = "test/unknown-origin.md"; const storageFile = workDir.join("vault", "test", "unknown-origin.md"); await runCliWithInputOrFail("original DB content\n", dbDir, "--settings", dbSettings, "put", path); const writeUnknownFile = async () => { - await Deno.writeTextFile(storageFile, "unrelated local content\n"); - await Deno.utime(storageFile, new Date(), new Date(Date.now() + 3600_000)); + await writeFile(storageFile, "unrelated local content\n"); + await utimes(storageFile, new Date(), new Date(Date.now() + 3600_000)); }; await writeUnknownFile(); await runMirror(); @@ -215,22 +218,22 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { for (const revision of [info.revision, info.conflicts]) { contents.set(await dbRun("cat-rev", path, revision), revision); } - assertEquals([...contents.keys()].sort(), ["original DB content\n", "unrelated local content\n"]); + assert.deepStrictEqual([...contents.keys()].sort(), ["original DB content\n", "unrelated local content\n"]); // Re-submit identical local bytes even if incoming reflection replaced // the file under writeDocumentsIfConflicted; no third branch is needed. await writeUnknownFile(); await runMirror(); const repeated = JSON.parse(await dbRun("info", path)); - assertEquals(repeated.revision, info.revision); - assertEquals(repeated.conflicts, info.conflicts); + assert.deepStrictEqual(repeated.revision, info.revision); + assert.deepStrictEqual(repeated.conflicts, info.conflicts); const localRevision = contents.get("unrelated local content\n")!; await runCliOrFail(dbDir, "--vault", vaultDir, "--settings", dbSettings, "resolve", path, localRevision); const resolved = JSON.parse(await dbRun("info", path)); - assertEquals(resolved.conflicts, "N/A"); - assertEquals(resolved.revision, localRevision); - assertEquals(await Deno.readTextFile(storageFile), "unrelated local content\n"); + assert.deepStrictEqual(resolved.conflicts, "N/A"); + assert.deepStrictEqual(resolved.revision, localRevision); + assert.deepStrictEqual(await readFile(storageFile, "utf8"), "unrelated local content\n"); }); }); @@ -240,7 +243,7 @@ Deno.test("mirror: storage <-> DB synchronisation", async (t) => { async function exists(path: string): Promise { try { - await Deno.stat(path); + await stat(path); return true; } catch { return false; diff --git a/src/apps/cli/testdeno/test-p2p-automatic-host.ts b/test/e2e-cli/test-p2p-automatic-host.ts similarity index 90% rename from src/apps/cli/testdeno/test-p2p-automatic-host.ts rename to test/e2e-cli/test-p2p-automatic-host.ts index f9b9c2a0..54221796 100644 --- a/src/apps/cli/testdeno/test-p2p-automatic-host.ts +++ b/test/e2e-cli/test-p2p-automatic-host.ts @@ -1,5 +1,9 @@ -import { assert, assertEquals } from "@std/assert"; -import { join } from "@std/path"; +import { test } from "node:test"; +import process from "node:process"; +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { spawnProcess, type ProcessStatus, type RunningProcess } from "./helpers/process.ts"; +import { join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; import { CLI_DIR } from "./helpers/cli.ts"; @@ -86,10 +90,13 @@ class AutomaticP2PHostProcess { #stdinWriter: WritableStreamDefaultWriter; #stdoutTask: Promise; #stderrTask: Promise; - #status: Promise; + #status: Promise; #stopped = false; - private constructor(readonly child: Deno.ChildProcess) { + readonly child: RunningProcess; + + private constructor(child: RunningProcess) { + this.child = child; this.#stdinWriter = child.stdin.getWriter(); this.#stdoutTask = this.#pump(child.stdout, (text) => this.#handleStdout(text)); this.#stderrTask = this.#pump(child.stderr, (text) => (this.#stderr += text)); @@ -119,14 +126,14 @@ class AutomaticP2PHostProcess { env[SEED_PATH_ENV] = seed.path; env[SEED_CONTENT_ENV] = seed.content; } - const child = new Deno.Command("node", { + const child = spawnProcess(process.execPath, { args: [join(CLI_DIR, "dist", "p2p-lifecycle-test.cjs"), vaultPath, "--settings", settingsPath, "p2p-host"], cwd: CLI_DIR, env, stdin: "piped", stdout: "piped", stderr: "piped", - }).spawn(); + }); const fixture = new AutomaticP2PHostProcess(child); try { await withTimeout(fixture.#ready, readyTimeoutMs, "waiting for the P2P automatic-host fixture to connect"); @@ -282,7 +289,7 @@ async function configureAutomaticSettings( turnServers ); await applyP2pTestTweaks(settingsPath, peerName, passphrase); - const settings = JSON.parse(await Deno.readTextFile(settingsPath)); + const settings = JSON.parse(await readFile(settingsPath, "utf8")); settings.P2P_AutoAccepting = 0; settings.P2P_AutoStart = false; settings.P2P_AutoBroadcast = true; @@ -290,21 +297,21 @@ async function configureAutomaticSettings( settings.P2P_AutoWatchPeers = autoWatchPeers; settings.P2P_AutoAcceptingPeers = autoAcceptingPeers; settings.P2P_AutoDenyingPeers = ""; - await Deno.writeTextFile(settingsPath, JSON.stringify(settings, null, 2)); + await writeFile(settingsPath, JSON.stringify(settings, null, 2)); } -Deno.test("p2p-host: restored automatic policy syncs clients through a persistent middle peer", async () => { - const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/"; - const peersTimeoutMs = Number(Deno.env.get("PEERS_TIMEOUT") ?? "20") * 1000; - const syncTimeoutMs = Number(Deno.env.get("SYNC_TIMEOUT") ?? "60") * 1000; +test("p2p-host: restored automatic policy syncs clients through a persistent middle peer", async () => { + const relay = process.env["RELAY"] ?? "ws://localhost:4000/"; + const peersTimeoutMs = Number(process.env["PEERS_TIMEOUT"] ?? "20") * 1000; + const syncTimeoutMs = Number(process.env["SYNC_TIMEOUT"] ?? "60") * 1000; const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; const roomId = `automatic-host-room-${nonce}`; const passphrase = `automatic-host-pass-${nonce}`; const hostPeerName = `p2p-auto-host-${nonce}`; const clientAPeerName = `p2p-auto-client-a-${nonce}`; const clientBPeerName = `p2p-auto-client-b-${nonce}`; - const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0"; - const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? "turn:127.0.0.1:3478" : "none"); + const useCoturn = process.env["LIVESYNC_USE_COTURN"] !== "0"; + const turnServers = process.env["TURN_SERVERS"] ?? (useCoturn ? "turn:127.0.0.1:3478" : "none"); await using workDir = await TempDir.create("livesync-cli-p2p-automatic-host"); const hostVault = workDir.join("vault-host"); @@ -314,9 +321,9 @@ Deno.test("p2p-host: restored automatic policy syncs clients through a persisten const clientASettings = workDir.join("settings-client-a.json"); const clientBSettings = workDir.join("settings-client-b.json"); await Promise.all([ - Deno.mkdir(hostVault, { recursive: true }), - Deno.mkdir(clientAVault, { recursive: true }), - Deno.mkdir(clientBVault, { recursive: true }), + mkdir(hostVault, { recursive: true }), + mkdir(clientAVault, { recursive: true }), + mkdir(clientBVault, { recursive: true }), ]); const relayStarted = await maybeStartLocalRelay(relay); @@ -357,7 +364,7 @@ Deno.test("p2p-host: restored automatic policy syncs clients through a persisten host = await AutomaticP2PHostProcess.start(hostVault, hostSettings, HOST_BASELINE, 20000); const initialHostState = await host.state(); assert(initialHostState.connected, "p2p-host did not explicitly connect while P2P_AutoStart was false"); - assertEquals( + assert.deepStrictEqual( initialHostState.policy, { P2P_Enabled: true, @@ -382,7 +389,7 @@ Deno.test("p2p-host: restored automatic policy syncs clients through a persisten await host.write(HOST_UPDATE.path, HOST_UPDATE.content); await clientA.waitForContent(HOST_UPDATE, syncTimeoutMs); - assertEquals(await clientA.stop(), 0, "Client A did not stop cleanly before going offline"); + assert.deepStrictEqual(await clientA.stop(), 0, "Client A did not stop cleanly before going offline"); clientA = undefined; clientB = await AutomaticP2PHostProcess.start(clientBVault, clientBSettings, undefined, 20000); @@ -396,14 +403,14 @@ Deno.test("p2p-host: restored automatic policy syncs clients through a persisten await clientB.write(CLIENT_B_UPDATE.path, CLIENT_B_UPDATE.content); await host.waitForContent(CLIENT_B_UPDATE, syncTimeoutMs); - assertEquals(await host.stop(), 0, "The persistent host did not stop cleanly for its restart check"); + assert.deepStrictEqual(await host.stop(), 0, "The persistent host did not stop cleanly for its restart check"); host = undefined; await clientB.write(CLIENT_B_DURING_OUTAGE.path, CLIENT_B_DURING_OUTAGE.content); host = await AutomaticP2PHostProcess.start(hostVault, hostSettings, undefined, 20000); const restartedHostState = await host.state(); assert(restartedHostState.connected, "The restarted p2p-host did not connect"); - assertEquals( + assert.deepStrictEqual( restartedHostState.policy, initialHostState.policy, "The restarted host did not restore its original P2P policy" diff --git a/src/apps/cli/testdeno/test-p2p-host.ts b/test/e2e-cli/test-p2p-host.ts similarity index 67% rename from src/apps/cli/testdeno/test-p2p-host.ts rename to test/e2e-cli/test-p2p-host.ts index 7c67c5ff..abd494ab 100644 --- a/src/apps/cli/testdeno/test-p2p-host.ts +++ b/test/e2e-cli/test-p2p-host.ts @@ -1,20 +1,22 @@ -import { assert } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir } from "node:fs/promises"; import { TempDir } from "./helpers/temp.ts"; import { initSettingsFile, applyP2pSettings } from "./helpers/settings.ts"; import { startP2pRelay, stopP2pRelay, isLocalP2pRelay } from "./helpers/docker.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; -Deno.test("p2p-host: starts and becomes ready", async () => { - const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/"; - const roomId = Deno.env.get("ROOM_ID") ?? `room-${Date.now()}`; - const passphrase = Deno.env.get("PASSPHRASE") ?? "test"; - const appId = Deno.env.get("APP_ID") ?? "self-hosted-livesync-cli-tests"; - const useInternalRelay = Deno.env.get("USE_INTERNAL_RELAY") !== "0"; +test("p2p-host: starts and becomes ready", async () => { + const relay = process.env["RELAY"] ?? "ws://localhost:4000/"; + const roomId = process.env["ROOM_ID"] ?? `room-${Date.now()}`; + const passphrase = process.env["PASSPHRASE"] ?? "test"; + const appId = process.env["APP_ID"] ?? "self-hosted-livesync-cli-tests"; + const useInternalRelay = process.env["USE_INTERNAL_RELAY"] !== "0"; await using workDir = await TempDir.create("livesync-cli-p2p-host"); const vaultDir = workDir.join("vault-host"); const settingsFile = workDir.join("settings-host.json"); - await Deno.mkdir(vaultDir, { recursive: true }); + await mkdir(vaultDir, { recursive: true }); let relayStarted = false; if (useInternalRelay && isLocalP2pRelay(relay)) { diff --git a/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts b/test/e2e-cli/test-p2p-peers-local-relay.ts similarity index 72% rename from src/apps/cli/testdeno/test-p2p-peers-local-relay.ts rename to test/e2e-cli/test-p2p-peers-local-relay.ts index e8b162e0..5862c442 100644 --- a/src/apps/cli/testdeno/test-p2p-peers-local-relay.ts +++ b/test/e2e-cli/test-p2p-peers-local-relay.ts @@ -1,4 +1,6 @@ -import { assert } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir } from "node:fs/promises"; import { TempDir } from "./helpers/temp.ts"; import { initSettingsFile, applyP2pSettings, applyP2pTestTweaks } from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; @@ -11,27 +13,27 @@ import { } from "./helpers/p2p.ts"; import { getOptimalLoopbackIp } from "./helpers/net.ts"; -Deno.test("p2p-peers: discovers host through local relay", async () => { +test("p2p-peers: discovers host through local relay", async () => { const loopbackIp = await getOptimalLoopbackIp(); const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp; - const relay = Deno.env.get("RELAY") ?? `ws://${loopbackHost}:4000/`; - const roomId = Deno.env.get("ROOM_ID") ?? `room-${Date.now()}`; - const passphrase = Deno.env.get("PASSPHRASE") ?? "test"; - const timeoutSeconds = Number(Deno.env.get("TIMEOUT_SECONDS") ?? "8"); + const relay = process.env["RELAY"] ?? `ws://${loopbackHost}:4000/`; + const roomId = process.env["ROOM_ID"] ?? `room-${Date.now()}`; + const passphrase = process.env["PASSPHRASE"] ?? "test"; + const timeoutSeconds = Number(process.env["TIMEOUT_SECONDS"] ?? "8"); const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; - const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; - const clientPeerName = Deno.env.get("CLIENT_PEER_NAME") ?? `p2p-client-${nonce}`; - const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0"; - const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); + const hostPeerName = process.env["HOST_PEER_NAME"] ?? `p2p-host-${nonce}`; + const clientPeerName = process.env["CLIENT_PEER_NAME"] ?? `p2p-client-${nonce}`; + const useCoturn = process.env["LIVESYNC_USE_COTURN"] !== "0"; + const turnServers = process.env["TURN_SERVERS"] ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); await using workDir = await TempDir.create("livesync-cli-p2p-peers-local-relay"); const hostVault = workDir.join("vault-host"); const hostSettings = workDir.join("settings-host.json"); const clientVault = workDir.join("vault"); const clientSettings = workDir.join("settings.json"); - await Deno.mkdir(hostVault, { recursive: true }); - await Deno.mkdir(clientVault, { recursive: true }); + await mkdir(hostVault, { recursive: true }); + await mkdir(clientVault, { recursive: true }); const relayStarted = await maybeStartLocalRelay(relay); const coturnStarted = await maybeStartCoturn(turnServers); diff --git a/test/e2e-cli/test-p2p-relay-disconnect.ts b/test/e2e-cli/test-p2p-relay-disconnect.ts new file mode 100644 index 00000000..074aaec8 --- /dev/null +++ b/test/e2e-cli/test-p2p-relay-disconnect.ts @@ -0,0 +1,29 @@ +import { test } from "node:test"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { runProcess } from "./helpers/process.ts"; +import assert from "node:assert/strict"; + +test("p2p lifecycle: explicit disconnect closes and pauses relay WebSockets", async () => { + const result = await runProcess(process.execPath, { + args: [fileURLToPath(new URL("./relay-disconnect-probe.mjs", import.meta.url))], + env: { + RELAY: process.env["RELAY"] ?? "ws://nostr-relay:7777/", + RELAY_TIMEOUT_MS: process.env["LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS"] ?? "15000", + }, + stdout: "piped", + stderr: "piped", + }); + const stdout = new TextDecoder().decode(result.stdout).trim(); + const stderr = new TextDecoder().decode(result.stderr).trim(); + + assert(result.success, `Relay disconnect probe failed\nstdout: ${stdout}\nstderr: ${stderr}`); + const report = JSON.parse(stdout.split("\n").at(-1) ?? "{}") as { + socketsClosed?: number; + stayedDisconnectedWhilePaused?: boolean; + reconnectedAfterResume?: boolean; + }; + assert((report.socketsClosed ?? 0) > 0, "The probe did not observe an open relay WebSocket"); + assert.deepStrictEqual(report.stayedDisconnectedWhilePaused, true); + assert.deepStrictEqual(report.reconnectedAfterResume, true); +}); diff --git a/src/apps/cli/testdeno/test-p2p-replicator-replacement.ts b/test/e2e-cli/test-p2p-replicator-replacement.ts similarity index 74% rename from src/apps/cli/testdeno/test-p2p-replicator-replacement.ts rename to test/e2e-cli/test-p2p-replicator-replacement.ts index 231337e5..28cc483c 100644 --- a/src/apps/cli/testdeno/test-p2p-replicator-replacement.ts +++ b/test/e2e-cli/test-p2p-replicator-replacement.ts @@ -1,11 +1,19 @@ -import { assert, assertEquals, assertStringIncludes } from "@std/assert"; -import { join } from "@std/path"; +import { test } from "node:test"; +import process from "node:process"; +import assert from "node:assert/strict"; +import { mkdir } from "node:fs/promises"; +import { runProcess } from "./helpers/process.ts"; +import { join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; import { initSettingsFile, applyP2pSettings, applyP2pTestTweaks } from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; import { maybeStartLocalRelay, stopLocalRelayIfStarted, maybeStartCoturn, stopCoturnIfStarted } from "./helpers/p2p.ts"; import { CLI_DIR, runCli, sanitiseCatStdout } from "./helpers/cli.ts"; +function assertStringIncludes(actual: string, expected: string, message?: string): void { + assert.ok(actual.includes(expected), message ?? `Expected ${JSON.stringify(actual)} to include ${JSON.stringify(expected)}`); +} + const NOTE_PATH = "p2p-replicator-replacement.md"; const NOTE_CONTENT = "Replicated after replacing the active P2P replicator."; @@ -15,7 +23,7 @@ async function runReplacementProbe( targetPeer: string, timeoutMs: number ): Promise<{ code: number; stdout: string; stderr: string }> { - const command = new Deno.Command("node", { + const result = await runProcess(process.execPath, { args: [ join(CLI_DIR, "dist", "p2p-lifecycle-test.cjs"), vaultPath, @@ -31,7 +39,6 @@ async function runReplacementProbe( stdout: "piped", stderr: "piped", }); - const result = await command.output(); return { code: result.code, stdout: new TextDecoder().decode(result.stdout), @@ -39,20 +46,20 @@ async function runReplacementProbe( }; } -Deno.test("p2p lifecycle: active-adapter replacement keeps real CLI communication on the stable service", async () => { - const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/"; - const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "20"); - const syncTimeout = Number(Deno.env.get("SYNC_TIMEOUT") ?? "60"); +test("p2p lifecycle: active-adapter replacement keeps real CLI communication on the stable service", async () => { + const relay = process.env["RELAY"] ?? "ws://localhost:4000/"; + const peersTimeout = Number(process.env["PEERS_TIMEOUT"] ?? "20"); + const syncTimeout = Number(process.env["SYNC_TIMEOUT"] ?? "60"); const probeTimeoutMs = Math.max(peersTimeout, syncTimeout) * 1000; const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; - const roomId = Deno.env.get("ROOM_ID") ?? `replacement-room-${nonce}`; - const passphrase = Deno.env.get("PASSPHRASE") ?? `replacement-pass-${nonce}`; + const roomId = process.env["ROOM_ID"] ?? `replacement-room-${nonce}`; + const passphrase = process.env["PASSPHRASE"] ?? `replacement-pass-${nonce}`; const appId = "self-hosted-livesync-cli-replacement-test"; const hostPeerName = `p2p-replacement-host-${nonce}`; const probePeerName = `p2p-replacement-probe-${nonce}`; const verifierPeerName = `p2p-replacement-verifier-${nonce}`; - const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0"; - const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? "turn:127.0.0.1:3478" : "none"); + const useCoturn = process.env["LIVESYNC_USE_COTURN"] !== "0"; + const turnServers = process.env["TURN_SERVERS"] ?? (useCoturn ? "turn:127.0.0.1:3478" : "none"); await using workDir = await TempDir.create("livesync-cli-p2p-replacement"); const hostVault = workDir.join("vault-host"); @@ -62,9 +69,9 @@ Deno.test("p2p lifecycle: active-adapter replacement keeps real CLI communicatio const probeSettings = workDir.join("settings-probe.json"); const verifierSettings = workDir.join("settings-verifier.json"); await Promise.all([ - Deno.mkdir(hostVault, { recursive: true }), - Deno.mkdir(probeVault, { recursive: true }), - Deno.mkdir(verifierVault, { recursive: true }), + mkdir(hostVault, { recursive: true }), + mkdir(probeVault, { recursive: true }), + mkdir(verifierVault, { recursive: true }), ]); const relayStarted = await maybeStartLocalRelay(relay); @@ -103,7 +110,7 @@ Deno.test("p2p lifecycle: active-adapter replacement keeps real CLI communicatio catResult.code === 0, `Verifier could not read ${NOTE_PATH}\nstdout: ${catResult.stdout}\nstderr: ${catResult.stderr}` ); - assertEquals(sanitiseCatStdout(catResult.stdout).trim(), NOTE_CONTENT); + assert.deepStrictEqual(sanitiseCatStdout(catResult.stdout).trim(), NOTE_CONTENT); } finally { await host.stop(); } diff --git a/src/apps/cli/testdeno/test-p2p-sync.ts b/test/e2e-cli/test-p2p-sync.ts similarity index 82% rename from src/apps/cli/testdeno/test-p2p-sync.ts rename to test/e2e-cli/test-p2p-sync.ts index 32f3d1ad..0392ef6f 100644 --- a/src/apps/cli/testdeno/test-p2p-sync.ts +++ b/test/e2e-cli/test-p2p-sync.ts @@ -1,4 +1,6 @@ -import { assert } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { TempDir } from "./helpers/temp.ts"; import { initSettingsFile, @@ -17,20 +19,20 @@ import { import { runCli, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; import { getOptimalLoopbackIp } from "./helpers/net.ts"; -Deno.test("p2p-sync: transfers with the same ID key and rejects a different document ID key", async () => { +test("p2p-sync: transfers with the same ID key and rejects a different document ID key", async () => { const loopbackIp = await getOptimalLoopbackIp(); const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp; - const relay = Deno.env.get("RELAY") ?? `ws://${loopbackHost}:4000/`; - const roomId = Deno.env.get("ROOM_ID") ?? `room-${Date.now()}`; - const passphrase = Deno.env.get("PASSPHRASE") ?? "test"; - const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "12"); - const syncTimeout = Number(Deno.env.get("SYNC_TIMEOUT") ?? "15"); + const relay = process.env["RELAY"] ?? `ws://${loopbackHost}:4000/`; + const roomId = process.env["ROOM_ID"] ?? `room-${Date.now()}`; + const passphrase = process.env["PASSPHRASE"] ?? "test"; + const peersTimeout = Number(process.env["PEERS_TIMEOUT"] ?? "12"); + const syncTimeout = Number(process.env["SYNC_TIMEOUT"] ?? "15"); const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; - const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; - const clientPeerName = Deno.env.get("CLIENT_PEER_NAME") ?? `p2p-client-${nonce}`; - const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0"; - const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); + const hostPeerName = process.env["HOST_PEER_NAME"] ?? `p2p-host-${nonce}`; + const clientPeerName = process.env["CLIENT_PEER_NAME"] ?? `p2p-client-${nonce}`; + const useCoturn = process.env["LIVESYNC_USE_COTURN"] !== "0"; + const turnServers = process.env["TURN_SERVERS"] ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); await using workDir = await TempDir.create("livesync-cli-p2p-sync"); const hostVault = workDir.join("vault-host"); @@ -39,9 +41,9 @@ Deno.test("p2p-sync: transfers with the same ID key and rejects a different docu const clientSettings = workDir.join("settings-sync.json"); const rejectedVault = workDir.join("vault-rejected"); const rejectedSettings = workDir.join("settings-rejected.json"); - await Deno.mkdir(hostVault, { recursive: true }); - await Deno.mkdir(clientVault, { recursive: true }); - await Deno.mkdir(rejectedVault, { recursive: true }); + await mkdir(hostVault, { recursive: true }); + await mkdir(clientVault, { recursive: true }); + await mkdir(rejectedVault, { recursive: true }); const relayStarted = await maybeStartLocalRelay(relay); const coturnStarted = await maybeStartCoturn(turnServers); @@ -84,15 +86,15 @@ Deno.test("p2p-sync: transfers with the same ID key and rejects a different docu [clientVault, clientSettings, "ab".repeat(32), "client"], [rejectedVault, rejectedSettings, "cd".repeat(32), "rejected"], ]) { - const settings = JSON.parse(await Deno.readTextFile(path)); + const settings = JSON.parse(await readFile(path, "utf8")); settings.idDerivationVersion = 1; settings.idDerivationKey = key; const sourcePath = workDir.join("setup-source-" + label + ".json"); - await Deno.writeTextFile(sourcePath, JSON.stringify(settings)); + await writeFile(sourcePath, JSON.stringify(settings)); const setupPassphrase = "independent-id-setup-passphrase"; const setupUri = await generateSetupUriFromSettings(sourcePath, setupPassphrase, true); await runCliWithInputOrFail(setupPassphrase + "\n", vault, "--settings", path, "setup", setupUri); - const persisted = JSON.parse(await Deno.readTextFile(path)); + const persisted = JSON.parse(await readFile(path, "utf8")); assert(persisted.idDerivationVersion === 1, "The Setup URI lost the ID derivation version."); assert(persisted.idDerivationKey === "", "The CLI stored the ID key in plain text."); assert( @@ -121,7 +123,7 @@ Deno.test("p2p-sync: transfers with the same ID key and rejects a different docu clientVault, clientSettings, peersTimeout, - Deno.env.get("TARGET_PEER") ?? hostPeerName + process.env["TARGET_PEER"] ?? hostPeerName ); const syncResult = await runCli( clientVault, diff --git a/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts b/test/e2e-cli/test-p2p-three-nodes-conflict.ts similarity index 86% rename from src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts rename to test/e2e-cli/test-p2p-three-nodes-conflict.ts index 45eab2c6..21ce9097 100644 --- a/src/apps/cli/testdeno/test-p2p-three-nodes-conflict.ts +++ b/test/e2e-cli/test-p2p-three-nodes-conflict.ts @@ -1,4 +1,6 @@ -import { assert } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir } from "node:fs/promises"; import { TempDir } from "./helpers/temp.ts"; import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; @@ -12,22 +14,22 @@ import { import { jsonStringField, runCliOrFail, runCliWithInputOrFail, sanitiseCatStdout } from "./helpers/cli.ts"; import { getOptimalLoopbackIp } from "./helpers/net.ts"; -Deno.test("p2p: three nodes detect and resolve conflicts", async () => { +test("p2p: three nodes detect and resolve conflicts", async () => { const loopbackIp = await getOptimalLoopbackIp(); const loopbackHost = loopbackIp === "::1" ? "[::1]" : loopbackIp; - const relay = Deno.env.get("RELAY") ?? `ws://${loopbackHost}:4000/`; - const roomId = Deno.env.get("ROOM_ID") ?? `room-${Date.now()}`; - const passphrase = Deno.env.get("PASSPHRASE") ?? "test"; + const relay = process.env["RELAY"] ?? `ws://${loopbackHost}:4000/`; + const roomId = process.env["ROOM_ID"] ?? `room-${Date.now()}`; + const passphrase = process.env["PASSPHRASE"] ?? "test"; const appId = "self-hosted-livesync-cli-tests"; - const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "10"); - const syncTimeout = Number(Deno.env.get("SYNC_TIMEOUT") ?? "15"); + const peersTimeout = Number(process.env["PEERS_TIMEOUT"] ?? "10"); + const syncTimeout = Number(process.env["SYNC_TIMEOUT"] ?? "15"); const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`; - const hostPeerName = Deno.env.get("HOST_PEER_NAME") ?? `p2p-host-${nonce}`; - const peerNameB = Deno.env.get("PEER_NAME_B") ?? `p2p-client-b-${nonce}`; - const peerNameC = Deno.env.get("PEER_NAME_C") ?? `p2p-client-c-${nonce}`; - const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0"; - const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); + const hostPeerName = process.env["HOST_PEER_NAME"] ?? `p2p-host-${nonce}`; + const peerNameB = process.env["PEER_NAME_B"] ?? `p2p-client-b-${nonce}`; + const peerNameC = process.env["PEER_NAME_C"] ?? `p2p-client-c-${nonce}`; + const useCoturn = process.env["LIVESYNC_USE_COTURN"] !== "0"; + const turnServers = process.env["TURN_SERVERS"] ?? (useCoturn ? `turn:${loopbackHost}:3478` : "none"); await using workDir = await TempDir.create("livesync-cli-p2p-3nodes"); const vaultA = workDir.join("vault-a"); @@ -36,9 +38,9 @@ Deno.test("p2p: three nodes detect and resolve conflicts", async () => { const settingsA = workDir.join("settings-a.json"); const settingsB = workDir.join("settings-b.json"); const settingsC = workDir.join("settings-c.json"); - await Deno.mkdir(vaultA, { recursive: true }); - await Deno.mkdir(vaultB, { recursive: true }); - await Deno.mkdir(vaultC, { recursive: true }); + await mkdir(vaultA, { recursive: true }); + await mkdir(vaultB, { recursive: true }); + await mkdir(vaultC, { recursive: true }); const relayStarted = await maybeStartLocalRelay(relay); const coturnStarted = await maybeStartCoturn(turnServers); diff --git a/src/apps/cli/testdeno/test-p2p-upload-download-repro.ts b/test/e2e-cli/test-p2p-upload-download-repro.ts similarity index 88% rename from src/apps/cli/testdeno/test-p2p-upload-download-repro.ts rename to test/e2e-cli/test-p2p-upload-download-repro.ts index ba07fd5c..f9b754e4 100644 --- a/src/apps/cli/testdeno/test-p2p-upload-download-repro.ts +++ b/test/e2e-cli/test-p2p-upload-download-repro.ts @@ -1,3 +1,5 @@ +import { test } from "node:test"; +import { mkdir, writeFile } from "node:fs/promises"; import { TempDir } from "./helpers/temp.ts"; import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; import { startCliInBackground } from "./helpers/backgroundCli.ts"; @@ -7,14 +9,14 @@ import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts"; async function writeFilledFile(path: string, size: number, byte: number): Promise { const data = new Uint8Array(size); data.fill(byte); - await Deno.writeFile(path, data); + await writeFile(path, data); } -Deno.test("p2p: upload/download reproduction scenario", async () => { - const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/"; - const appId = Deno.env.get("APP_ID") ?? "self-hosted-livesync-cli-tests"; - const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "20"); - const syncTimeout = Number(Deno.env.get("SYNC_TIMEOUT") ?? "240"); +test("p2p: upload/download reproduction scenario", async () => { + const relay = process.env["RELAY"] ?? "ws://localhost:4000/"; + const appId = process.env["APP_ID"] ?? "self-hosted-livesync-cli-tests"; + const peersTimeout = Number(process.env["PEERS_TIMEOUT"] ?? "20"); + const syncTimeout = Number(process.env["SYNC_TIMEOUT"] ?? "240"); const roomId = `p2p-room-${Date.now()}`; const passphrase = `p2p-pass-${Date.now()}`; @@ -26,7 +28,7 @@ Deno.test("p2p: upload/download reproduction scenario", async () => { const settingsUp = workDir.join("settings-up.json"); const settingsDown = workDir.join("settings-down.json"); for (const dir of [vaultHost, vaultUp, vaultDown]) { - await Deno.mkdir(dir, { recursive: true }); + await mkdir(dir, { recursive: true }); } const relayStarted = await maybeStartLocalRelay(relay); @@ -48,10 +50,10 @@ Deno.test("p2p: upload/download reproduction scenario", async () => { const diffA = workDir.join("test-diff-1.md"); const diffB = workDir.join("test-diff-2.md"); const diffC = workDir.join("test-diff-3.md"); - await Deno.writeTextFile(storeText, "Hello, World!\n"); - await Deno.writeTextFile(diffA, "Content A\n"); - await Deno.writeTextFile(diffB, "Content B\n"); - await Deno.writeTextFile(diffC, "Content C\n"); + await writeFile(storeText, "Hello, World!\n"); + await writeFile(diffA, "Content A\n"); + await writeFile(diffB, "Content B\n"); + await writeFile(diffC, "Content C\n"); await runCliOrFail(vaultUp, "--settings", settingsUp, "push", storeText, "p2p/store-file.md"); await runCliOrFail(vaultUp, "--settings", settingsUp, "push", diffA, "p2p/test-diff-1.md"); await runCliOrFail(vaultUp, "--settings", settingsUp, "push", diffB, "p2p/test-diff-2.md"); @@ -61,8 +63,8 @@ Deno.test("p2p: upload/download reproduction scenario", async () => { const large1m = workDir.join("large-1m.txt"); const binary100k = workDir.join("binary-100k.bin"); const binary5m = workDir.join("binary-5m.bin"); - await Deno.writeTextFile(large100k, "a".repeat(100000)); - await Deno.writeTextFile(large1m, "b".repeat(1000000)); + await writeFile(large100k, "a".repeat(100000)); + await writeFile(large1m, "b".repeat(1000000)); await writeFilledFile(binary100k, 100000, 0x5a); await writeFilledFile(binary5m, 5000000, 0x7c); await runCliOrFail(vaultUp, "--settings", settingsUp, "push", large100k, "p2p/large-100000.md"); diff --git a/src/apps/cli/testdeno/test-push-pull.ts b/test/e2e-cli/test-push-pull.ts similarity index 65% rename from src/apps/cli/testdeno/test-push-pull.ts rename to test/e2e-cli/test-push-pull.ts index b5dd61be..3cabe004 100644 --- a/src/apps/cli/testdeno/test-push-pull.ts +++ b/test/e2e-cli/test-push-pull.ts @@ -1,44 +1,47 @@ /** - * Deno port of test-push-pull-linux.sh + * Node.js port of test-push-pull-linux.sh * * Requires CouchDB connection details either via environment variables or a * .test.env file. If neither is present the test logs a warning and the * CLI will likely fail at the push step. * * Run: - * deno test -A test-push-pull.ts + * node --test test-push-pull.ts * * With explicit CouchDB: * COUCHDB_URI=http://127.0.0.1:5984 \ * COUCHDB_USER=admin \ * COUCHDB_PASSWORD=password \ * COUCHDB_DBNAME=livesync-test \ - * deno test -A test-push-pull.ts + * node --test test-push-pull.ts */ -import { join } from "@std/path"; -import { assertEquals } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; + +import { join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; import { runCliOrFail } from "./helpers/cli.ts"; import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts"; import { startCouchdb, stopCouchdb } from "./helpers/docker.ts"; -const REMOTE_PATH = Deno.env.get("REMOTE_PATH") ?? "test/push-pull.txt"; +const REMOTE_PATH = process.env["REMOTE_PATH"] ?? "test/push-pull.txt"; -Deno.test("push/pull roundtrip", async () => { +test("push/pull roundtrip", async () => { await using workDir = await TempDir.create("livesync-cli-push-pull"); const settingsFile = workDir.join("data.json"); const vaultDir = workDir.join("vault"); - await Deno.mkdir(join(vaultDir, "test"), { recursive: true }); + await mkdir(join(vaultDir, "test"), { recursive: true }); - const uri = Deno.env.get("COUCHDB_URI") ?? "http://127.0.0.1:5989/"; - const user = Deno.env.get("COUCHDB_USER") ?? "admin"; - const password = Deno.env.get("COUCHDB_PASSWORD") ?? "testpassword"; - const dbname = Deno.env.get("COUCHDB_DBNAME") ?? `push-pull-${Date.now()}`; + const uri = process.env["COUCHDB_URI"] ?? "http://127.0.0.1:5989/"; + const user = process.env["COUCHDB_USER"] ?? "admin"; + const password = process.env["COUCHDB_PASSWORD"] ?? "testpassword"; + const dbname = process.env["COUCHDB_DBNAME"] ?? `push-pull-${Date.now()}`; - const shouldStartDocker = Deno.env.get("LIVESYNC_START_DOCKER") !== "0"; - const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1"; + const shouldStartDocker = process.env["LIVESYNC_START_DOCKER"] !== "0"; + const keepDocker = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1"; if (shouldStartDocker) { await startCouchdb(uri, user, password, dbname); @@ -59,7 +62,7 @@ Deno.test("push/pull roundtrip", async () => { const srcFile = workDir.join("push-source.txt"); const pulledFile = workDir.join("pull-result.txt"); const content = `push-pull-test ${new Date().toISOString()}\n`; - await Deno.writeTextFile(srcFile, content); + await writeFile(srcFile, content); console.log(`[INFO] push -> ${REMOTE_PATH}`); await runCliOrFail(vaultDir, "--settings", settingsFile, "push", srcFile, REMOTE_PATH); @@ -67,8 +70,8 @@ Deno.test("push/pull roundtrip", async () => { console.log(`[INFO] pull <- ${REMOTE_PATH}`); await runCliOrFail(vaultDir, "--settings", settingsFile, "pull", REMOTE_PATH, pulledFile); - const pulled = await Deno.readTextFile(pulledFile); - assertEquals(content, pulled, "push/pull roundtrip content mismatch"); + const pulled = await readFile(pulledFile, "utf8"); + assert.deepStrictEqual(content, pulled, "push/pull roundtrip content mismatch"); console.log("[PASS] push/pull roundtrip matched"); } finally { if (shouldStartDocker && !keepDocker) { diff --git a/src/apps/cli/testdeno/test-remote-administration-exit-codes.ts b/test/e2e-cli/test-remote-administration-exit-codes.ts similarity index 74% rename from src/apps/cli/testdeno/test-remote-administration-exit-codes.ts rename to test/e2e-cli/test-remote-administration-exit-codes.ts index ccb20e6b..75f332f7 100644 --- a/src/apps/cli/testdeno/test-remote-administration-exit-codes.ts +++ b/test/e2e-cli/test-remote-administration-exit-codes.ts @@ -1,18 +1,24 @@ -import { assertEquals, assertStringIncludes } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir } from "node:fs/promises"; import { TempDir } from "./helpers/temp.ts"; import { runCli } from "./helpers/cli.ts"; import { applyCouchdbSettings, applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts"; +function assertStringIncludes(actual: string, expected: string, message?: string): void { + assert.ok(actual.includes(expected), message ?? `Expected ${JSON.stringify(actual)} to include ${JSON.stringify(expected)}`); +} + async function prepareFixture(prefix: string) { const workDir = await TempDir.create(prefix); const settingsFile = workDir.join("settings.json"); const databaseDir = workDir.join("database"); - await Deno.mkdir(databaseDir, { recursive: true }); + await mkdir(databaseDir, { recursive: true }); await initSettingsFile(settingsFile); return { workDir, settingsFile, databaseDir }; } -Deno.test("remote administration process exit policy distinguishes returned verification failure", async () => { +test("remote administration process exit policy distinguishes returned verification failure", async () => { const fixture = await prepareFixture("livesync-cli-remote-admin-exit"); await using workDir = fixture.workDir; const { settingsFile, databaseDir } = fixture; @@ -29,7 +35,7 @@ Deno.test("remote administration process exit policy distinguishes returned veri await applyP2pTestTweaks(settingsFile, "remote-admin-exit-device", "remote-admin-exit-passphrase"); const defaultFailure = await runCli(databaseDir, "--settings", settingsFile, "mark-resolved"); - assertEquals(defaultFailure.code, 1, defaultFailure.combined); + assert.deepStrictEqual(defaultFailure.code, 1, defaultFailure.combined); assertStringIncludes( defaultFailure.combined, "[Verification] Remote administration is unavailable for this provider." @@ -43,7 +49,7 @@ Deno.test("remote administration process exit policy distinguishes returned veri "--compat-remote-admin-exit-zero", "mark-resolved" ); - assertEquals(compatibilitySuccess.code, 0, compatibilitySuccess.combined); + assert.deepStrictEqual(compatibilitySuccess.code, 0, compatibilitySuccess.combined); assertStringIncludes( compatibilitySuccess.combined, "[Verification] Remote administration is unavailable for this provider." @@ -51,7 +57,7 @@ Deno.test("remote administration process exit policy distinguishes returned veri assertStringIncludes(compatibilitySuccess.combined, "[Done] Command 'mark-resolved' completed"); }); -Deno.test("remote administration compatibility does not hide a thrown mutation failure", async () => { +test("remote administration compatibility does not hide a thrown mutation failure", async () => { const fixture = await prepareFixture("livesync-cli-remote-admin-mutation"); await using workDir = fixture.workDir; const { settingsFile, databaseDir } = fixture; @@ -71,7 +77,7 @@ Deno.test("remote administration compatibility does not hide a thrown mutation f "--compat-remote-admin-exit-zero", "mark-resolved" ); - assertEquals(mutationFailure.code, 1, mutationFailure.combined); + assert.deepStrictEqual(mutationFailure.code, 1, mutationFailure.combined); assertStringIncludes(mutationFailure.combined, "[Command] mark-resolved"); assertStringIncludes(mutationFailure.combined, "[Error] Failed to start:"); }); diff --git a/src/apps/cli/testdeno/test-remote-commands.ts b/test/e2e-cli/test-remote-commands.ts similarity index 87% rename from src/apps/cli/testdeno/test-remote-commands.ts rename to test/e2e-cli/test-remote-commands.ts index 6040ab11..64678470 100644 --- a/src/apps/cli/testdeno/test-remote-commands.ts +++ b/test/e2e-cli/test-remote-commands.ts @@ -1,5 +1,5 @@ /** - * Deno port of test-remote-commands-linux.sh + * Node.js port of test-remote-commands-linux.sh * * Tests remote management commands: remote-status, lock-remote, unlock-remote, * and mark-resolved. @@ -14,10 +14,13 @@ * current device is accepted. * * Run: - * deno test -A test-remote-commands.ts + * node --test test-remote-commands.ts */ -import { join } from "@std/path"; +import { test } from "node:test"; +import { mkdir } from "node:fs/promises"; + +import { join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; import { runCli, assertContains } from "./helpers/cli.ts"; import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts"; @@ -31,21 +34,21 @@ async function runCliCombinedOrFail(...args: string[]): Promise { return res.combined; } -Deno.test("remote management commands", async () => { +test("remote management commands", async () => { await using workDir = await TempDir.create("livesync-cli-remote-cmds"); const settingsFile = workDir.join("settings.json"); const vaultDir = workDir.join("vault"); - await Deno.mkdir(vaultDir, { recursive: true }); + await mkdir(vaultDir, { recursive: true }); - const uri = Deno.env.get("COUCHDB_URI") ?? "http://127.0.0.1:5989/"; - const user = Deno.env.get("COUCHDB_USER") ?? "admin"; - const password = Deno.env.get("COUCHDB_PASSWORD") ?? "testpassword"; + const uri = process.env["COUCHDB_URI"] ?? "http://127.0.0.1:5989/"; + const user = process.env["COUCHDB_USER"] ?? "admin"; + const password = process.env["COUCHDB_PASSWORD"] ?? "testpassword"; const dbSuffix = `${Date.now()}-${Math.floor(Math.random() * 10000)}`; - const dbname = Deno.env.get("COUCHDB_DBNAME") ?? `remotes-${dbSuffix}`; + const dbname = process.env["COUCHDB_DBNAME"] ?? `remotes-${dbSuffix}`; - const shouldStartDocker = Deno.env.get("LIVESYNC_START_DOCKER") !== "0"; - const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1"; + const shouldStartDocker = process.env["LIVESYNC_START_DOCKER"] !== "0"; + const keepDocker = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1"; if (shouldStartDocker) { await startCouchdb(uri, user, password, dbname); diff --git a/src/apps/cli/testdeno/test-settings-writeback.ts b/test/e2e-cli/test-settings-writeback.ts similarity index 54% rename from src/apps/cli/testdeno/test-settings-writeback.ts rename to test/e2e-cli/test-settings-writeback.ts index 4e3c4adf..ff37eca9 100644 --- a/src/apps/cli/testdeno/test-settings-writeback.ts +++ b/test/e2e-cli/test-settings-writeback.ts @@ -1,4 +1,6 @@ -import { assert, assertEquals } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { TempDir } from "./helpers/temp.ts"; import { runCli } from "./helpers/cli.ts"; import { initSettingsFile } from "./helpers/settings.ts"; @@ -7,24 +9,24 @@ async function prepareSettingsFixture(prefix: string) { const workDir = await TempDir.create(prefix); const settingsFile = workDir.join("settings.json"); const databaseDir = workDir.join("database"); - await Deno.mkdir(databaseDir, { recursive: true }); + await mkdir(databaseDir, { recursive: true }); await initSettingsFile(settingsFile); return { workDir, settingsFile, databaseDir }; } -Deno.test("settings-changing commands persist durable settings without CLI runtime suspension", async () => { +test("settings-changing commands persist durable settings without CLI runtime suspension", async () => { const fixture = await prepareSettingsFixture("livesync-cli-settings-command"); await using workDir = fixture.workDir; const { settingsFile, databaseDir } = fixture; - const settings = JSON.parse(await Deno.readTextFile(settingsFile)); + const settings = JSON.parse(await readFile(settingsFile, "utf8")); settings.liveSync = true; settings.syncOnStart = true; settings.periodicReplication = true; settings.P2P_Enabled = true; settings.P2P_AutoStart = true; settings.P2P_AutoBroadcast = true; - await Deno.writeTextFile(settingsFile, JSON.stringify(settings, null, 2)); + await writeFile(settingsFile, JSON.stringify(settings, null, 2)); const result = await runCli( databaseDir, @@ -34,22 +36,22 @@ Deno.test("settings-changing commands persist durable settings without CLI runti "test-remote", "sls+https://user:pass@example.com/database" ); - assertEquals(result.code, 0, result.combined); + assert.deepStrictEqual(result.code, 0, result.combined); const firstRemoteId = result.stdout.trim().split("\t")[0]; assert(firstRemoteId, `remote-add did not return an ID: ${result.combined}`); - let persisted = JSON.parse(await Deno.readTextFile(settingsFile)); - let remotes = Object.values(persisted.remoteConfigurations ?? {}) as Array<{ name?: string }>; + let persisted = JSON.parse(await readFile(settingsFile, "utf8")); + const remotesByName = Object.values(persisted.remoteConfigurations ?? {}) as Array<{ name?: string }>; assert( - remotes.some((remote) => remote.name === "test-remote"), + remotesByName.some((remote) => remote.name === "test-remote"), "remote-add did not persist the new profile" ); - assertEquals(persisted.liveSync, true); - assertEquals(persisted.syncOnStart, true); - assertEquals(persisted.periodicReplication, true); - assertEquals(persisted.P2P_Enabled, true); - assertEquals(persisted.P2P_AutoStart, true); - assertEquals(persisted.P2P_AutoBroadcast, true); + assert.deepStrictEqual(persisted.liveSync, true); + assert.deepStrictEqual(persisted.syncOnStart, true); + assert.deepStrictEqual(persisted.periodicReplication, true); + assert.deepStrictEqual(persisted.P2P_Enabled, true); + assert.deepStrictEqual(persisted.P2P_AutoStart, true); + assert.deepStrictEqual(persisted.P2P_AutoBroadcast, true); const secondAdd = await runCli( databaseDir, @@ -59,14 +61,14 @@ Deno.test("settings-changing commands persist durable settings without CLI runti "second-remote", "sls+https://other:secret@example.net/second" ); - assertEquals(secondAdd.code, 0, secondAdd.combined); + assert.deepStrictEqual(secondAdd.code, 0, secondAdd.combined); const secondRemoteId = secondAdd.stdout.trim().split("\t")[0]; assert(secondRemoteId, `second remote-add did not return an ID: ${secondAdd.combined}`); const activate = await runCli(databaseDir, "--settings", settingsFile, "remote-activate", secondRemoteId); - assertEquals(activate.code, 0, activate.combined); - persisted = JSON.parse(await Deno.readTextFile(settingsFile)); - assertEquals(persisted.activeConfigurationId, secondRemoteId); + assert.deepStrictEqual(activate.code, 0, activate.combined); + persisted = JSON.parse(await readFile(settingsFile, "utf8")); + assert.deepStrictEqual(persisted.activeConfigurationId, secondRemoteId); const set = await runCli( databaseDir, @@ -76,59 +78,59 @@ Deno.test("settings-changing commands persist durable settings without CLI runti secondRemoteId, "sls+https://replacement:secret@example.org/replaced" ); - assertEquals(set.code, 0, set.combined); + assert.deepStrictEqual(set.code, 0, set.combined); const exported = await runCli(databaseDir, "--settings", settingsFile, "remote-export", secondRemoteId); - assertEquals(exported.code, 0, exported.combined); + assert.deepStrictEqual(exported.code, 0, exported.combined); assert(exported.stdout.includes("replacement"), "remote-set did not persist the replacement URI"); const remove = await runCli(databaseDir, "--settings", settingsFile, "remote-rm", secondRemoteId); - assertEquals(remove.code, 0, remove.combined); - persisted = JSON.parse(await Deno.readTextFile(settingsFile)); - remotes = Object.values(persisted.remoteConfigurations ?? {}) as Array<{ id?: string }>; - assert(!remotes.some((remote) => remote.id === secondRemoteId), "remote-rm did not persist the removal"); - assertEquals(persisted.activeConfigurationId, firstRemoteId); + assert.deepStrictEqual(remove.code, 0, remove.combined); + persisted = JSON.parse(await readFile(settingsFile, "utf8")); + const remotesById = Object.values(persisted.remoteConfigurations ?? {}) as Array<{ id?: string }>; + assert(!remotesById.some((remote) => remote.id === secondRemoteId), "remote-rm did not persist the removal"); + assert.deepStrictEqual(persisted.activeConfigurationId, firstRemoteId); }); -Deno.test("ordinary commands keep the settings file unchanged by default", async () => { +test("ordinary commands keep the settings file unchanged by default", async () => { const fixture = await prepareSettingsFixture("livesync-cli-settings-readonly"); await using workDir = fixture.workDir; const { settingsFile, databaseDir } = fixture; - const settings = JSON.parse(await Deno.readTextFile(settingsFile)); + const settings = JSON.parse(await readFile(settingsFile, "utf8")); settings.settingVersion = 9; const original = JSON.stringify(settings, null, 2); - await Deno.writeTextFile(settingsFile, original); + await writeFile(settingsFile, original); const result = await runCli(databaseDir, "--settings", settingsFile, "ls"); - assertEquals(result.code, 0, result.combined); - assertEquals(await Deno.readTextFile(settingsFile), original); + assert.deepStrictEqual(result.code, 0, result.combined); + assert.deepStrictEqual(await readFile(settingsFile, "utf8"), original); }); -Deno.test("--write-settings persists durable start-up setting changes", async () => { +test("--write-settings persists durable start-up setting changes", async () => { const fixture = await prepareSettingsFixture("livesync-cli-settings-explicit"); await using workDir = fixture.workDir; const { settingsFile, databaseDir } = fixture; - const settings = JSON.parse(await Deno.readTextFile(settingsFile)); + const settings = JSON.parse(await readFile(settingsFile, "utf8")); settings.settingVersion = 9; delete settings.useIndexedDBAdapter; - await Deno.writeTextFile(settingsFile, JSON.stringify(settings, null, 2)); + await writeFile(settingsFile, JSON.stringify(settings, null, 2)); const result = await runCli(databaseDir, "--settings", settingsFile, "--write-settings", "ls"); - assertEquals(result.code, 0, result.combined); + assert.deepStrictEqual(result.code, 0, result.combined); - const persisted = JSON.parse(await Deno.readTextFile(settingsFile)); - assertEquals(persisted.settingVersion, 10); + const persisted = JSON.parse(await readFile(settingsFile, "utf8")); + assert.deepStrictEqual(persisted.settingVersion, 10); assert(!("useIndexedDBAdapter" in persisted), "the CLI-only adapter override was written to the settings file"); }); -Deno.test("failed settings-changing commands leave the settings file unchanged", async () => { +test("failed settings-changing commands leave the settings file unchanged", async () => { const fixture = await prepareSettingsFixture("livesync-cli-settings-failure"); await using workDir = fixture.workDir; const { settingsFile, databaseDir } = fixture; - const original = await Deno.readTextFile(settingsFile); + const original = await readFile(settingsFile, "utf8"); const result = await runCli(databaseDir, "--settings", settingsFile, "remote-rm", "missing-remote"); assert(result.code !== 0, "remote-rm unexpectedly succeeded"); - assertEquals(await Deno.readTextFile(settingsFile), original); + assert.deepStrictEqual(await readFile(settingsFile, "utf8"), original); }); diff --git a/src/apps/cli/testdeno/test-setup-put-cat.ts b/test/e2e-cli/test-setup-put-cat.ts similarity index 76% rename from src/apps/cli/testdeno/test-setup-put-cat.ts rename to test/e2e-cli/test-setup-put-cat.ts index 1df861e4..c1d8d142 100644 --- a/src/apps/cli/testdeno/test-setup-put-cat.ts +++ b/test/e2e-cli/test-setup-put-cat.ts @@ -1,29 +1,32 @@ /** - * Deno port of test-setup-put-cat-linux.sh + * Node.js port of test-setup-put-cat-linux.sh * * Tests all local-DB file operations that require no external remote: * setup / * push / cat / ls / info / rm / resolve / cat-rev / pull-rev * * Run (no external services needed): - * deno test -A test-setup-put-cat.ts + * node --test test-setup-put-cat.ts */ -import { join } from "@std/path"; -import { assertEquals, assert } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; + +import { join } from "node:path"; import { TempDir } from "./helpers/temp.ts"; import { runCli, runCliOrFail, runCliWithInput, sanitiseCatStdout } from "./helpers/cli.ts"; import { generateSetupUriFromSettings, initSettingsFile } from "./helpers/settings.ts"; -const REMOTE_PATH = Deno.env.get("REMOTE_PATH") ?? "test/setup-put-cat.txt"; -const SETUP_PASSPHRASE = Deno.env.get("SETUP_PASSPHRASE") ?? "setup-passphrase"; +const REMOTE_PATH = process.env["REMOTE_PATH"] ?? "test/setup-put-cat.txt"; +const SETUP_PASSPHRASE = process.env["SETUP_PASSPHRASE"] ?? "setup-passphrase"; -Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev / pull-rev", async (t) => { +test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev / pull-rev", async (t) => { await using workDir = await TempDir.create("livesync-cli-setup-put-cat"); const settingsFile = workDir.join("data.json"); const vaultDir = workDir.join("vault"); - await Deno.mkdir(join(vaultDir, "test"), { recursive: true }); + await mkdir(join(vaultDir, "test"), { recursive: true }); await initSettingsFile(settingsFile); @@ -41,8 +44,8 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev setupResult.combined.includes("[Command] setup ->"), `setup command did not execute expected code path\n${setupResult.combined}` ); - const persistedSetup = JSON.parse(await Deno.readTextFile(settingsFile)); - assertEquals(persistedSetup.isConfigured, true, "setup did not persist the configured state"); + const persistedSetup = JSON.parse(await readFile(settingsFile, "utf8")); + assert.deepStrictEqual(persistedSetup.isConfigured, true, "setup did not persist the configured state"); assert( typeof persistedSetup.encryptedCouchDBConnection === "string" && persistedSetup.encryptedCouchDBConnection.length > 0, @@ -54,10 +57,10 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev // ------------------------------------------------------------------ // push / cat roundtrip // ------------------------------------------------------------------ - await t.step("push/cat roundtrip", async () => { + await t.test("push/cat roundtrip", async () => { const srcFile = workDir.join("put-source.txt"); const content = `setup-put-cat-test ${new Date().toISOString()}\nline-2\n`; - await Deno.writeTextFile(srcFile, content); + await writeFile(srcFile, content); console.log(`[INFO] push -> ${REMOTE_PATH}`); await runCliWithInput(content, vaultDir, "--settings", settingsFile, "put", REMOTE_PATH); @@ -66,14 +69,29 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev const rawOutput = await run("cat", REMOTE_PATH); const catOutput = sanitiseCatStdout(rawOutput); - assertEquals(content, catOutput, "push/cat roundtrip content mismatch"); + assert.deepStrictEqual(content, catOutput, "push/cat roundtrip content mismatch"); console.log("[PASS] push/cat roundtrip matched"); }); + await t.test("push/pull binary roundtrip", async () => { + const source = workDir.join("binary-source.bin"); + const restored = workDir.join("binary-restored.bin"); + const remotePath = "test/blob-roundtrip.bin"; + const content = Uint8Array.from({ length: 1025 }, (_, index) => index % 256); + await writeFile(source, content); + + await run("push", source, remotePath); + await run("pull", remotePath, restored); + + assert.deepStrictEqual(new Uint8Array(await readFile(restored)), content, "Binary push/pull must preserve every byte"); + const info = JSON.parse(await run("info", remotePath)) as Record; + assert.deepStrictEqual(info.size, content.byteLength, "Binary Metadata must retain the original byte length"); + }); + // ------------------------------------------------------------------ // ls: single file // ------------------------------------------------------------------ - await t.step("ls output format (single file)", async () => { + await t.test("ls output format (single file)", async () => { const lsOutput = await run("ls", REMOTE_PATH); const line = lsOutput .trim() @@ -82,7 +100,7 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev assert(line, `ls output did not include ${REMOTE_PATH}`); const [lsPath, lsSize, lsMtime, lsRev] = line.split("\t"); - assertEquals(lsPath, REMOTE_PATH, "ls path column mismatch"); + assert.deepStrictEqual(lsPath, REMOTE_PATH, "ls path column mismatch"); assert(/^\d+$/.test(lsSize), `ls size not numeric: ${lsSize}`); assert(/^\d+$/.test(lsMtime), `ls mtime not numeric: ${lsMtime}`); assert(lsRev?.length > 0, "ls revision column is empty"); @@ -92,7 +110,7 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev // ------------------------------------------------------------------ // ls: prefix filter and sort order // ------------------------------------------------------------------ - await t.step("ls prefix filter and sort order", async () => { + await t.test("ls prefix filter and sort order", async () => { await runCliWithInput("file-a\n", vaultDir, "--settings", settingsFile, "put", "test/a-first.txt"); await runCliWithInput("file-z\n", vaultDir, "--settings", settingsFile, "put", "test/z-last.txt"); @@ -119,16 +137,16 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev // ------------------------------------------------------------------ // ls: no-match prefix returns empty output // ------------------------------------------------------------------ - await t.step("ls no-match prefix returns empty", async () => { + await t.test("ls no-match prefix returns empty", async () => { const lsOut = await run("ls", "no-such-prefix/"); - assertEquals(lsOut.trim(), "", "ls no-match prefix should produce empty output"); + assert.deepStrictEqual(lsOut.trim(), "", "ls no-match prefix should produce empty output"); console.log("[PASS] ls no-match prefix matched"); }); // ------------------------------------------------------------------ // info: JSON output format // ------------------------------------------------------------------ - await t.step("info output JSON format", async () => { + await t.test("info output JSON format", async () => { const infoOut = await run("info", REMOTE_PATH); let data: Record; try { @@ -136,18 +154,18 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev } catch { throw new Error(`info output is not valid JSON:\n${infoOut}`); } - assertEquals(data.path, REMOTE_PATH, "info .path mismatch"); - assertEquals(data.filename, REMOTE_PATH.split("/").at(-1), "info .filename mismatch"); + assert.deepStrictEqual(data.path, REMOTE_PATH, "info .path mismatch"); + assert.deepStrictEqual(data.filename, REMOTE_PATH.split("/").at(-1), "info .filename mismatch"); assert(typeof data.size === "number" && data.size >= 0, `info .size invalid: ${data.size}`); assert(typeof data.chunks === "number" && data.chunks >= 1, `info .chunks invalid: ${data.chunks}`); - assertEquals(data.conflicts, "N/A", "info .conflicts should be N/A"); + assert.deepStrictEqual(data.conflicts, "N/A", "info .conflicts should be N/A"); console.log("[PASS] info output format matched"); }); // ------------------------------------------------------------------ // info: non-existent path exits non-zero // ------------------------------------------------------------------ - await t.step("info non-existent path returns non-zero", async () => { + await t.test("info non-existent path returns non-zero", async () => { const r = await runCli(vaultDir, "--settings", settingsFile, "info", "no-such-file.md"); assert(r.code !== 0, "info on non-existent file should exit non-zero"); console.log("[PASS] info non-existent path returns non-zero"); @@ -156,7 +174,7 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev // ------------------------------------------------------------------ // rm: removes file from ls and makes cat fail // ------------------------------------------------------------------ - await t.step("rm removes target from ls and cat", async () => { + await t.test("rm removes target from ls and cat", async () => { await run("rm", "test/z-last.txt"); const catResult = await runCli(vaultDir, "--settings", settingsFile, "cat", "test/z-last.txt"); @@ -170,7 +188,7 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev // ------------------------------------------------------------------ // resolve: accepts current revision, rejects invalid revision // ------------------------------------------------------------------ - await t.step("resolve: valid and invalid revisions", async () => { + await t.test("resolve: valid and invalid revisions", async () => { const lsLine = (await run("ls", "test/a-first.txt")).trim().split("\n")[0]; assert(lsLine, "could not fetch revision for resolve test"); const rev = lsLine.split("\t")[3]; @@ -187,7 +205,7 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev // ------------------------------------------------------------------ // cat-rev / pull-rev: retrieve a past revision // ------------------------------------------------------------------ - await t.step("cat-rev / pull-rev: retrieve past revision", async () => { + await t.test("cat-rev / pull-rev: retrieve past revision", async () => { const revPath = "test/revision-history.txt"; await runCliWithInput("revision-v1\n", vaultDir, "--settings", settingsFile, "put", revPath); await runCliWithInput("revision-v2\n", vaultDir, "--settings", settingsFile, "put", revPath); @@ -211,7 +229,7 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev const pullRevFile = workDir.join("rev-pull-output.txt"); await run("pull-rev", revPath, pullRevFile, pastRev); - const pullRevContent = await Deno.readTextFile(pullRevFile); + const pullRevContent = await readFile(pullRevFile, "utf8"); assert( pullRevContent === "revision-v1\n" || pullRevContent === "revision-v2\n", `pull-rev output did not match expected past revision:\n${pullRevContent}` diff --git a/src/apps/cli/testdeno/test-sync-locked-remote.ts b/test/e2e-cli/test-sync-locked-remote.ts similarity index 84% rename from src/apps/cli/testdeno/test-sync-locked-remote.ts rename to test/e2e-cli/test-sync-locked-remote.ts index d8b2e3d8..099b85e8 100644 --- a/src/apps/cli/testdeno/test-sync-locked-remote.ts +++ b/test/e2e-cli/test-sync-locked-remote.ts @@ -1,27 +1,34 @@ /** - * Deno port of test-sync-locked-remote-linux.sh + * Node.js port of test-sync-locked-remote-linux.sh * * Verifies CLI sync behaviour when the remote milestone document is unlocked * versus locked. */ -import { assert, assertStringIncludes } from "@std/assert"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir } from "node:fs/promises"; + import { TempDir } from "./helpers/temp.ts"; import { runCli } from "./helpers/cli.ts"; import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts"; import { createCouchdbDatabase, startCouchdb, stopCouchdb, updateCouchdbDoc } from "./helpers/docker.ts"; +function assertStringIncludes(actual: string, expected: string, message?: string): void { + assert.ok(actual.includes(expected), message ?? `Expected ${JSON.stringify(actual)} to include ${JSON.stringify(expected)}`); +} + const MILESTONE_DOC = "_local/obsydian_livesync_milestone"; function requireEnv(...keys: string[]): string { for (const key of keys) { - const value = Deno.env.get(key)?.trim(); + const value = process.env[key]?.trim(); if (value) return value; } throw new Error(`Required env var is missing: ${keys.join(" or ")}`); } -Deno.test("sync: actionable error against locked remote DB", async () => { +test("sync: actionable error against locked remote DB", async () => { const couchdbUri = requireEnv("COUCHDB_URI", "hostname").replace(/\/$/, ""); const couchdbUser = requireEnv("COUCHDB_USER", "username"); const couchdbPassword = requireEnv("COUCHDB_PASSWORD", "password"); @@ -31,10 +38,10 @@ Deno.test("sync: actionable error against locked remote DB", async () => { await using workDir = await TempDir.create("livesync-cli-locked-test"); const vaultDir = workDir.join("vault"); const settingsFile = workDir.join("settings.json"); - await Deno.mkdir(vaultDir, { recursive: true }); + await mkdir(vaultDir, { recursive: true }); - const shouldStartDocker = Deno.env.get("LIVESYNC_START_DOCKER") !== "0"; - const keepDocker = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1"; + const shouldStartDocker = process.env["LIVESYNC_START_DOCKER"] !== "0"; + const keepDocker = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1"; if (shouldStartDocker) { console.log(`[INFO] starting CouchDB and creating test database: ${dbname}`); diff --git a/src/apps/cli/testdeno/test-sync-two-local-databases.ts b/test/e2e-cli/test-sync-two-local-databases.ts similarity index 81% rename from src/apps/cli/testdeno/test-sync-two-local-databases.ts rename to test/e2e-cli/test-sync-two-local-databases.ts index 5717d405..7bd90dbd 100644 --- a/src/apps/cli/testdeno/test-sync-two-local-databases.ts +++ b/test/e2e-cli/test-sync-two-local-databases.ts @@ -1,5 +1,5 @@ /** - * Deno port of test-sync-two-local-databases-linux.sh + * Node.js port of test-sync-two-local-databases-linux.sh * * Tests two-vault synchronisation via CouchDB including conflict detection * and resolution. @@ -12,7 +12,7 @@ * CouchDB instance instead. * * Run: - * deno test -A test-sync-two-local-databases.ts + * node --test test-sync-two-local-databases.ts * * With an existing CouchDB: * COUCHDB_URI=http://127.0.0.1:5984 \ @@ -20,10 +20,13 @@ * COUCHDB_PASSWORD=password \ * COUCHDB_DBNAME=livesync-test \ * LIVESYNC_START_DOCKER=0 \ - * deno test -A test-sync-two-local-databases.ts + * node --test test-sync-two-local-databases.ts */ -import { assertEquals, assert } from "@std/assert"; +import { test, type TestContext } from "node:test"; +import assert from "node:assert/strict"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; + import { TempDir } from "./helpers/temp.ts"; import { runCliOrFail, jsonFieldIsNa } from "./helpers/cli.ts"; import { applyCouchdbSettings, initSettingsFile } from "./helpers/settings.ts"; @@ -39,7 +42,7 @@ async function resolveConfig(): Promise<{ password: string; baseDbname: string; } | null> { - const env = Deno.env.toObject(); + const env = { ...process.env }; const uri = (env["COUCHDB_URI"] ?? env["hostname"] ?? "").replace(/\/$/, ""); const user = env["COUCHDB_USER"] ?? env["username"] ?? ""; @@ -51,25 +54,23 @@ async function resolveConfig(): Promise<{ } const config = await resolveConfig(); -const START_DOCKER = Deno.env.get("LIVESYNC_START_DOCKER") !== "0"; -const KEEP_DOCKER = Deno.env.get("LIVESYNC_DEBUG_KEEP_DOCKER") === "1"; -const SYNC_RETRY = Number(Deno.env.get("LIVESYNC_SYNC_RETRY") ?? "8"); +const START_DOCKER = process.env["LIVESYNC_START_DOCKER"] !== "0"; +const KEEP_DOCKER = process.env["LIVESYNC_DEBUG_KEEP_DOCKER"] === "1"; +const SYNC_RETRY = Number(process.env["LIVESYNC_SYNC_RETRY"] ?? "8"); // Provide a sane default for flaky remote connectivity in Docker-on-WSL // environments. Users can override explicitly if needed. -if (!Deno.env.has("LIVESYNC_CLI_RETRY")) { - Deno.env.set("LIVESYNC_CLI_RETRY", "2"); +if (!Object.hasOwn(process.env, "LIVESYNC_CLI_RETRY")) { + (process.env["LIVESYNC_CLI_RETRY"] = "2"); } // --------------------------------------------------------------------------- // Test suite // --------------------------------------------------------------------------- -Deno.test( - { - name: "sync two local databases: sync + conflict detection + resolution", - ignore: config === null, - }, +test( + "sync two local databases: sync + conflict detection + resolution", + { skip: config === null }, async (t) => { if (!config) return; // narrowing for TypeScript @@ -104,7 +105,7 @@ Deno.test( // --------------------------------------------------------------------------- async function runSuite( - t: Deno.TestContext, + t: TestContext, workDir: TempDir, config: { uri: string; user: string; password: string }, dbname: string @@ -130,8 +131,8 @@ async function runSuite( const vaultB = workDir.join("vault-b"); const settingsA = workDir.join("a-settings.json"); const settingsB = workDir.join("b-settings.json"); - await Deno.mkdir(vaultA, { recursive: true }); - await Deno.mkdir(vaultB, { recursive: true }); + await mkdir(vaultA, { recursive: true }); + await mkdir(vaultB, { recursive: true }); await initSettingsFile(settingsA); await initSettingsFile(settingsB); @@ -152,45 +153,45 @@ async function runSuite( // ------------------------------------------------------------------ // Case 1: A creates file, B reads after sync // ------------------------------------------------------------------ - await t.step("case 1: A creates file -> B can read after sync", async () => { + await t.test("case 1: A creates file -> B can read after sync", async () => { const srcA = workDir.join("from-a-src.txt"); - await Deno.writeTextFile(srcA, "from-a\n"); + await writeFile(srcA, "from-a\n"); await runA("push", srcA, "shared/from-a.txt"); await syncA(); await syncB(); const value = (await catB("shared/from-a.txt")).replace(/\r\n/g, "\n").trimEnd(); - assertEquals(value, "from-a", "B could not read file created on A"); + assert.deepStrictEqual(value, "from-a", "B could not read file created on A"); console.log("[PASS] case 1 passed"); }); // ------------------------------------------------------------------ // Case 2: B creates file, A reads after sync // ------------------------------------------------------------------ - await t.step("case 2: B creates file -> A can read after sync", async () => { + await t.test("case 2: B creates file -> A can read after sync", async () => { const srcB = workDir.join("from-b-src.txt"); - await Deno.writeTextFile(srcB, "from-b\n"); + await writeFile(srcB, "from-b\n"); await runB("push", srcB, "shared/from-b.txt"); await syncB(); await syncA(); const value = (await catA("shared/from-b.txt")).replace(/\r\n/g, "\n").trimEnd(); - assertEquals(value, "from-b", "A could not read file created on B"); + assert.deepStrictEqual(value, "from-b", "A could not read file created on B"); console.log("[PASS] case 2 passed"); }); // ------------------------------------------------------------------ // Case 3: concurrent edits create a conflict // ------------------------------------------------------------------ - await t.step("case 3: concurrent edits create conflict", async () => { + await t.test("case 3: concurrent edits create conflict", async () => { const baseSrc = workDir.join("base-src.txt"); - await Deno.writeTextFile(baseSrc, "base\n"); + await writeFile(baseSrc, "base\n"); await runA("push", baseSrc, "shared/conflicted.txt"); await syncA(); await syncB(); const aEdit = workDir.join("edit-a.txt"); const bEdit = workDir.join("edit-b.txt"); - await Deno.writeTextFile(aEdit, "edit-from-a\n"); - await Deno.writeTextFile(bEdit, "edit-from-b\n"); + await writeFile(aEdit, "edit-from-a\n"); + await writeFile(bEdit, "edit-from-b\n"); await runA("push", aEdit, "shared/conflicted.txt"); await runB("push", bEdit, "shared/conflicted.txt"); @@ -201,10 +202,10 @@ async function runSuite( for (const side of ["a", "b"] as const) { if (side === "a") await syncA(); else await syncB(); - await Deno.writeTextFile(infoFileA, await runA("info", "shared/conflicted.txt")); - await Deno.writeTextFile(infoFileB, await runB("info", "shared/conflicted.txt")); - const da = JSON.parse(await Deno.readTextFile(infoFileA)) as Record; - const db = JSON.parse(await Deno.readTextFile(infoFileB)) as Record; + await writeFile(infoFileA, await runA("info", "shared/conflicted.txt")); + await writeFile(infoFileB, await runB("info", "shared/conflicted.txt")); + const da = JSON.parse(await readFile(infoFileA, "utf8")) as Record; + const db = JSON.parse(await readFile(infoFileB, "utf8")) as Record; if (!jsonFieldIsNa(da, "conflicts") || !jsonFieldIsNa(db, "conflicts")) { conflictDetected = true; break; @@ -217,14 +218,14 @@ async function runSuite( // ------------------------------------------------------------------ // Case 4: resolve on A, verify B has no conflict after sync // ------------------------------------------------------------------ - await t.step("case 4: resolve on A propagates to B", async () => { + await t.test("case 4: resolve on A propagates to B", async () => { const infoFileA = workDir.join("info-a-resolve.json"); const infoFileB = workDir.join("info-b-resolve.json"); // Ensure A sees the conflict for (let i = 0; i < 5; i++) { const raw = await runA("info", "shared/conflicted.txt"); - await Deno.writeTextFile(infoFileA, raw); + await writeFile(infoFileA, raw); const da = JSON.parse(raw) as Record; if (!jsonFieldIsNa(da, "conflicts")) break; await syncB(); @@ -232,7 +233,7 @@ async function runSuite( } const rawA = await runA("info", "shared/conflicted.txt"); - await Deno.writeTextFile(infoFileA, rawA); + await writeFile(infoFileA, rawA); const dataA = JSON.parse(rawA) as Record; assert(!jsonFieldIsNa(dataA, "conflicts"), "A does not see conflict, cannot resolve from A only"); @@ -247,8 +248,8 @@ async function runSuite( await syncB(); const rawA2 = await runA("info", "shared/conflicted.txt"); const rawB2 = await runB("info", "shared/conflicted.txt"); - await Deno.writeTextFile(infoFileA, rawA2); - await Deno.writeTextFile(infoFileB, rawB2); + await writeFile(infoFileA, rawA2); + await writeFile(infoFileB, rawB2); const da2 = JSON.parse(rawA2) as Record; const db2 = JSON.parse(rawB2) as Record; if (jsonFieldIsNa(da2, "conflicts") && jsonFieldIsNa(db2, "conflicts")) { @@ -265,7 +266,7 @@ async function runSuite( const contentA = (await catA("shared/conflicted.txt")).replace(/\r\n/g, "\n"); const contentB = (await catB("shared/conflicted.txt")).replace(/\r\n/g, "\n"); - assertEquals(contentA, contentB, "resolved content mismatch between A and B"); + assert.deepStrictEqual(contentA, contentB, "resolved content mismatch between A and B"); console.log("[PASS] case 4 passed"); console.log("[PASS] all sync/resolve scenarios passed"); }); diff --git a/test/e2e-cli/test_dev_node.md b/test/e2e-cli/test_dev_node.md new file mode 100644 index 00000000..2f006878 --- /dev/null +++ b/test/e2e-cli/test_dev_node.md @@ -0,0 +1,121 @@ +# CLI Node.js Test Development Notes + +The CLI test suite under `test/e2e-cli/` uses Node.js 26 native TypeScript execution and the built-in `node:test` runner. The standard service-backed suite is selected by `ci-tasks.json`. P2P scenarios use the repository Compose entry point so that networking, signalling, and the runner environment are reproducible. Existing Bash tests under `src/apps/cli/test/` remain available as compatibility references. + +## Goals and structure + +- Run CLI subprocess and filesystem scenarios on Windows, macOS, and Linux. +- Use Node.js standard modules and shared helpers in place of a separate test runtime. +- Keep the default service-backed suite and canonical Compose P2P suite distinct. + +```text +test/e2e-cli/ + tasks.json + ci-tasks.json + run-task.ts + run-ci-suite.ts + run-compose-p2p.ts + run-coverage.ts + helpers/ + test-*.ts +``` + +Node executes erasable TypeScript syntax directly. The scoped `test/e2e-cli/tsconfig.json` checks that test files stay within that runtime boundary. The task runner uses one test worker at a time and loads the CLI package `.test.env` file when present. + +## Task manifest + +`tasks.json` preserves named test and benchmark entry points as structured records. A record can name test files and an optional test-name filter, a TypeScript script, an ordered list of other tasks, or a command with arguments for the retained benchmark shell scripts. Shell commands are restricted to those legacy benchmark tasks. + +`ci-tasks.json` is the source of truth for the default service-backed CI task list. Keep it aligned with the `test:ci` task runner. `test:local` groups setup, mirror, and daemon smoke tasks which do not require Docker. + +## Test coverage + +- `test-setup-put-cat.ts` covers setup URI generation and core CLI commands without a remote service. +- `test-mirror.ts` covers seven mirror scenarios, including unknown revision provenance, conflict preservation, deduplication, and resolution. +- `test-daemon.ts` and `test-daemon-startup.ts` cover daemon ignore rules and startup behaviour. +- `test-push-pull.ts`, `test-sync-two-local-databases.ts`, and `test-sync-locked-remote.ts` cover remote synchronisation and locking. +- `test-decoupled-vault.ts` covers a vault directory which is separate from the database directory. +- `test-remote-commands.ts`, `test-remote-administration-exit-codes.ts`, and `test-settings-writeback.ts` cover remote management and settings behaviour. +- P2P tests cover host startup, peer discovery, synchronisation, automatic host policy, replacement, relay disconnection, multi-node conflicts, and upload/download. The Compose suite runs the canonical synchronisation, replacement, and relay-disconnection scenarios. +- `test-e2e-two-vaults-couchdb.ts` and `test-e2e-two-vaults-matrix.ts` cover two-vault CouchDB and CouchDB/Object Storage cases. + +## Running tests + +Run the canonical workflows from the repository root. These npm scripts build the CLI before the suite where needed: + +```bash +npm run test:e2e:cli +npm run test:e2e:cli:p2p +npm run test:e2e:cli:all +``` + +Run an individual named task after building the CLI: + +```bash +npm run test:e2e:task --workspace self-hosted-livesync-cli -- test:mirror +``` + +The local smoke set is also available as `test:local`. It runs setup, mirror, and daemon tests without starting Docker services. To run the mirror file directly from its directory: + +```bash +cd test/e2e-cli +node --env-file-if-exists=../../src/apps/cli/.test.env --test --test-concurrency=1 test-mirror.ts +``` + +Collect CLI body coverage with the local smoke suite, or pass `test:ci` to cover the standard service-backed suite: + +```bash +npm run test:e2e:coverage --workspace self-hosted-livesync-cli +npm run test:e2e:coverage --workspace self-hosted-livesync-cli -- test:ci +``` + +The runner defaults to `test:local`, which does not require services. It writes HTML, JSON, and text reports under `coverage/cli-e2e/`. The runner enables CLI bundle source maps only for the coverage build with `LIVESYNC_CLI_COVERAGE=1`; ordinary builds do not enable them. + +The CLI Node.js workflow runs this local smoke coverage on Linux and publishes the HTML and JSON reports as the `cli-e2e-local-coverage` artifact, retained for seven days. This artifact covers `test:local` only and is separate from unit-test and service-backed/P2P coverage. + +## Environment variables + +### CouchDB + +- `COUCHDB_URI` +- `COUCHDB_USER` +- `COUCHDB_PASSWORD` +- `COUCHDB_DBNAME` + +The equivalent keys in `.test.env` are `hostname`, `username`, `password`, and `dbname`. + +### Behaviour switches + +- `LIVESYNC_START_DOCKER=0`: use an existing CouchDB service. +- `REMOTE_PATH`: override the target path for selected tests. +- `LIVESYNC_TEST_TEE=1`: stream CLI stdout and stderr during execution. +- `LIVESYNC_DOCKER_TEE=1`: stream Docker stdout and stderr. +- `LIVESYNC_CLI_RETRY=`: retry transient network failures. +- `LIVESYNC_DEBUG_KEEP_DOCKER=1`: keep `couchdb-test` after a test completes. + +### Docker command selection + +`helpers/docker.ts` supports `LIVESYNC_DOCKER_MODE=auto` by default. On Windows it tries `wsl docker` first, then `docker`; on other platforms it tries `docker` first, then `wsl docker`. Set `LIVESYNC_DOCKER_MODE=native` or `LIVESYNC_DOCKER_MODE=wsl` to select one explicitly. `LIVESYNC_DOCKER_COMMAND` takes priority and can specify a custom command, such as `wsl docker`. + +### P2P + +- `RELAY` +- `ROOM_ID` +- `PASSPHRASE` +- `APP_ID` +- `PEERS_TIMEOUT` +- `SYNC_TIMEOUT` +- `USE_INTERNAL_RELAY=0|1` +- `TIMEOUT_SECONDS` + +## Continuous integration + +The `.github/workflows/cli-node-tests.yml` workflow runs for changes to the CLI or its shared runtime inputs. Its Linux `prepare` job builds the CLI once and distributes that bundle to the Linux service-backed test jobs. The `local-smoke` job runs automatically on pushes and non-draft pull requests, and also when `test:local` is selected manually; it builds the CLI once on each of Ubuntu, macOS, and Windows and starts no Docker services. + +The separate `.github/workflows/cli-p2p-tests.yml` workflow runs the canonical Compose P2P suite for CLI and `test/bench-network/` changes. A change confined to Compose fixtures does not start the non-P2P matrix. + +The `.github/workflows/cli-docker-validation.yml` workflow still builds the production CLI bundle with Node.js 22, then validates Docker images on native amd64 and arm64 runners. That release-image runtime remains independent of the Node.js version used by CLI E2E tests. + +## Maintenance policy + +Keep scenarios in Node test files and reuse helpers from `helpers/`. Preserve the authoritative CI task list in `ci-tasks.json`. Retain the Bash suites and benchmark wrappers until their corresponding migration has been separately agreed and completed. diff --git a/test/e2e-cli/tsconfig.json b/test/e2e-cli/tsconfig.json new file mode 100644 index 00000000..971b0dcd --- /dev/null +++ b/test/e2e-cli/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ESNext", "DOM", "DOM.Iterable"], + "types": ["node"], + "strict": true, + "skipLibCheck": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "erasableSyntaxOnly": true, + "resolveJsonModule": true + }, + "include": ["**/*.ts"] +} diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md index f47ad6e6..1e4fc3a4 100644 --- a/test/e2e-obsidian/README.md +++ b/test/e2e-obsidian/README.md @@ -50,6 +50,8 @@ Set `showVerboseLog: true` only in isolated plug-in data when a focused investig Collect evidence before cleanup, and keep process, Vault, profile, and remote-fixture cleanup in `finally`. After `app.emulateMobile(true)`, use the active CDP renderer for fixture operations because Obsidian may remove desktop-only CLI commands. Visually inspect screenshots before copying selected images into user documentation; a passing locator assertion does not establish that a dialogue is readable or unobstructed. +The dialogue scenario simulates the host keyboard-height variable while the mobile CouchDB setup dialogue is open. It verifies that the space after the final action does not grow with the keyboard height and restores the original variable and scroll position afterwards. It also checks that the warning about saving without a connection test scrolls with the form and leaves room for at least two input rows above the fixed actions. This layout fixture does not open a physical software keyboard. + ## Local Setup Set `OBSIDIAN_BINARY` when Obsidian is not installed in a standard location. Set `OBSIDIAN_CLI` as well when its companion executable is outside the built-in discovery paths. @@ -84,6 +86,7 @@ npm run test:e2e:obsidian:focused -- folder-batch npm run test:e2e:obsidian:focused -- security-seed-reconnect npm run test:e2e:obsidian:focused -- couchdb-http-cache npm run test:e2e:obsidian:focused -- replication-status +npm run test:e2e:obsidian:focused -- sync-guard ``` The wrapper accepts only maintained real-Obsidian scenario names; run it with `--help` for the current list. It deliberately does not manage CouchDB, Object Storage, or the P2P signalling relay. Start the required fixture first, or use the complete service-managed suite. @@ -163,7 +166,7 @@ The Harness also measures ID generation with fixed in-memory data on desktop and `test:e2e:obsidian:configuration-encryption-sync` requires only the CouchDB fixture configured through `.test.env` or the process environment. It changes the configuration protection key through the real settings interface, restarts the source Vault, and sends a note with E2EE enabled. A fresh Vault receives the encrypted settings without a saved device key, cancels and retries recovery, and restarts with the validated key. Both Vaults then exchange exact note bytes through CouchDB. The scenario checks the restored connection credentials, E2EE passphrase, ID key, remote Metadata, and Chunks, and removes its temporary Vaults, profiles, and unique remote database. -`test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, the onboarding invitation, Svelte dialogue mounting, revision repair, settings UI, configuration encryption and its CouchDB synchronisation workflow, the Review Harness, the P2P status pane, Vault reflection, CouchDB upload and manual setup, CLI-to-Obsidian synchronisation, Object Storage upload and Setup URI and QR round trips, P2P Setup URI round-trip, startup scan, provisioned CouchDB Setup URI, two-vault synchronisation, Hidden File Sync, Customisation Sync, internal Metadata Doctor, and setting Markdown export in sequence. Start the local CouchDB, RustFS, and P2P relay fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run. +`test:e2e:obsidian:local-suite` builds the plug-in and, unless `LIVESYNC_CLI_COMMAND` selects an external CLI, the local LiveSync CLI. It then runs discovery, smoke, the onboarding invitation, Svelte dialogue mounting, revision repair, settings UI, configuration encryption and its CouchDB synchronisation workflow, the Review Harness, the P2P status pane, Vault reflection, CouchDB upload and manual setup, CLI-to-Obsidian synchronisation, Object Storage upload and Setup URI and QR round trips, P2P Setup URI round-trip, startup scan, provisioned CouchDB Setup URI, two-vault synchronisation, SyncGuard catch-up protection, Hidden File Sync, Customisation Sync, internal Metadata Doctor, and setting Markdown export in sequence. Start the local CouchDB, RustFS, and P2P relay fixtures before running it, or use `test:e2e:obsidian:local-suite:services` to let the wrapper stop leftover fixtures, start fresh fixtures, and stop them again after the run. `test:e2e:obsidian:couchdb-upload` reuses the CouchDB variables from `.test.env` or the process environment. It expects a reachable CouchDB service, creates a unique database, starts from configured plug-in data without the device-local compatibility marker, and verifies that the marker is initialised without a compatibility dialogue, reminder, or runtime pause. It then creates a note in real Obsidian, commits it into the local database, runs one-shot synchronisation, and verifies that the remote database contains both the Metadata document and its Chunks. @@ -171,6 +174,8 @@ The same workflow checks the two remote-activity status boundaries. It first hol `test:e2e:obsidian:couchdb-http-cache` checks replication after rebuilding an encrypted CouchDB remote while an earlier HTTP response remains cached. It enables internal Metadata encryption with Hidden File Sync and Customisation Sync, verifies that the Rebuild replaces the Security Seed and ciphertext while retaining the document revision, and requires the replication pre-check and subsequent sends to succeed without clearing the browser cache. It uses the CouchDB fixture variables, an isolated Vault and profile, and a unique remote database. The raw command uses the current built plug-in by default; `E2E_OBSIDIAN_ARTIFACT_ROOT` can select an exact earlier build for regression comparison. +Run `npm run test:e2e:obsidian:focused -- couchdb-http-cache --request-api` to warm Obsidian's Internal API cache and perform the same workflow with **Use Internal API** enabled. Both modes require an explicit `no-store` read to return the rebuilt ciphertext. The default mode also retains the stale browser-cache assertion; the Internal API mode permits reads during Rebuild to refresh that cache. + `npm run test:e2e:obsidian:focused -- chunk-fetch-retry` checks delayed Chunk availability through a real CouchDB service and Obsidian. It creates a Metadata-only remote fixture, starts ordinary one-shot replication with `readChunksOnline`, and inserts the missing Chunk only after a real fetch has returned an empty result. A pass-through observer records the replicator's call times and results without substituting responses or adding waits. The fixture sets the existing minimum request interval to 500 ms to keep the real retry status observable even if finite completion expedites the final probe. The actual status bar must show zero initial requests (`🛄`) and one retry (`🔁`), and both counts must return to zero after delivery ends. On-demand replication excludes Chunk documents from the ordinary pull, so the delayed Chunk must arrive through the observed fetch and produce the exact Vault content. If finite replication was already inactive when the initial lookup began, the scenario requires a retry at least two seconds later and no physical request slot occupied during backoff. If finite replication ends during the initial lookup or the following backoff, the retry must instead be a post-completion final probe before the two-second delay would expire. This distinction is determined from the observed finite-count transitions, not an assumed ordering between replication and HTTP completion. @@ -267,6 +272,12 @@ This proves in real Obsidian the plug-in behaviour shared by supported platforms `test:e2e:obsidian:hidden-file-snippet-sync` runs a two-vault hidden file round-trip. It verifies creation and deletion of a real `.obsidian/snippets/*.css` file, automatic JSON conflict merging for a hidden file with the merged result propagated by a second synchronisation, manual JSON Resolve dialogue application through Obsidian's UI, and per-device target patterns where one vault ignores a hidden file that the other vault synchronises. That target-pattern case keeps a normal-file allow-list for Markdown and PDF files enabled on both devices, then requires exact hidden JSON content after the receiving device includes it. Initial enablement must open one user-visible progress Notice before the enabled setting is saved, then retain that Notice while its nested rebuild and scan phases continue in the ordinary log. The configured fixture starts with a current CouchDB remote profile, so migration from legacy remote settings remains the responsibility of the upgrade scenarios and cannot add unrelated Notices to this check. It also covers [issue #555](https://github.com/vrtmrz/obsidian-livesync/issues/555) by requiring several plug-in and settings changes to share one separate action Notice whose controls remain usable in mobile layouts; a manually dismissed group must not repeat its acknowledged rows when a later change arrives. +The JSON conflict cases enter through `resolveConflictOnInternalFiles()`, exercising the analysis queue and the serial manual-resolution queue. They check the stored content and remaining conflict revisions as well as the Vault files; the manual case must remain unfinished until a choice is made. + +The conflict cases also verify same-file dialogue replacement and received resolutions. A newer check closes the older JSON comparison, reads the latest revisions, and preserves both leaves until selection. A resolution delivered to the registered optional-file replication handler closes only the affected dialogue, drains its queue, and retains the winner and Vault bytes; an unrelated file is a control. This isolates the Obsidian receive boundary from peer and transport setup. Use `--case=replacement` or `--case=external-resolution` to run one of these cases alone, with the CouchDB fixture already running. + +Separate Vault pairs begin with Hidden File Sync disabled and enable it through the visible `Merge`, `Fetch`, and `Overwrite` buttons. Each pair contains a newer local file, a newer database file, and files present on only one side. The workflow verifies the selected contents in the files and database, the saved enablement setting, persistence after restart, and convergence on the other device. `--case=conflicts` runs only the conflict cases; `--case=enablement` runs only these enablement cases. Add `--mode=Merge`, `--mode=Fetch`, or `--mode=Overwrite` to select one enablement method. + `test:e2e:obsidian:hidden-file-selectors` stores and restores selected JS, CSS, and JSON files with a normal-file allow-list set, and compares both directions with an empty allow-list. It uses one temporary Vault and a local database, and also runs in the complete local suite. Run it through the focused wrapper to build the current bundle first; no remote service is required. Set `E2E_HIDDEN_FILE_SELECTORS_OUTPUT` to choose the JSON report path. Before enablement, the scenario directly requests initialisation while Hidden File Sync is disabled. It requires a refused result, unchanged local files, Metadata revisions, and processed markers, and no initialisation progress Notice. @@ -291,6 +302,8 @@ The workflow also opens the Customisation Sync dialogue. `--case=visibility` che `test:e2e:obsidian:received-change-readiness` uses two sequential real Obsidian sessions and isolated CouchDB databases. The source creates ordinary notes and their Chunks; the target starts continuous replication, resets readiness through the public lifecycle service, and receives those documents while its Vault files remain absent. Marking the target ready twice must emit one readiness event and reflect the queued content. A second note remains absent after readiness while database reflecting is explicitly suspended, then appears when that setting is resumed. This focused scenario is outside `test:e2e:obsidian:local-suite`. +`test:e2e:obsidian:sync-guard` uploads six notes and their Chunks from real Obsidian to an isolated CouchDB database, then receives them in two fresh sequential Vaults. It pauses the real receive callback below and at the device-local threshold to verify that the workspace is protected only after enough notes arrive. A held real Vault reflection must retain the overlay after network catch-up completes, beyond the ordinary quiet period, before automatic release and exact content checks. The second target verifies that keyboard input to the focused editor is blocked, clicks **Continue without waiting**, edits an ordinary note through the visible editor while catch-up remains pending, and then verifies every incoming note and the retained local edit after transfer and reflection resume. The scenario restores its runtime hooks in `finally`, retains stage screenshots, and runs in the local suite. It requires CouchDB; no mobile operating-system lifecycle is simulated. + `test:e2e:obsidian:remote-feature-change` starts real Obsidian with continuous CouchDB replication, then changes the remote version document from generation 12 to generation 13 with an unknown feature. It waits for the control document to reach the local database and the active Replicator to retire, checks that another replication is refused, and verifies that an already accepted Vault note remains intact. After restarting the same Vault, the current remote declaration still blocks finite replication and the actual continuous connection attempt. No KV feature history is involved. It is a focused test outside `test:e2e:obsidian:local-suite`; recovery with a future compatible client remains a separate validation boundary. `test:e2e:obsidian:internal-metadata-migration` enables internal Metadata encryption through the settings UI without Rebuild. It checks unchanged plaintext and rewritten encrypted Hidden File Sync and Customisation Sync Metadata in CouchDB, stable document IDs, mismatch rejection on a second device, and file restoration after aligning settings. It then turns the preference OFF, runs Fast Fetch, and compares content loaded from both Metadata representations and their Chunks while retaining the remote feature declaration. These focused tests use the local CouchDB fixture and are outside `test:e2e:obsidian:local-suite`. diff --git a/test/e2e-obsidian/scripts/conflict-dialog-policy.ts b/test/e2e-obsidian/scripts/conflict-dialog-policy.ts index ed53222a..62638685 100644 --- a/test/e2e-obsidian/scripts/conflict-dialog-policy.ts +++ b/test/e2e-obsidian/scripts/conflict-dialog-policy.ts @@ -38,7 +38,10 @@ type ObsidianTestApp = { | { core?: { services?: { - conflict?: { ensureAllProcessed(): Promise }; + conflict?: { + ensureAllProcessed(): Promise; + conflictProcessQueueCount: { value: number }; + }; }; }; } @@ -607,11 +610,40 @@ async function main(): Promise { await requestRepeatedConflictChecks(cliBinary, repeatedSession.cliEnv, repeatedPath, 3); await page.waitForFunction((element) => !element.isConnected, original, { timeout: uiTimeoutMs }); await modal.waitFor({ state: "visible", timeout: uiTimeoutMs }); - await page.waitForTimeout(250); + // All enqueue calls have settled. Only the final resolver may remain + // while its dialogue waits for a response; earlier checks also count. + await page.waitForFunction( + (targetPath) => { + const conflict = (globalThis as ObsidianTestGlobal).app?.plugins?.plugins?.["obsidian-livesync"] + ?.core?.services?.conflict; + const visible = Array.from(document.querySelectorAll(".modal-container")).filter( + (element) => + element.getClientRects().length > 0 && + element.querySelector(".modal-title")?.textContent?.includes("Conflicting changes") && + element.textContent?.includes(targetPath) + ); + return conflict?.conflictProcessQueueCount.value === 1 && visible.length === 1; + }, + repeatedPath, + { timeout: uiTimeoutMs } + ); const visibleCount = await conflictDialogue(page, repeatedPath).evaluateAll( (elements) => elements.filter((element) => element.getClientRects().length > 0).length ); if (visibleCount !== 1) { + const dialogues = await page.locator(".modal-container").evaluateAll((elements) => + elements.map((element) => ({ + visible: element.getClientRects().length > 0, + text: element.textContent?.slice(0, 1200), + })) + ); + const fixture = await readConflictFixture(cliBinary, repeatedSession.cliEnv, repeatedPath); + console.error(JSON.stringify({ sameFileReplacementFailure: { visibleCount, dialogues, fixture } })); + await captureObsidianPage( + repeatedSession.remoteDebuggingPort, + "conflict-dialog-same-file-replacement.failure.png", + async () => {} + ); throw new Error(`Expected one newest same-file dialogue, but found ${visibleCount}`); } }); diff --git a/test/e2e-obsidian/scripts/couchdb-http-cache.ts b/test/e2e-obsidian/scripts/couchdb-http-cache.ts index 19f7faf3..25a1ec81 100644 --- a/test/e2e-obsidian/scripts/couchdb-http-cache.ts +++ b/test/e2e-obsidian/scripts/couchdb-http-cache.ts @@ -28,6 +28,8 @@ import { createTemporaryVault } from "../runner/vault.ts"; process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; +const useRequestAPI = process.argv.includes("--request-api"); + const customPath = ".obsidian/snippets/http-cache.css"; const nextCustomPath = ".obsidian/snippets/http-cache-new.css"; const settings = { @@ -45,7 +47,7 @@ const settings = { syncInternalFilesInterval: 0, syncInternalFilesTargetPatterns: "^\\.http-cache(?:/|$)", watchInternalFileChanges: false, - useRequestAPI: false, + useRequestAPI, autoAcceptCompatibleTweak: false, doctorProcessedVersion: DoctorRegulation.version, deviceAndVaultName: "http-cache-source", @@ -74,10 +76,11 @@ async function main(): Promise { .storeCustomizationFiles(path,core.services.setting.getDeviceAndVaultName()); return JSON.stringify(true);`); }; - const browserRead = async (id: string, cache: RequestCache): Promise => { + const readCachedDocument = async (id: string, cache: RequestCache): Promise => { const url = `${couchDb.uri}/${encodeURIComponent(dbName)}/${encodeURIComponent(id)}`; const authorization = `Basic ${Buffer.from(`${couchDb.username}:${couchDb.password}`).toString("base64")}`; - return await evaluate(`const response=await fetch(${JSON.stringify(url)},{ + const fetchFunction = useRequestAPI ? "core.services.API.nativeFetch.bind(core.services.API)" : "fetch"; + return await evaluate(`const response=await ${fetchFunction}(${JSON.stringify(url)},{ headers:{authorization:${JSON.stringify(authorization)}},credentials:'include',cache:${JSON.stringify(cache)}}); if(!response.ok)throw new Error('Fixture read failed: '+response.status); return JSON.stringify(await response.json());`); @@ -110,7 +113,7 @@ async function main(): Promise { assert.ok(original?.path, "The encrypted ordinary Metadata fixture is missing."); const initialParameters = await fetchCouchDbDocument(couchDb, dbName, DOCID_SYNC_PARAMETERS); // Populate the HTTP cache as an earlier plug-in version would have done. - assert.deepEqual(await browserRead(original._id, "reload"), original); + assert.deepEqual(await readCachedDocument(original._id, "reload"), original); await evaluate(`await core.services.setting.applyPartial({encryptInternalMetadata:true},true); await core.services.control.applySettings(); @@ -128,11 +131,13 @@ async function main(): Promise { assert.notEqual(rebuiltParameters.pbkdf2salt, initialParameters.pbkdf2salt, "The Seed did not change."); assert.equal(rebuilt._rev, original._rev, "The document revision was not retained."); assert.notEqual(rebuilt.path, original.path, "The remote ciphertext did not change."); - assert.equal( - (await browserRead(original._id, "default")).path, - original.path, - "The stale browser HTTP cache fixture is no longer present." - ); + if (!useRequestAPI) { + assert.equal( + (await readCachedDocument(original._id, "default")).path, + original.path, + "The stale browser HTTP cache fixture is no longer present." + ); + } await evaluate(`await core.services.setting.applyPartial({usePluginSync:true,syncInternalFiles:true},true); await core.services.control.applySettings();return JSON.stringify(true);`); @@ -143,9 +148,12 @@ async function main(): Promise { if(opened)await opened.close();return JSON.stringify({admitted:!!opened}); }catch(error){return JSON.stringify({admitted:false,error:error.name});}`); assert.ok(admission.admitted, `Replication admission failed: ${JSON.stringify(admission)}`); + assert.equal((await readCachedDocument(original._id, "no-store")).path, rebuilt.path); await pushLocalChanges(cliBinary, session.cliEnv); await pushLocalChanges(cliBinary, session.cliEnv); - console.log("Rebuild and subsequent replication succeeded with stale ciphertext still in the HTTP cache."); + console.log( + `Rebuild and subsequent replication succeeded after warming the ${useRequestAPI ? "Internal API" : "browser"} HTTP cache.` + ); } finally { await session?.app.stop(); await vault.dispose(); diff --git a/test/e2e-obsidian/scripts/dialog-mounts.ts b/test/e2e-obsidian/scripts/dialog-mounts.ts index 60d7cf1e..d65fe153 100644 --- a/test/e2e-obsidian/scripts/dialog-mounts.ts +++ b/test/e2e-obsidian/scripts/dialog-mounts.ts @@ -1,13 +1,19 @@ import { readFile } from "node:fs/promises"; import { join } from "node:path"; -import type { Page } from "playwright"; +import type { Locator, Page } from "playwright"; import { decodeSettingsFromSetupURI } from "@vrtmrz/livesync-commonlib/setup-uri"; import { decryptString } from "@vrtmrz/livesync-commonlib/compat/encryption/stringEncryption"; import { SETTING_KEY_P2P_DEVICE_NAME } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { $msg } from "../../../src/common/translation.ts"; import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; import { createE2eCouchDbPluginData, waitForLiveSyncCoreReady } from "../runner/liveSyncWorkflow.ts"; -import { assertMobileDialogueLayout, assertMobileNoticeLayout, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; +import { + assertMobileDialogueLayout, + assertMobileNoticeLayout, + iPhoneSafeArea, + mobileViewport, + setObsidianMobileTestMode, +} from "../runner/mobileUi.ts"; import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; import { captureObsidianDialogue, @@ -400,6 +406,185 @@ async function verifyRemoteSizeNoticeAndDialogue(): Promise<{ }; } +async function assertDialogueActionAlignment(modal: Locator, mode: DialogueMode, name: string): Promise { + const groups = await modal.locator(".dialog-host .button-group").evaluateAll((elements) => { + return elements + .filter((element) => element.getClientRects().length > 0) + .map((element) => { + const group = element.getBoundingClientRect(); + const buttons = Array.from(element.children) + .filter((child) => child instanceof HTMLButtonElement && child.getClientRects().length > 0) + .map((child) => { + const rect = child.getBoundingClientRect(); + return { left: rect.left, right: rect.right, width: rect.width }; + }); + return { group: { left: group.left, right: group.right, width: group.width }, buttons }; + }) + .filter((group) => group.buttons.length > 0); + }); + if (groups.length === 0) { + throw new Error(`Expected visible action buttons in ${name} (${mode}).`); + } + console.log(JSON.stringify({ event: "dialogue-action-alignment", name, mode, groups })); + for (const { group, buttons } of groups) { + for (const button of buttons) { + if (Math.abs(button.left - group.left) > 1 || Math.abs(button.right - group.right) > 1) { + throw new Error( + `Expected ${name} (${mode}) action buttons to align with both edges of their content column: ${JSON.stringify({ group, button })}` + ); + } + } + } +} + +async function assertNoExtraKeyboardSpace(page: Page, modal: Locator): Promise { + const original = await page.evaluate(() => ({ + value: document.body.style.getPropertyValue("--keyboard-height"), + priority: document.body.style.getPropertyPriority("--keyboard-height"), + })); + const originalScroll = await modal.locator(".modal-content").evaluate((element) => element.scrollTop); + const keyboardHeight = await page.evaluate(() => Math.round(innerHeight * 0.4)); + let baseline: number | undefined; + try { + for (const height of [0, keyboardHeight, 0]) { + await page.evaluate((value) => document.body.style.setProperty("--keyboard-height", `${value}px`), height); + await modal.locator(".modal-content").evaluate((element) => { + element.scrollTop = element.scrollHeight; + }); + const gap = await modal.locator(".dialog-host").evaluate((host) => { + const buttons = Array.from(host.querySelectorAll("button")).filter( + (button) => button.getClientRects().length > 0 + ); + const last = buttons[buttons.length - 1]; + if (last === undefined) throw new Error("The dialogue has no action button."); + return host.getBoundingClientRect().bottom - last.getBoundingClientRect().bottom; + }); + console.log(JSON.stringify({ event: "dialogue-keyboard-spacing", height, gap, baseline })); + baseline ??= gap; + if (Math.abs(gap - baseline) > 1) { + throw new Error( + `Expected the keyboard not to add blank content after the final action: ${JSON.stringify({ height, gap, baseline })}` + ); + } + } + } finally { + await page.evaluate(({ value, priority }) => { + if (value === "") document.body.style.removeProperty("--keyboard-height"); + else document.body.style.setProperty("--keyboard-height", value, priority); + }, original); + await modal.locator(".modal-content").evaluate((element, top) => { + element.scrollTop = top; + }, originalScroll); + } +} + +async function assertCouchDBKeyboardFormSpace(page: Page, modal: Locator): Promise { + const originalViewport = await page.evaluate(() => ({ width: innerWidth, height: innerHeight })); + const properties = ["--keyboard-height", "--safe-area-inset-top", "--safe-area-inset-bottom"]; + const original = await page.evaluate( + (names) => + names.map((name) => ({ + name, + value: document.body.style.getPropertyValue(name), + priority: document.body.style.getPropertyPriority(name), + })), + properties + ); + const content = modal.locator(".modal-content"); + const originalScroll = await content.evaluate((element) => element.scrollTop); + const keyboardHeight = Math.round(mobileViewport.height * 0.42); + try { + await page.setViewportSize(mobileViewport); + await page.evaluate( + ({ keyboardHeight, top }) => { + document.body.style.setProperty("--keyboard-height", `${keyboardHeight}px`); + document.body.style.setProperty("--safe-area-inset-bottom", `${keyboardHeight}px`); + document.body.style.setProperty("--safe-area-inset-top", `${top}px`); + }, + { keyboardHeight, top: iPhoneSafeArea.top } + ); + await modal.locator('input[name="couchdb-username"]').focus(); + await page.waitForTimeout(350); + await page.screenshot({ + path: join( + process.env.E2E_OBSIDIAN_DIAGNOSTICS_DIR ?? "/tmp/obsidian-livesync-e2e", + "setup-couchdb-keyboard-form.png" + ), + animations: "disabled", + style: ".notice-container { visibility: hidden !important; }", + }); + const warning = modal.locator(".sls-info-note").filter({ + hasText: + "Saving without a successful connection test keeps this profile, but automatic synchronisation may fail until the connection is corrected.", + }); + if ((await warning.count()) !== 1) throw new Error("Expected the untested-save warning to remain available."); + const pinned = await warning.evaluate((element) => { + for (let parent = element.parentElement; parent !== null; parent = parent.parentElement) { + if (parent.classList.contains("modal-content")) break; + const position = getComputedStyle(parent).position; + if (position === "sticky" || position === "fixed") return true; + } + return false; + }); + if (pinned) throw new Error("The untested-save warning still occupies the fixed action area."); + const bounds = await modal.evaluate((element) => { + const host = element.querySelector(".dialog-host"); + const content = element.querySelector(".modal-content"); + const actionGroups = host?.querySelectorAll(":scope > .button-group"); + const footerElement = actionGroups?.item(actionGroups.length - 1); + const rowElement = host?.querySelector(".row"); + if (!host || !content || !footerElement || !rowElement) + throw new Error("CouchDB form layout is unavailable."); + const footer = footerElement.getBoundingClientRect(); + const row = rowElement.getBoundingClientRect(); + const actions = Array.from(footerElement.querySelectorAll("button"), (button) => + button.getBoundingClientRect() + ); + const footerTop = Math.min(footer.top, ...actions.map((action) => action.top)); + const footerBottom = Math.max(footer.bottom, ...actions.map((action) => action.bottom)); + const modalElement = element.querySelector(".modal"); + if (!modalElement) throw new Error("The CouchDB modal is unavailable."); + const modalRect = modalElement.getBoundingClientRect(); + const contentRect = content.getBoundingClientRect(); + const headerBottom = + element.querySelector(".modal-header")?.getBoundingClientRect().bottom ?? modalRect.top; + const formTop = Math.max(contentRect.top, modalRect.top, headerBottom); + return { + width: innerWidth, + height: innerHeight, + formTop, + footerTop, + formHeight: Math.min(footerTop, contentRect.bottom, modalRect.bottom) - formTop, + rowHeight: row.height + (parseFloat(getComputedStyle(host).rowGap) || 0), + footerBottom, + actionBounds: actions.map((action) => ({ top: action.top, bottom: action.bottom })), + }; + }); + console.log(JSON.stringify({ event: "couchdb-keyboard-form-space", keyboardHeight, bounds })); + if ( + bounds.width !== mobileViewport.width || + bounds.height !== mobileViewport.height || + bounds.formHeight < bounds.rowHeight * 2 || + bounds.footerBottom > bounds.height - keyboardHeight + 1 + ) { + throw new Error( + `Expected room for at least two input rows above keyboard actions: ${JSON.stringify(bounds)}` + ); + } + } finally { + await page.evaluate((entries) => { + for (const { name, value, priority } of entries) { + if (value === "") document.body.style.removeProperty(name); + else document.body.style.setProperty(name, value, priority); + } + }, original); + await page.setViewportSize(originalViewport); + await content.evaluate((element, top) => { + element.scrollTop = top; + }, originalScroll); + } +} + async function verifyRemoteSelectionDialogue(mode: DialogueMode): Promise { await openRemoteSelectionDialogue(); const screenshotPath = await captureObsidianDialogue( @@ -446,6 +631,7 @@ async function verifyRemoteSelectionDialogue(mode: DialogueMode): Promise { ); screenshotPaths.push(await captureStep(page, "06-after-search-next-match")); + const highlightDiff = modal.locator('.history-highlight-diff-label input[type="checkbox"]'); + await highlightDiff.check(); + const diffBlocks = modal.locator(".history-added, .history-deleted"); + await diffBlocks.first().waitFor({ state: "visible", timeout: 10000 }); + const diffIndicator = modal.locator(".diff-nav-indicator"); + const diffCount = await diffBlocks.count(); + const assertDiffSelection = async (index: number, count: number) => { + assertEqual( + (await diffIndicator.innerText()).trim(), + `${index + 1}/${count}`, + "Unexpected diff indicator." + ); + const focused = await diffBlocks.evaluateAll((elements) => + elements.flatMap((element, index) => (element.classList.contains("diff-focused") ? [index] : [])) + ); + assertEqual( + JSON.stringify(focused), + JSON.stringify([index]), + "Diff focus did not move to exactly one block." + ); + }; + await assertDiffSelection(0, diffCount); + const diffPrev = modal.locator(".diff-nav-btn").first(); + const diffNext = modal.locator(".diff-nav-btn").last(); + await diffPrev.click(); + await assertDiffSelection(diffCount - 1, diffCount); + await diffNext.click(); + await assertDiffSelection(0, diffCount); + for (let index = 0; index < diffCount; index++) await diffNext.click(); + await assertDiffSelection(0, diffCount); + + const oldBlock = await diffBlocks.first().elementHandle(); + if (oldBlock === null) throw new Error("Could not retain the previous revision's diff block"); + const changeRevision = (await nextBtn.isDisabled()) ? prevBtn : nextBtn; + await changeRevision.click(); + await page.waitForFunction((element) => !element.isConnected, oldBlock, { timeout: 10000 }); + await diffBlocks.first().waitFor({ state: "visible", timeout: 10000 }); + const replacementDiffCount = await diffBlocks.count(); + await assertDiffSelection(0, replacementDiffCount); + await diffPrev.click(); + await assertDiffSelection(replacementDiffCount - 1, replacementDiffCount); + screenshotPaths.push(await captureStep(page, "07-diff-navigation-after-revision-change")); + return [ + `diffNavigation: ${diffCount} blocks; replacement revision: ${replacementDiffCount} blocks`, `initialIndicator: ${initialIndicator}`, `afterPrevIndicator: ${afterPrevIndicator}`, `afterNextIndicator: ${afterNextIndicator}`, diff --git a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts index 2a9cde9d..4105aeca 100644 --- a/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts +++ b/test/e2e-obsidian/scripts/hidden-file-snippet-sync.ts @@ -1,5 +1,6 @@ -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rm, utimes, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; +import { decodeBinary } from "octagonal-wheels/binary"; import { VERSIONING_DOCID } from "@vrtmrz/livesync-commonlib/compat/common/types"; import { ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION } from "@vrtmrz/livesync-commonlib/replication"; import { @@ -38,6 +39,7 @@ import { captureJsonResolveDialogue, clickJsonResolveOption, obsidianRemoteDebuggingPort, + openLiveSyncSettings, withObsidianPage, } from "../runner/ui.ts"; import { iPhoneSafeArea, setObsidianMobileTestMode } from "../runner/mobileUi.ts"; @@ -60,6 +62,12 @@ const snippetContent = [ const mergeJsonPath = ".obsidian/livesync-e2e-merge.json"; const manualMergeJsonPath = ".obsidian/livesync-e2e-manual-merge.json"; +const replacementMergeJsonPath = ".obsidian/livesync-e2e-manual-replacement.json"; +const externalResolutionTargetPath = ".obsidian/livesync-e2e-external-resolution.json"; +const externalResolutionUnrelatedPath = ".obsidian/livesync-e2e-external-unrelated.json"; +const hiddenConflictStateKey = "__livesyncE2EHiddenConflictResolution"; +const hiddenConflictComparisonStateKey = "__livesyncE2EHiddenConflictComparison"; +const hiddenConflictModalStateKey = "__livesyncE2EHiddenConflictModal"; const targetPath = ".obsidian/livesync-targeted/only-a.json"; const hiddenFileCliTimeoutMs = Number(process.env.E2E_OBSIDIAN_HIDDEN_FILE_CLI_TIMEOUT_MS ?? 90000); const hiddenFileInitialisationStateKey = "__livesyncE2EHiddenFileInitialisation"; @@ -85,6 +93,8 @@ type RunnerContext = { couchDb: CouchDbConfig; dbName: string; activeSessions: Set; + temporaryVaults: Set; + diagnosticPaths: string[]; }; async function writeVaultFile(vaultPath: string, path: string, content: string): Promise { @@ -313,74 +323,186 @@ async function resolveHiddenConflicts(cliBinary: string, env: NodeJS.ProcessEnv) ); } -async function autoMergeHiddenJsonConflict(cliBinary: string, env: NodeJS.ProcessEnv, path: string): Promise { - await evalObsidianJson( +async function startHiddenConflictResolution(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + await evalObsidianJson( cliBinary, [ - "(async()=>{", - `const path=${JSON.stringify(path)};`, - "const prefixedPath=`i:${path}`;", - "const core=app.plugins.plugins['obsidian-livesync'].core;", - "const addOn=core.getAddOn('HiddenFileSync');", - "let doc=false;", - "for await (const entry of core.localDatabase.findEntries('i:','i;',{conflicts:true})){", - " if(entry.path===prefixedPath){ doc=entry; break; }", - "}", - "if(!doc) throw new Error(`Could not find hidden conflict candidate: ${path}`);", - "if(!doc._conflicts?.length) throw new Error(`Hidden file has no conflicts: ${path}`);", - "const conflicts=doc._conflicts.sort((a,b)=>Number(a.split('-')[0])-Number(b.split('-')[0]));", - "const conflictedRev=conflicts[0];", - "const conflictedRevNo=Number(conflictedRev.split('-')[0]);", - "const revFrom=await core.localDatabase.getRaw(doc._id,{revs_info:true});", - "const commonBase=(revFrom._revs_info||[])", - " .filter((rev)=>rev.status==='available'&&Number(rev.rev.split('-')[0])rev.rev)[0]||'';", - "const result=await core.localDatabase.managers.conflictManager.mergeObject(", - " doc.path, commonBase, doc._rev, conflictedRev", + "(()=>{", + `const key=${JSON.stringify(hiddenConflictStateKey)};`, + "const state={done:false,error:null};", + "globalThis[key]=state;", + `const comparison=globalThis[${JSON.stringify(hiddenConflictComparisonStateKey)}];`, + "if(comparison)comparison.operation=state;", + "const addOn=app.plugins.plugins['obsidian-livesync'].core.getAddOn('HiddenFileSync');", + "void addOn.resolveConflictOnInternalFiles().then(", + "()=>{state.done=true;},", + "error=>{state.error=String(error);state.done=true;}", ");", - "if(!result){", - " throw new Error(`Hidden JSON conflict was not auto-mergeable: ${path}; base=${commonBase}; current=${doc._rev}; conflict=${conflictedRev}`);", - "}", - "await addOn.ensureDir(path);", - "const stat=await addOn.writeFile(path,result);", - "if(!stat) throw new Error(`Could not write merged hidden file: ${path}`);", - "await addOn.storeInternalFileToDatabase({path,mtime:stat.mtime,ctime:stat.ctime,size:stat.size},true);", - "await core.localDatabase.removeRevision(doc._id,conflictedRev);", - "await addOn.extractInternalFileFromDatabase(path);", - "await addOn.scanAllDatabaseChanges(true);", - "return JSON.stringify({ok:true,merged:JSON.parse(result)});", + "return JSON.stringify({started:true});", "})()", ].join(""), env ); } -async function openHiddenJsonResolveModal(cliBinary: string, env: NodeJS.ProcessEnv, path: string): Promise { +type HiddenJsonComparisonCall = { + leftRev: string | null; + rightRev: string | null; + done: boolean; + result: unknown; + error: string | null; +}; + +type HiddenJsonComparisonObservation = { + pipelineDone: boolean; + calls: HiddenJsonComparisonCall[]; +}; + +async function installHiddenJsonComparisonObserver(cliBinary: string, env: NodeJS.ProcessEnv): Promise { await evalObsidianJson( cliBinary, [ - "(async()=>{", - `const path=${JSON.stringify(path)};`, - "const prefixedPath=`i:${path}`;", - "const core=app.plugins.plugins['obsidian-livesync'].core;", - "const addOn=core.getAddOn('HiddenFileSync');", - "let doc=false;", - "for await (const entry of core.localDatabase.findEntries('i:','i;',{conflicts:true})){", - " if(entry.path===prefixedPath){ doc=entry; break; }", - "}", - "if(!doc?._conflicts?.length) throw new Error(`Could not find hidden JSON conflict: ${path}`);", - "const conflicts=doc._conflicts.sort((a,b)=>Number(a.split('-')[0])-Number(b.split('-')[0]));", - "const docA=await core.localDatabase.getDBEntry(prefixedPath,{rev:doc._rev});", - "const docB=await core.localDatabase.getDBEntry(prefixedPath,{rev:conflicts[0]});", - "if(docA===false||docB===false) throw new Error(`Could not load conflicted hidden JSON entries: ${path}`);", - "void addOn.showJSONMergeDialogAndMerge(docA,docB);", - "return JSON.stringify({ok:true});", + "(()=>{", + `const key=${JSON.stringify(hiddenConflictComparisonStateKey)};`, + "const addOn=app.plugins.plugins['obsidian-livesync'].core.getAddOn('HiddenFileSync');", + "const original=addOn.showJSONMergeDialogAndMerge;", + "if(typeof original!=='function')throw new Error('Hidden File Sync JSON comparison method is unavailable');", + "const state={calls:[],original,operation:null};", + "state.wrapper=function(left,right){", + "const call={leftRev:left?._rev??null,rightRev:right?._rev??null,done:false,result:null,error:null};", + "state.calls.push(call);", + "const result=state.original.call(this,left,right);", + "Promise.resolve(result).then(value=>{call.done=true;call.result=typeof value==='boolean'||value===null?value:String(value);},error=>{call.done=true;call.error=String(error);});", + "return result;};", + "addOn.showJSONMergeDialogAndMerge=state.wrapper;", + "globalThis[key]=state;return JSON.stringify({installed:true});", "})()", ].join(""), env ); } +async function readHiddenJsonComparisonObservation( + cliBinary: string, + env: NodeJS.ProcessEnv +): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(()=>{", + `const state=globalThis[${JSON.stringify(hiddenConflictComparisonStateKey)}];`, + "const operation=state?.operation;", + "if(!state||!operation)throw new Error('Hidden JSON comparison observation is unavailable');", + "return JSON.stringify({pipelineDone:operation.done,calls:state.calls});", + "})()", + ].join(""), + env + ); +} + +async function waitForHiddenConflictResolution(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + `const key=${JSON.stringify(hiddenConflictStateKey)};`, + "const state=globalThis[key];", + "if(!state)throw new Error('Hidden conflict resolution was not started');", + "const deadline=Date.now()+60000;", + "while(!state.done&&Date.now()setTimeout(resolve,50));", + "if(!state.done)throw new Error('Hidden conflict pipeline did not complete');", + "if(state.error)throw new Error(state.error);", + "delete globalThis[key];", + "return JSON.stringify({done:true});", + "})()", + ].join(""), + env, + hiddenFileCliTimeoutMs + ); +} + +async function assertResolvedHiddenContent( + cliBinary: string, + env: NodeJS.ProcessEnv, + path: string, + expectedContent: string +): Promise { + const stored = await evalObsidianJson<{ + rev: string; + currentRev: string; + conflicts: string[]; + datatype: string; + data: string | string[]; + }>( + cliBinary, + [ + "(async()=>{", + `const path=${JSON.stringify(path)};`, + "const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase;", + "const entry=await db.getDBEntry(`i:${path}`);", + "if(!entry)throw new Error(`Resolved hidden file is missing: ${path}`);", + "const current=await db.getRaw(entry._id,{conflicts:true});", + "return JSON.stringify({rev:entry._rev,currentRev:current._rev,conflicts:current._conflicts||[],datatype:entry.datatype,data:entry.data});", + "})()", + ].join(""), + env + ); + assertEqual(stored.rev, stored.currentRev, `Hidden result changed while reading ${path}.`); + assertEqual(stored.conflicts.length, 0, `Hidden conflict remains for ${path}.`); + const content = + stored.datatype === "plain" + ? typeof stored.data === "string" + ? stored.data + : stored.data.join("") + : new TextDecoder().decode(decodeBinary(stored.data)); + assertEqual(content, expectedContent, `Stored hidden content differs from the Vault file: ${path}.`); +} + +type HiddenConflictLeaf = { + rev: string; + datatype: string; + data: string | string[]; +}; + +async function readHiddenConflictLeaves( + cliBinary: string, + env: NodeJS.ProcessEnv, + path: string +): Promise<{ currentRev: string; conflictRevs: string[]; leaves: HiddenConflictLeaf[] }> { + return await evalObsidianJson<{ + currentRev: string; + conflictRevs: string[]; + leaves: HiddenConflictLeaf[]; + }>( + cliBinary, + [ + "(async()=>{", + `const path=${JSON.stringify(path)};`, + "const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase;", + "const entry=await db.getDBEntry(`i:${path}`);", + "if(!entry)throw new Error(`Hidden conflict entry is missing: ${path}`);", + "const current=await db.getRaw(entry._id,{conflicts:true});", + "const conflictRevs=current._conflicts||[];", + "const leaves=await Promise.all([current._rev,...conflictRevs].map(async rev=>{", + "const doc=await db.getDBEntry(`i:${path}`,{rev},false,true,true);", + "if(!doc)throw new Error(`Could not load hidden conflict revision ${rev}: ${path}`);", + "return {rev:doc._rev,datatype:doc.datatype,data:doc.data};}));", + "if(leaves.some(leaf=>!leaf.datatype||leaf.data===undefined))throw new Error(`Could not read every hidden conflict leaf: ${path}`);", + "return JSON.stringify({currentRev:current._rev,conflictRevs,leaves});", + "})()", + ].join(""), + env + ); +} + +function decodeHiddenConflictLeaf(leaf: HiddenConflictLeaf): string { + return leaf.datatype === "plain" + ? typeof leaf.data === "string" + ? leaf.data + : leaf.data.join("") + : new TextDecoder().decode(decodeBinary(leaf.data)); +} + async function storeHiddenFileAsConflict( cliBinary: string, env: NodeJS.ProcessEnv, @@ -437,6 +559,15 @@ async function createHiddenJsonConflict( await writeVaultFile(vault.path, path, right); await storeHiddenFileAsConflict(context.cliBinary, session.cliEnv, path, baseEntry.rev); + const conflicts = await evalObsidianJson( + context.cliBinary, + `(async()=>{ + const entry=await app.plugins.plugins['obsidian-livesync'].core.localDatabase.getDBEntryMeta(${JSON.stringify(`i:${path}`)},{conflicts:true}); + return JSON.stringify(entry?entry._conflicts?.length||0:0); + })()`, + session.cliEnv + ); + assertEqual(conflicts, 1, "The hidden conflict fixture did not retain two live revisions."); } async function startConfiguredSession( @@ -601,11 +732,12 @@ async function runJsonConflictRoundTrip( let session = await startConfiguredSession(context, vaultB); await createHiddenJsonConflict(context, session, vaultB, mergeJsonPath, base, left, right); - await autoMergeHiddenJsonConflict(context.cliBinary, session.cliEnv, mergeJsonPath); + await resolveHiddenConflicts(context.cliBinary, session.cliEnv); await pushLocalChanges(context.cliBinary, session.cliEnv); const mergedOnB = await waitForPathContent(vaultB.path, mergeJsonPath, (content) => hasJsonValues(content, { fromA: true, fromB: true }) ); + await assertResolvedHiddenContent(context.cliBinary, session.cliEnv, mergeJsonPath, mergedOnB); await stopSession(context, session); session = await startConfiguredSession(context, vaultA); @@ -613,6 +745,7 @@ async function runJsonConflictRoundTrip( const mergedOnA = await waitForPathContent(vaultA.path, mergeJsonPath, (content) => hasJsonValues(content, { fromA: true, fromB: true }) ); + await assertResolvedHiddenContent(context.cliBinary, session.cliEnv, mergeJsonPath, mergedOnA); await stopSession(context, session); assertEqual(mergedOnA, mergedOnB, "Merged hidden JSON content was not consistent across both vaults."); @@ -626,13 +759,21 @@ async function runJsonManualConflictResolution(context: RunnerContext, vault: Te const session = await startConfiguredSession(context, vault); await createHiddenJsonConflict(context, session, vault, manualMergeJsonPath, base, left, right); - await openHiddenJsonResolveModal(context.cliBinary, session.cliEnv, manualMergeJsonPath); + await startHiddenConflictResolution(context.cliBinary, session.cliEnv); const screenshotPath = await captureJsonResolveDialogue(obsidianRemoteDebuggingPort()); + const completedBeforeSelection = await evalObsidianJson( + context.cliBinary, + `JSON.stringify(globalThis[${JSON.stringify(hiddenConflictStateKey)}].done)`, + session.cliEnv + ); + assertEqual(completedBeforeSelection, false, "The hidden conflict pipeline completed before selection."); await clickJsonResolveOption(obsidianRemoteDebuggingPort(), "AB"); + await waitForHiddenConflictResolution(context.cliBinary, session.cliEnv); const merged = await waitForPathContent(vault.path, manualMergeJsonPath, (content) => hasJsonValues(content, { shared: "right", fromA: true, fromB: true }) ); + await assertResolvedHiddenContent(context.cliBinary, session.cliEnv, manualMergeJsonPath, merged); await stopSession(context, session); const parsed = JSON.parse(merged); @@ -642,6 +783,493 @@ async function runJsonManualConflictResolution(context: RunnerContext, vault: Te console.log(`Hidden JSON conflict modal applied the selected merged result. Screenshot: ${screenshotPath}`); } +async function runJsonManualConflictReplacement(context: RunnerContext, vault: TemporaryVault): Promise { + const base = JSON.stringify({ shared: "base" }, null, 4) + "\n"; + const left = JSON.stringify({ shared: "left", fromA: true }, null, 4) + "\n"; + const right = JSON.stringify({ shared: "right", fromB: true }, null, 4) + "\n"; + const latestMarker = `hidden-json-replacement-${context.dbName}`; + const latest = JSON.stringify({ shared: "latest", fromA: true, fromB: true, latestMarker }, null, 4) + "\n"; + const session = await startConfiguredSession(context, vault); + + await createHiddenJsonConflict(context, session, vault, replacementMergeJsonPath, base, left, right); + await installHiddenJsonComparisonObserver(context.cliBinary, session.cliEnv); + + await startHiddenConflictResolution(context.cliBinary, session.cliEnv); + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + const firstOption = page.locator('label:has(input[name="disp"][value="AB"])').last(); + await firstOption.waitFor({ state: "visible", timeout: 10000 }); + await firstOption.evaluate((element, key) => { + const firstModal = element.closest(".modal-container"); + if (!(firstModal instanceof HTMLElement)) + throw new Error("The first JSON conflict dialogue has no modal container"); + (globalThis as unknown as Record)[key] = { firstModal }; + }, hiddenConflictModalStateKey); + }); + + const firstDialogueState = await readHiddenJsonComparisonObservation(context.cliBinary, session.cliEnv); + assertEqual( + firstDialogueState.pipelineDone, + false, + "The hidden conflict pipeline completed before replacing its first dialogue." + ); + assertEqual(firstDialogueState.calls.length, 1, "The first hidden JSON comparison was not observed exactly once."); + assertEqual( + firstDialogueState.calls[0].done, + false, + "The first hidden JSON comparison completed before the replacement request." + ); + + await writeVaultFile(vault.path, replacementMergeJsonPath, latest); + const updatedWinner = await evalObsidianJson<{ rev: string }>( + context.cliBinary, + [ + "(async()=>{", + `const path=${JSON.stringify(replacementMergeJsonPath)};`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const addOn=core.getAddOn('HiddenFileSync');", + "const db=core.localDatabase;", + "const entry=await db.getDBEntry(`i:${path}`);", + "if(!entry)throw new Error(`Hidden conflict winner is missing: ${path}`);", + "const current=await db.getRaw(entry._id,{conflicts:true});", + "const conflictRevs=current._conflicts||[];", + "if(conflictRevs.length!==1)throw new Error(`Expected one competing hidden leaf before replacement: ${path}`);", + "const conflicts=await Promise.all(conflictRevs.map(rev=>db.getDBEntry(`i:${path}`,{rev},false,true,true)));", + "if(conflicts.some(doc=>!doc))throw new Error(`Could not read the competing hidden revision: ${path}`);", + "const mtime=Math.max(Number(entry.mtime)||0,...conflicts.map(doc=>Number(doc.mtime)||0))+10;", + "const fileInfo=await addOn.loadFileWithInfo(path);", + "if(fileInfo.deleted)throw new Error(`Hidden replacement file was unexpectedly deleted: ${path}`);", + "const baseData=await addOn.__loadBaseSaveData(path,true);", + "if(baseData===false)throw new Error(`Could not load hidden replacement save data: ${path}`);", + "const saveData={...baseData,data:fileInfo.body,mtime,ctime:fileInfo.stat.ctime,size:fileInfo.stat.size,children:[],deleted:false,type:baseData.datatype};", + "const result=await db.putDBEntry(saveData,false,entry._rev);", + "if(!result?.ok)throw new Error(`Could not advance the current hidden conflict winner: ${path}`);", + "return JSON.stringify({rev:result.rev});", + "})()", + ].join(""), + session.cliEnv + ); + const beforeQueue = await readHiddenConflictLeaves(context.cliBinary, session.cliEnv, replacementMergeJsonPath); + assertEqual( + beforeQueue.currentRev, + updatedWinner.rev, + "The latest hidden conflict revision is not the current winner." + ); + assertEqual( + beforeQueue.conflictRevs.length, + 1, + "Advancing the current winner changed the competing hidden leaf count." + ); + assertEqual( + beforeQueue.leaves.length, + 2, + "Advancing the current winner did not retain both hidden conflict leaves." + ); + const beforeQueueContents = beforeQueue.leaves.map(decodeHiddenConflictLeaf); + assertEqual( + beforeQueueContents.includes(latest), + true, + "The current hidden winner does not contain the latest JSON content." + ); + assertEqual( + beforeQueueContents.some((content) => content === left || content === right), + true, + "The competing hidden leaf content is not readable after advancing the current winner." + ); + + await evalObsidianJson( + context.cliBinary, + [ + "(()=>{", + `const path=${JSON.stringify(replacementMergeJsonPath)};`, + "const addOn=app.plugins.plugins['obsidian-livesync'].core.getAddOn('HiddenFileSync');", + "addOn.queueConflictCheck(path);", + "return JSON.stringify({queued:true});", + "})()", + ].join(""), + session.cliEnv + ); + + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + await page.waitForFunction( + (key) => { + const state = (globalThis as unknown as Record)[key]; + if (!state || state.firstModal.isConnected) return false; + const visibleLabels = Array.from( + document.querySelectorAll('label:has(input[name="disp"])') + ).filter( + (label) => label.getClientRects().length > 0 && getComputedStyle(label).visibility !== "hidden" + ); + const roots = new Set(visibleLabels.map((label) => label.closest(".modal-container"))); + return visibleLabels.length >= 2 && roots.size === 1; + }, + hiddenConflictModalStateKey, + { timeout: 10000 } + ); + const visibleDialogueCount = await page.evaluate(() => { + const visibleLabels = Array.from( + document.querySelectorAll('label:has(input[name="disp"])') + ).filter((label) => label.getClientRects().length > 0 && getComputedStyle(label).visibility !== "hidden"); + return new Set(visibleLabels.map((label) => label.closest(".modal-container"))).size; + }); + assertEqual( + visibleDialogueCount, + 1, + "Replacing a hidden comparison left more than one JSON conflict dialogue visible." + ); + }); + + const replacementDialogueState = await readHiddenJsonComparisonObservation(context.cliBinary, session.cliEnv); + assertEqual( + replacementDialogueState.pipelineDone, + false, + "The hidden conflict pipeline completed before the replacement dialogue was applied." + ); + assertEqual( + replacementDialogueState.calls.length, + 2, + "The latest hidden conflict request did not open a second comparison." + ); + assertEqual( + replacementDialogueState.calls[0].done, + true, + "The first hidden JSON comparison did not complete when replaced." + ); + assertEqual(replacementDialogueState.calls[0].result, false, "The first hidden JSON comparison was not cancelled."); + assertEqual( + [replacementDialogueState.calls[1].leftRev, replacementDialogueState.calls[1].rightRev].includes( + updatedWinner.rev + ), + true, + "The replacement JSON comparison did not include the latest current revision." + ); + assertEqual( + replacementDialogueState.calls[1].done, + false, + "The replacement JSON comparison completed before the user selected it." + ); + + const afterCancellation = await readHiddenConflictLeaves( + context.cliBinary, + session.cliEnv, + replacementMergeJsonPath + ); + assertEqual( + afterCancellation.currentRev, + updatedWinner.rev, + "Cancelling the first dialogue changed the latest current revision." + ); + assertEqual( + afterCancellation.conflictRevs.length, + 1, + "Cancelling the first dialogue removed the competing hidden leaf." + ); + assertEqual( + afterCancellation.leaves.length, + 2, + "Cancelling the first dialogue did not preserve both hidden conflict leaves." + ); + const afterCancellationContents = afterCancellation.leaves.map(decodeHiddenConflictLeaf); + assertEqual( + afterCancellationContents.includes(latest), + true, + "The latest hidden content became unreadable after cancelling the first dialogue." + ); + assertEqual( + afterCancellationContents.some((content) => content === left || content === right), + true, + "The competing hidden content became unreadable after cancelling the first dialogue." + ); + + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + const optionAB = page.locator('label:has(input[name="disp"][value="AB"])').last(); + await optionAB.waitFor({ state: "visible", timeout: 10000 }); + await optionAB.click(); + const selected = await page.locator('input[name="disp"][value="AB"]').last().isChecked(); + assertEqual(selected, true, "The replacement JSON dialogue did not select AB."); + await page.waitForFunction( + (marker) => { + const options = Array.from( + document.querySelectorAll('input[name="disp"][value="AB"]') + ); + const option = options + .reverse() + .find((candidate) => (candidate.closest("label")?.getClientRects().length ?? 0) > 0); + const modal = option?.closest(".modal-container"); + if (!(modal instanceof HTMLElement)) return false; + const values = Array.from( + modal.querySelectorAll("input,textarea") + ).map((element) => element.value); + return `${modal.innerText}\n${values.join("\n")}`.includes(marker); + }, + latestMarker, + { timeout: 10000 } + ); + await page.getByRole("button", { name: "Apply" }).last().click(); + }); + await waitForHiddenConflictResolution(context.cliBinary, session.cliEnv); + + const resolved = await waitForPathContent(vault.path, replacementMergeJsonPath, (content) => + hasJsonValues(content, { shared: "latest", fromA: true, fromB: true, latestMarker }) + ); + await assertResolvedHiddenContent(context.cliBinary, session.cliEnv, replacementMergeJsonPath, resolved); + const parsed = JSON.parse(resolved) as Record; + assertEqual(parsed.latestMarker, latestMarker, "The applied JSON result lost the latest replacement marker."); + + const finalDialogueState = await readHiddenJsonComparisonObservation(context.cliBinary, session.cliEnv); + assertEqual(finalDialogueState.pipelineDone, true, "The replacement hidden conflict pipeline did not finish."); + assertEqual( + finalDialogueState.calls[0].result, + false, + "The replaced hidden JSON comparison did not remain cancelled." + ); + assertEqual(finalDialogueState.calls[1].done, true, "The replacement hidden JSON comparison did not complete."); + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + const finalState = await page.evaluate((key) => { + const state = (globalThis as unknown as Record)[key]; + const visibleLabels = Array.from( + document.querySelectorAll('label:has(input[name="disp"])') + ).filter((label) => label.getClientRects().length > 0 && getComputedStyle(label).visibility !== "hidden"); + return { + firstConnected: state?.firstModal.isConnected ?? false, + visibleDialogueCount: new Set(visibleLabels.map((label) => label.closest(".modal-container"))).size, + }; + }, hiddenConflictModalStateKey); + assertEqual( + finalState.firstConnected, + false, + "The replaced JSON conflict dialogue remained connected after resolution." + ); + assertEqual( + finalState.visibleDialogueCount, + 0, + "A stale JSON conflict dialogue remained open after resolution." + ); + }); + + await stopSession(context, session); + console.log( + "Hidden JSON conflict replacement cancelled the stale dialogue and applied the latest revision and contents." + ); +} + +async function runJsonManualExternalResolution(context: RunnerContext, vault: TemporaryVault): Promise { + const base = JSON.stringify({ shared: "base" }, null, 4) + "\n"; + const left = JSON.stringify({ shared: "left", fromA: true }, null, 4) + "\n"; + const right = JSON.stringify({ shared: "right", fromB: true }, null, 4) + "\n"; + const unrelatedContent = JSON.stringify({ externalResolution: "unrelated" }, null, 4) + "\n"; + const session = await startConfiguredSession(context, vault); + + await writeVaultFile(vault.path, externalResolutionUnrelatedPath, unrelatedContent); + await scanHiddenStorage(context.cliBinary, session.cliEnv); + const unrelatedEntry = await waitForLocalDatabaseEntry( + context.cliBinary, + session.cliEnv, + externalResolutionUnrelatedPath, + { + hidden: true, + } + ); + await createHiddenJsonConflict(context, session, vault, externalResolutionTargetPath, base, left, right); + const initialLeaves = await readHiddenConflictLeaves( + context.cliBinary, + session.cliEnv, + externalResolutionTargetPath + ); + assertEqual( + initialLeaves.conflictRevs.length, + 1, + "The external-resolution target did not have one competing leaf." + ); + assertEqual(initialLeaves.leaves.length, 2, "The external-resolution target did not retain two live leaves."); + const winnerContent = decodeHiddenConflictLeaf( + initialLeaves.leaves.find((leaf) => leaf.rev === initialLeaves.currentRev) ?? initialLeaves.leaves[0] + ); + assertEqual( + [left, right].includes(winnerContent), + true, + "The initial hidden conflict winner content is not readable." + ); + await writeVaultFile(vault.path, externalResolutionTargetPath, winnerContent); + + await installHiddenJsonComparisonObserver(context.cliBinary, session.cliEnv); + await startHiddenConflictResolution(context.cliBinary, session.cliEnv); + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + const firstOption = page.locator('label:has(input[name="disp"][value="AB"])').last(); + await firstOption.waitFor({ state: "visible", timeout: 10000 }); + await firstOption.evaluate((element, key) => { + const firstModal = element.closest(".modal-container"); + if (!(firstModal instanceof HTMLElement)) + throw new Error("The external-resolution JSON dialogue has no modal container"); + (globalThis as unknown as Record)[key] = { firstModal }; + }, hiddenConflictModalStateKey); + }); + + const initialObservation = await readHiddenJsonComparisonObservation(context.cliBinary, session.cliEnv); + assertEqual( + initialObservation.pipelineDone, + false, + "The external-resolution conflict pipeline completed before a result arrived." + ); + assertEqual( + initialObservation.calls.length, + 1, + "The external-resolution target did not open exactly one comparison." + ); + assertEqual( + initialObservation.calls[0].done, + false, + "The external-resolution dialogue completed before a result arrived." + ); + + const unrelatedResolution = await evalObsidianJson<{ rev: string }>( + context.cliBinary, + [ + "(async()=>{", + `const path=${JSON.stringify(externalResolutionUnrelatedPath)};`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const meta=await core.localDatabase.getDBEntryMeta(`i:${path}`,{conflicts:true},true);", + "if(!meta)throw new Error(`Unrelated hidden metadata is missing: ${path}`);", + "if((meta._conflicts||[]).length!==0)throw new Error(`Unrelated hidden metadata unexpectedly has conflicts: ${path}`);", + "await core.services.replication.processOptionalSynchroniseResult(meta);", + "return JSON.stringify({rev:meta._rev});", + "})()", + ].join(""), + session.cliEnv, + hiddenFileCliTimeoutMs + ); + assertEqual( + unrelatedResolution.rev, + unrelatedEntry.rev, + "The unrelated hidden fixture did not have the expected single revision." + ); + const afterUnrelatedResult = await readHiddenJsonComparisonObservation(context.cliBinary, session.cliEnv); + assertEqual( + afterUnrelatedResult.pipelineDone, + false, + "A result for another hidden path completed the target conflict pipeline." + ); + assertEqual(afterUnrelatedResult.calls.length, 1, "A result for another path opened another target comparison."); + assertEqual( + afterUnrelatedResult.calls[0].done, + false, + "A result for another path closed the target JSON comparison." + ); + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + const modalState = await page.evaluate((key) => { + const state = (globalThis as unknown as Record)[key]; + const visibleLabels = Array.from( + document.querySelectorAll('label:has(input[name="disp"])') + ).filter((label) => label.getClientRects().length > 0 && getComputedStyle(label).visibility !== "hidden"); + return { + firstConnected: state?.firstModal.isConnected ?? false, + visibleDialogueCount: new Set(visibleLabels.map((label) => label.closest(".modal-container"))).size, + }; + }, hiddenConflictModalStateKey); + assertEqual(modalState.firstConnected, true, "An unrelated hidden result detached the target JSON dialogue."); + assertEqual( + modalState.visibleDialogueCount, + 1, + "An unrelated hidden result changed the visible target dialogue count." + ); + }); + + const targetResolution = await evalObsidianJson<{ winnerRev: string; removedRev: string }>( + context.cliBinary, + [ + "(async()=>{", + `const path=${JSON.stringify(externalResolutionTargetPath)};`, + "const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase;", + "const entry=await db.getDBEntry(`i:${path}`);", + "if(!entry)throw new Error(`Hidden conflict entry is missing: ${path}`);", + "const current=await db.getRaw(entry._id,{conflicts:true});", + "const conflictRevs=current._conflicts||[];", + "if(conflictRevs.length!==1)throw new Error(`Expected one hidden conflict leaf before external resolution: ${path}`);", + "const removedRev=conflictRevs[0];", + "await db.removeRevision(entry._id,removedRev);", + "const meta=await db.getDBEntryMeta(`i:${path}`,{conflicts:true},true);", + "if(!meta)throw new Error(`Resolved hidden metadata is missing: ${path}`);", + "if((meta._conflicts||[]).length!==0)throw new Error(`External hidden resolution still has conflicts: ${path}`);", + "await app.plugins.plugins['obsidian-livesync'].core.services.replication.processOptionalSynchroniseResult(meta);", + "return JSON.stringify({winnerRev:meta._rev,removedRev});", + "})()", + ].join(""), + session.cliEnv, + hiddenFileCliTimeoutMs + ); + await waitForHiddenConflictResolution(context.cliBinary, session.cliEnv); + + const resolvedLeaves = await readHiddenConflictLeaves( + context.cliBinary, + session.cliEnv, + externalResolutionTargetPath + ); + assertEqual( + resolvedLeaves.currentRev, + targetResolution.winnerRev, + "External hidden resolution changed the selected winner revision." + ); + assertEqual( + resolvedLeaves.currentRev, + initialLeaves.currentRev, + "External hidden resolution changed the current winner revision." + ); + assertEqual( + targetResolution.removedRev, + initialLeaves.conflictRevs[0], + "External hidden resolution removed a different leaf than the competing revision." + ); + assertEqual( + resolvedLeaves.conflictRevs.length, + 0, + "The externally resolved hidden conflict remains in the database." + ); + assertEqual( + resolvedLeaves.leaves.length, + 1, + "External hidden resolution did not leave exactly one readable revision." + ); + assertEqual( + decodeHiddenConflictLeaf(resolvedLeaves.leaves[0]), + winnerContent, + "External hidden resolution changed the current winner bytes." + ); + const vaultWinner = await waitForPathContent( + vault.path, + externalResolutionTargetPath, + (content) => content === winnerContent + ); + await assertResolvedHiddenContent(context.cliBinary, session.cliEnv, externalResolutionTargetPath, vaultWinner); + + const finalObservation = await readHiddenJsonComparisonObservation(context.cliBinary, session.cliEnv); + assertEqual(finalObservation.pipelineDone, true, "The externally resolved hidden conflict pipeline did not drain."); + assertEqual(finalObservation.calls.length, 1, "External resolution opened a second JSON comparison."); + assertEqual(finalObservation.calls[0].done, true, "The external-resolution JSON comparison did not complete."); + assertEqual(finalObservation.calls[0].result, false, "The external-resolution JSON dialogue was not cancelled."); + await withObsidianPage(obsidianRemoteDebuggingPort(), async (page) => { + const modalState = await page.evaluate((key) => { + const state = (globalThis as unknown as Record)[key]; + const visibleLabels = Array.from( + document.querySelectorAll('label:has(input[name="disp"])') + ).filter((label) => label.getClientRects().length > 0 && getComputedStyle(label).visibility !== "hidden"); + return { + firstConnected: state?.firstModal.isConnected ?? false, + visibleDialogueCount: new Set(visibleLabels.map((label) => label.closest(".modal-container"))).size, + }; + }, hiddenConflictModalStateKey); + assertEqual(modalState.firstConnected, false, "The externally resolved JSON dialogue remained connected."); + assertEqual( + modalState.visibleDialogueCount, + 0, + "The old hidden conflict dialogue reopened after external resolution." + ); + }); + + await stopSession(context, session); + console.log( + "Hidden JSON conflict dialogue closed for an external same-path resolution and preserved the selected bytes." + ); +} + async function runTargetMismatch( context: RunnerContext, vaultA: TemporaryVault, @@ -1033,7 +1661,220 @@ async function runConfigurationNoticeGrouping(context: RunnerContext, vault: Tem } } +type HiddenEnablementMode = "Merge" | "Fetch" | "Overwrite"; + +async function enableHiddenSyncThroughSettings( + context: RunnerContext, + session: ObsidianLiveSyncSession, + vault: TemporaryVault, + mode: HiddenEnablementMode +): Promise { + const stateKey = "__livesyncE2EHiddenEnablement"; + await evalObsidianJson( + context.cliBinary, + `(()=>{ + const core=app.plugins.plugins['obsidian-livesync'].core; + if(core.settings.syncInternalFiles)throw new Error('Hidden File Sync must start disabled'); + const setting=core.services.setting; + const original=setting.enableOptionalFeature; + const state={done:false,error:null,mode:null,restore:()=>{setting.enableOptionalFeature=original;}}; + globalThis[${JSON.stringify(stateKey)}]=state; + setting.enableOptionalFeature=async function(mode){ + state.mode=mode; + try{return await original.call(this,mode);} + catch(error){state.error=String(error);throw error;} + finally{state.done=true;state.restore();} + }; + return JSON.stringify(true); + })()`, + session.cliEnv + ); + try { + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + const navigator = await openLiveSyncSettings(page); + const settingsPage = await navigator.openPage("Sync Settings"); + const row = settingsPage.locator(".setting-item").filter({ + has: navigator.page.getByRole("button", { name: "Merge", exact: true }), + }); + await row.getByRole("button", { name: mode, exact: true }).click(); + }); + const result = await evalObsidianJson<{ mode: string; enabled: boolean }>( + context.cliBinary, + ` + (async()=>{ + const state=globalThis[${JSON.stringify(stateKey)}]; + const deadline=Date.now()+60000; + while(!state.done&&Date.now()setTimeout(resolve,50)); + if(!state.done)throw new Error('Hidden File Sync enablement did not complete'); + if(state.error)throw new Error(state.error); + const core=app.plugins.plugins['obsidian-livesync'].core; + return JSON.stringify({mode:state.mode,enabled:core.settings.syncInternalFiles}); + })() + `, + session.cliEnv, + hiddenFileCliTimeoutMs + ); + assertEqual(result.mode, mode.toUpperCase(), "The settings action invoked a different enablement mode."); + assertEqual(result.enabled, true, "Hidden File Sync was not enabled by its settings action."); + const saved = JSON.parse(await readVaultFile(vault.path, ".obsidian/plugins/obsidian-livesync/data.json")); + assertEqual(saved.syncInternalFiles, true, "Hidden File Sync enablement was not persisted."); + } finally { + await evalObsidianJson( + context.cliBinary, + `(()=>{ + globalThis[${JSON.stringify(stateKey)}]?.restore(); + delete globalThis[${JSON.stringify(stateKey)}]; + return JSON.stringify(true); + })()`, + session.cliEnv + ); + } +} + +async function startPersistedSession(context: RunnerContext, vault: TemporaryVault): Promise { + const session = await startObsidianLiveSyncSession({ binary: context.binary, cliBinary: context.cliBinary, vault }); + context.activeSessions.add(session); + await waitForLiveSyncCoreReady(context.cliBinary, session.cliEnv); + const enabled = await evalObsidianJson( + context.cliBinary, + "JSON.stringify(app.plugins.plugins['obsidian-livesync'].core.settings.syncInternalFiles)", + session.cliEnv + ); + assertEqual(enabled, true, "Hidden File Sync enablement did not survive restart."); + return session; +} + +async function verifyHiddenFiles( + context: RunnerContext, + session: ObsidianLiveSyncSession, + vault: TemporaryVault, + expected: Record +): Promise { + for (const [path, content] of Object.entries(expected)) { + await waitForPathContent(vault.path, path, (actual) => actual === content); + await assertResolvedHiddenContent(context.cliBinary, session.cliEnv, path, content); + } +} + +async function runHiddenEnablement(context: RunnerContext, mode: HiddenEnablementMode): Promise { + const source = await createTemporaryVault(); + context.temporaryVaults.add(source); + const target = await createTemporaryVault(); + context.temporaryVaults.add(target); + const root = `.obsidian/livesync-enable-${mode.toLowerCase()}`; + const localNewer = `${root}/local-newer.json`; + const remoteNewer = `${root}/remote-newer.json`; + const remoteOnly = `${root}/remote-only.json`; + const localOnly = `${root}/local-only.json`; + context.diagnosticPaths = [localNewer, remoteNewer, remoteOnly, localOnly]; + const remoteContent = JSON.stringify({ source: "remote", mode }) + "\n"; + const localContent = JSON.stringify({ source: "local", mode }) + "\n"; + const timestamp = Math.floor(Date.now() / 1000) - 120; + for (const path of [localNewer, remoteNewer, remoteOnly]) { + await writeVaultFile(source.path, path, remoteContent); + await utimes(join(source.path, path), timestamp + 10, timestamp + 10); + } + const settings = { + syncInternalFiles: false, + useAdvancedMode: true, + syncInternalFilesTargetPatterns: `^\\.obsidian(?:$|/livesync-enable-${mode.toLowerCase()}(?:/|$))`, + }; + let session = await startConfiguredSession(context, source, settings); + await enableHiddenSyncThroughSettings(context, session, source, "Overwrite"); + await verifyHiddenFiles(context, session, source, { + [localNewer]: remoteContent, + [remoteNewer]: remoteContent, + [remoteOnly]: remoteContent, + }); + await pushLocalChanges(context.cliBinary, session.cliEnv); + const uploaded: LocalDatabaseEntry[] = []; + for (const path of [localNewer, remoteNewer, remoteOnly]) { + uploaded.push(await waitForLocalDatabaseEntry(context.cliBinary, session.cliEnv, path, { hidden: true })); + } + await waitForCouchDbDocs(context.couchDb, context.dbName, (docs) => + uploaded.every((entry) => docs.some((doc) => doc._id === entry.id && doc._rev === entry.rev)) + ); + await stopSession(context, session); + + for (const [path, time] of [ + [localNewer, timestamp + 30], + [remoteNewer, timestamp], + [localOnly, timestamp + 30], + ] as const) { + await writeVaultFile(target.path, path, localContent); + await utimes(join(target.path, path), time, time); + } + session = await startConfiguredSession(context, target, settings); + await pushLocalChanges(context.cliBinary, session.cliEnv); + for (const path of [localNewer, remoteNewer]) { + assertEqual(await readVaultFile(target.path, path), localContent, "Disabled sync replaced a local fixture."); + } + const receivedRevisions = await evalObsidianJson( + context.cliBinary, + `(async()=>{ + const db=app.plugins.plugins['obsidian-livesync'].core.localDatabase; + const revisions=[]; + for(const id of ${JSON.stringify(uploaded.map((entry) => entry.id))}) { + const doc=await db.getRaw(id); + revisions.push(doc._rev); + } + return JSON.stringify(revisions); + })()`, + session.cliEnv + ); + assertEqual( + JSON.stringify(receivedRevisions), + JSON.stringify(uploaded.map((entry) => entry.rev)), + "Disabled Hidden File Sync did not retain the uploaded revisions in its database." + ); + assertEqual(await pathExists(target.path, remoteOnly), false, "Disabled sync reflected a remote-only file."); + + await enableHiddenSyncThroughSettings(context, session, target, mode); + const expected = { + [localNewer]: mode === "Fetch" ? remoteContent : localContent, + [remoteNewer]: mode === "Overwrite" ? localContent : remoteContent, + [remoteOnly]: remoteContent, + [localOnly]: localContent, + }; + await verifyHiddenFiles(context, session, target, expected); + await pushLocalChanges(context.cliBinary, session.cliEnv); + await stopSession(context, session); + + session = await startPersistedSession(context, target); + await verifyHiddenFiles(context, session, target, expected); + await stopSession(context, session); + + session = await startPersistedSession(context, source); + await pullAndApplyHiddenFiles(context, session); + await verifyHiddenFiles(context, session, source, expected); + await stopSession(context, session); + console.log( + `Hidden File Sync ${mode} preserved the selected contents in files and the database, after restart, and on the other device.` + ); +} + async function main(): Promise { + const selectedCase = process.argv.find((argument) => argument.startsWith("--case="))?.slice(7); + if ( + selectedCase && + selectedCase !== "conflicts" && + selectedCase !== "replacement" && + selectedCase !== "external-resolution" && + selectedCase !== "enablement" + ) { + throw new Error("Use --case=conflicts, --case=replacement, --case=external-resolution, or --case=enablement."); + } + const conflictsOnly = selectedCase === "conflicts"; + const replacementOnly = selectedCase === "replacement"; + const externalResolutionOnly = selectedCase === "external-resolution"; + const enablementOnly = selectedCase === "enablement"; + const selectedMode = process.argv.find((argument) => argument.startsWith("--mode="))?.slice(7); + const enablementModes = (["Merge", "Fetch", "Overwrite"] as const).filter( + (mode) => !selectedMode || selectedMode === mode + ); + if (selectedMode && (!enablementOnly || enablementModes.length !== 1)) { + throw new Error("Use --case=enablement with --mode=Merge, --mode=Fetch, or --mode=Overwrite."); + } const binary = requireObsidianBinary(); const cli = discoverObsidianCli(); if (!cli.binary) { @@ -1044,7 +1885,22 @@ async function main(): Promise { const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "hidden-roundtrip"); const vaultA = await createTemporaryVault(); const vaultB = await createTemporaryVault(); - const context: RunnerContext = { binary, cliBinary: cli.binary, couchDb, dbName, activeSessions: new Set() }; + const context: RunnerContext = { + binary, + cliBinary: cli.binary, + couchDb, + dbName, + activeSessions: new Set(), + temporaryVaults: new Set([vaultA, vaultB]), + diagnosticPaths: [ + mergeJsonPath, + manualMergeJsonPath, + replacementMergeJsonPath, + externalResolutionTargetPath, + externalResolutionUnrelatedPath, + snippetPath, + ], + }; try { await assertCouchDbReachable(couchDb); @@ -1055,17 +1911,62 @@ async function main(): Promise { console.log(`Temporary vault B: ${vaultB.path}`); console.log(`Temporary CouchDB database: ${dbName}`); - await runCreateRoundTrip(context, vaultA, vaultB); - await runDeleteRoundTrip(context, vaultA, vaultB); - await runJsonConflictRoundTrip(context, vaultA, vaultB); - await runJsonManualConflictResolution(context, vaultB); - await runTargetMismatch(context, vaultA, vaultB); - await runInitialisationNoticeGrouping(context, vaultB); - await runConfigurationNoticeGrouping(context, vaultB); + if (!conflictsOnly && !replacementOnly && !externalResolutionOnly && !enablementOnly) { + await runCreateRoundTrip(context, vaultA, vaultB); + await runDeleteRoundTrip(context, vaultA, vaultB); + } + if (!enablementOnly && !replacementOnly && !externalResolutionOnly) { + await runJsonConflictRoundTrip(context, vaultA, vaultB); + await runJsonManualConflictResolution(context, vaultB); + } + if (!enablementOnly && !externalResolutionOnly) { + await runJsonManualConflictReplacement(context, vaultB); + } + if (!enablementOnly && !replacementOnly) { + await runJsonManualExternalResolution(context, vaultB); + } + if (!conflictsOnly && !replacementOnly && !externalResolutionOnly && !enablementOnly) { + await runTargetMismatch(context, vaultA, vaultB); + await runInitialisationNoticeGrouping(context, vaultB); + await runConfigurationNoticeGrouping(context, vaultB); + } + if (!conflictsOnly && !replacementOnly && !externalResolutionOnly) { + for (const mode of enablementModes) await runHiddenEnablement(context, mode); + } + } catch (error) { + for (const session of context.activeSessions) { + await captureObsidianPage( + session.remoteDebuggingPort, + "hidden-file-workflow-failure.png", + async () => {} + ).catch((captureError: unknown) => console.error(String(captureError))); + await evalObsidianJson( + cli.binary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const addOn=core.getAddOn('HiddenFileSync');", + `const paths=${JSON.stringify(context.diagnosticPaths)};`, + "const files=await addOn.scanInternalFileNames();", + "const observations=[];", + "for(const path of paths){", + "const stat=await core.storageAccess.statHidden(path);", + "const entry=await core.localDatabase.getDBEntryMeta(`i:${path}`,{conflicts:true});", + "observations.push({path,selected:files.includes(path),target:await addOn.isTargetFile(path),stat,processed:addOn.getLastProcessedFileKey(path),entry:entry?{rev:entry._rev,conflicts:entry._conflicts||[]}:false});", + "}", + `const operation=globalThis[${JSON.stringify(hiddenConflictStateKey)}];`, + "return JSON.stringify({enabled:core.settings.syncInternalFiles,ready:core.localDatabase.isReady,remaining:addOn.conflictResolutionProcessor.totalRemaining,operation,observations});", + "})()", + ].join(""), + session.cliEnv + ) + .then((snapshot) => console.error("Hidden-file failure state:", JSON.stringify(snapshot))) + .catch((snapshotError: unknown) => console.error(String(snapshotError))); + } + throw error; } finally { for (const session of context.activeSessions) await stopSession(context, session); - await vaultA.dispose(); - await vaultB.dispose(); + for (const vault of context.temporaryVaults) await vault.dispose(); if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") { await deleteCouchDbDatabase(couchDb, dbName).catch((error: unknown) => { console.warn(error instanceof Error ? error.message : error); diff --git a/test/e2e-obsidian/scripts/local-suite.ts b/test/e2e-obsidian/scripts/local-suite.ts index ee953160..99893534 100644 --- a/test/e2e-obsidian/scripts/local-suite.ts +++ b/test/e2e-obsidian/scripts/local-suite.ts @@ -58,6 +58,7 @@ const testSteps: Step[] = [ { name: "startup scan", args: ["run", "test:e2e:obsidian:startup-scan"] }, { name: "provisioned Setup URI workflow", args: ["run", "test:e2e:obsidian:setup-uri-workflow"] }, { name: "two-vault synchronisation", args: ["run", "test:e2e:obsidian:two-vault-sync"] }, + { name: "SyncGuard catch-up protection", args: ["run", "test:e2e:obsidian:sync-guard"] }, { name: "hidden file selectors", args: ["run", "test:e2e:obsidian:hidden-file-selectors"] }, { name: "hidden file snippet synchronisation", args: ["run", "test:e2e:obsidian:hidden-file-snippet-sync"] }, { name: "Customisation Sync", args: ["run", "test:e2e:obsidian:customisation-sync"] }, diff --git a/test/e2e-obsidian/scripts/log-lifecycle.ts b/test/e2e-obsidian/scripts/log-lifecycle.ts new file mode 100644 index 00000000..a126da2d --- /dev/null +++ b/test/e2e-obsidian/scripts/log-lifecycle.ts @@ -0,0 +1,373 @@ +import { createHash, randomUUID } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { LOG_LEVEL_URGENT } from "octagonal-wheels/common/logger"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { requireObsidianBinary, requireObsidianCli } from "../runner/environment.ts"; +import { + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + waitForLiveSyncCoreReady, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; + +process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ??= "60000"; + +type FixtureLog = { + message: string; + level?: number; + key?: string; +}; + +type ShutdownObservation = { + closeCalls: number; + disposeCalls: number; + disposeCompleted: boolean; + fixtureLogs: FixtureLog[]; + events: string[]; +}; + +async function evaluate(cliBinary: string, session: ObsidianLiveSyncSession, body: string): Promise { + return await evalObsidianJson( + cliBinary, + `(async()=>{ + const require=(condition,message)=>{if(!condition)throw new Error(message);}; + const waitFor=async(predicate,message)=>{ + const until=Date.now()+15000; + while(!predicate()) { + if(Date.now()>until)throw new Error(message); + await new Promise(resolve=>setTimeout(resolve,20)); + } + }; + ${body} + })()`, + session.cliEnv + ); +} + +function createPluginData(dbName: string): Record { + return createE2eCouchDbPluginData( + { uri: "http://127.0.0.1:1", username: "unused", password: "unused", dbName }, + { + syncInternalFiles: true, + usePluginSync: true, + usePluginSyncV2: true, + deviceAndVaultName: "log-lifecycle-device", + autoSweepPlugins: false, + autoSweepPluginsPeriodic: false, + syncInternalFilesInterval: 0, + liveSync: false, + periodicReplication: false, + syncOnStart: false, + syncOnSave: false, + syncOnFileOpen: false, + syncOnEditorSave: false, + syncAfterMerge: false, + showVerboseLog: true, + } + ); +} + +async function startSession( + binary: string, + cliBinary: string, + vault: TemporaryVault +): Promise { + const session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + pluginData: createPluginData(`log-lifecycle-${vault.name}`), + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + lifecycle: { + beforeLaunch: async () => { + const registryPath = join(vault.userDataPath, "obsidian.json"); + const registry = JSON.parse(await readFile(registryPath, "utf8")); + registry.updateDisabled = true; + await writeFile(registryPath, JSON.stringify(registry)); + }, + }, + }); + return session; +} + +function createShutdownScript(options: { + duringMarker: string; + failureMarker: string; + reloadMarker: string; + injectFailure: boolean; + urgentLevel: number; +}): string { + const fixtures = JSON.stringify(options); + return ` + const fixtures=${fixtures}; + const plugin=app.plugins.plugins["obsidian-livesync"]; + const core=plugin?.core; + require(core,"The LiveSync plug-in did not expose its core"); + const module=core.modules.find(candidate=>candidate.constructor.name==="ModuleLog"); + require(module,"The loaded core did not contain ModuleLog"); + const api=core.services.API; + const database=core.services.database.localDatabaseDirect; + require(database,"The loaded core did not expose its local database"); + + let releaseCloseGate; + const closeGate=new Promise(resolve=>{releaseCloseGate=resolve;}); + const state={ + core, + api, + module, + fixtureLogs:[], + events:[], + closeStarted:false, + closeCalls:0, + disposeCalls:0, + disposeCompleted:false, + closeGateReleased:false, + disableError:"", + }; + app.__livesyncLogLifecycleState=state; + + const originalAddLog=module.__addLog; + module.__addLog=function(message,level,key) { + if(typeof message==="string" && [ + fixtures.duringMarker, + fixtures.failureMarker, + fixtures.reloadMarker + ].some(marker=>marker && message.includes(marker))) { + state.fixtureLogs.push({message,level,key}); + const marker=[fixtures.duringMarker,fixtures.failureMarker,fixtures.reloadMarker] + .find(candidate=>candidate && message.includes(candidate)); + state.events.push("log:"+marker); + } + return originalAddLog.apply(this,arguments); + }; + + const resource=module.logResource; + const originalDispose=resource.dispose; + resource.dispose=async function() { + state.disposeCalls++; + state.events.push("dispose-start"); + try { + return await originalDispose.apply(this,arguments); + } finally { + state.disposeCompleted=true; + state.events.push("dispose-complete"); + } + }; + + const originalClose=database.close; + database.close=async function() { + state.closeCalls++; + if(state.closeCalls===1) { + state.closeStarted=true; + state.events.push("close-start"); + await closeGate; + } + const result=await originalClose.apply(this,arguments); + if(state.closeCalls===1) { + state.events.push("close-finished"); + if(fixtures.injectFailure) { + state.events.push("close-failure-injected"); + throw new Error(fixtures.failureMarker); + } + } + return result; + }; + + state.restore=()=>{ + module.__addLog=originalAddLog; + resource.dispose=originalDispose; + database.close=originalClose; + }; + + const run=async()=>{ + state.disablePromise=app.plugins.disablePlugin("obsidian-livesync"); + state.disablePromise.catch(error=>{state.disableError=String(error);}); + await waitFor(()=>state.closeStarted,"The real local database close did not start"); + require(state.disposeCalls===0 && !state.disposeCompleted, + "The log resource was released before database shutdown completed"); + api.addLog(fixtures.duringMarker,undefined,"e2e-log-lifecycle-during-close"); + await waitFor( + ()=>state.fixtureLogs.some(entry=>entry.message.includes(fixtures.duringMarker)), + "ModuleLog did not receive the during-close log" + ); + require(state.disposeCalls===0,"The log resource was released while database close was gated"); + releaseCloseGate(); + state.closeGateReleased=true; + await waitFor(()=>state.disposeCompleted,"The log resource was not released after database close"); + + require(state.closeCalls===1,"The local database close was not observed exactly once"); + require(state.disposeCalls===1,"The log resource was not disposed exactly once"); + require(state.events.indexOf("close-finished")>=0 && + state.events.indexOf("close-finished")entry.message.includes(fixtures.failureMarker)) + : undefined; + if(fixtures.injectFailure) { + require(failedLog,"The injected post-close failure was not logged"); + require(failedLog.key==="shutdown","The shutdown failure log used the wrong key"); + require(failedLog.level===fixtures.urgentLevel,"The shutdown failure log was not urgent"); + require( + state.events.indexOf("close-finished")state.disposeCompleted,"Cleanup timed out waiting for log resource disposal"); + } catch {} + } + } + `; +} + +async function runCase(options: { + binary: string; + cliBinary: string; + caseName: string; + injectFailure: boolean; +}): Promise> { + const vault = await createTemporaryVault(`obsidian-log-lifecycle-${options.caseName}-`); + let session: ObsidianLiveSyncSession | undefined; + const markers = { + duringMarker: `log-lifecycle-${options.caseName}-during-${randomUUID()}`, + failureMarker: options.injectFailure + ? `log-lifecycle-${options.caseName}-injected-close-failure-${randomUUID()}` + : "", + reloadMarker: options.injectFailure ? "" : `log-lifecycle-${options.caseName}-reload-${randomUUID()}`, + injectFailure: options.injectFailure, + urgentLevel: LOG_LEVEL_URGENT, + }; + + try { + session = await startSession(options.binary, options.cliBinary, vault); + await waitForLiveSyncCoreReady(options.cliBinary, session.cliEnv); + const shutdown = await evaluate(options.cliBinary, session, createShutdownScript(markers)); + if (options.injectFailure) { + return { caseName: options.caseName, shutdown }; + } + + await evaluate( + options.cliBinary, + session, + `await app.plugins.enablePlugin("obsidian-livesync"); return JSON.stringify({enabled:true});` + ); + await waitForLiveSyncCoreReady(options.cliBinary, session.cliEnv); + const reload = await evaluate>( + options.cliBinary, + session, + ` + const state=app.__livesyncLogLifecycleState; + require(state,"The old core observation was not retained across plug-in reload"); + const plugin=app.plugins.plugins["obsidian-livesync"]; + const core=plugin?.core; + require(core && core!==state.core,"Enabling the plug-in did not create a new core"); + const module=core.modules.find(candidate=>candidate.constructor.name==="ModuleLog"); + require(module,"The reloaded core did not contain ModuleLog"); + const originalAddLog=module.__addLog; + const observed=[]; + module.__addLog=function(message,level,key) { + if(typeof message==="string" && message.includes(${JSON.stringify(markers.reloadMarker)})) { + observed.push({message,level,key}); + } + return originalAddLog.apply(this,arguments); + }; + try { + core.services.API.addLog(${JSON.stringify(markers.reloadMarker)},undefined,"e2e-log-lifecycle-reload"); + await waitFor(()=>observed.length>0,"The reloaded ModuleLog did not receive its API log"); + require(observed.length===1,"The reloaded ModuleLog received the fixture log more than once"); + require( + !state.fixtureLogs.some(entry=>entry.message.includes(${JSON.stringify(markers.reloadMarker)})), + "The old ModuleLog observer received the reloaded core's fixture log" + ); + let oldHandlerDetached=false; + try { + state.api.addLog("e2e-log-lifecycle-reloaded-old-api"); + } catch(error) { + oldHandlerDetached=String(error).includes("not assigned"); + } + require(oldHandlerDetached,"The old API log handler was reassigned after reload"); + return JSON.stringify({ + coreChanged:core!==state.core, + observedByNewModule:observed.length, + observedByOldModule:state.fixtureLogs.filter( + entry=>entry.message.includes(${JSON.stringify(markers.reloadMarker)}) + ).length + }); + } finally { + module.__addLog=originalAddLog; + state.restore(); + delete app.__livesyncLogLifecycleState; + } + ` + ); + return { caseName: options.caseName, shutdown, reload }; + } finally { + if (session) { + try { + await evaluate( + options.cliBinary, + session, + ` + const state=app.__livesyncLogLifecycleState; + state?.restore?.(); + delete app.__livesyncLogLifecycleState; + return JSON.stringify({cleaned:true}); + ` + ); + } catch {} + try { + await session.app.stop(); + } finally { + await vault.dispose(); + } + } else { + await vault.dispose(); + } + } +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cliBinary = requireObsidianCli(); + const bundleHash = createHash("sha256") + .update(new Uint8Array(await readFile("main.js"))) + .digest("hex"); + + const observations = [ + await runCase({ binary, cliBinary, caseName: "normal", injectFailure: false }), + await runCase({ binary, cliBinary, caseName: "controlled-failure", injectFailure: true }), + ]; + console.log(JSON.stringify({ ok: true, bundleHash, observations }, null, 2)); +} + +await main(); diff --git a/test/e2e-obsidian/scripts/run-focused.ts b/test/e2e-obsidian/scripts/run-focused.ts index ffb805ed..2bf37fdf 100644 --- a/test/e2e-obsidian/scripts/run-focused.ts +++ b/test/e2e-obsidian/scripts/run-focused.ts @@ -49,8 +49,10 @@ const focusedScenarios = new Set([ "customisation-sync", "customisation-catalogue-reload", "optional-file-save-completion", + "log-lifecycle", "livesync-batch", "received-change-readiness", + "sync-guard", "internal-metadata-doctor", "setting-markdown-export", "setting-markdown-roundtrip", diff --git a/test/e2e-obsidian/scripts/sync-guard.ts b/test/e2e-obsidian/scripts/sync-guard.ts new file mode 100644 index 00000000..b70fa36e --- /dev/null +++ b/test/e2e-obsidian/scripts/sync-guard.ts @@ -0,0 +1,498 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { evalObsidianJson } from "../runner/cli.ts"; +import { + assertCouchDbReachable, + createCouchDbDatabase, + deleteCouchDbDatabase, + loadCouchDbConfig, + makeUniqueDatabaseName, + waitForCouchDbDocs, + type CouchDbConfig, +} from "../runner/couchdb.ts"; +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { + assertEqual, + createE2eCouchDbPluginData, + createE2eObsidianDeviceLocalState, + prepareRemote, + pushLocalChanges, + waitForLiveSyncCoreReady, + waitForLocalDatabaseEntry, + type LocalDatabaseEntry, +} from "../runner/liveSyncWorkflow.ts"; +import { startObsidianLiveSyncSession, type ObsidianLiveSyncSession } from "../runner/session.ts"; +import { captureObsidianPage, withObsidianPage } from "../runner/ui.ts"; +import { createTemporaryVault, type TemporaryVault } from "../runner/vault.ts"; + +const observerKey = "__livesyncE2eSyncGuard"; +const minimumFiles = 3; +const notes = Array.from({ length: 6 }, (_, index) => ({ + path: `E2E/sync-guard/incoming-${index}.md`, + content: [ + `# Incoming SyncGuard note ${index}`, + "", + ...Array.from( + { length: 6 }, + (_, paragraph) => + `Paragraph ${paragraph} of note ${index} travels through real CouchDB Metadata and Chunks.` + ), + "", + ].join("\n"), +})); +const workspacePath = "local-workspace.md"; +const workspaceContent = "# Local workspace\n\nLocal work can continue after bypass.\n"; +const workspaceEdit = "Written after bypass.\n"; + +type GuardObservation = { + phase: string; + observedPhases: string[]; + finiteActivity: number; + forwardedIds: string[]; + transferStoppedAt: number | null; + reflectionHeld: boolean; + storageApplying: number; + parsing: number; + reflecting: number; + startError: string | null; + visible: boolean; + stage: string; + count: string; +}; + +async function readObservation(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + return await evalObsidianJson( + cliBinary, + [ + "(()=>{", + `const state=globalThis[${JSON.stringify(observerKey)}];`, + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "const overlay=document.querySelector('.sls-sync-guard');", + "return JSON.stringify({", + "phase:core.services.replicator.replicationStatics.value.couchDB?.phase??'idle',", + "observedPhases:state.observedPhases,finiteActivity:core.services.replicator.finiteReplicationActivityCount.value,", + "forwardedIds:[...state.forwardedIds],transferStoppedAt:state.transferStoppedAt,", + "reflectionHeld:state.reflectionHeld,storageApplying:core.services.replication.storageApplyingCount.value,", + "parsing:state.parsing,reflecting:state.reflecting,startError:state.startError,", + "visible:!!overlay&&overlay.getBoundingClientRect().width>0,", + "stage:overlay?.querySelector('.sls-sync-guard-stage')?.textContent??'',", + "count:overlay?.querySelector('.sls-sync-guard-count')?.textContent??'',", + "});})()", + ].join(""), + env + ); +} + +function assertCatchUpPending(state: GuardObservation): void { + assertEqual(state.observedPhases.includes("catch-up"), true, "The provider never entered CouchDB catch-up."); + assertEqual( + state.observedPhases.includes("continuous"), + false, + "Catch-up completed before the held receive callback." + ); + assertEqual(state.finiteActivity > 0, true, "The held receive callback lost its finite replication activity."); + assertEqual(state.parsing > 0, true, "The real receive callback was no longer pending."); + assertEqual( + state.phase === "catch-up" || state.phase === "idle", + true, + `Unexpected phase while the finite catch-up still owns its receive callback: ${state.phase}` + ); +} + +async function waitForObservation( + cliBinary: string, + env: NodeJS.ProcessEnv, + description: string, + predicate: (state: GuardObservation) => boolean +): Promise { + const deadline = Date.now() + Number(process.env.E2E_OBSIDIAN_REMOTE_ACTIVITY_TIMEOUT_MS ?? 30000); + let state: GuardObservation; + do { + state = await readObservation(cliBinary, env); + if (state.startError) throw new Error(`Continuous replication failed: ${state.startError}`); + if (predicate(state)) return state; + await new Promise((resolve) => setTimeout(resolve, 100)); + } while (Date.now() < deadline); + throw new Error(`Timed out waiting for ${description}: ${JSON.stringify(state)}`); +} + +async function installGates( + cliBinary: string, + env: NodeJS.ProcessEnv, + incomingEntries: LocalDatabaseEntry[] +): Promise { + await evalObsidianJson( + cliBinary, + [ + "(()=>{", + `const key=${JSON.stringify(observerKey)};`, + "if(globalThis[key]) throw new Error('SyncGuard gates are already installed');", + "const services=app.plugins.plugins['obsidian-livesync'].core.services;", + "const replication=services.replication;", + `const incomingIds=new Set(${JSON.stringify(incomingEntries.map((entry) => entry.id))});`, + `const heldPath=${JSON.stringify(notes[0].path)};`, + `const minimumFiles=${minimumFiles};`, + "const state={forwardedIds:new Set(),transferStoppedAt:null,reflectionHeld:false,", + "parsing:0,reflecting:0,startError:null,releases:[],restore:null,observedPhases:[]};", + "const observePhase=()=>{const phase=services.replicator.replicationStatics.value.couchDB?.phase??'idle';", + "if(state.observedPhases.at(-1)!==phase) state.observedPhases.push(phase);};", + "services.replicator.replicationStatics.onChanged(observePhase);observePhase();", + "const gate=()=>{let release;const promise=new Promise((resolve)=>{release=resolve;});", + "state.releases.push(release);return {promise,release};};", + "const below=gate(),threshold=gate(),reflection=gate();", + "state.releaseBelow=below.release;state.releaseThreshold=threshold.release;state.releaseReflection=reflection.release;", + "const originalParse=replication.parseSynchroniseResult;", + "const originalReflect=replication.processSynchroniseResult;", + // Forward only documents supplied by the real pull. Pausing its awaited + // callback retains the provider's own catch-up lifetime and counters. + "const parse=Object.assign(async(documents)=>{", + "state.parsing++;try{let accepted=true;for(const document of documents){", + "accepted=(await originalParse([document]))&&accepted;", + "if(!incomingIds.has(document._id)||state.forwardedIds.has(document._id)) continue;", + "state.forwardedIds.add(document._id);", + "if(state.forwardedIds.size===minimumFiles-1){state.transferStoppedAt=minimumFiles-1;await below.promise;state.transferStoppedAt=null;}", + "if(state.forwardedIds.size===minimumFiles){state.transferStoppedAt=minimumFiles;await threshold.promise;state.transferStoppedAt=null;}", + "}return accepted;}finally{state.parsing--;}", + "},originalParse);", + // Result processing has already incremented storageApplyingCount when + // this real reflection enters. The test never assigns queue counts. + "const reflect=Object.assign(async(entry)=>{", + "state.reflecting++;try{if(entry.path===heldPath){state.reflectionHeld=true;await reflection.promise;state.reflectionHeld=false;}", + "return await originalReflect(entry);}finally{state.reflecting--;}", + "},originalReflect);", + "replication.parseSynchroniseResult=parse;replication.processSynchroniseResult=reflect;", + "state.restore=()=>{", + "services.replicator.replicationStatics.offChanged(observePhase);", + "if(replication.parseSynchroniseResult===parse) replication.parseSynchroniseResult=originalParse;", + "if(replication.processSynchroniseResult===reflect) replication.processSynchroniseResult=originalReflect;", + "for(const release of state.releases) release();};", + "globalThis[key]=state;return JSON.stringify(true);})()", + ].join(""), + env + ); +} + +async function releaseGate( + cliBinary: string, + env: NodeJS.ProcessEnv, + gate: "Below" | "Threshold" | "Reflection" +): Promise { + await evalObsidianJson( + cliBinary, + `(()=>{globalThis[${JSON.stringify(observerKey)}].release${gate}();return JSON.stringify(true);})()`, + env + ); +} + +async function removeGates(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + await evalObsidianJson( + cliBinary, + `(()=>{globalThis[${JSON.stringify(observerKey)}]?.restore();return JSON.stringify(true);})()`, + env + ); + await waitForObservation( + cliBinary, + env, + "released receive and reflection hooks", + (state) => state.parsing === 0 && state.reflecting === 0 + ); + await evalObsidianJson( + cliBinary, + `(()=>{delete globalThis[${JSON.stringify(observerKey)}];return JSON.stringify(true);})()`, + env + ); +} + +async function startCatchUp(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + await evalObsidianJson( + cliBinary, + [ + "(async()=>{", + "const core=app.plugins.plugins['obsidian-livesync'].core;", + "await core.services.setting.setSyncGuardSetting('syncGuardEnabled',true);", + `await core.services.setting.setSyncGuardSetting('syncGuardMinimumFiles',${minimumFiles});`, + `const state=globalThis[${JSON.stringify(observerKey)}];`, + "state.startTask=(async()=>{", + "await core.services.setting.applyExternalSettings({liveSync:true},true);", + "await core.services.control.applySettings();", + "})().catch((error)=>{state.startError=String(error);});", + "return JSON.stringify(core.services.setting.syncGuardSettings);})()", + ].join(""), + env + ); +} + +async function assertVaultPathMissing(vault: TemporaryVault, path: string): Promise { + try { + await readFile(join(vault.path, path), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + throw new Error(`The held reflection wrote its Vault note before release: ${path}`); +} + +async function waitForVaultContent(vault: TemporaryVault, path: string, expected: string): Promise { + const deadline = Date.now() + Number(process.env.E2E_OBSIDIAN_FILE_TIMEOUT_MS ?? 15000); + let actual: string | null = null; + do { + try { + actual = await readFile(join(vault.path, path), "utf8"); + if (actual === expected) return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } while (Date.now() < deadline); + throw new Error(`Vault content did not converge for ${path}: ${JSON.stringify(actual)}`); +} + +async function assertIncomingContents(cliBinary: string, env: NodeJS.ProcessEnv, vault: TemporaryVault): Promise { + for (const note of notes) await waitForVaultContent(vault, note.path, note.content); + const contents = await evalObsidianJson( + cliBinary, + `(async()=>{const paths=${JSON.stringify(notes.map((note) => note.path))};return JSON.stringify(await Promise.all(paths.map(async(path)=>{const file=app.vault.getAbstractFileByPath(path);return file?await app.vault.read(file):null;})));})()`, + env + ); + notes.forEach((note, index) => + assertEqual(contents[index], note.content, `Obsidian read back incorrect content at ${note.path}.`) + ); +} + +async function createWorkspaceNote(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + await evalObsidianJson( + cliBinary, + `(async()=>{const file=await app.vault.create(${JSON.stringify(workspacePath)},${JSON.stringify(workspaceContent)});const leaf=app.workspace.getLeaf(false);await leaf.openFile(file,{state:{mode:'source'}});leaf.view.editor.focus();return JSON.stringify(true);})()`, + env + ); +} + +async function runTarget( + binary: string, + cliBinary: string, + couchDb: CouchDbConfig, + dbName: string, + incomingEntries: LocalDatabaseEntry[], + bypass: boolean +): Promise { + const label = bypass ? "bypass" : "automatic"; + const vault = await createTemporaryVault(`obsidian-livesync-sync-guard-${label}-`); + let session: ObsidianLiveSyncSession | undefined; + try { + session = await startObsidianLiveSyncSession({ + binary, + cliBinary, + vault, + pluginData: createE2eCouchDbPluginData({ ...couchDb, dbName }), + localStorageEntries: createE2eObsidianDeviceLocalState(vault.name), + }); + const { cliEnv } = session; + await waitForLiveSyncCoreReady(cliBinary, cliEnv); + await prepareRemote(cliBinary, cliEnv); + if (bypass) await createWorkspaceNote(cliBinary, cliEnv); + await installGates(cliBinary, cliEnv, incomingEntries); + await startCatchUp(cliBinary, cliEnv); + + const below = await waitForObservation( + cliBinary, + cliEnv, + "real incoming notes below the threshold", + (state) => state.transferStoppedAt === minimumFiles - 1 + ); + assertCatchUpPending(below); + assertEqual(below.forwardedIds.length, minimumFiles - 1, "Unexpected incoming note count below the threshold."); + assertEqual(below.visible, false, "SyncGuard blocked the workspace before the configured threshold."); + await releaseGate(cliBinary, cliEnv, "Below"); + + const threshold = await waitForObservation( + cliBinary, + cliEnv, + "SyncGuard at the configured threshold", + (state) => state.transferStoppedAt === minimumFiles && state.visible + ); + assertCatchUpPending(threshold); + assertEqual( + threshold.forwardedIds.length, + minimumFiles, + "SyncGuard did not activate at the configured threshold." + ); + assertEqual( + threshold.count, + `At least ${minimumFiles} changed files received`, + "SyncGuard showed an incorrect received-file count." + ); + assertEqual(threshold.stage, "Receiving changes from the server…", "SyncGuard did not show transfer progress."); + console.log( + await captureObsidianPage( + session.remoteDebuggingPort, + `sync-guard-${label}-threshold.png`, + async (page) => { + await page + .getByRole("button", { name: "Continue without waiting", exact: true }) + .waitFor({ state: "visible" }); + } + ) + ); + + if (bypass) { + const editorFocused = await evalObsidianJson( + cliBinary, + "(()=>{const editor=app.workspace.activeLeaf.view.editor;editor.focus();return JSON.stringify(editor.hasFocus());})()", + cliEnv + ); + assertEqual(editorFocused, true, "The guarded keyboard check did not focus the editor."); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.keyboard.type("This input must stay blocked."); + }); + const guardedContent = await evalObsidianJson( + cliBinary, + "(()=>JSON.stringify(app.workspace.activeLeaf.view.editor.getValue()))()", + cliEnv + ); + assertEqual(guardedContent, workspaceContent, "The protected editor accepted keyboard input."); + await withObsidianPage(session.remoteDebuggingPort, async (page) => { + await page.getByRole("button", { name: "Continue without waiting", exact: true }).click(); + await page.locator(".sls-sync-guard").waitFor({ state: "hidden" }); + const editor = page.locator(".workspace-leaf.mod-active .cm-content"); + await editor.click(); + await page.keyboard.press("ControlOrMeta+End"); + await page.keyboard.type(workspaceEdit); + }); + await waitForVaultContent(vault, workspacePath, workspaceContent + workspaceEdit); + const continuing = await readObservation(cliBinary, cliEnv); + assertCatchUpPending(continuing); + assertEqual( + continuing.transferStoppedAt, + minimumFiles, + "The receive callback was not pending during bypass." + ); + assertEqual(continuing.visible, false, "The bypass did not restore the workspace."); + await releaseGate(cliBinary, cliEnv, "Threshold"); + await waitForObservation( + cliBinary, + cliEnv, + "continuous replication after bypass", + (state) => state.phase === "continuous" && state.reflectionHeld + ); + await assertVaultPathMissing(vault, notes[0].path); + assertEqual( + (await readObservation(cliBinary, cliEnv)).visible, + false, + "The guard reopened during the bypassed catch-up." + ); + } else { + await releaseGate(cliBinary, cliEnv, "Threshold"); + const applying = await waitForObservation( + cliBinary, + cliEnv, + "network completion with a held Vault reflection", + (state) => state.phase === "continuous" && state.reflectionHeld && state.storageApplying > 0 + ); + assertEqual( + applying.forwardedIds.length, + notes.length, + "The real catch-up did not deliver every incoming note." + ); + assertEqual(applying.visible, true, "SyncGuard released before the held Vault reflection finished."); + assertEqual( + applying.stage, + "Applying received changes to local notes…", + "SyncGuard did not show local application progress." + ); + await assertVaultPathMissing(vault, notes[0].path); + // Exceed the guard's ordinary quiet period while real application work remains pending. + await new Promise((resolve) => setTimeout(resolve, 2500)); + const stillApplying = await readObservation(cliBinary, cliEnv); + assertEqual(stillApplying.visible, true, "SyncGuard released while real reflection was still pending."); + assertEqual(stillApplying.reflectionHeld, true, "The reflection gate was released unexpectedly."); + console.log( + await captureObsidianPage(session.remoteDebuggingPort, "sync-guard-applying.png", async (page) => { + await page.locator(".sls-sync-guard").waitFor({ state: "visible" }); + }) + ); + } + + await releaseGate(cliBinary, cliEnv, "Reflection"); + await assertIncomingContents(cliBinary, cliEnv, vault); + await waitForObservation( + cliBinary, + cliEnv, + "workspace release after exact Vault reflection", + (state) => !state.visible && state.phase === "continuous" && state.storageApplying === 0 + ); + if (bypass) await waitForVaultContent(vault, workspacePath, workspaceContent + workspaceEdit); + console.log( + `SyncGuard ${label}: threshold ${minimumFiles}, ${notes.length} exact incoming notes, workspace released.` + ); + } catch (error) { + if (session) { + await captureObsidianPage( + session.remoteDebuggingPort, + `sync-guard-${label}-failure.png`, + async () => {} + ).catch(() => undefined); + const observation = await readObservation(cliBinary, session.cliEnv).catch(() => null); + console.error(`SyncGuard ${label} failure state: ${JSON.stringify(observation)}`); + } + throw error; + } finally { + if (session) { + await removeGates(cliBinary, session.cliEnv).catch((error: unknown) => { + console.warn(`SyncGuard hook cleanup: ${error instanceof Error ? error.message : String(error)}`); + }); + await session.app.stop(); + } + await vault.dispose(); + } +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); + const couchDb = await loadCouchDbConfig(); + const dbName = makeUniqueDatabaseName(couchDb.dbPrefix, "sync-guard"); + const sourceVault = await createTemporaryVault("obsidian-livesync-sync-guard-source-"); + let source: ObsidianLiveSyncSession | undefined; + try { + await assertCouchDbReachable(couchDb); + await createCouchDbDatabase(couchDb, dbName); + source = await startObsidianLiveSyncSession({ + binary, + cliBinary: cli.binary, + vault: sourceVault, + pluginData: createE2eCouchDbPluginData({ ...couchDb, dbName }), + localStorageEntries: createE2eObsidianDeviceLocalState(sourceVault.name), + }); + await waitForLiveSyncCoreReady(cli.binary, source.cliEnv); + await prepareRemote(cli.binary, source.cliEnv); + await evalObsidianJson( + cli.binary, + `(async()=>{const notes=${JSON.stringify(notes)};await app.vault.createFolder('E2E');await app.vault.createFolder('E2E/sync-guard');for(const note of notes){const file=await app.vault.create(note.path,note.content);if(await app.vault.read(file)!==note.content) throw new Error('Source note read-back failed');}return JSON.stringify(true);})()`, + source.cliEnv + ); + const entries: LocalDatabaseEntry[] = []; + for (const note of notes) entries.push(await waitForLocalDatabaseEntry(cli.binary, source.cliEnv, note.path)); + if (entries.some((entry) => entry.children.length === 0)) + throw new Error("Source notes did not produce real Chunks."); + await pushLocalChanges(cli.binary, source.cliEnv); + await waitForCouchDbDocs(couchDb, dbName, (documents) => { + const ids = new Set(documents.map((document) => document._id)); + return entries.every((entry) => ids.has(entry.id) && entry.children.every((id) => ids.has(id))); + }); + await source.app.stop(); + source = undefined; + + await runTarget(binary, cli.binary, couchDb, dbName, entries, false); + await runTarget(binary, cli.binary, couchDb, dbName, entries, true); + } finally { + if (source) await source.app.stop(); + await sourceVault.dispose(); + if (process.env.E2E_OBSIDIAN_KEEP_COUCHDB !== "true") await deleteCouchDbDatabase(couchDb, dbName); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exit(1); +}); diff --git a/tsconfig.json b/tsconfig.json index 4e159699..e01d0f5e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,6 +30,7 @@ "src/**/*.test.ts", "**/_test/**", "test/browser-apps", + "test/e2e-cli", "test/styles/*.deno.ts", "utilsdeno" ] diff --git a/updates.md b/updates.md index 0165c12a..fea046f5 100644 --- a/updates.md +++ b/updates.md @@ -12,27 +12,135 @@ Earlier releases remain available in the 1.0 release history, the 1.0 preview hi ## Unreleased -### Fixes +### Fixed -- A new Join or scheduled Fetch asks for data-handling choices again, even after a previous Fetch changed the local database suffix. - - Fast Setup retains interrupted choices in an editable Markdown code block in the fetch flag. On restart, we confirm the current remote and saved choices, choose again, or cancel; a remaining flag never reapplies choices silently. - - If the saved selection cannot be read, we can choose new options for that attempt without repairing or rewriting the flag. Completing or cancelling the operation removes the whole flag. - - After updating during an interrupted Fetch, confirm the choices once more. Older separately stored choices are ignored; downloaded data and the existing checkpoint are not removed by this change. - -### Synchronisation - -- Added an optional, device-local editing guard for substantial CouchDB LiveSync catch-up transfers. When enough incoming notes are detected, it covers the workspace until the transfer and local application queues have settled. The overlay can always be bypassed, and ordinary small transfers remain uninterrupted. - -- We can see CouchDB change-feed comparison progress separately for push and pull. - - The `⏩↑` and `⏩↓` counts show entries compared in each direction; the upload and download totals remain separate. - - Existing CSS snippets which target `livesync-stat-ff` to hide comparison progress need to target both `livesync-stat-ff-push` and `livesync-stat-ff-pull`. See the [status display guide](docs/tips/status-display.md). - -### Translations +#### Translations - The notice shown when a translation is available now displays the **Display Language** setting label correctly in English, French, Hebrew, Russian, and Simplified Chinese. (#1298) Thank you to @cwatanab for the contribution! - Hebrew interface messages also display their referenced labels correctly. -- Simplified Chinese translations have been expanded across settings, setup, recovery, and configuration messages. (#1258) Thank you to @kannazuki-c for the contribution! +## 1.0.44 + +1st November, 2026 + +Recently, I have been concentrating on maintenance and improvements to individual features. I began to wonder whether I had been focusing a little too closely on the details and could give more attention to the overall picture. That led me to a somewhat broader round of changes, including reworking parts of the architecture. Along the way, I was able to bring several implementations of the same concepts together, introduce shared abstractions, and tidy up the codebase a little. + +Please let me know if any problems arise from these changes. + +### Fixed + +#### Synchronisation + +- Files containing a raw unit separator (`U+001F`) are no longer truncated when received through S3-compatible Object Storage. (#1303) + - Update every participating client to prevent further truncation. Updating alone does not repair copies already damaged. + - Before recovery, preserve verified intact copies, unique unsynchronised edits, and any required history, then follow the [recovery procedure](docs/recovery.md#recover-files-truncated-during-object-storage-reception). + - Rebuild and Fetch leave Hidden File Sync and Customisation Sync disabled. Initialise them deliberately after ordinary file recovery is complete. + +- We can continue CouchDB synchronisation after rebuilding an encrypted remote when **Use Internal API** is enabled. (#1234) + - Cached responses encrypted with the previous Security Seed no longer interfere with subsequent reads. + +#### Conflict resolution + +- A queued conflict check no longer gets stuck waiting for a response to an outdated comparison of the same file. + +#### Hidden File Sync and Customisation Sync + +- Newer local hidden files are no longer omitted when initialising Hidden File Sync with `Merge`. +- Refreshing the Customisation Sync catalogue no longer brings back outdated items from an earlier scan. + +### Improved + +#### Hidden File Sync + +- We can review the latest hidden-file revisions when changes arrive during a manual JSON comparison. + - The earlier comparison closes, and the latest revisions are checked again. If another device has resolved the conflict, the comparison closes without asking for another selection. + +### Sunset + +#### Recovery + +- Recovery from old cleanup locks is now unified around the standard recovery dialogue, including on devices using the legacy IndexedDB adapter. + - Use Fetch to reconstruct the local database from the remote, unlock the device when both databases are already aligned, or dismiss the dialogue. + - These choices replace the legacy in-place `Cleanup` operation. Garbage Collection V3 remains available. + - Before Fetch, preserve unsynchronised work and any required local history, as it replaces the local database. + +### CLI + +#### Fixed + +- We can continue using the reconstructed database after CLI `fetch` or `rebuild`, avoiding the empty-database restart and file deletion reported in #1304. + - Subsequent commands retain the selected database even without `--write-settings`. Interrupted reconstruction blocks ordinary synchronisation until recovery. + - Older runtime state with no recorded owner requires explicit reconstruction from a verified source. Back up unique edits before choosing Fetch or Rebuild. + +- The CLI mirror starts correctly on Windows when the Vault path is omitted and the database directory is also used as the Vault. + +### Miscellaneous + +- Installation and update downloads are smaller. +- Database initialisation handles encrypted change notifications more reliably. (Commonlib #191) + +### Acknowledgements + +Thank you for your contributions! + +- @AdrianNicu, for improving decryption handling during database initialisation (Commonlib #191). + +## 1.0.43 + +30th October, 2026 + +### Synchronisation + +#### New Feature + +- We can avoid accidentally editing out-of-date notes while a large CouchDB catch-up is in progress. (#1241, #1275) + - Turn on **Enable SyncGuard** under **General Settings** → **Appearance** to cover the workspace until incoming changes have been received and applied to the Vault. + - Disabled by default. During CouchDB LiveSync mode start-up or resume, protection begins after a configurable number of distinct incoming notes is detected (default: 20). + - **Continue without waiting** releases the workspace, including after a failed or interrupted transfer. Synchronisation can continue, and some notes may still be out of date. + - The settings apply only to this device. + - Small catch-up transfers and ordinary live changes remain uninterrupted. Some changes can arrive before the guard activates; it does not prevent other plug-ins or external tools from editing files. + +#### Improved + +- We can tell that CouchDB synchronisation is progressing while it compares existing revisions, even before any files are uploaded. (#1293, #1294) + - Separate `⏩↑` and `⏩↓` counts show the change-feed entries compared for push and pull. These are comparison counts; the upload and download totals remain separate. + - Existing CSS snippets which target `livesync-stat-ff` to hide comparison progress need to target both `livesync-stat-ff-push` and `livesync-stat-ff-pull`. The status display guide includes updated examples. + +### Setup and recovery + +#### Fixed + +- We can choose how a new Join or scheduled Fetch handles our existing files, without silently reusing choices from an earlier Fetch. (#1244, #1296) + - Each new operation asks again, including after a previous Fetch changed the local database suffix. Fast Setup retains interrupted choices in an editable code block in the fetch flag. + - On restart, we confirm the current remote and saved choices, choose again, or cancel; a remaining flag never reapplies choices silently. + - If the saved selection cannot be read, we can choose new options for that attempt without repairing or rewriting the flag. Completing or cancelling the operation removes the whole flag. + - After updating during an interrupted Fetch, confirm the choices once more. Older separately stored choices are ignored; downloaded data and the existing checkpoint are not removed by this change. + +### Dialogues + +#### Fixed + +- Confirmation and setup actions are easier to follow, and keyboard focus remains visible. (#1297) + - Buttons align with the content above them, with room for confirmation buttons' focus outlines. +- Mobile dialogues retain Obsidian's standard spacing and leave more usable space above the software keyboard. (#1297) + +### Translations + +- More settings, setup, recovery, and configuration messages are available in Simplified Chinese. (#1258) + - Expanded the translation catalogue to include previously untranslated messages and shared-library labels. + +### Miscellaneous + +- Docker setup now handles shell-sensitive characters in health-check credentials correctly, avoiding false authentication failures which could prevent CouchDB initialisation. (#1295) + +### Acknowledgements + +Thank you for your contributions! + +- @Akilaydin, for adding the optional SyncGuard workspace protection (#1275). +- @ARCJ137442, for adding separate push and pull comparison progress displays (#1294). +- @Jovlang, for correcting Docker health-check credential quoting and authenticated readiness checks (#1295). +- @kannazuki-c, for expanding the Simplified Chinese translations (#1258). ## 1.0.42 @@ -172,87 +280,3 @@ I have been finding troubleshooting difficult, so I decided to go back to basics - The five reporting tools are grouped separately from checks, recovery, and reset operations. Name and format choices remain visible, and the report preview resizes around the controls. - We can request and provide targeted troubleshooting evidence through the issue-reporting guide. - Direct links cover Integrity, Usage, pseudonym lookup, and follow-up reports, with request examples for issue discussions. - -## 1.0.38 - -17th October, 2026 - -### Synchronisation - -#### Added - -- We can opt into NFC path identity for CouchDB synchronisation between the CLI and Obsidian. This prevents decomposed or mixed Unicode file names from becoming inaccessible through a receiving device's normalised path. (#1259) - - Existing Vaults retain their current identity rules. Migration requires updated clients, a Rebuild from a complete authoritative Vault, and a Fetch on the other devices. - - Reconstruction stops when the supported host file listing or local Metadata exposes ambiguous files or parent folders under NFC and the configured case handling. The CLI retains existing physical file names. P2P and Object Storage do not support the new mode. - -#### Fixed - -- We can synchronise file deletions and renames when **Do not keep metadata of deleted files.** is enabled. - - A file restored during the final deletion check remains in the receiving Vault. -- We can resolve configuration mismatches when **Encrypt internal file Properties** is enabled but its encryption prerequisites are inactive. (#1180) - - Compatible chunk settings can be aligned automatically, or reviewed through `Sync now`. Applying those settings preserves our inactive encryption preference. -- We can receive synchronised updates to unchanged files even after the local database has discarded their recorded revision, avoiding unnecessary conflicts. (#1207) - - Saved content evidence survives revision-history pruning. Genuine local edits remain protected, and older records are upgraded only after their content is verified. - -### Maintenance - -- The Review Harness can compare SHA-256 and xxHash64 file-content hashing on the current device, with fixed inputs from 4 KiB to 50 MiB and separate initialisation and warm-up measurements. - -## 1.0.37 - -16th October, 2026 - -### Security - -- Fixed an issue where synchronised changes could reach files outside the intended synchronisation scope under certain conditions. Updating is recommended. - -### CLI - -#### Fixed - -- We can edit files through the CLI without resetting their creation time where the filesystem supplies it. (#1243) - - Filesystems without birth-time support retain the existing change-time fallback. File replacement and restoration of a synchronised creation time remain subject to filesystem limitations. -- `p2p-host` restores configured P2P automatic synchronisation, following, and change announcements after initialisation. (#1247) - - Headless peer acceptance follows the configured policy. Single-operation commands retain their automatic synchronisation suspension. -- We can select CLI-only Docker patches by their full release tag, such as `1.0.36-patch1-cli`, while keeping the Obsidian plug-in version. - - Patch and RC images receive version and SHA-qualified tags. The `latest` and major-minor tags remain reserved for ordinary stable releases. - -### Acknowledgements - -Thank you for your contributions! - -- @kimjansheden, for reporting the issue and contributing the fix and regression tests. - -## 1.0.36 - -13th October, 2026 - -### Configuration Encryption - -#### Fixed - -- We can change the protection of sensitive configuration with Apply and keep our saved settings readable after a restart. (#1238, #1245) - - The selected protection mode and passphrase are saved together. If Apply fails, our edits remain available to retry. -- We can enter a missing or unusable custom configuration passphrase at start-up to unlock our protected settings. (#1239, #1245) - - Cancelling the recovery dialogue preserves the encrypted configuration and stops that start-up attempt. Headless command-line start-up cannot complete the dialogue when recovery input is required. - -### Localisation - -#### Fixed - -- We can read more settings, maintenance controls, and Fast Setup instructions in Spanish. (#1213, #1214) - - Remote database lock warnings now use the existing translations consistently. - - Fast Setup follows the selected display language while preserving saved choices across restarts and language changes. -- We can read more settings, setup instructions, and messages in Simplified Chinese. (#1242) - -### Maintenance - -- Development tooling now uses compatible dependency versions with available security fixes. (#1245) - - Repository-owned build and import alias helpers remove the vulnerable `braces` dependency. - -### Acknowledgements - -Thank you for your contributions! - -- @zeedif (#1213, #1214) -- @lingting0 (#1242) diff --git a/utils/ci-pages.unit.spec.ts b/utils/ci-pages.unit.spec.ts index fbece795..38853f6c 100644 --- a/utils/ci-pages.unit.spec.ts +++ b/utils/ci-pages.unit.spec.ts @@ -57,7 +57,7 @@ describe("CI workflow path filters", () => { "utils/ci-pages.unit.spec.ts", "utils/cli-ci.unit.spec.ts", ".github/workflows/cli-docker.yml", - ".github/workflows/cli-deno-tests.yml", + ".github/workflows/cli-node-tests.yml", ".github/workflows/cli-p2p-tests.yml", ".github/workflows/cli-docker-validation.yml", ]; diff --git a/utils/cli-ci.unit.spec.ts b/utils/cli-ci.unit.spec.ts index 500de60e..dd60faf0 100644 --- a/utils/cli-ci.unit.spec.ts +++ b/utils/cli-ci.unit.spec.ts @@ -19,7 +19,8 @@ type Workflow = { if?: string; permissions?: Record; "runs-on"?: string; - strategy?: { matrix?: { include?: Array<{ arch: string; runner: string }> } }; + strategy?: { matrix?: { include?: Array<{ arch: string; runner: string }>; os?: string[] } }; + services?: Record; } >; }; @@ -35,12 +36,12 @@ function matches(paths: string[], file: string): boolean { return included; } function localTasks(): string[] { - const runner = readFileSync(new URL("src/apps/cli/testdeno/run-ci-suite.ts", root), "utf8"); + const runner = readFileSync(new URL("test/e2e-cli/run-ci-suite.ts", root), "utf8"); expect(runner).toContain('import TASKS from "./ci-tasks.json" with { type: "json" }'); - return JSON.parse(readFileSync(new URL("src/apps/cli/testdeno/ci-tasks.json", root), "utf8")) as string[]; + return JSON.parse(readFileSync(new URL("test/e2e-cli/ci-tasks.json", root), "utf8")) as string[]; } function selectedTasks(task: string): string[] { - const select = workflow("cli-deno-tests.yml").jobs.prepare.steps?.find((step) => step.id === "select"); + const select = workflow("cli-node-tests.yml").jobs.prepare.steps?.find((step) => step.id === "select"); expect(select?.run).toBeDefined(); const directory = mkdtempSync(join(tmpdir(), "livesync-ci-matrix-")); const output = join(directory, "output"); @@ -67,19 +68,20 @@ describe("CLI CI selection", () => { expect(selectedTasks("test:ci")).toEqual( expect.arrayContaining(["test:settings-writeback", "test:remote-administration-exit-codes"]) ); - const tasks = JSON.parse(readFileSync(new URL("src/apps/cli/testdeno/deno.json", root), "utf8")).tasks; - expect(localTasks().filter((task) => !(task in tasks))).toEqual([]); + const tasks = JSON.parse(readFileSync(new URL("test/e2e-cli/tasks.json", root), "utf8")); + expect(localTasks().filter((task) => !Object.hasOwn(tasks, task))).toEqual([]); + expect(tasks["test:local"]).toEqual({ tasks: ["test:setup-put-cat", "test:mirror", "test:daemon"] }); expect(new Set(localTasks()).size).toBe(localTasks().length); }); it("keeps the selected manual subsets", () => { - expect(selectedTasks("test:local")).toEqual(["test:setup-put-cat", "test:mirror", "test:daemon"]); + expect(selectedTasks("test:local")).toEqual(["test:local"]); expect(selectedTasks("test:e2e-matrix")).toEqual( localTasks().filter((task) => task.startsWith("test:e2e-matrix:")) ); }); it.each([ - ["cli-deno-tests.yml", "push"], - ["cli-deno-tests.yml", "pull_request"], + ["cli-node-tests.yml", "push"], + ["cli-node-tests.yml", "pull_request"], ["cli-p2p-tests.yml", "push"], ["cli-p2p-tests.yml", "pull_request"], ["cli-docker.yml", "push"], @@ -105,16 +107,72 @@ describe("CLI CI selection", () => { expect(inputs.filter((file) => !matches(paths, file))).toEqual([]); expect(matches(paths, "src/modules/features/ModuleObsidianSettingTab.ts")).toBe(false); expect(matches(paths, "docs/settings.md")).toBe(false); - expect(matches(paths, "src/apps/cli/testdeno/CONTRIBUTING_TESTS.md")).toBe(false); }); + it.each([ + ["cli-node-tests.yml", "push"], + ["cli-node-tests.yml", "pull_request"], + ["cli-p2p-tests.yml", "push"], + ["cli-p2p-tests.yml", "pull_request"], + ])("runs CLI E2E workflows for suite changes and skips suite documentation in %s on %s", (name, event) => { + const paths = workflow(name).on[event].paths ?? []; + expect(matches(paths, "test/e2e-cli/test-setup-put-cat.ts")).toBe(true); + expect(matches(paths, "test/e2e-cli/CONTRIBUTING_TESTS.md")).toBe(false); + }); + it("runs Compose alone for a change to its fixture", () => { const file = "test/bench-network/compose.yml"; expect(matches(workflow("cli-p2p-tests.yml").on.pull_request.paths ?? [], file)).toBe(true); - expect(matches(workflow("cli-deno-tests.yml").on.pull_request.paths ?? [], file)).toBe(false); + expect(matches(workflow("cli-node-tests.yml").on.pull_request.paths ?? [], file)).toBe(false); expect(matches(workflow("cli-docker-validation.yml").on.pull_request.paths ?? [], file)).toBe(false); }); + it("uses Node 26 for CLI tests and runs local smoke tests on three platforms without Docker", () => { + const { prepare, test, "local-smoke": localSmoke } = workflow("cli-node-tests.yml").jobs; + for (const job of [prepare, test, localSmoke]) { + const setupNode = job.steps?.find((step) => step.uses?.startsWith("actions/setup-node@")); + expect(setupNode?.with?.["node-version"]).toBe("26.x"); + expect(job.steps?.some((step) => step.uses?.startsWith("denoland/setup-deno@"))).toBe(false); + } + expect(localSmoke.if).toBe( + "${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && (github.event_name != 'workflow_dispatch' || inputs.test_task == 'test:local') }}" + ); + expect(localSmoke.strategy?.matrix?.os).toEqual(["ubuntu-latest", "macos-latest", "windows-latest"]); + expect(localSmoke["runs-on"]).toBe("${{ matrix.os }}"); + expect(localSmoke.services).toBeUndefined(); + expect(localSmoke.steps?.some((step) => step.run?.includes("docker"))).toBe(false); + expect(localSmoke.steps?.filter((step) => step.run?.includes("npm run build --workspace self-hosted-livesync-cli"))).toHaveLength(1); + expect(localSmoke.steps?.find((step) => step.run?.includes("test:local"))?.run).toContain("node ../../../test/e2e-cli/run-task.ts test:local"); + expect(test.steps?.find((step) => step.run?.includes("node run-task.ts"))?.run).toContain('node run-task.ts "$TASK"'); + }); + + it("collects local smoke coverage once on Linux and retains its reports for seven days", () => { + const steps = workflow("cli-node-tests.yml").jobs["local-smoke"].steps ?? []; + const coverage = steps.filter((step) => step.run?.includes("test:e2e:coverage")); + expect(coverage).toHaveLength(1); + expect(coverage[0].run).toBe("npm run test:e2e:coverage --workspace self-hosted-livesync-cli"); + expect(coverage[0].if).toBe("${{ matrix.os == 'ubuntu-latest' }}"); + for (const step of steps.filter( + (step) => step.run?.includes("npm run build --workspace") || step.run?.includes("run-task.ts test:local") + )) { + expect(step.if).toBe("${{ matrix.os != 'ubuntu-latest' }}"); + } + const uploads = steps.filter((step) => step.uses?.startsWith("actions/upload-artifact@")); + expect(uploads).toHaveLength(1); + const upload = uploads[0]; + expect(upload.if).toBe("${{ always() && matrix.os == 'ubuntu-latest' }}"); + expect(upload.with?.name).toBe("cli-e2e-local-coverage"); + expect(upload.with?.["retention-days"]).toBe(7); + expect(upload.with?.["if-no-files-found"]).toBe("warn"); + const paths = String(upload.with?.path).trim().split("\n"); + expect(matches(paths, "coverage/cli-e2e/index.html")).toBe(true); + expect(matches(paths, "coverage/cli-e2e/src/apps/cli/main.ts.html")).toBe(true); + expect(matches(paths, "coverage/cli-e2e/base.css")).toBe(true); + expect(matches(paths, "coverage/cli-e2e/coverage-final.json")).toBe(true); + expect(matches(paths, "coverage/cli-e2e/raw/coverage-123.json")).toBe(false); + expect(matches(paths, "coverage/unit/index.html")).toBe(false); + }); + it("passes a single CLI build to every standard test job", () => { - const { prepare, test } = workflow("cli-deno-tests.yml").jobs; + const { prepare, test } = workflow("cli-node-tests.yml").jobs; expect(test.needs).toBe("prepare"); expect(prepare.steps?.filter((step) => step.run === "npm run build")).toHaveLength(1); expect(test.steps?.some((step) => step.run?.includes("npm run build"))).toBe(false); diff --git a/utilsdeno/README.md b/utilsdeno/README.md index d0b2a76e..d7d358c7 100644 --- a/utilsdeno/README.md +++ b/utilsdeno/README.md @@ -95,6 +95,6 @@ Ensures that internal plug-in import statements are standardised to the `@/` ali ## Safety and Exclusions -* **Tests Excluded**: All scripts automatically skip files located in `_test/` or `testdeno/` folders, as well as files ending with `.spec.ts` or `.test.ts`. +* **Source Scope and Tests**: `refactor-styles.ts` and `refactor-globals.ts` process files only inside the project `src/` directory. They skip files in `_test/` folders and files ending with `.spec.ts` or `.test.ts`. The CLI E2E suite is under `test/e2e-cli/`, outside that source scope. * **Package Boundary**: These tools operate on this repository only. Changes to Commonlib belong in its own repository and must be validated with its package checks. * **Verification**: Always run `npm run check` and `npm run test:unit` after performing refactoring tasks to verify that type safety and tests remain intact. diff --git a/utilsdeno/refactor-globals.ts b/utilsdeno/refactor-globals.ts index ea54a1f4..870fc56b 100644 --- a/utilsdeno/refactor-globals.ts +++ b/utilsdeno/refactor-globals.ts @@ -67,8 +67,7 @@ for (const sourceFile of project.getSourceFiles()) { if ( posixFilePath.endsWith(".spec.ts") || posixFilePath.endsWith(".test.ts") || - posixFilePath.includes("/_test/") || - posixFilePath.includes("/testdeno/") + posixFilePath.includes("/_test/") ) { continue; } diff --git a/utilsdeno/refactor-styles.ts b/utilsdeno/refactor-styles.ts index 95f46d82..35d14685 100644 --- a/utilsdeno/refactor-styles.ts +++ b/utilsdeno/refactor-styles.ts @@ -102,8 +102,7 @@ for (const sourceFile of project.getSourceFiles()) { if ( posixFilePath.endsWith(".spec.ts") || posixFilePath.endsWith(".test.ts") || - posixFilePath.includes("/_test/") || - posixFilePath.includes("/testdeno/") + posixFilePath.includes("/_test/") ) { continue; } diff --git a/versions.json b/versions.json index 8ad7a151..669aeb93 100644 --- a/versions.json +++ b/versions.json @@ -53,5 +53,7 @@ "1.0.39": "1.7.2", "1.0.40": "1.7.2", "1.0.41": "1.7.2", - "1.0.42": "1.7.2" + "1.0.42": "1.7.2", + "1.0.43": "1.7.2", + "1.0.44": "1.7.2" } diff --git a/vitest.config.integration.ts b/vitest.config.integration.ts index 6ad9e109..f5c92ba7 100644 --- a/vitest.config.integration.ts +++ b/vitest.config.integration.ts @@ -19,7 +19,7 @@ export default mergeConfig( logHeapUsage: true, name: "integration-tests", include: ["**/*.integration.spec.ts", "**/*.integration.test.ts"], - exclude: ["test/**", "src/apps/**/testdeno/**"], + exclude: ["test/**"], }, }) ); diff --git a/vitest.config.unit.ts b/vitest.config.unit.ts index 09ba5865..aab252f6 100644 --- a/vitest.config.unit.ts +++ b/vitest.config.unit.ts @@ -24,7 +24,7 @@ export default mergeConfig( // maxConcurrency: 2, name: "unit-tests", include: ["**/*unit.test.ts", "**/*.unit.spec.ts"], - exclude: ["node_modules/**", ".devmemo.local/**", "src/apps/**/testdeno/**"], + exclude: ["node_modules/**", ".devmemo.local/**", "test/e2e-cli/**"], coverage: { include: ["src/**/*.ts"], exclude: [ @@ -33,7 +33,6 @@ export default mergeConfig( "**/*.unit.spec.ts", "test/**", "**/_*", - "src/apps/**/testdeno/**", "**/*_obsolete.ts", ...importOnlyFiles, ...obsidianRuntimeOnlyFiles,