diff --git a/docs/adr/2026_06_real_obsidian_e2e.md b/docs/adr/2026_06_real_obsidian_e2e.md new file mode 100644 index 0000000..7fa8045 --- /dev/null +++ b/docs/adr/2026_06_real_obsidian_e2e.md @@ -0,0 +1,216 @@ +# Architectural Decision Record: Real Obsidian End-to-End Test Runner + +## Status + +Proposed / Spike Implemented + +## Release + +Not yet. Planned after the serviceFeature refactoring branch is reviewed. + +## Context + +The current end-to-end tests run through Vitest browser mode and a mocked Obsidian environment in `test/harness`. This has been useful for exercising synchronisation flows without launching Obsidian, but it is no longer a reliable final signal for plug-in behaviour. + +The main issues are: + +- The harness reimplements a large part of the Obsidian API surface, including vault files, workspace events, settings, and lifecycle behaviour. This mock can drift from real Obsidian behaviour without failing. +- The tests run inside a browser-style environment, while the desktop plug-in runs inside Obsidian's Electron environment with its own application lifecycle, storage paths, command registry, and event ordering. +- Several high-value regressions are about integration boundaries: boot-up sequence timing, real vault file reflection, Obsidian command registration, settings persistence, restart prompts, and file watcher behaviour. These are precisely the areas where a mock harness gives weak confidence. +- Maintaining the harness competes with maintaining the plug-in. Adding behaviour to the plug-in often requires teaching the mock another Obsidian detail before the actual regression can be tested. + +The current harness should therefore stop being treated as the primary E2E layer. + +## Decision + +Introduce a new E2E layer that launches real Obsidian with temporary vaults and the built Self-hosted LiveSync plug-in installed into those vaults. + +The long-term test pyramid should be: + +1. Unit tests for deterministic operations and serviceFeature boundaries. +2. Integration tests for CouchDB, Object Storage, P2P services, database operations, and replication protocols. +3. Real Obsidian E2E tests for boot-up sequence, vault reflection, command registration, settings dialogues, restart scheduling, and user-visible workflows. + +The existing `test/harness` should be demoted to a transitional compatibility layer. It may remain temporarily while the real Obsidian runner reaches parity for critical flows, but new high-level E2E coverage should target the real runner. + +## Non-Goals + +- Do not replace unit or integration tests with slow UI tests. +- Do not keep extending the Obsidian mock to cover new Obsidian APIs unless a short-term compatibility bridge is required. +- Do not require real Obsidian E2E for every pull request initially. The first CI integration should be opt-in or nightly until stability is proven. +- Do not test every setting dialogue through UI clicks if the behaviour is already covered by unit or integration tests. Use UI automation only for workflows whose risk is in real Obsidian integration. + +## Proposed Architecture + +### Runner + +Create a dedicated runner under `test/e2e-obsidian/`. + +The runner should: + +- Create one or more temporary vault directories. +- Build the plug-in once with `npm run build` or a narrower production build command. +- Install `main.js`, `manifest.json`, and `styles.css` when present into `.obsidian/plugins/obsidian-livesync/`. +- Prepare `.obsidian/community-plugins.json` and `.obsidian/plugins/obsidian-livesync/data.json` as needed. +- Launch Obsidian against the temporary vault. +- Wait until the plug-in reports readiness through a deterministic probe. +- Drive assertions through a narrow control channel rather than fragile visual selectors wherever possible. +- Dispose of Obsidian and temporary vaults after each scenario. + +### Obsidian Launch + +The preferred desktop target is the installed Obsidian application. The launch mechanism should be platform-specific but hidden behind a small adapter: + +- Linux: launch the Obsidian executable with a vault path or Obsidian URI, depending on what is most reliable. If an AppImage is used and FUSE is not available, extract it with `--appimage-extract` and launch the extracted `squashfs-root/obsidian` binary. +- macOS: launch the app bundle through `open` or the executable inside the bundle. +- Windows: launch the installed executable or the registered application protocol. + +The first implementation can support Linux only if that is the local and CI target. Cross-platform support can be added after the runner contract is stable. + +In headless Linux environments, launch through `xvfb-run`, pass Electron flags such as `--no-sandbox` and `--disable-gpu`, and isolate `HOME`, `XDG_CONFIG_HOME`, and `--user-data-dir` per temporary vault. + +### Control Channel + +The runner needs a stable way to observe readiness and issue test commands. Prefer a test-only plug-in bridge compiled only in test builds or enabled only by an environment variable. + +Possible bridge options: + +- The official Obsidian CLI, using the installed `obsidian-cli` helper to open vaults, reload the plug-in, run `eval`, and call developer commands. +- A local HTTP/WebSocket bridge bound to `127.0.0.1` with a random port and token. +- A file-based bridge in the vault, where Obsidian writes status files and consumes command files. +- A DevTools protocol bridge if Obsidian exposes a stable debugging port in the test environment. + +The first implementation uses Obsidian's CLI for orchestration and readiness checks. The CLI handles vault opening through `obsidian://open?path=...`, enables community plug-ins through `app.plugins.setEnable(true)`, reloads Self-hosted LiveSync through `plugin:reload id=obsidian-livesync`, and verifies that `app.plugins.plugins['obsidian-livesync']` is loaded. + +This keeps E2E-only behaviour out of the production plug-in bundle. The runner should not require Self-hosted LiveSync to write marker files or expose a test server merely to prove that Obsidian loaded it. + +The DevTools protocol remains useful for diagnostics. Obsidian's CLI exposes developer commands such as `dev:cdp`, `dev:errors`, and `dev:console`, so the runner should prefer the CLI path first and fall back to direct DevTools attachment only if the CLI cannot provide the required signal. + +### Test Data and Services + +Keep the existing Docker scripts for CouchDB, MinIO, and P2P services. The real Obsidian runner should reuse these service fixtures instead of creating another service orchestration stack. + +Each test should use unique database names, bucket prefixes, vault names, and P2P room IDs. This prevents tests from depending on cleanup and makes interrupted runs less harmful. + +## Migration Plan + +### Phase 0: Discovery + +- Confirm how Obsidian can be launched reliably on the local development environment. +- Confirm whether Obsidian accepts a vault path directly, requires an Obsidian URI, or needs a pre-existing vault registry. +- Identify where Obsidian stores per-user state in the test environment and decide how to isolate it. +- Decide whether the first bridge is file-based or HTTP/WebSocket. + +Initial discovery on Linux ARM64 found that: + +- `Obsidian-1.12.7-arm64.AppImage` requires `libfuse.so.2` for direct AppImage execution. +- Extracting the AppImage with `--appimage-extract` works without FUSE. +- Launching the extracted `squashfs-root/obsidian` binary under `xvfb-run` with isolated user data stays alive for the smoke timeout. +- No missing shared libraries were reported by `ldd` for the extracted binary in the tested environment. +- Obsidian's CLI is disabled unless the global `obsidian.json` contains `cli: true`. +- Passing only `.obsidian/community-plugins.json` is not enough to load community plug-ins on Obsidian 1.12. The runner also has to enable the global community plug-in switch through `app.plugins.setEnable(true)`. +- The reliable launch sequence is: start Obsidian, send `obsidian://open?path=...` through `obsidian-cli`, wait until the vault-side CLI exposes the plug-in catalogue, enable community plug-ins, reload Self-hosted LiveSync, and verify plug-in readiness through `obsidian-cli eval`. + +### Phase 1: Smoke Runner + +- Add `test/e2e-obsidian/runner` utilities for temporary vault creation, plug-in installation, launch, readiness wait, and cleanup. +- Add one smoke test: + - launch Obsidian with an empty vault, + - load Self-hosted LiveSync, + - wait for the boot-up sequence to become ready, + - read the plug-in version or status through the control channel, + - close Obsidian cleanly. +- Add an npm script such as `test:e2e:obsidian`. + +Current implementation status: + +- Added `test/e2e-obsidian/runner` helpers for Obsidian discovery, CLI discovery, temporary vault creation, plug-in installation, process launch, CLI execution, and readiness polling. +- Added `test:e2e:obsidian:discover`, `test:e2e:obsidian:cli-help`, `test:e2e:obsidian:smoke`, and `test:e2e:obsidian:install-appimage`. +- Added a manual AppImage installer that downloads Obsidian `1.12.7` for `arm64` or `x86_64`, stores it under `_testdata/obsidian`, and extracts it for FUSE-free execution. +- Confirmed the smoke runner on Linux ARM64 with the extracted Obsidian `1.12.7` AppImage, `xvfb-run`, and the built Self-hosted LiveSync bundle. +- Confirmed the runner can enable the Obsidian CLI through isolated `obsidian.json` state, open the temporary vault through `obsidian-cli`, enable community plug-ins through `app.plugins.setEnable(true)`, reload Self-hosted LiveSync, and verify readiness through `obsidian-cli eval`. +- Removed the first test-only ready-marker bridge from the plug-in bundle. The current runner observes readiness from outside the plug-in through Obsidian's own CLI, so normal user vaults do not receive E2E marker files. + +Current verification: + +- `npm run tsc-check` passes. +- `npm run build` passes with existing Svelte warnings. +- `npm run test:e2e:obsidian:discover` finds `_testdata/obsidian/squashfs-root/obsidian` when the extracted AppImage is present. +- `E2E_OBSIDIAN_SMOKE_TIMEOUT_MS=1000 npm run test:e2e:obsidian:smoke` passes locally. +- `npm run test:e2e:obsidian:install-appimage` reuses the existing AppImage and extracted binary when they are already present. + +Known limits: + +- The smoke runner currently proves only one-vault launch and plug-in load readiness. It does not yet exercise synchronisation, settings persistence, restart behaviour, or database writes. +- Cross-platform support is still discovery-level. The working path has been validated on Linux ARM64. +- CI wiring is not yet implemented. CI should use `OBSIDIAN_BINARY` or a cached `_testdata/obsidian/squashfs-root` rather than downloading the AppImage on every run. + +### Phase 2: First Real Workflow + +- Add a one-vault local workflow: + - configure a temporary CouchDB database, + - create a note in the real vault, + - wait for metadata and chunks to be stored, + - restart Obsidian, + - verify that the plug-in loads and the note remains consistent. + +This validates real boot-up, settings persistence, vault file access, database writes, and restart-sensitive state. + +### Phase 3: Two-Vault Synchronisation + +- Launch two Obsidian instances with two temporary vaults. +- Configure both against the same temporary remote database. +- Create, modify, rename, and delete notes in one vault. +- Verify reflection in the other vault. +- Cover encrypted and non-encrypted configurations separately. + +### Phase 4: Harness Retirement + +- Mark `test/harness` as deprecated in documentation. +- Stop adding new tests to `test/suite` unless they are explicitly transitional. +- Move critical existing scenarios from `test/suite` to real Obsidian E2E or lower-level integration tests. +- Remove the harness only after the new runner covers the critical boot-up and synchronisation workflows. + +## CI Strategy + +Start with local-only execution. After the smoke runner is stable: + +- Run the smoke test in CI on Linux. +- Keep full two-vault synchronisation scenarios as nightly or manually triggered jobs until runtime and flakiness are understood. +- Do not download the Obsidian AppImage on every CI run. Use a pre-installed Obsidian binary, a CI cache for `_testdata/obsidian/squashfs-root`, or a manually triggered preparation job. +- Capture Obsidian logs, plug-in logs, vault snapshots, and service logs on failure. +- Fail fast on launch failures, readiness timeouts, and cleanup failures with clear diagnostics. + +## Risks and Mitigations + +- **Obsidian licensing and installation**: CI may need a cached installer or a pre-installed binary. Keep the runner capable of using `OBSIDIAN_BINARY`. +- **Flakiness from UI timing**: Prefer a control channel and service-level probes over visual selectors. +- **Multiple instances**: Obsidian may not support multiple independent instances cleanly on all platforms. Start with one-instance smoke tests, then validate two-instance behaviour on Linux before expanding scope. +- **State leakage**: Isolate vault directories, Obsidian user data, remote database names, and bridge tokens per test. +- **Security of E2E controls**: Keep readiness and control outside the production plug-in bundle. Prefer Obsidian CLI probes over E2E-only plug-in code. +- **Runtime cost**: Keep the default PR gate small. Move slow synchronisation matrices to scheduled jobs. + +## Open Questions + +- Which launch mechanism is most reliable for Obsidian on Linux in this repository's CI environment? +- Can two Obsidian instances run with isolated user data at the same time? +- Do future scenarios need a richer control channel than Obsidian CLI, or can CLI `eval` and developer commands cover the required workflows? +- Should any future E2E-only plug-in code live in a separate test build, or should the production bundle remain free of E2E controls? +- Which existing `test/suite` scenarios are critical enough to port before deprecating the harness? + +## Initial Implementation Checklist + +1. Add an Obsidian launch discovery script that prints the detected executable, version, and launch mode. +2. Add temporary vault and plug-in installation helpers. +3. Add CLI-based plug-in readiness polling. +4. Add `test:e2e:obsidian:smoke` for one-vault plug-in load. +5. Document required local environment variables, especially `OBSIDIAN_BINARY`. +6. Port one CouchDB-backed workflow after the smoke test is stable. +7. Mark `test/harness` as transitional and block new broad E2E work from targeting it. + +## Consequences + +- Real Obsidian E2E becomes the source of truth for plug-in lifecycle and vault integration. +- Unit and integration tests remain the primary fast feedback loops. +- The old browser harness can be deleted once the new runner covers the critical workflows. +- The project will gain slower but higher-confidence tests for the behaviours most likely to differ between mocks and Obsidian itself. diff --git a/package.json b/package.json index 0d13b79..09f2be1 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,10 @@ "test:unit:coverage": "vitest run --config vitest.config.unit.ts --coverage", "test:install-playwright": "npx playwright install chromium", "test:install-dependencies": "npm run test:install-playwright", + "test:e2e:obsidian:install-appimage": "tsx test/e2e-obsidian/scripts/install-appimage.ts", + "test:e2e:obsidian:discover": "tsx test/e2e-obsidian/scripts/discover.ts", + "test:e2e:obsidian:cli-help": "tsx test/e2e-obsidian/scripts/cli-help.ts", + "test:e2e:obsidian:smoke": "tsx test/e2e-obsidian/scripts/smoke.ts", "test:coverage": "vitest run --coverage", "test:docker-couchdb:up": "npx dotenv-cli -e .env -e .test.env -- ./test/shell/couchdb-start.sh", "test:docker-couchdb:init": "npx dotenv-cli -e .env -e .test.env -- ./test/shell/couchdb-init.sh", diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index 5579ae2..574a64a 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -1,634 +1,634 @@ -# Self-hosted LiveSync CLI -Command-line version of Self-hosted LiveSync plug-in for syncing vaults without Obsidian. - -## Features - -- ✅ Sync Obsidian vaults using CouchDB without running Obsidian -- ✅ Compatible with Self-hosted LiveSync plug-in settings -- ✅ Supports all core sync features (encryption, conflict resolution, etc.) -- ✅ Lightweight and headless operation -- ✅ Cross-platform (Windows, macOS, Linux) - -## Architecture - -This CLI version is built using the same core as the Obsidian plug-in: - -``` -CLI Main - └─ LiveSyncBaseCore - ├─ NodeServiceHub (All services without Obsidian dependencies) - └─ ServiceModules (wired by initialiseServiceModulesCLI) - ├─ FileAccessCLI (Node.js FileSystemAdapter) - ├─ StorageEventManagerCLI - ├─ ServiceFileAccessCLI - ├─ ServiceDatabaseFileAccessCLI - ├─ ServiceFileHandler - └─ ServiceRebuilder -``` - -### Key Components - -1. **Node.js FileSystem Adapter** (`adapters/`) - - Platform-agnostic file operations using Node.js `fs/promises` - - Implements same interface as Obsidian's file system - -2. **Service Modules** (`serviceModules/`) - - Initialised by `initialiseServiceModulesCLI` - - All core sync functionality preserved - -3. **Service Hub and Settings Services** (`services/`) - - `NodeServiceHub` provides the CLI service context - - Node-specific settings and key-value services are provided without Obsidian dependencies - -4. **Main Entry Point** (`main.ts`) - - Command-line interface - - Settings management (JSON file) - - Graceful shutdown handling - -## Usage - -The CLI operates on a **database directory** which contains PouchDB data and settings. - -> [!NOTE] -> `livesync-cli` is the alias for the CLI executable. Please replace with the actual command of your installation (e.g. `npm run --silent cli --` or `docker run ...`). - -```bash -livesync-cli [database-path] [command] [args...] -``` - - -### Arguments - -- `database-path`: Path to the directory where `.livesync` folder and `settings.json` are (or will be) located. - - Note: In previous versions, this was referred to as the "vault" path. Now it is clearly distinguished from the actual vault (the directory containing your `.md` files). -- `--vault ` / `-V `: (daemon/mirror only) Path to the vault directory containing `.md` files. - - Allows the PouchDB database directory and the actual vault directory to be different locations. - - For `mirror` command, the positional `[vault-path]` argument takes precedence over `--vault`. - -### Commands - -- `sync`: Run one replication cycle with the remote CouchDB. -- `mirror [vault-path]`: Bidirectional sync between the local database and a local directory (**the actual vault**). - - If `vault-path` is provided, the CLI will synchronise the database with files in the vault directory. - - If `vault-path` is omitted, it defaults to `database-path` (compatibility mode). - - Use this command to keep your local `.md` files in sync with the database. -- `ls [prefix]`: List files currently stored in the local database. -- `push `: Push a local file `` into the database at path ``. -- `pull `: Pull a file `` from the database into local file ``. -- `cat `: Read a file from the database and write to stdout. -- `put `: Read from stdin and write to the database path ``. -- `remote-add `: Add a remote configuration from a connection string. -- `remote-rm `: Remove a remote configuration by ID. -- `remote-ls`: List remote configurations (`id`, `name`, `active/inactive`, redacted URI). -- `remote-export `: Export the stored connection string by remote ID. -- `remote-set `: Replace the stored connection string by remote ID. -- `remote-activate `: Activate a remote configuration by ID. -- `mark-resolved [remote-id]`: Resolve remote synchronisation status. -- `unlock-remote [remote-id]`: Unlock the remote database. -- `lock-remote [remote-id]`: Lock the remote database. -- `remote-status [remote-id]`: Show remote database status. -- `init-settings [file]`: Create a default settings file. - -### Examples - -```bash -# Basic sync with remote -livesync-cli ./my-db sync - -# Mirroring to your actual Obsidian vault -livesync-cli ./my-db mirror /path/to/obsidian-vault - -# Manual file operations -livesync-cli ./my-db push ./note.md folder/note.md -livesync-cli ./my-db pull folder/note.md ./note.md -``` - -## Installation - -### Build from source - -```bash -# Clone with submodules, because the shared core lives in src/lib -git clone --recurse-submodules -cd obsidian-livesync - -# If you already cloned without submodules, run this once instead -git submodule update --init --recursive - -# Install dependencies from the repository root -npm install - -# Build the CLI from the repository root -npm run build -w self-hosted-livesync-cli - -# Or from the package directory -cd src/apps/cli -npm run build -``` - -If `src/lib` is missing, the build process stops early with a targeted message instead of a low-level Vite `ENOENT` error. - -Run the CLI: - -```bash -# Run with npm workspace script (from repository root) -npm run cli -w self-hosted-livesync-cli -- [database-path] [command] [args...] - -# Or from the package directory -cd src/apps/cli -npm run cli -- [database-path] [command] [args...] - -# Run the built executable directly -node src/apps/cli/dist/index.cjs [database-path] [command] [args...] -``` - -### Docker - -A Docker image is provided for headless / server deployments. Build from the repository root: - -```bash -docker build -f src/apps/cli/Dockerfile -t livesync-cli . -``` - -Run: - -```bash -# Sync with CouchDB -docker run --rm -v /path/to/your/db:/data livesync-cli sync - -# Mirror to a specific vault directory -docker run --rm -v /path/to/your/db:/data -v /path/to/your/vault:/vault livesync-cli mirror /vault - -# List files in the local database -docker run --rm -v /path/to/your/db:/data livesync-cli ls -``` - -The database directory is mounted at `/data` by default. Override with `-e LIVESYNC_DB_PATH=/other/path`. - -#### P2P (WebRTC) and Docker networking - -The P2P replicator (`p2p-host`, `p2p-sync`, `p2p-peers`) uses WebRTC and generates -three kinds of ICE candidates. The default Docker bridge network affects which -candidates are usable: - -| Candidate type | Description | Bridge network | -| -------------- | ---------------------------------- | -------------------------- | -| `host` | Container bridge IP (`172.17.x.x`) | Unreachable from LAN peers | -| `srflx` | Host public IP via STUN reflection | Works over the internet | -| `relay` | Traffic relayed via TURN server | Always reachable | - -**LAN P2P on Linux** — use `--network host` so that the real host IP is -advertised as the `host` candidate: - -```bash -docker run --rm --network host -v /path/to/your/vault:/data livesync-cli p2p-host -``` - -Note: also fix the alias to include `--network host` if you want to use `livesync-cli` for P2P commands. - -> `--network host` is not available on Docker Desktop for macOS or Windows. - -**LAN P2P on macOS / Windows Docker Desktop** — configure a TURN server in the -settings file (`P2P_turnServers`, `P2P_turnUsername`, `P2P_turnCredential`). -All P2P traffic will then be relayed through the TURN server, bypassing the -bridge-network limitation. - -**Internet P2P** — the default bridge network is sufficient. The `srflx` -candidate carries the host's public IP and peers can connect normally. - -**CouchDB sync only (no P2P)** — no special network configuration is required. - - -### Adding `livesync-cli` alias - -To use the `livesync-cli` command globally, you can add an alias to your shell configuration file (e.g., `.zshrc` or `.bashrc`). - -If you are using `npm run`, add the following line: - -```bash -alias livesync-cli='npm run --silent --prefix /path/to/repository/src/apps/cli cli --' -# or -alias livesync-cli="npm run --silent --prefix $PWD cli --" -``` - -Alternatively, if you want to use the built executable directly: - -```bash -alias livesync-cli='node /path/to/repository/src/apps/cli/dist/index.cjs' -or -alias livesync-cli="node $PWD/dist/index.cjs" -``` - -If you prefer using Docker: - -```bash -alias livesync-cli='docker run --rm -v /path/to/your/db:/data livesync-cli' -``` - -After adding the alias, restart your shell or run `source ~/.zshrc` (or `.bashrc`). - -## Usage - -### Basic Usage - -As you know, the CLI is designed to be used in a headless environment. Hence all operations are performed against a local vault directory and a settings file. Here are some example commands: - -```bash -# Sync local database with CouchDB (no files will be changed). -livesync-cli /path/to/your-local-database --settings /path/to/settings.json sync - -# Push files to local database -livesync-cli /path/to/your-local-database --settings /path/to/settings.json push /your/storage/file.md /vault/path/file.md - -# Pull files from local database -livesync-cli /path/to/your-local-database --settings /path/to/settings.json pull /vault/path/file.md /your/storage/file.md - -# Verbose logging -livesync-cli /path/to/your-local-database --settings /path/to/settings.json --verbose - -# Apply setup URI to settings file (settings only; does not run synchronisation) -livesync-cli /path/to/your-local-database --settings /path/to/settings.json setup "obsidian://setuplivesync?settings=..." - -# Put text from stdin into local database -echo "Hello from stdin" | livesync-cli /path/to/your-local-database --settings /path/to/settings.json put /vault/path/file.md - -# Output a file from local database to stdout -livesync-cli /path/to/your-local-database --settings /path/to/settings.json cat /vault/path/file.md - -# Output a specific revision of a file from local database -livesync-cli /path/to/your-local-database --settings /path/to/settings.json cat-rev /vault/path/file.md 3-abcdef - -# Pull a specific revision of a file from local database to local storage -livesync-cli /path/to/your-local-database --settings /path/to/settings.json pull-rev /vault/path/file.md /your/storage/file.old.md 3-abcdef - -# List files in local database -livesync-cli /path/to/your-local-database --settings /path/to/settings.json ls /vault/path/ - -# Show metadata for a file in local database -livesync-cli /path/to/your-local-database --settings /path/to/settings.json info /vault/path/file.md - -# Mark a file as deleted in local database -livesync-cli /path/to/your-local-database --settings /path/to/settings.json rm /vault/path/file.md - -# Resolve conflict by keeping a specific revision -livesync-cli /path/to/your-local-database --settings /path/to/settings.json resolve /vault/path/file.md 3-abcdef - -# Add, list, activate, and remove remote configurations -livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-add main "sls+https://user:pass@example.com/db" -livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-ls -livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-export remote-abc123 -livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-set remote-abc123 "sls+p2p://room-abc?passphrase=secret" -livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-activate remote-abc123 -livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-rm remote-abc123 - -# Lock, unlock, resolve, and view status of remote database -livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-status remote-abc123 -livesync-cli /path/to/your-local-database --settings /path/to/settings.json lock-remote remote-abc123 -livesync-cli /path/to/your-local-database --settings /path/to/settings.json mark-resolved remote-abc123 -livesync-cli /path/to/your-local-database --settings /path/to/settings.json unlock-remote remote-abc123 -``` - -### Configuration - -The CLI uses the same settings format as the Obsidian plug-in. Create a `.livesync/settings.json` file in your vault directory: - -```json -{ - "couchDB_URI": "http://localhost:5984", - "couchDB_USER": "admin", - "couchDB_PASSWORD": "password", - "couchDB_DBNAME": "obsidian-livesync", - "liveSync": true, - "syncOnSave": true, - "syncOnStart": true, - "encrypt": true, - "passphrase": "your-encryption-passphrase", - "usePluginSync": false, - "isConfigured": true -} -``` - -**Minimum required settings:** - -- `couchDB_URI`: CouchDB server URL -- `couchDB_USER`: CouchDB username -- `couchDB_PASSWORD`: CouchDB password -- `couchDB_DBNAME`: Database name -- `isConfigured`: Set to `true` after configuration - -### Command-line Reference - -``` -Usage: - livesync-cli [options] [command-args] - livesync-cli init-settings [path] - -Arguments: - database-path Path to the local database directory (required except for init-settings) - -Options: - --settings, -s Path to settings file (default: .livesync/settings.json in local database directory) - --force, -f Overwrite existing file on init-settings - --verbose, -v Enable verbose logging - --debug, -d Enable debug logging (includes verbose) - --interval , -i (daemon only) Poll CouchDB every N seconds instead of using the _changes feed - --vault , -V (daemon/mirror) Path to vault directory, decoupled from database-path - --help, -h Show this help message - -Commands: - daemon (default) Run mirror scan then continuously sync CouchDB <-> local filesystem - init-settings [path] Create settings JSON from DEFAULT_SETTINGS - sync Run one replication cycle and exit - p2p-peers Show discovered peers as [peer] - p2p-sync Synchronise with specified peer-id or peer-name - p2p-host Start P2P host mode and wait until interrupted (Ctrl+C) - push Push local file into local database path - pull Pull file from local database into local file - pull-rev Pull specific revision into local file - setup Apply setup URI to settings file - put Read text from standard input and write to local database path - cat Write latest file content from local database to standard output - cat-rev Write specific revision content from local database to standard output - ls [prefix] List files as pathsizemtimerevision[*] - info Show file metadata including current and past revisions, conflicts, and chunk list - rm Mark file as deleted in local database - resolve Resolve conflict by keeping the specified revision - mirror [vaultPath] Mirror database contents to the local file system - (vaultPath positional arg > --vault flag > database-path) -``` - -Run via npm script: - -```bash -npm run --silent cli -- [database-path] [options] [command] [command-args] -``` - -#### Detailed Command Descriptions - -##### ls -`ls` lists files in the local database with optional prefix filtering. Output format is: - -```vault/path/file.mdsizemtimerevision[*] -``` -Note: `*` indicates if the file has conflicts. - -##### p2p-peers - -`p2p-peers ` waits for the specified number of seconds, then prints each discovered peer on a separate line: - -```text -[peer] -``` - -Use this command to select a target for `p2p-sync`. - -##### p2p-sync - -`p2p-sync ` discovers peers up to the specified timeout and synchronises with the selected peer. - -- `` accepts either `peer-id` or `peer-name` from `p2p-peers` output. -- On success, the command prints a completion message to standard error and exits with status code `0`. -- On failure, the command prints an error message and exits non-zero. - -##### p2p-host - -`p2p-host` starts the local P2P host and keeps running until interrupted. - -- Other peers can discover and synchronise with this host while it is running. -- Stop the host with `Ctrl+C`. -- In CLI mode, behaviour is non-interactive and acceptance follows settings. - -##### info - -`info` output fields: - -- `id`: Document ID -- `revision`: Current revision -- `conflicts`: Conflicted revisions, or `N/A` -- `filename`: Basename of path -- `path`: Vault-relative path -- `size`: Size in bytes -- `revisions`: Available non-current revisions -- `chunks`: Number of chunk IDs -- `children`: Chunk ID list - -##### mirror - -`mirror` is a command that synchronises your storage with your local vault. It is essentially a process that runs upon startup in Obsidian. - -In other words, it performs the following actions: - -1. **Precondition checks** — Aborts early if any of the following conditions are not met: - - Settings must be configured (`isConfigured: true`). - - File watching must not be suspended (`suspendFileWatching: false`). - - Remediation mode must be inactive (`maxMTimeForReflectEvents: 0`). - -2. **State restoration** — On subsequent runs (after the first successful scan), restores the previous storage state before proceeding. - -3. **Expired deletion cleanup** — If `automaticallyDeleteMetadataOfDeletedFiles` is set to a positive number of days, any document that is marked deleted and whose `mtime` is older than the retention period is permanently removed from the local database. - -4. **File collection** — Enumerates files from two sources: - - **Storage**: all files under the vault path that pass `isTargetFile`. - - **Local database**: all normal documents (fetched with conflict information) whose paths are valid and pass `isTargetFile`. - - Both collections build case-insensitive ↔ case-sensitive path maps, controlled by `handleFilenameCaseSensitive`. - -5. **Categorisation and synchronisation** — The union of both file sets is split into three groups and processed concurrently (up to 10 files at a time): - - | Group | Condition | Action | - | ----------------------------- | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | - | **UPDATE DATABASE** | File exists in storage only | Store the file into the local database. | - | **UPDATE STORAGE** | File exists in database only | If the entry is active (not deleted) and not conflicted, restore the file from the database to storage. Deleted entries and conflicted entries are skipped. | - | **SYNC DATABASE AND STORAGE** | File exists in both | Compare `mtime` freshness. If storage is newer → write to database (`STORAGE → DB`). If database is newer → restore to storage (`STORAGE ← DB`). If equal → do nothing. Conflicted documents and files exceeding the size limit are always skipped. | - -6. **Initialisation flag** — On the very first successful run, writes `initialized = true` to the key-value database so that subsequent runs can restore state in step 2. - -Note: `mirror` does not respect file deletions. If a file is deleted in storage, it will be restored on the next `mirror` run. To delete a file, use the `rm` command instead. This is a little inconvenient, but it is intentional behaviour (if we handle this automatically in `mirror`, we should be against a ton of edge cases). - -##### daemon - -`daemon` is the default command when no command is specified. It runs an initial mirror scan and then continuously syncs changes in both directions: - -- **CouchDB → local filesystem**: via the `_changes` feed (LiveSync mode, default) or periodic polling (`--interval N`). -- **local filesystem → CouchDB**: via chokidar file watching. Any file created, modified, or deleted in the vault directory is pushed to CouchDB. - -In **LiveSync mode** the `_changes` feed delivers remote changes as they arrive, with sub-second latency. In **polling mode** (`--interval N`) the CLI polls CouchDB every N seconds. Use polling mode if your CouchDB instance does not support long-lived HTTP connections, or if you need predictable network usage. - -The daemon exits cleanly on `SIGINT` or `SIGTERM`. - -```bash -# LiveSync mode (default — _changes feed, near-real-time) -livesync-cli /path/to/vault - -# Polling mode — poll every 60 seconds -livesync-cli /path/to/vault --interval 60 -``` - -### .livesync/ignore - -Place a `.livesync/ignore` file in your vault root to exclude files from sync in both directions (local → CouchDB and CouchDB → local). - -**Format:** - -- Lines beginning with `#` are comments. -- Blank lines are ignored. -- All other lines are [minimatch](https://github.com/isaacs/minimatch) glob patterns, relative to the vault root. -- The directive `import: .gitignore` (exactly this string) reads `.gitignore` from the vault root and merges its non-comment, non-blank lines into the ignore rules. -- Negation patterns (lines starting with `!`) are not supported and will cause an error on load. - -**Example `.livesync/ignore`:** - -``` -# Ignore temporary files -*.tmp -*.swp - -# Ignore build output -build/ -dist/ - -# Merge patterns from .gitignore -import: .gitignore -``` - -Patterns apply in both directions: the chokidar watcher will not emit events for matched files, and the `isTargetFile` filter will exclude them from CouchDB → local sync. - -Changes to this file require a daemon restart to take effect. - -### Systemd Installation - -The `deploy/` directory contains a systemd unit template and an install script. - -**Automated install (user service, recommended):** - -```bash -bash src/apps/cli/deploy/install.sh --vault /path/to/vault -``` - -**With polling interval:** - -```bash -bash src/apps/cli/deploy/install.sh --vault /path/to/vault --interval 60 -``` - -**System-wide install** (requires root / sudo for `/etc/systemd/system/`): - -```bash -bash src/apps/cli/deploy/install.sh --system --vault /path/to/vault -``` - -The script: -1. Builds the CLI (`npm install` + `npm run build`). -2. Installs the binary to `~/.local/bin/livesync-cli` (user) or `/usr/local/bin/livesync-cli` (system). -3. Writes the unit file to `~/.config/systemd/user/livesync-cli.service` (user) or `/etc/systemd/system/livesync-cli.service` (system). -4. Runs `systemctl [--user] daemon-reload && systemctl [--user] enable --now livesync-cli`. - -**Manual setup** — if you prefer to manage the unit yourself, copy `deploy/livesync-cli.service`, replace `LIVESYNC_BIN` and `LIVESYNC_VAULT_PATH` with the actual binary path and vault path, then install to the appropriate systemd directory. - -### Planned options: - -- `--immediate`: Perform sync after the command (e.g. `push`, `pull`, `put`, `rm`). -- `serve`: Start CLI in server mode, exposing REST APIs for remote, and batch operations. -- `cause-conflicted `: Mark a file as conflicted without changing its content, to trigger conflict resolution in Obsidian. - -## Use Cases - -### 1. Bootstrap a new headless vault - -Create default settings, apply a setup URI, then run one sync cycle. - -```bash -livesync-cli -- init-settings /data/livesync-settings.json -printf '%s\n' "$SETUP_PASSPHRASE" | livesync-cli -- /data/vault --settings /data/livesync-settings.json setup "$SETUP_URI" -livesync-cli -- /data/vault --settings /data/livesync-settings.json sync -``` - -### 2. Scripted import and export - -Push local files into the database from automation, and pull them back for export or backup. - -```bash -livesync-cli -- /data/vault --settings /data/livesync-settings.json push ./note.md notes/note.md -livesync-cli -- /data/vault --settings /data/livesync-settings.json pull notes/note.md ./exports/note.md -``` - -### 3. Revision inspection and restore - -List metadata, find an older revision, then restore it by content (`cat-rev`) or file output (`pull-rev`). - -```bash -livesync-cli -- /data/vault --settings /data/livesync-settings.json info notes/note.md -livesync-cli -- /data/vault --settings /data/livesync-settings.json cat-rev notes/note.md 3-abcdef -livesync-cli -- /data/vault --settings /data/livesync-settings.json pull-rev notes/note.md ./restore/note.old.md 3-abcdef -``` - -### 4. Conflict and cleanup workflow - -Inspect conflicted revisions, resolve by keeping one revision, then delete obsolete files. - -```bash -livesync-cli -- /data/vault --settings /data/livesync-settings.json info notes/note.md -livesync-cli -- /data/vault --settings /data/livesync-settings.json resolve notes/note.md 3-abcdef -livesync-cli -- /data/vault --settings /data/livesync-settings.json rm notes/obsolete.md -``` - -### 5. CI smoke test for content round-trip - -Validate that `put`/`cat` is behaving as expected in a pipeline. - -```bash -echo "hello-ci" | livesync-cli -- /data/vault --settings /data/livesync-settings.json put ci/test.md -livesync-cli -- /data/vault --settings /data/livesync-settings.json cat ci/test.md -``` - -## Development - -### Project Structure - -``` -src/apps/cli/ -├── commands/ # Command dispatcher and command utilities -│ ├── runCommand.ts -│ ├── runCommand.unit.spec.ts -│ ├── types.ts -│ ├── utils.ts -│ └── utils.unit.spec.ts -├── adapters/ # Node.js FileSystem Adapter -│ ├── NodeConversionAdapter.ts -│ ├── NodeFileSystemAdapter.ts -│ ├── NodePathAdapter.ts -│ ├── NodeStorageAdapter.ts -│ ├── NodeStorageAdapter.unit.spec.ts -│ ├── NodeTypeGuardAdapter.ts -│ ├── NodeTypes.ts -│ └── NodeVaultAdapter.ts -├── lib/ -│ └── pouchdb-node.ts -├── managers/ # CLI-specific managers -│ ├── CLIStorageEventManagerAdapter.ts -│ └── StorageEventManagerCLI.ts -├── serviceModules/ # Service modules (ported from main.ts) -│ ├── CLIServiceModules.ts -│ ├── DatabaseFileAccess.ts -│ ├── FileAccessCLI.ts -│ └── ServiceFileAccessImpl.ts -├── services/ -│ ├── NodeKeyValueDBService.ts -│ ├── NodeServiceHub.ts -│ └── NodeSettingService.ts -├── test/ -│ ├── test-e2e-two-vaults-common.sh -│ ├── test-e2e-two-vaults-matrix.sh -│ ├── test-e2e-two-vaults-with-docker-linux.sh -│ ├── test-push-pull-linux.sh -│ ├── test-setup-put-cat-linux.sh -│ └── test-sync-two-local-databases-linux.sh -├── .gitignore -├── entrypoint.ts # CLI executable entry point (shebang) -├── main.ts # CLI entry point -├── main.unit.spec.ts -├── package.json -├── README.md # This file -├── tsconfig.json -├── util/ # Test and local utility scripts -└── vite.config.ts -``` +# Self-hosted LiveSync CLI +Command-line version of Self-hosted LiveSync plug-in for syncing vaults without Obsidian. + +## Features + +- ✅ Sync Obsidian vaults using CouchDB without running Obsidian +- ✅ Compatible with Self-hosted LiveSync plug-in settings +- ✅ Supports all core sync features (encryption, conflict resolution, etc.) +- ✅ Lightweight and headless operation +- ✅ Cross-platform (Windows, macOS, Linux) + +## Architecture + +This CLI version is built using the same core as the Obsidian plug-in: + +``` +CLI Main + └─ LiveSyncBaseCore + ├─ NodeServiceHub (All services without Obsidian dependencies) + └─ ServiceModules (wired by initialiseServiceModulesCLI) + ├─ FileAccessCLI (Node.js FileSystemAdapter) + ├─ StorageEventManagerCLI + ├─ ServiceFileAccessCLI + ├─ ServiceDatabaseFileAccessCLI + ├─ ServiceFileHandler + └─ ServiceRebuilder +``` + +### Key Components + +1. **Node.js FileSystem Adapter** (`adapters/`) + - Platform-agnostic file operations using Node.js `fs/promises` + - Implements same interface as Obsidian's file system + +2. **Service Modules** (`serviceModules/`) + - Initialised by `initialiseServiceModulesCLI` + - All core sync functionality preserved + +3. **Service Hub and Settings Services** (`services/`) + - `NodeServiceHub` provides the CLI service context + - Node-specific settings and key-value services are provided without Obsidian dependencies + +4. **Main Entry Point** (`main.ts`) + - Command-line interface + - Settings management (JSON file) + - Graceful shutdown handling + +## Usage + +The CLI operates on a **database directory** which contains PouchDB data and settings. + +> [!NOTE] +> `livesync-cli` is the alias for the CLI executable. Please replace with the actual command of your installation (e.g. `npm run --silent cli --` or `docker run ...`). + +```bash +livesync-cli [database-path] [command] [args...] +``` + + +### Arguments + +- `database-path`: Path to the directory where `.livesync` folder and `settings.json` are (or will be) located. + - Note: In previous versions, this was referred to as the "vault" path. Now it is clearly distinguished from the actual vault (the directory containing your `.md` files). +- `--vault ` / `-V `: (daemon/mirror only) Path to the vault directory containing `.md` files. + - Allows the PouchDB database directory and the actual vault directory to be different locations. + - For `mirror` command, the positional `[vault-path]` argument takes precedence over `--vault`. + +### Commands + +- `sync`: Run one replication cycle with the remote CouchDB. +- `mirror [vault-path]`: Bidirectional sync between the local database and a local directory (**the actual vault**). + - If `vault-path` is provided, the CLI will synchronise the database with files in the vault directory. + - If `vault-path` is omitted, it defaults to `database-path` (compatibility mode). + - Use this command to keep your local `.md` files in sync with the database. +- `ls [prefix]`: List files currently stored in the local database. +- `push `: Push a local file `` into the database at path ``. +- `pull `: Pull a file `` from the database into local file ``. +- `cat `: Read a file from the database and write to stdout. +- `put `: Read from stdin and write to the database path ``. +- `remote-add `: Add a remote configuration from a connection string. +- `remote-rm `: Remove a remote configuration by ID. +- `remote-ls`: List remote configurations (`id`, `name`, `active/inactive`, redacted URI). +- `remote-export `: Export the stored connection string by remote ID. +- `remote-set `: Replace the stored connection string by remote ID. +- `remote-activate `: Activate a remote configuration by ID. +- `mark-resolved [remote-id]`: Resolve remote synchronisation status. +- `unlock-remote [remote-id]`: Unlock the remote database. +- `lock-remote [remote-id]`: Lock the remote database. +- `remote-status [remote-id]`: Show remote database status. +- `init-settings [file]`: Create a default settings file. + +### Examples + +```bash +# Basic sync with remote +livesync-cli ./my-db sync + +# Mirroring to your actual Obsidian vault +livesync-cli ./my-db mirror /path/to/obsidian-vault + +# Manual file operations +livesync-cli ./my-db push ./note.md folder/note.md +livesync-cli ./my-db pull folder/note.md ./note.md +``` + +## Installation + +### Build from source + +```bash +# Clone with submodules, because the shared core lives in src/lib +git clone --recurse-submodules +cd obsidian-livesync + +# If you already cloned without submodules, run this once instead +git submodule update --init --recursive + +# Install dependencies from the repository root +npm install + +# Build the CLI from the repository root +npm run build -w self-hosted-livesync-cli + +# Or from the package directory +cd src/apps/cli +npm run build +``` + +If `src/lib` is missing, the build process stops early with a targeted message instead of a low-level Vite `ENOENT` error. + +Run the CLI: + +```bash +# Run with npm workspace script (from repository root) +npm run cli -w self-hosted-livesync-cli -- [database-path] [command] [args...] + +# Or from the package directory +cd src/apps/cli +npm run cli -- [database-path] [command] [args...] + +# Run the built executable directly +node src/apps/cli/dist/index.cjs [database-path] [command] [args...] +``` + +### Docker + +A Docker image is provided for headless / server deployments. Build from the repository root: + +```bash +docker build -f src/apps/cli/Dockerfile -t livesync-cli . +``` + +Run: + +```bash +# Sync with CouchDB +docker run --rm -v /path/to/your/db:/data livesync-cli sync + +# Mirror to a specific vault directory +docker run --rm -v /path/to/your/db:/data -v /path/to/your/vault:/vault livesync-cli mirror /vault + +# List files in the local database +docker run --rm -v /path/to/your/db:/data livesync-cli ls +``` + +The database directory is mounted at `/data` by default. Override with `-e LIVESYNC_DB_PATH=/other/path`. + +#### P2P (WebRTC) and Docker networking + +The P2P replicator (`p2p-host`, `p2p-sync`, `p2p-peers`) uses WebRTC and generates +three kinds of ICE candidates. The default Docker bridge network affects which +candidates are usable: + +| Candidate type | Description | Bridge network | +| -------------- | ---------------------------------- | -------------------------- | +| `host` | Container bridge IP (`172.17.x.x`) | Unreachable from LAN peers | +| `srflx` | Host public IP via STUN reflection | Works over the internet | +| `relay` | Traffic relayed via TURN server | Always reachable | + +**LAN P2P on Linux** — use `--network host` so that the real host IP is +advertised as the `host` candidate: + +```bash +docker run --rm --network host -v /path/to/your/vault:/data livesync-cli p2p-host +``` + +Note: also fix the alias to include `--network host` if you want to use `livesync-cli` for P2P commands. + +> `--network host` is not available on Docker Desktop for macOS or Windows. + +**LAN P2P on macOS / Windows Docker Desktop** — configure a TURN server in the +settings file (`P2P_turnServers`, `P2P_turnUsername`, `P2P_turnCredential`). +All P2P traffic will then be relayed through the TURN server, bypassing the +bridge-network limitation. + +**Internet P2P** — the default bridge network is sufficient. The `srflx` +candidate carries the host's public IP and peers can connect normally. + +**CouchDB sync only (no P2P)** — no special network configuration is required. + + +### Adding `livesync-cli` alias + +To use the `livesync-cli` command globally, you can add an alias to your shell configuration file (e.g., `.zshrc` or `.bashrc`). + +If you are using `npm run`, add the following line: + +```bash +alias livesync-cli='npm run --silent --prefix /path/to/repository/src/apps/cli cli --' +# or +alias livesync-cli="npm run --silent --prefix $PWD cli --" +``` + +Alternatively, if you want to use the built executable directly: + +```bash +alias livesync-cli='node /path/to/repository/src/apps/cli/dist/index.cjs' +or +alias livesync-cli="node $PWD/dist/index.cjs" +``` + +If you prefer using Docker: + +```bash +alias livesync-cli='docker run --rm -v /path/to/your/db:/data livesync-cli' +``` + +After adding the alias, restart your shell or run `source ~/.zshrc` (or `.bashrc`). + +## Usage + +### Basic Usage + +As you know, the CLI is designed to be used in a headless environment. Hence all operations are performed against a local vault directory and a settings file. Here are some example commands: + +```bash +# Sync local database with CouchDB (no files will be changed). +livesync-cli /path/to/your-local-database --settings /path/to/settings.json sync + +# Push files to local database +livesync-cli /path/to/your-local-database --settings /path/to/settings.json push /your/storage/file.md /vault/path/file.md + +# Pull files from local database +livesync-cli /path/to/your-local-database --settings /path/to/settings.json pull /vault/path/file.md /your/storage/file.md + +# Verbose logging +livesync-cli /path/to/your-local-database --settings /path/to/settings.json --verbose + +# Apply setup URI to settings file (settings only; does not run synchronisation) +livesync-cli /path/to/your-local-database --settings /path/to/settings.json setup "obsidian://setuplivesync?settings=..." + +# Put text from stdin into local database +echo "Hello from stdin" | livesync-cli /path/to/your-local-database --settings /path/to/settings.json put /vault/path/file.md + +# Output a file from local database to stdout +livesync-cli /path/to/your-local-database --settings /path/to/settings.json cat /vault/path/file.md + +# Output a specific revision of a file from local database +livesync-cli /path/to/your-local-database --settings /path/to/settings.json cat-rev /vault/path/file.md 3-abcdef + +# Pull a specific revision of a file from local database to local storage +livesync-cli /path/to/your-local-database --settings /path/to/settings.json pull-rev /vault/path/file.md /your/storage/file.old.md 3-abcdef + +# List files in local database +livesync-cli /path/to/your-local-database --settings /path/to/settings.json ls /vault/path/ + +# Show metadata for a file in local database +livesync-cli /path/to/your-local-database --settings /path/to/settings.json info /vault/path/file.md + +# Mark a file as deleted in local database +livesync-cli /path/to/your-local-database --settings /path/to/settings.json rm /vault/path/file.md + +# Resolve conflict by keeping a specific revision +livesync-cli /path/to/your-local-database --settings /path/to/settings.json resolve /vault/path/file.md 3-abcdef + +# Add, list, activate, and remove remote configurations +livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-add main "sls+https://user:pass@example.com/db" +livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-ls +livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-export remote-abc123 +livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-set remote-abc123 "sls+p2p://room-abc?passphrase=secret" +livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-activate remote-abc123 +livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-rm remote-abc123 + +# Lock, unlock, resolve, and view status of remote database +livesync-cli /path/to/your-local-database --settings /path/to/settings.json remote-status remote-abc123 +livesync-cli /path/to/your-local-database --settings /path/to/settings.json lock-remote remote-abc123 +livesync-cli /path/to/your-local-database --settings /path/to/settings.json mark-resolved remote-abc123 +livesync-cli /path/to/your-local-database --settings /path/to/settings.json unlock-remote remote-abc123 +``` + +### Configuration + +The CLI uses the same settings format as the Obsidian plug-in. Create a `.livesync/settings.json` file in your vault directory: + +```json +{ + "couchDB_URI": "http://localhost:5984", + "couchDB_USER": "admin", + "couchDB_PASSWORD": "password", + "couchDB_DBNAME": "obsidian-livesync", + "liveSync": true, + "syncOnSave": true, + "syncOnStart": true, + "encrypt": true, + "passphrase": "your-encryption-passphrase", + "usePluginSync": false, + "isConfigured": true +} +``` + +**Minimum required settings:** + +- `couchDB_URI`: CouchDB server URL +- `couchDB_USER`: CouchDB username +- `couchDB_PASSWORD`: CouchDB password +- `couchDB_DBNAME`: Database name +- `isConfigured`: Set to `true` after configuration + +### Command-line Reference + +``` +Usage: + livesync-cli [options] [command-args] + livesync-cli init-settings [path] + +Arguments: + database-path Path to the local database directory (required except for init-settings) + +Options: + --settings, -s Path to settings file (default: .livesync/settings.json in local database directory) + --force, -f Overwrite existing file on init-settings + --verbose, -v Enable verbose logging + --debug, -d Enable debug logging (includes verbose) + --interval , -i (daemon only) Poll CouchDB every N seconds instead of using the _changes feed + --vault , -V (daemon/mirror) Path to vault directory, decoupled from database-path + --help, -h Show this help message + +Commands: + daemon (default) Run mirror scan then continuously sync CouchDB <-> local filesystem + init-settings [path] Create settings JSON from DEFAULT_SETTINGS + sync Run one replication cycle and exit + p2p-peers Show discovered peers as [peer] + p2p-sync Synchronise with specified peer-id or peer-name + p2p-host Start P2P host mode and wait until interrupted (Ctrl+C) + push Push local file into local database path + pull Pull file from local database into local file + pull-rev Pull specific revision into local file + setup Apply setup URI to settings file + put Read text from standard input and write to local database path + cat Write latest file content from local database to standard output + cat-rev Write specific revision content from local database to standard output + ls [prefix] List files as pathsizemtimerevision[*] + info Show file metadata including current and past revisions, conflicts, and chunk list + rm Mark file as deleted in local database + resolve Resolve conflict by keeping the specified revision + mirror [vaultPath] Mirror database contents to the local file system + (vaultPath positional arg > --vault flag > database-path) +``` + +Run via npm script: + +```bash +npm run --silent cli -- [database-path] [options] [command] [command-args] +``` + +#### Detailed Command Descriptions + +##### ls +`ls` lists files in the local database with optional prefix filtering. Output format is: + +```vault/path/file.mdsizemtimerevision[*] +``` +Note: `*` indicates if the file has conflicts. + +##### p2p-peers + +`p2p-peers ` waits for the specified number of seconds, then prints each discovered peer on a separate line: + +```text +[peer] +``` + +Use this command to select a target for `p2p-sync`. + +##### p2p-sync + +`p2p-sync ` discovers peers up to the specified timeout and synchronises with the selected peer. + +- `` accepts either `peer-id` or `peer-name` from `p2p-peers` output. +- On success, the command prints a completion message to standard error and exits with status code `0`. +- On failure, the command prints an error message and exits non-zero. + +##### p2p-host + +`p2p-host` starts the local P2P host and keeps running until interrupted. + +- Other peers can discover and synchronise with this host while it is running. +- Stop the host with `Ctrl+C`. +- In CLI mode, behaviour is non-interactive and acceptance follows settings. + +##### info + +`info` output fields: + +- `id`: Document ID +- `revision`: Current revision +- `conflicts`: Conflicted revisions, or `N/A` +- `filename`: Basename of path +- `path`: Vault-relative path +- `size`: Size in bytes +- `revisions`: Available non-current revisions +- `chunks`: Number of chunk IDs +- `children`: Chunk ID list + +##### mirror + +`mirror` is a command that synchronises your storage with your local vault. It is essentially a process that runs upon startup in Obsidian. + +In other words, it performs the following actions: + +1. **Precondition checks** — Aborts early if any of the following conditions are not met: + - Settings must be configured (`isConfigured: true`). + - File watching must not be suspended (`suspendFileWatching: false`). + - Remediation mode must be inactive (`maxMTimeForReflectEvents: 0`). + +2. **State restoration** — On subsequent runs (after the first successful scan), restores the previous storage state before proceeding. + +3. **Expired deletion cleanup** — If `automaticallyDeleteMetadataOfDeletedFiles` is set to a positive number of days, any document that is marked deleted and whose `mtime` is older than the retention period is permanently removed from the local database. + +4. **File collection** — Enumerates files from two sources: + - **Storage**: all files under the vault path that pass `isTargetFile`. + - **Local database**: all normal documents (fetched with conflict information) whose paths are valid and pass `isTargetFile`. + - Both collections build case-insensitive ↔ case-sensitive path maps, controlled by `handleFilenameCaseSensitive`. + +5. **Categorisation and synchronisation** — The union of both file sets is split into three groups and processed concurrently (up to 10 files at a time): + + | Group | Condition | Action | + | ----------------------------- | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + | **UPDATE DATABASE** | File exists in storage only | Store the file into the local database. | + | **UPDATE STORAGE** | File exists in database only | If the entry is active (not deleted) and not conflicted, restore the file from the database to storage. Deleted entries and conflicted entries are skipped. | + | **SYNC DATABASE AND STORAGE** | File exists in both | Compare `mtime` freshness. If storage is newer → write to database (`STORAGE → DB`). If database is newer → restore to storage (`STORAGE ← DB`). If equal → do nothing. Conflicted documents and files exceeding the size limit are always skipped. | + +6. **Initialisation flag** — On the very first successful run, writes `initialized = true` to the key-value database so that subsequent runs can restore state in step 2. + +Note: `mirror` does not respect file deletions. If a file is deleted in storage, it will be restored on the next `mirror` run. To delete a file, use the `rm` command instead. This is a little inconvenient, but it is intentional behaviour (if we handle this automatically in `mirror`, we should be against a ton of edge cases). + +##### daemon + +`daemon` is the default command when no command is specified. It runs an initial mirror scan and then continuously syncs changes in both directions: + +- **CouchDB → local filesystem**: via the `_changes` feed (LiveSync mode, default) or periodic polling (`--interval N`). +- **local filesystem → CouchDB**: via chokidar file watching. Any file created, modified, or deleted in the vault directory is pushed to CouchDB. + +In **LiveSync mode** the `_changes` feed delivers remote changes as they arrive, with sub-second latency. In **polling mode** (`--interval N`) the CLI polls CouchDB every N seconds. Use polling mode if your CouchDB instance does not support long-lived HTTP connections, or if you need predictable network usage. + +The daemon exits cleanly on `SIGINT` or `SIGTERM`. + +```bash +# LiveSync mode (default — _changes feed, near-real-time) +livesync-cli /path/to/vault + +# Polling mode — poll every 60 seconds +livesync-cli /path/to/vault --interval 60 +``` + +### .livesync/ignore + +Place a `.livesync/ignore` file in your vault root to exclude files from sync in both directions (local → CouchDB and CouchDB → local). + +**Format:** + +- Lines beginning with `#` are comments. +- Blank lines are ignored. +- All other lines are [minimatch](https://github.com/isaacs/minimatch) glob patterns, relative to the vault root. +- The directive `import: .gitignore` (exactly this string) reads `.gitignore` from the vault root and merges its non-comment, non-blank lines into the ignore rules. +- Negation patterns (lines starting with `!`) are not supported and will cause an error on load. + +**Example `.livesync/ignore`:** + +``` +# Ignore temporary files +*.tmp +*.swp + +# Ignore build output +build/ +dist/ + +# Merge patterns from .gitignore +import: .gitignore +``` + +Patterns apply in both directions: the chokidar watcher will not emit events for matched files, and the `isTargetFile` filter will exclude them from CouchDB → local sync. + +Changes to this file require a daemon restart to take effect. + +### Systemd Installation + +The `deploy/` directory contains a systemd unit template and an install script. + +**Automated install (user service, recommended):** + +```bash +bash src/apps/cli/deploy/install.sh --vault /path/to/vault +``` + +**With polling interval:** + +```bash +bash src/apps/cli/deploy/install.sh --vault /path/to/vault --interval 60 +``` + +**System-wide install** (requires root / sudo for `/etc/systemd/system/`): + +```bash +bash src/apps/cli/deploy/install.sh --system --vault /path/to/vault +``` + +The script: +1. Builds the CLI (`npm install` + `npm run build`). +2. Installs the binary to `~/.local/bin/livesync-cli` (user) or `/usr/local/bin/livesync-cli` (system). +3. Writes the unit file to `~/.config/systemd/user/livesync-cli.service` (user) or `/etc/systemd/system/livesync-cli.service` (system). +4. Runs `systemctl [--user] daemon-reload && systemctl [--user] enable --now livesync-cli`. + +**Manual setup** — if you prefer to manage the unit yourself, copy `deploy/livesync-cli.service`, replace `LIVESYNC_BIN` and `LIVESYNC_VAULT_PATH` with the actual binary path and vault path, then install to the appropriate systemd directory. + +### Planned options: + +- `--immediate`: Perform sync after the command (e.g. `push`, `pull`, `put`, `rm`). +- `serve`: Start CLI in server mode, exposing REST APIs for remote, and batch operations. +- `cause-conflicted `: Mark a file as conflicted without changing its content, to trigger conflict resolution in Obsidian. + +## Use Cases + +### 1. Bootstrap a new headless vault + +Create default settings, apply a setup URI, then run one sync cycle. + +```bash +livesync-cli -- init-settings /data/livesync-settings.json +printf '%s\n' "$SETUP_PASSPHRASE" | livesync-cli -- /data/vault --settings /data/livesync-settings.json setup "$SETUP_URI" +livesync-cli -- /data/vault --settings /data/livesync-settings.json sync +``` + +### 2. Scripted import and export + +Push local files into the database from automation, and pull them back for export or backup. + +```bash +livesync-cli -- /data/vault --settings /data/livesync-settings.json push ./note.md notes/note.md +livesync-cli -- /data/vault --settings /data/livesync-settings.json pull notes/note.md ./exports/note.md +``` + +### 3. Revision inspection and restore + +List metadata, find an older revision, then restore it by content (`cat-rev`) or file output (`pull-rev`). + +```bash +livesync-cli -- /data/vault --settings /data/livesync-settings.json info notes/note.md +livesync-cli -- /data/vault --settings /data/livesync-settings.json cat-rev notes/note.md 3-abcdef +livesync-cli -- /data/vault --settings /data/livesync-settings.json pull-rev notes/note.md ./restore/note.old.md 3-abcdef +``` + +### 4. Conflict and cleanup workflow + +Inspect conflicted revisions, resolve by keeping one revision, then delete obsolete files. + +```bash +livesync-cli -- /data/vault --settings /data/livesync-settings.json info notes/note.md +livesync-cli -- /data/vault --settings /data/livesync-settings.json resolve notes/note.md 3-abcdef +livesync-cli -- /data/vault --settings /data/livesync-settings.json rm notes/obsolete.md +``` + +### 5. CI smoke test for content round-trip + +Validate that `put`/`cat` is behaving as expected in a pipeline. + +```bash +echo "hello-ci" | livesync-cli -- /data/vault --settings /data/livesync-settings.json put ci/test.md +livesync-cli -- /data/vault --settings /data/livesync-settings.json cat ci/test.md +``` + +## Development + +### Project Structure + +``` +src/apps/cli/ +├── commands/ # Command dispatcher and command utilities +│ ├── runCommand.ts +│ ├── runCommand.unit.spec.ts +│ ├── types.ts +│ ├── utils.ts +│ └── utils.unit.spec.ts +├── adapters/ # Node.js FileSystem Adapter +│ ├── NodeConversionAdapter.ts +│ ├── NodeFileSystemAdapter.ts +│ ├── NodePathAdapter.ts +│ ├── NodeStorageAdapter.ts +│ ├── NodeStorageAdapter.unit.spec.ts +│ ├── NodeTypeGuardAdapter.ts +│ ├── NodeTypes.ts +│ └── NodeVaultAdapter.ts +├── lib/ +│ └── pouchdb-node.ts +├── managers/ # CLI-specific managers +│ ├── CLIStorageEventManagerAdapter.ts +│ └── StorageEventManagerCLI.ts +├── serviceModules/ # Service modules (ported from main.ts) +│ ├── CLIServiceModules.ts +│ ├── DatabaseFileAccess.ts +│ ├── FileAccessCLI.ts +│ └── ServiceFileAccessImpl.ts +├── services/ +│ ├── NodeKeyValueDBService.ts +│ ├── NodeServiceHub.ts +│ └── NodeSettingService.ts +├── test/ +│ ├── test-e2e-two-vaults-common.sh +│ ├── test-e2e-two-vaults-matrix.sh +│ ├── test-e2e-two-vaults-with-docker-linux.sh +│ ├── test-push-pull-linux.sh +│ ├── test-setup-put-cat-linux.sh +│ └── test-sync-two-local-databases-linux.sh +├── .gitignore +├── entrypoint.ts # CLI executable entry point (shebang) +├── main.ts # CLI entry point +├── main.unit.spec.ts +├── package.json +├── README.md # This file +├── tsconfig.json +├── util/ # Test and local utility scripts +└── vite.config.ts +``` diff --git a/test/e2e-obsidian/README.md b/test/e2e-obsidian/README.md new file mode 100644 index 0000000..25ffd96 --- /dev/null +++ b/test/e2e-obsidian/README.md @@ -0,0 +1,65 @@ +# Real Obsidian E2E Runner + +This directory contains the experimental real Obsidian end-to-end runner. + +The current smoke runner verifies only the launch path: + +1. create a temporary vault, +2. install the built Self-hosted LiveSync plug-in artifacts, +3. launch real Obsidian, +4. open the temporary vault through `obsidian-cli`, +5. enable Obsidian community plug-ins for the temporary app profile, +6. reload Self-hosted LiveSync through `obsidian-cli`, +7. verify through `obsidian-cli eval` that the plug-in is loaded, +8. terminate Obsidian and remove the temporary vault. + +The runner does not require Self-hosted LiveSync to expose an E2E-only bridge. Readiness is checked from outside the plug-in through Obsidian's own CLI. + +Obsidian 1.12 stores the global community plug-in switch outside `.obsidian/community-plugins.json`. The smoke runner enables it through `app.plugins.setEnable(true)` after the vault window is available. + +## Local Setup + +Set `OBSIDIAN_BINARY` when Obsidian is not installed in a standard location. + +For an AppImage on Linux without FUSE, use the helper script: + +```bash +npm run test:e2e:obsidian:install-appimage +``` + +The script downloads Obsidian `1.12.7` for the current architecture, stores it in `_testdata/obsidian`, and extracts it to `_testdata/obsidian/squashfs-root`. The runner checks `_testdata/obsidian/squashfs-root/obsidian` before the AppImage path. + +Do not download the AppImage on every CI run. Prefer one of these approaches: + +- set `OBSIDIAN_BINARY` to a pre-installed Obsidian executable, +- restore `_testdata/obsidian/squashfs-root` from a CI cache, or +- run `test:e2e:obsidian:install-appimage` only in a manually triggered preparation job. + +## Commands + +```bash +npm run test:e2e:obsidian:install-appimage +npm run test:e2e:obsidian:discover +npm run test:e2e:obsidian:cli-help -- vaults verbose +npm run test:e2e:obsidian:smoke +``` + +Useful environment variables: + +- `OBSIDIAN_BINARY`: explicit Obsidian executable path. +- `E2E_OBSIDIAN_VERSION`: Obsidian AppImage version for `test:e2e:obsidian:install-appimage`; default is `1.12.7`. +- `E2E_OBSIDIAN_APPIMAGE_ARCH`: AppImage architecture override, such as `arm64` or `x86_64`. +- `E2E_OBSIDIAN_APPIMAGE_URL`: explicit AppImage URL override. +- `E2E_OBSIDIAN_DOWNLOAD_DIR`: AppImage download and extraction directory; default is `_testdata/obsidian`. +- `E2E_OBSIDIAN_FORCE_DOWNLOAD=true`: re-download the AppImage even when it exists. +- `E2E_OBSIDIAN_SKIP_EXTRACT=true`: download the AppImage without extracting it. +- `E2E_OBSIDIAN_SMOKE_TIMEOUT_MS`: smoke timeout in milliseconds. +- `E2E_OBSIDIAN_READY_TIMEOUT_MS`: plug-in readiness timeout in milliseconds. +- `E2E_OBSIDIAN_CLI_READY_TIMEOUT_MS`: timeout for waiting until the vault-side Obsidian CLI exposes the plug-in catalogue. +- `E2E_OBSIDIAN_CLI_TIMEOUT_MS`: timeout for each `obsidian-cli` invocation. +- `E2E_OBSIDIAN_STARTUP_GRACE_MS`: early process-exit detection window in milliseconds. +- `E2E_OBSIDIAN_KEEP_VAULT=true`: keep the temporary vault for inspection. +- `E2E_OBSIDIAN_USE_XVFB=false`: disable automatic `xvfb-run` on headless Linux. +- `E2E_OBSIDIAN_ARGS`: override the default Obsidian launch arguments. + +On headless Linux, the runner automatically uses `/usr/bin/xvfb-run` when no `DISPLAY` or `WAYLAND_DISPLAY` is present. diff --git a/test/e2e-obsidian/runner/cli.ts b/test/e2e-obsidian/runner/cli.ts new file mode 100644 index 0000000..f1766b8 --- /dev/null +++ b/test/e2e-obsidian/runner/cli.ts @@ -0,0 +1,62 @@ +import { spawn } from "node:child_process"; + +export type ObsidianCliResult = { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +}; + +export async function runObsidianCli( + cliBinary: string, + args: string[], + env: NodeJS.ProcessEnv = process.env, + timeoutMs = Number(process.env.E2E_OBSIDIAN_CLI_TIMEOUT_MS ?? 10000) +): Promise { + return await new Promise((resolve, reject) => { + const child = spawn(cliBinary, args, { + stdio: ["ignore", "pipe", "pipe"], + env, + }); + let stdout = ""; + let stderr = ""; + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`Obsidian CLI timed out: ${cliBinary} ${args.join(" ")}`)); + }, timeoutMs); + + child.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + child.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.on("exit", (code, signal) => { + clearTimeout(timeout); + resolve({ code, signal, stdout, stderr }); + }); + }); +} + +export async function openVaultWithObsidianCli( + cliBinary: string, + vaultPath: string, + env: NodeJS.ProcessEnv = process.env +): Promise { + const result = await runObsidianCli(cliBinary, [`obsidian://open?path=${encodeURIComponent(vaultPath)}`], env); + if (result.code !== 0) { + throw new Error( + [ + `Failed to open Obsidian vault through CLI. code=${result.code}, signal=${result.signal}`, + result.stdout ? `stdout:\n${result.stdout}` : undefined, + result.stderr ? `stderr:\n${result.stderr}` : undefined, + ] + .filter(Boolean) + .join("\n") + ); + } +} diff --git a/test/e2e-obsidian/runner/environment.ts b/test/e2e-obsidian/runner/environment.ts new file mode 100644 index 0000000..46b08a6 --- /dev/null +++ b/test/e2e-obsidian/runner/environment.ts @@ -0,0 +1,149 @@ +import { accessSync, constants, existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { platform } from "node:process"; + +export type ObsidianDiscoveryResult = { + binary?: string; + source?: string; + checked: string[]; +}; + +const defaultCandidatesByPlatform: Record = { + aix: [], + android: [], + darwin: [ + "/Applications/Obsidian.app/Contents/MacOS/Obsidian", + "/Applications/Obsidian.app/Contents/MacOS/obsidian", + ], + freebsd: [], + haiku: [], + linux: [ + "_testdata/obsidian/squashfs-root/obsidian", + "_testdata/obsidian/squashfs-root/AppRun", + "_testdata/obsidian/Obsidian-1.12.7-arm64.AppImage", + "_testdata/obsidian/Obsidian-1.12.7-x86_64.AppImage", + "/usr/bin/obsidian", + "/usr/local/bin/obsidian", + "/snap/bin/obsidian", + "/opt/Obsidian/obsidian", + "/opt/obsidian/obsidian", + "/app/bin/obsidian", + ], + openbsd: [], + sunos: [], + win32: ["C:\\Program Files\\Obsidian\\Obsidian.exe", "C:\\Program Files (x86)\\Obsidian\\Obsidian.exe"], + cygwin: [], + netbsd: [], +}; + +const defaultCliCandidatesByPlatform: Record = { + aix: [], + android: [], + darwin: [ + "/Applications/Obsidian.app/Contents/MacOS/obsidian-cli", + "/Applications/Obsidian.app/Contents/Resources/obsidian-cli", + ], + freebsd: [], + haiku: [], + linux: [ + "_testdata/obsidian/squashfs-root/obsidian-cli", + "/usr/bin/obsidian-cli", + "/usr/local/bin/obsidian-cli", + "/snap/bin/obsidian-cli", + "/opt/Obsidian/obsidian-cli", + "/opt/obsidian/obsidian-cli", + ], + openbsd: [], + sunos: [], + win32: ["C:\\Program Files\\Obsidian\\obsidian-cli.exe", "C:\\Program Files (x86)\\Obsidian\\obsidian-cli.exe"], + cygwin: [], + netbsd: [], +}; + +function isUsableFile(path: string): boolean { + const resolvedPath = resolve(path); + if (!existsSync(resolvedPath)) { + return false; + } + if (platform === "win32") { + return true; + } + try { + accessSync(resolvedPath, constants.X_OK); + return true; + } catch { + return false; + } +} + +export function discoverObsidianBinary(env: NodeJS.ProcessEnv = process.env): ObsidianDiscoveryResult { + const checked: string[] = []; + const envBinary = env.OBSIDIAN_BINARY?.trim(); + if (envBinary) { + checked.push(envBinary); + if (isUsableFile(envBinary)) { + return { + binary: resolve(envBinary), + source: "OBSIDIAN_BINARY", + checked, + }; + } + } + + const candidates = defaultCandidatesByPlatform[platform] ?? []; + for (const candidate of candidates) { + checked.push(candidate); + if (isUsableFile(candidate)) { + return { + binary: resolve(candidate), + source: "default-path", + checked, + }; + } + } + + return { checked }; +} + +export function requireObsidianBinary(env: NodeJS.ProcessEnv = process.env): string { + const result = discoverObsidianBinary(env); + if (!result.binary) { + throw new Error( + [ + "Could not find an Obsidian executable.", + "Set OBSIDIAN_BINARY to the installed Obsidian executable path.", + `Checked paths: ${result.checked.length > 0 ? result.checked.join(", ") : "(none)"}`, + ].join("\n") + ); + } + return result.binary; +} + +export function discoverObsidianCli(env: NodeJS.ProcessEnv = process.env): ObsidianDiscoveryResult { + const checked: string[] = []; + const envBinary = env.OBSIDIAN_CLI?.trim(); + if (envBinary) { + checked.push(envBinary); + if (isUsableFile(envBinary)) { + return { + binary: resolve(envBinary), + source: "OBSIDIAN_CLI", + checked, + }; + } + } + + const candidates = defaultCliCandidatesByPlatform[platform] ?? []; + for (const candidate of candidates) { + checked.push(candidate); + if (isUsableFile(candidate)) { + return { + binary: resolve(candidate), + source: "default-path", + checked, + }; + } + } + + return { checked }; +} diff --git a/test/e2e-obsidian/runner/launch.ts b/test/e2e-obsidian/runner/launch.ts new file mode 100644 index 0000000..8a9b602 --- /dev/null +++ b/test/e2e-obsidian/runner/launch.ts @@ -0,0 +1,119 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { existsSync } from "node:fs"; +import { dirname } from "node:path"; +import { platform } from "node:process"; + +export type ObsidianProcess = { + process: ChildProcess; + stop: () => Promise; +}; + +export type LaunchObsidianOptions = { + binary: string; + vaultPath: string; + homePath?: string; + xdgConfigPath?: string; + userDataPath?: string; + startupGraceMs?: number; +}; + +function splitArgs(args: string): string[] { + return args.split(" ").filter((arg) => arg.length > 0); +} + +function launchArgs(options: LaunchObsidianOptions): string[] { + const explicitArgs = process.env.E2E_OBSIDIAN_ARGS; + if (explicitArgs) { + return splitArgs(explicitArgs); + } + return [ + "--no-sandbox", + "--disable-gpu", + "--disable-software-rasterizer", + ...(process.env.E2E_OBSIDIAN_USE_USER_DATA_DIR === "true" && options.userDataPath + ? [`--user-data-dir=${options.userDataPath}`] + : []), + ]; +} + +function shouldUseXvfb(): boolean { + if (process.env.E2E_OBSIDIAN_USE_XVFB === "false") { + return false; + } + if (process.env.DISPLAY || process.env.WAYLAND_DISPLAY) { + return false; + } + return platform === "linux" && existsSync("/usr/bin/xvfb-run"); +} + +export async function launchObsidian(options: LaunchObsidianOptions): Promise { + const startupGraceMs = options.startupGraceMs ?? 1000; + const args = launchArgs(options); + const useXvfb = shouldUseXvfb(); + const command = useXvfb ? "/usr/bin/xvfb-run" : options.binary; + const commandArgs = useXvfb ? ["-a", options.binary, ...args] : args; + const child = spawn(command, commandArgs, { + cwd: dirname(options.binary), + detached: true, + stdio: ["ignore", "pipe", "pipe"], + env: { + ...process.env, + ...(options.homePath ? { HOME: options.homePath } : {}), + ...(options.xdgConfigPath ? { XDG_CONFIG_HOME: options.xdgConfigPath } : {}), + OBSIDIAN_DISABLE_GPU: process.env.OBSIDIAN_DISABLE_GPU ?? "1", + }, + }); + + let stderr = ""; + let stdout = ""; + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + child.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + + const exitPromise = once(child, "exit").then(([code, signal]) => ({ code, signal })); + const timer = new Promise<"timeout">((resolve) => { + setTimeout(() => resolve("timeout"), startupGraceMs); + }); + const firstResult = await Promise.race([exitPromise, timer]); + if (firstResult !== "timeout") { + throw new Error( + [ + `Obsidian exited before the smoke timeout. code=${firstResult.code}, signal=${firstResult.signal}`, + stdout ? `stdout:\n${stdout}` : undefined, + stderr ? `stderr:\n${stderr}` : undefined, + ] + .filter(Boolean) + .join("\n") + ); + } + + return { + process: child, + stop: async () => { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + if (child.pid) { + process.kill(-child.pid, "SIGTERM"); + } else { + child.kill("SIGTERM"); + } + const stopTimer = new Promise<"timeout">((resolve) => { + setTimeout(() => resolve("timeout"), 5000); + }); + const stopResult = await Promise.race([exitPromise, stopTimer]); + if (stopResult === "timeout") { + if (child.pid) { + process.kill(-child.pid, "SIGKILL"); + } else { + child.kill("SIGKILL"); + } + await exitPromise; + } + }, + }; +} diff --git a/test/e2e-obsidian/runner/pluginInstaller.ts b/test/e2e-obsidian/runner/pluginInstaller.ts new file mode 100644 index 0000000..db28cc9 --- /dev/null +++ b/test/e2e-obsidian/runner/pluginInstaller.ts @@ -0,0 +1,39 @@ +import { copyFile, mkdir, writeFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { join, resolve } from "node:path"; + +export type PluginInstallResult = { + pluginDir: string; + copied: string[]; +}; + +const pluginId = "obsidian-livesync"; + +export async function installBuiltPlugin(vaultPath: string, rootDir = process.cwd()): Promise { + const pluginDir = join(vaultPath, ".obsidian", "plugins", pluginId); + const copied: string[] = []; + await mkdir(pluginDir, { recursive: true }); + + const requiredArtifacts = ["main.js", "manifest.json"]; + for (const artifact of requiredArtifacts) { + const source = resolve(rootDir, artifact); + if (!existsSync(source)) { + throw new Error(`Required plug-in artifact is missing: ${source}`); + } + await copyFile(source, join(pluginDir, artifact)); + copied.push(artifact); + } + + const optionalArtifacts = ["styles.css"]; + for (const artifact of optionalArtifacts) { + const source = resolve(rootDir, artifact); + if (!existsSync(source)) { + continue; + } + await copyFile(source, join(pluginDir, artifact)); + copied.push(artifact); + } + + await writeFile(join(vaultPath, ".obsidian", "community-plugins.json"), JSON.stringify([pluginId], null, 4)); + return { pluginDir, copied }; +} diff --git a/test/e2e-obsidian/runner/readiness.ts b/test/e2e-obsidian/runner/readiness.ts new file mode 100644 index 0000000..d56c2af --- /dev/null +++ b/test/e2e-obsidian/runner/readiness.ts @@ -0,0 +1,52 @@ +import { runObsidianCli } from "./cli.ts"; + +export type PluginReadiness = { + status: "ready"; + pluginId: string; + pluginVersion: string; + vaultName: string; +}; + +function parseEvalJson(stdout: string): unknown { + const marker = "=> "; + const markerIndex = stdout.indexOf(marker); + const text = markerIndex >= 0 ? stdout.slice(markerIndex + marker.length) : stdout; + return JSON.parse(text.trim()); +} + +export async function waitForPluginReady( + cliBinary: string, + env: NodeJS.ProcessEnv, + timeoutMs = Number(process.env.E2E_OBSIDIAN_READY_TIMEOUT_MS ?? 20000) +): Promise { + const deadline = Date.now() + timeoutMs; + let lastOutput = ""; + while (Date.now() < deadline) { + const result = await runObsidianCli( + cliBinary, + [ + "eval", + [ + "code=(async()=>JSON.stringify({", + "status:!!app.plugins.plugins['obsidian-livesync']?'ready':'pending',", + "pluginId:'obsidian-livesync',", + "pluginVersion:app.plugins.manifests['obsidian-livesync']?.version,", + "vaultName:app.vault.getName()", + "}))()", + ].join(""), + ], + env + ); + lastOutput = [result.stdout, result.stderr].filter(Boolean).join("\n"); + try { + const readiness = parseEvalJson(result.stdout) as PluginReadiness; + if (readiness.status === "ready") { + return readiness; + } + } catch { + // Keep polling until Obsidian exposes the vault-side CLI and plug-in state. + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for Self-hosted LiveSync readiness through Obsidian CLI.\n${lastOutput}`); +} diff --git a/test/e2e-obsidian/runner/vault.ts b/test/e2e-obsidian/runner/vault.ts new file mode 100644 index 0000000..4f151b4 --- /dev/null +++ b/test/e2e-obsidian/runner/vault.ts @@ -0,0 +1,72 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +export type TemporaryVault = { + path: string; + name: string; + homePath: string; + xdgConfigPath: string; + userDataPath: string; + dispose: () => Promise; +}; + +export async function createTemporaryVault(prefix = "obsidian-livesync-e2e-"): Promise { + const vaultPath = await mkdtemp(join(tmpdir(), prefix)); + const name = vaultPath.split(/[\\/]/).pop() ?? "obsidian-livesync-e2e"; + await mkdir(join(vaultPath, ".obsidian"), { recursive: true }); + const homePath = join(vaultPath, ".obsidian", "e2e-home"); + const xdgConfigPath = join(vaultPath, ".obsidian", "e2e-xdg-config"); + const userDataPath = join(vaultPath, ".obsidian", "e2e-user-data"); + await mkdir(homePath, { recursive: true }); + await mkdir(xdgConfigPath, { recursive: true }); + await mkdir(userDataPath, { recursive: true }); + await writeFile( + join(vaultPath, ".obsidian", "app.json"), + JSON.stringify({ legacyEditor: false, safeMode: false }, null, 4) + ); + await writeObsidianVaultRegistry(vaultPath, name, homePath, xdgConfigPath, userDataPath); + + return { + path: vaultPath, + name, + homePath, + xdgConfigPath, + userDataPath, + dispose: async () => { + if (process.env.E2E_OBSIDIAN_KEEP_VAULT === "true") { + console.log(`Keeping temporary vault: ${vaultPath}`); + return; + } + await rm(vaultPath, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + }, + }; +} + +async function writeObsidianVaultRegistry( + vaultPath: string, + vaultName: string, + homePath: string, + xdgConfigPath: string, + userDataPath: string +): Promise { + const vaultId = `livesync-e2e-${Date.now()}`; + const registry = { + cli: true, + vaults: { + [vaultId]: { + path: vaultPath, + ts: Date.now(), + open: true, + name: vaultName, + }, + }, + }; + const registryText = JSON.stringify(registry, null, 4); + for (const configRoot of [join(homePath, ".config"), xdgConfigPath]) { + const obsidianConfigDir = join(configRoot, "obsidian"); + await mkdir(obsidianConfigDir, { recursive: true }); + await writeFile(join(obsidianConfigDir, "obsidian.json"), registryText); + } + await writeFile(join(userDataPath, "obsidian.json"), registryText); +} diff --git a/test/e2e-obsidian/scripts/cli-help.ts b/test/e2e-obsidian/scripts/cli-help.ts new file mode 100644 index 0000000..f48de33 --- /dev/null +++ b/test/e2e-obsidian/scripts/cli-help.ts @@ -0,0 +1,51 @@ +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { launchObsidian } from "../runner/launch.ts"; +import { runObsidianCli } from "../runner/cli.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; +import { installBuiltPlugin } from "../runner/pluginInstaller.ts"; + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) { + throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); + } + const vault = await createTemporaryVault(); + let app; + try { + await installBuiltPlugin(vault.path); + app = await launchObsidian({ + binary, + vaultPath: vault.path, + homePath: vault.homePath, + xdgConfigPath: vault.xdgConfigPath, + userDataPath: vault.userDataPath, + }); + const cliEnv = { + ...process.env, + HOME: vault.homePath, + XDG_CONFIG_HOME: vault.xdgConfigPath, + }; + await runObsidianCli(cli.binary, [`obsidian://open?path=${encodeURIComponent(vault.path)}`], cliEnv); + await new Promise((resolve) => setTimeout(resolve, 3000)); + if (process.env.E2E_OBSIDIAN_RELOAD_PLUGIN === "true") { + await runObsidianCli(cli.binary, ["eval", "code=(async()=>app.plugins.setEnable(true))()"], cliEnv); + await runObsidianCli(cli.binary, ["plugin:reload", "id=obsidian-livesync"], cliEnv); + } + const cliArgs = process.argv.slice(2); + const result = await runObsidianCli(cli.binary, cliArgs.length > 0 ? cliArgs : ["--help"], cliEnv); + console.log(result.stdout); + console.error(result.stderr); + process.exitCode = result.code ?? 1; + } finally { + if (app) { + await app.stop(); + } + await vault.dispose(); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exit(1); +}); diff --git a/test/e2e-obsidian/scripts/discover.ts b/test/e2e-obsidian/scripts/discover.ts new file mode 100644 index 0000000..3b18888 --- /dev/null +++ b/test/e2e-obsidian/scripts/discover.ts @@ -0,0 +1,13 @@ +import { discoverObsidianBinary } from "../runner/environment.ts"; + +const result = discoverObsidianBinary(); +if (result.binary) { + console.log(`Obsidian executable: ${result.binary}`); + console.log(`Source: ${result.source}`); + process.exit(0); +} + +console.error("Obsidian executable was not found."); +console.error("Set OBSIDIAN_BINARY to the installed Obsidian executable path."); +console.error(`Checked paths: ${result.checked.length > 0 ? result.checked.join(", ") : "(none)"}`); +process.exit(1); diff --git a/test/e2e-obsidian/scripts/install-appimage.ts b/test/e2e-obsidian/scripts/install-appimage.ts new file mode 100644 index 0000000..72c7cab --- /dev/null +++ b/test/e2e-obsidian/scripts/install-appimage.ts @@ -0,0 +1,121 @@ +import { createWriteStream, existsSync } from "node:fs"; +import { chmod, mkdir } from "node:fs/promises"; +import { get } from "node:https"; +import { arch } from "node:process"; +import { basename, join, resolve } from "node:path"; +import { spawn } from "node:child_process"; + +const defaultVersion = "1.12.7"; + +function appImageArch(): string { + const requestedArch = process.env.E2E_OBSIDIAN_APPIMAGE_ARCH?.trim(); + if (requestedArch) { + return requestedArch; + } + if (arch === "arm64") { + return "arm64"; + } + if (arch === "x64") { + return "x86_64"; + } + throw new Error(`Unsupported architecture for Obsidian AppImage: ${arch}`); +} + +function appImageUrl(version: string, imageArch: string): string { + return `https://github.com/obsidianmd/obsidian-releases/releases/download/v${version}/Obsidian-${version}-${imageArch}.AppImage`; +} + +function download(url: string, destination: string, redirectsLeft = 5): Promise { + return new Promise((resolveDownload, reject) => { + const request = get(url, (response) => { + const statusCode = response.statusCode ?? 0; + const location = response.headers.location; + if (statusCode >= 300 && statusCode < 400 && location) { + response.resume(); + if (redirectsLeft <= 0) { + reject(new Error(`Too many redirects while downloading ${url}`)); + return; + } + download(new URL(location, url).toString(), destination, redirectsLeft - 1) + .then(resolveDownload) + .catch(reject); + return; + } + if (statusCode !== 200) { + response.resume(); + reject(new Error(`Failed to download ${url}: HTTP ${statusCode}`)); + return; + } + + const file = createWriteStream(destination, { mode: 0o755 }); + response.pipe(file); + file.on("finish", () => { + file.close((error) => { + if (error) { + reject(error); + } else { + resolveDownload(); + } + }); + }); + file.on("error", reject); + }); + request.on("error", reject); + }); +} + +function extractAppImage(appImagePath: string, cwd: string): Promise { + return new Promise((resolveExtract, reject) => { + const child = spawn(appImagePath, ["--appimage-extract"], { + cwd, + stdio: "inherit", + }); + child.on("error", reject); + child.on("exit", (code, signal) => { + if (code === 0) { + resolveExtract(); + return; + } + reject(new Error(`AppImage extraction failed. code=${code}, signal=${signal}`)); + }); + }); +} + +async function main(): Promise { + const version = process.env.E2E_OBSIDIAN_VERSION?.trim() || defaultVersion; + const imageArch = appImageArch(); + const targetDir = resolve(process.env.E2E_OBSIDIAN_DOWNLOAD_DIR?.trim() || "_testdata/obsidian"); + const url = process.env.E2E_OBSIDIAN_APPIMAGE_URL?.trim() || appImageUrl(version, imageArch); + const appImagePath = join(targetDir, basename(new URL(url).pathname)); + const extractedBinary = join(targetDir, "squashfs-root", "obsidian"); + const forceDownload = process.env.E2E_OBSIDIAN_FORCE_DOWNLOAD === "true"; + const skipExtract = process.env.E2E_OBSIDIAN_SKIP_EXTRACT === "true"; + + await mkdir(targetDir, { recursive: true }); + + if (!existsSync(appImagePath) || forceDownload) { + console.log(`Downloading Obsidian AppImage: ${url}`); + console.log(`Destination: ${appImagePath}`); + await download(url, appImagePath); + await chmod(appImagePath, 0o755); + } else { + console.log(`Using existing Obsidian AppImage: ${appImagePath}`); + } + + if (!skipExtract) { + if (existsSync(extractedBinary)) { + console.log(`Using existing extracted Obsidian binary: ${extractedBinary}`); + } else { + console.log(`Extracting Obsidian AppImage in ${targetDir}`); + await extractAppImage(appImagePath, targetDir); + console.log(`Extracted Obsidian binary: ${extractedBinary}`); + } + } + + console.log(`Set OBSIDIAN_BINARY=${extractedBinary} to use the extracted binary explicitly.`); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exit(1); +}); diff --git a/test/e2e-obsidian/scripts/smoke.ts b/test/e2e-obsidian/scripts/smoke.ts new file mode 100644 index 0000000..ad9bf8e --- /dev/null +++ b/test/e2e-obsidian/scripts/smoke.ts @@ -0,0 +1,96 @@ +import { discoverObsidianCli, requireObsidianBinary } from "../runner/environment.ts"; +import { launchObsidian } from "../runner/launch.ts"; +import { installBuiltPlugin } from "../runner/pluginInstaller.ts"; +import { waitForPluginReady } from "../runner/readiness.ts"; +import { createTemporaryVault } from "../runner/vault.ts"; +import { openVaultWithObsidianCli, runObsidianCli } from "../runner/cli.ts"; + +async function waitForPluginCatalogue(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + const deadline = Date.now() + Number(process.env.E2E_OBSIDIAN_CLI_READY_TIMEOUT_MS ?? 15000); + let lastOutput = ""; + while (Date.now() < deadline) { + const result = await runObsidianCli(cliBinary, ["plugins", "filter=community"], env); + lastOutput = [result.stdout, result.stderr].filter(Boolean).join("\n"); + if (result.stdout.includes("obsidian-livesync")) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`Timed out waiting for Obsidian plug-in catalogue through CLI.\n${lastOutput}`); +} + +async function enableCommunityPlugins(cliBinary: string, env: NodeJS.ProcessEnv): Promise { + const result = await runObsidianCli(cliBinary, ["eval", "code=(async()=>app.plugins.setEnable(true))()"], env); + if (result.code !== 0 || result.stdout.includes("Error:")) { + throw new Error( + [ + `Failed to enable Obsidian community plug-ins through CLI. code=${result.code}, signal=${result.signal}`, + result.stdout ? `stdout:\n${result.stdout}` : undefined, + result.stderr ? `stderr:\n${result.stderr}` : undefined, + ] + .filter(Boolean) + .join("\n") + ); + } +} + +async function main(): Promise { + const binary = requireObsidianBinary(); + const cli = discoverObsidianCli(); + if (!cli.binary) { + throw new Error(`Could not find obsidian-cli. Checked paths: ${cli.checked.join(", ")}`); + } + const vault = await createTemporaryVault(); + let app; + try { + const install = await installBuiltPlugin(vault.path); + console.log(`Using Obsidian executable: ${binary}`); + console.log(`Temporary vault: ${vault.path}`); + console.log(`Installed plug-in artifacts: ${install.copied.join(", ")}`); + + app = await launchObsidian({ + binary, + vaultPath: vault.path, + homePath: vault.homePath, + xdgConfigPath: vault.xdgConfigPath, + userDataPath: vault.userDataPath, + startupGraceMs: Number(process.env.E2E_OBSIDIAN_STARTUP_GRACE_MS ?? 1000), + }); + const cliEnv = { + ...process.env, + HOME: vault.homePath, + XDG_CONFIG_HOME: vault.xdgConfigPath, + }; + await openVaultWithObsidianCli(cli.binary, vault.path, cliEnv); + await waitForPluginCatalogue(cli.binary, cliEnv); + await enableCommunityPlugins(cli.binary, cliEnv); + const reload = await runObsidianCli(cli.binary, ["plugin:reload", "id=obsidian-livesync"], cliEnv); + if (reload.code !== 0 || !reload.stdout.includes("Reloaded: obsidian-livesync")) { + throw new Error( + [ + `Failed to reload Self-hosted LiveSync through Obsidian CLI. code=${reload.code}, signal=${reload.signal}`, + reload.stdout ? `stdout:\n${reload.stdout}` : undefined, + reload.stderr ? `stderr:\n${reload.stderr}` : undefined, + ] + .filter(Boolean) + .join("\n") + ); + } + const readiness = await waitForPluginReady(cli.binary, cliEnv); + console.log( + `Obsidian plug-in ready: ${readiness.pluginId}@${readiness.pluginVersion} in ${readiness.vaultName}` + ); + await new Promise((resolve) => setTimeout(resolve, Number(process.env.E2E_OBSIDIAN_SMOKE_TIMEOUT_MS ?? 1000))); + console.log("Obsidian stayed alive after the plug-in readiness check."); + } finally { + if (app) { + await app.stop(); + } + await vault.dispose(); + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : error); + process.exit(1); +});