diff --git a/package-lock.json b/package-lock.json index 56a150c..4c2fba3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "fflate": "^0.8.2", "idb": "^8.0.3", "markdown-it": "^14.1.1", + "micromatch": "^4.0.0", "minimatch": "^10.2.2", "octagonal-wheels": "^0.1.45", "pouchdb-adapter-leveldb": "^9.0.0", @@ -39,6 +40,7 @@ "@types/deno": "^2.5.0", "@types/diff-match-patch": "^1.0.36", "@types/markdown-it": "^14.1.2", + "@types/micromatch": "^4.0.10", "@types/node": "^24.10.13", "@types/pouchdb": "^6.4.2", "@types/pouchdb-adapter-http": "^6.1.6", @@ -4298,6 +4300,13 @@ "@babel/types": "^7.0.0" } }, + "node_modules/@types/braces": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.5.tgz", + "integrity": "sha512-SQFof9H+LXeWNz8wDe7oN5zu7ket0qwMu5vZubW4GCJ8Kkeh6nBWUz87+KTz/G3Kqsrp0j/W253XJb3KMEeg3w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -4417,6 +4426,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/micromatch": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/@types/micromatch/-/micromatch-4.0.10.tgz", + "integrity": "sha512-5jOhFDElqr4DKTrTEbnW8DZ4Hz5LRUEmyrGpCMrD/NphYv3nUnaF08xmSLx1rGGnyEs/kFnhiw6dCgcDqMr5PQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/braces": "*" + } + }, "node_modules/@types/minimatch": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", @@ -6119,7 +6138,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -8248,7 +8266,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -9351,7 +9368,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -10401,7 +10417,6 @@ "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.3", @@ -11111,7 +11126,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -13345,7 +13359,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" diff --git a/package.json b/package.json index 92eff77..bc96572 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "@types/deno": "^2.5.0", "@types/diff-match-patch": "^1.0.36", "@types/markdown-it": "^14.1.2", + "@types/micromatch": "^4.0.10", "@types/node": "^24.10.13", "@types/pouchdb": "^6.4.2", "@types/pouchdb-adapter-http": "^6.1.6", @@ -127,18 +128,19 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.808.0", - "chokidar": "^4.0.0", "@smithy/fetch-http-handler": "^5.3.10", "@smithy/md5-js": "^4.2.9", "@smithy/middleware-apply-body-checksum": "^4.3.9", "@smithy/protocol-http": "^5.3.9", "@smithy/querystring-builder": "^4.2.9", "@trystero-p2p/nostr": "^0.23.0", + "chokidar": "^4.0.0", "commander": "^14.0.3", "diff-match-patch": "^1.0.5", "fflate": "^0.8.2", "idb": "^8.0.3", "markdown-it": "^14.1.1", + "micromatch": "^4.0.0", "minimatch": "^10.2.2", "octagonal-wheels": "^0.1.45", "pouchdb-adapter-leveldb": "^9.0.0", diff --git a/src/apps/cli/README.md b/src/apps/cli/README.md index 6bd0082..d45f985 100644 --- a/src/apps/cli/README.md +++ b/src/apps/cli/README.md @@ -92,39 +92,39 @@ livesync-cli ./my-db pull folder/note.md ./note.md ## Installation -### Build from source - -```bash -# Clone with submodules, because the shared core lives in src/lib -git clone --recurse-submodules -cd obsidian-livesync - -# If you already cloned without submodules, run this once instead -git submodule update --init --recursive - -# Install dependencies from the repository root -npm install - -# Build the CLI from its package directory -cd src/apps/cli -npm run build -``` - -If `src/lib` is missing, `npm run build` now stops early with a targeted message -instead of a low-level Vite `ENOENT` error. +### Build from source -Run the CLI: - -```bash -# Run with npm script (from repository root) -npm run --silent cli -- [database-path] [command] [args...] +```bash +# Clone with submodules, because the shared core lives in src/lib +git clone --recurse-submodules +cd obsidian-livesync + +# If you already cloned without submodules, run this once instead +git submodule update --init --recursive + +# Install dependencies from the repository root +npm install + +# Build the CLI from its package directory +cd src/apps/cli +npm run build +``` + +If `src/lib` is missing, `npm run build` now stops early with a targeted message +instead of a low-level Vite `ENOENT` error. + +Run the CLI: + +```bash +# Run with npm script (from repository root) +npm run --silent cli -- [database-path] [command] [args...] # Run the built executable directly node src/apps/cli/dist/index.cjs [database-path] [command] [args...] ``` -### Docker - -A Docker image is provided for headless / server deployments. Build from the repository root: +### Docker + +A Docker image is provided for headless / server deployments. Build from the repository root: ```bash docker build -f src/apps/cli/Dockerfile -t livesync-cli . @@ -297,9 +297,11 @@ Options: --force, -f Overwrite existing file on init-settings --verbose, -v Enable verbose logging --debug, -d Enable debug logging (includes verbose) - --help, -h Show help message + --interval , -i (daemon only) Poll CouchDB every N seconds instead of using the _changes feed + --help, -h Show this help message Commands: + daemon (default) Run mirror scan then continuously sync CouchDB <-> local filesystem init-settings [path] Create settings JSON from DEFAULT_SETTINGS sync Run one replication cycle and exit p2p-peers Show discovered peers as [peer] @@ -406,6 +408,86 @@ In other words, it performs the following actions: Note: `mirror` does not respect file deletions. If a file is deleted in storage, it will be restored on the next `mirror` run. To delete a file, use the `rm` command instead. This is a little inconvenient, but it is intentional behaviour (if we handle this automatically in `mirror`, we should be against a ton of edge cases). +##### daemon + +`daemon` is the default command when no command is specified. It runs an initial mirror scan and then continuously syncs changes in both directions: + +- **CouchDB → local filesystem**: via the `_changes` feed (LiveSync mode, default) or periodic polling (`--interval N`). +- **local filesystem → CouchDB**: via chokidar file watching. Any file created, modified, or deleted in the vault directory is pushed to CouchDB. + +In **LiveSync mode** the `_changes` feed delivers remote changes as they arrive, with sub-second latency. In **polling mode** (`--interval N`) the CLI polls CouchDB every N seconds. Use polling mode if your CouchDB instance does not support long-lived HTTP connections, or if you need predictable network usage. + +The daemon exits cleanly on `SIGINT` or `SIGTERM`. + +```bash +# LiveSync mode (default — _changes feed, near-real-time) +livesync-cli /path/to/vault + +# Polling mode — poll every 60 seconds +livesync-cli /path/to/vault --interval 60 +``` + +### .livesync/ignore + +Place a `.livesync/ignore` file in your vault root to exclude files from sync in both directions (local → CouchDB and CouchDB → local). + +**Format:** + +- Lines beginning with `#` are comments. +- Blank lines are ignored. +- All other lines are [minimatch](https://github.com/isaacs/minimatch) glob patterns, relative to the vault root. +- The directive `import: .gitignore` (exactly this string) reads `.gitignore` from the vault root and merges its non-comment, non-blank lines into the ignore rules. +- Negation patterns (lines starting with `!`) are not supported and will cause an error on load. + +**Example `.livesync/ignore`:** + +``` +# Ignore temporary files +*.tmp +*.swp + +# Ignore build output +build/ +dist/ + +# Merge patterns from .gitignore +import: .gitignore +``` + +Patterns apply in both directions: the chokidar watcher will not emit events for matched files, and the `isTargetFile` filter will exclude them from CouchDB → local sync. + +Changes to this file require a daemon restart to take effect. + +### Systemd Installation + +The `deploy/` directory contains a systemd unit template and an install script. + +**Automated install (user service, recommended):** + +```bash +bash src/apps/cli/deploy/install.sh --vault /path/to/vault +``` + +**With polling interval:** + +```bash +bash src/apps/cli/deploy/install.sh --vault /path/to/vault --interval 60 +``` + +**System-wide install** (requires root / sudo for `/etc/systemd/system/`): + +```bash +bash src/apps/cli/deploy/install.sh --system --vault /path/to/vault +``` + +The script: +1. Builds the CLI (`npm install` + `npm run build`). +2. Installs the binary to `~/.local/bin/livesync-cli` (user) or `/usr/local/bin/livesync-cli` (system). +3. Writes the unit file to `~/.config/systemd/user/livesync-cli.service` (user) or `/etc/systemd/system/livesync-cli.service` (system). +4. Runs `systemctl [--user] daemon-reload && systemctl [--user] enable --now livesync-cli`. + +**Manual setup** — if you prefer to manage the unit yourself, copy `deploy/livesync-cli.service`, replace `LIVESYNC_BIN` and `LIVESYNC_VAULT_PATH` with the actual binary path and vault path, then install to the appropriate systemd directory. + ### Planned options: - `--immediate`: Perform sync after the command (e.g. `push`, `pull`, `put`, `rm`). diff --git a/src/apps/cli/commands/runCommand.ts b/src/apps/cli/commands/runCommand.ts index 5e1af90..9acb19d 100644 --- a/src/apps/cli/commands/runCommand.ts +++ b/src/apps/cli/commands/runCommand.ts @@ -22,6 +22,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext // accept whatever configuration the remote has. await core.services.setting.applyPartial({ disableCheckingConfigMismatch: true }, true); + // 1. Replicate CouchDB → local PouchDB so the mirror scan has content to work with. log("Replicating from CouchDB..."); const replResult = await core.services.replication.replicate(true); diff --git a/src/apps/cli/deploy/install.sh b/src/apps/cli/deploy/install.sh new file mode 100755 index 0000000..d0d3a2e --- /dev/null +++ b/src/apps/cli/deploy/install.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# install.sh — install livesync-cli as a systemd service +# +# Usage: +# install.sh [--user] [--system] [--vault ] [--interval ] +# +# Defaults: user install, prompts for vault path if not supplied. +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/../../.." && pwd)" +CLI_DIR="$REPO_ROOT/src/apps/cli" +SERVICE_TEMPLATE="$SCRIPT_DIR/livesync-cli.service" + +# ── Argument parsing ──────────────────────────────────────────────────────── +INSTALL_MODE="user" +VAULT_PATH="" +INTERVAL="" +FORCE=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --user) + INSTALL_MODE="user" + shift + ;; + --system) + INSTALL_MODE="system" + shift + ;; + --vault) + if [[ -z "${2:-}" ]]; then + echo "Error: --vault requires a path argument" >&2 + exit 1 + fi + VAULT_PATH="$2" + shift 2 + ;; + --interval) + if [[ -z "${2:-}" ]]; then + echo "Error: --interval requires a numeric argument" >&2 + exit 1 + fi + INTERVAL="$2" + if ! [[ "$INTERVAL" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: --interval requires a positive integer, got '$INTERVAL'" >&2 + exit 1 + fi + shift 2 + ;; + --force|-f) + FORCE=1 + shift + ;; + --help|-h) + cat <] [--interval ] [--force] + + --user Install as a user systemd service (default, ~/.config/systemd/user/) + --system Install as a system systemd service (/etc/systemd/system/) + --vault Path to the vault directory (prompted if omitted) + --interval Poll CouchDB every N seconds instead of using the _changes feed + --force Overwrite existing service unit without prompting +EOF + exit 0 + ;; + *) + echo "Error: Unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +# ── Vault path ────────────────────────────────────────────────────────────── +if [[ -z "$VAULT_PATH" ]]; then + if [ ! -t 0 ]; then + echo "Error: --vault is required in non-interactive mode" >&2 + exit 1 + fi + printf 'Vault path: ' + read -r VAULT_PATH +fi + +_orig_vault="$VAULT_PATH" +if ! VAULT_PATH="$(cd -- "$VAULT_PATH" 2>/dev/null && pwd)"; then + echo "Error: vault directory does not exist: $_orig_vault" >&2 + exit 1 +fi + +echo "[INFO] Vault: $VAULT_PATH" +echo "[INFO] Install mode: $INSTALL_MODE" + +# ── Build ──────────────────────────────────────────────────────────────────── +echo "[INFO] Building CLI from $REPO_ROOT..." +(cd "$REPO_ROOT" && npm install --silent) +(cd "$CLI_DIR" && npm run build) + +BUILT_CJS="$CLI_DIR/dist/index.cjs" +if [[ ! -f "$BUILT_CJS" ]]; then + echo "Error: build output not found: $BUILT_CJS" >&2 + exit 1 +fi + +# ── Install binary ─────────────────────────────────────────────────────────── +if [[ "$INSTALL_MODE" == "user" ]]; then + BIN_DIR="$HOME/.local/bin" + UNIT_DIR="$HOME/.config/systemd/user" + SYSTEMCTL_FLAGS="--user" +else + BIN_DIR="/usr/local/bin" + UNIT_DIR="/etc/systemd/system" + SYSTEMCTL_FLAGS="" +fi + +mkdir -p "$BIN_DIR" + +LIVESYNC_BIN="$BIN_DIR/livesync-cli" +LIVESYNC_JS="$BIN_DIR/livesync-cli.js" + +# Copy the CJS bundle so the wrapper is self-contained and independent of the +# build directory location. +cp "$BUILT_CJS" "$LIVESYNC_JS" + +# Write a bash wrapper that invokes node on the installed bundle. +cat > "$LIVESYNC_BIN" <&2 + exit 1 + fi + printf 'Service unit already exists at %s. Overwrite? [y/N]: ' "$UNIT_PATH" + read -r CONFIRM + case "$CONFIRM" in + [yY]|[yY][eE][sS]) : ;; + *) + echo "[INFO] Aborted. Existing unit left in place." + exit 0 + ;; + esac +fi + +# In awk gsub(), '&' in the replacement means "matched text"; escape any literal '&' +# in path variables before passing them as awk replacement strings. +AWK_BIN="${LIVESYNC_BIN//&/\\&}" +AWK_VAULT="${VAULT_PATH//&/\\&}" +awk -v bin="$AWK_BIN" -v vault="$AWK_VAULT" -v exec_start="ExecStart=$EXEC_START" \ + '/^ExecStart=/ { print exec_start; next } {gsub("LIVESYNC_BIN", bin); gsub("LIVESYNC_VAULT_PATH", vault); print}' \ + "$SERVICE_TEMPLATE" > "$UNIT_PATH" + +echo "[INFO] Installed unit: $UNIT_PATH" + +# ── Enable service ─────────────────────────────────────────────────────────── +if ! command -v systemctl >/dev/null 2>&1; then + echo "[WARN] systemctl not found — skipping service activation" + echo "[INFO] To enable manually, copy $UNIT_PATH to the correct systemd directory and run:" + echo " systemctl $SYSTEMCTL_FLAGS daemon-reload" + echo " systemctl $SYSTEMCTL_FLAGS enable --now livesync-cli" + exit 0 +fi + +# shellcheck disable=SC2086 +systemctl $SYSTEMCTL_FLAGS daemon-reload +# shellcheck disable=SC2086 +systemctl $SYSTEMCTL_FLAGS enable --now livesync-cli + +echo "" +echo "[Done] livesync-cli service installed and started." +echo "" +# shellcheck disable=SC2086 +systemctl $SYSTEMCTL_FLAGS status livesync-cli --no-pager || true diff --git a/src/apps/cli/deploy/livesync-cli.service b/src/apps/cli/deploy/livesync-cli.service new file mode 100644 index 0000000..b76e786 --- /dev/null +++ b/src/apps/cli/deploy/livesync-cli.service @@ -0,0 +1,17 @@ +[Unit] +Description=Self-hosted LiveSync CLI Daemon +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=LIVESYNC_BIN LIVESYNC_VAULT_PATH +Restart=on-failure +RestartSec=10 +TimeoutStartSec=300 +StandardOutput=journal +StandardError=journal +LimitNOFILE=65536 + +[Install] +WantedBy=default.target diff --git a/src/apps/cli/main.ts b/src/apps/cli/main.ts index 07ef657..535d137 100644 --- a/src/apps/cli/main.ts +++ b/src/apps/cli/main.ts @@ -26,6 +26,7 @@ import { VALID_COMMANDS } from "./commands/types"; import type { CLICommand, CLIOptions } from "./commands/types"; import { getPathFromUXFileInfo } from "@lib/common/typeUtils"; import { stripAllPrefixes } from "@lib/string_and_binary/path"; +import { IgnoreRules } from "./serviceModules/IgnoreRules"; const SETTINGS_FILE = ".livesync/settings.json"; ensureGlobalNodeLocalStorage(); @@ -221,6 +222,9 @@ async function createDefaultSettingsFile(options: CLIOptions) { export async function main() { const options = parseArgs(); + if (options.interval && options.command !== "daemon") { + console.error(`Warning: --interval is only used in daemon mode, ignored for '${options.command}'`); + } const avoidStdoutNoise = options.command === "cat" || options.command === "cat-rev" || @@ -275,13 +279,13 @@ export async function main() { // For daemon and mirror mode, load ignore rules before the core is constructed so that // chokidar's ignored option is populated when beginWatch() fires during onLoad(). const watchEnabled = options.command === "daemon"; - const vaultPathForIgnoreRules = + const vaultPath = options.command === "mirror" && options.commandArgs[0] ? path.resolve(options.commandArgs[0]) : databasePath; let ignoreRules: IgnoreRules | undefined; if (options.command === "daemon" || options.command === "mirror") { - ignoreRules = new IgnoreRules(vaultPathForIgnoreRules); + ignoreRules = new IgnoreRules(vaultPath); await ignoreRules.load(); } @@ -365,11 +369,7 @@ export async function main() { const core = new LiveSyncBaseCore( serviceHubInstance, (core: LiveSyncBaseCore, serviceHub: InjectableServiceHub) => { - const mirrorVaultPath = - options.command === "mirror" && options.commandArgs[0] - ? path.resolve(options.commandArgs[0]) - : databasePath; - return initialiseServiceModulesCLI(mirrorVaultPath, core, serviceHub); + return initialiseServiceModulesCLI(vaultPath, core, serviceHub, ignoreRules, watchEnabled); }, (core) => [ // No modules need to be registered for P2P replication in CLI. Directly using Replicators in p2p.ts @@ -385,8 +385,25 @@ export async function main() { if (parts.some((part) => part.startsWith("."))) { return await Promise.resolve(false); } + // PouchDB LevelDB database directory lives in the vault directory. + if (parts[0]?.endsWith("-livesync-v2")) { + return await Promise.resolve(false); + } return await Promise.resolve(true); }, -1 /* highest priority */); + + // Apply user-defined ignore rules for daemon mode (lower priority, runs after dotfile check). + if (ignoreRules) { + const rules = ignoreRules; + core.services.vault.isTargetFile.addHandler(async (target) => { + const targetPath = stripAllPrefixes(getPathFromUXFileInfo(target)); + if (rules.shouldIgnore(targetPath)) { + return false; + } + // undefined = pass through to next handler in chain + return undefined; + }, 0); + } } ); diff --git a/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts b/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts index ea6e31e..9abc5fd 100644 --- a/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts +++ b/src/apps/cli/managers/CLIStorageEventManagerAdapter.ts @@ -15,6 +15,7 @@ import type { Stats } from "fs"; import * as fs from "fs/promises"; import * as path from "path"; import { watch as chokidarWatch, type FSWatcher } from "chokidar"; +import type { IgnoreRules } from "../serviceModules/IgnoreRules"; /** * CLI-specific type guard adapter @@ -96,7 +97,7 @@ class CLIConverterAdapter implements IStorageEventConverterAdapter { class CLIWatchAdapter implements IStorageEventWatchAdapter { private _watcher: FSWatcher | undefined; - constructor(private basePath: string, private watchEnabled: boolean = false) {} + constructor(private basePath: string, private ignoreRules?: IgnoreRules, private watchEnabled: boolean = false) {} private _toNodeFile(filePath: string, stats: Stats | undefined): NodeFile { return { @@ -110,18 +111,6 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter { }; } - private _toNodeFileStub(filePath: string): NodeFile { - return { - path: path.relative(this.basePath, filePath) as FilePath, - stat: { - ctime: Date.now(), - mtime: Date.now(), - size: 0, - type: "file", - }, - }; - } - private _toNodeFolder(dirPath: string): NodeFolder { return { path: path.relative(this.basePath, dirPath) as FilePath, @@ -131,10 +120,19 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter { async beginWatch(handlers: IStorageEventWatchHandlers): Promise { if (!this.watchEnabled) return; + const baseIgnored: Array boolean)> = [ + /(^|[/\\])\./, + /(^|[/\\])[^/\\]*-livesync-v2([/\\]|$)/, + ]; + // Bind rules to a local const before the closure — chokidar v4 requires a + // MatchFunction, not glob strings, for custom patterns. + const rules = this.ignoreRules; + const ignored = rules + ? [...baseIgnored, (p: string) => rules.shouldIgnore(path.relative(this.basePath, p))] + : baseIgnored; + const watcher = chokidarWatch(this.basePath, { - ignored: [ - /(^|[/\\])\./, - ], + ignored, ignoreInitial: true, persistent: true, awaitWriteFinish: { @@ -154,7 +152,7 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter { }); watcher.on("unlink", (filePath) => { - const nodeFile = this._toNodeFileStub(filePath); + const nodeFile = this._toNodeFile(filePath, undefined); handlers.onDelete(nodeFile); }); @@ -199,10 +197,10 @@ export class CLIStorageEventManagerAdapter implements IStorageEventManagerAdapte readonly status: CLIStatusAdapter; readonly converter: CLIConverterAdapter; - constructor(basePath: string, watchEnabled: boolean = false) { + constructor(basePath: string, ignoreRules?: IgnoreRules, watchEnabled: boolean = false) { this.typeGuard = new CLITypeGuardAdapter(); this.persistence = new CLIPersistenceAdapter(basePath); - this.watch = new CLIWatchAdapter(basePath, watchEnabled); + this.watch = new CLIWatchAdapter(basePath, ignoreRules, watchEnabled); this.status = new CLIStatusAdapter(); this.converter = new CLIConverterAdapter(); } diff --git a/src/apps/cli/managers/StorageEventManagerCLI.ts b/src/apps/cli/managers/StorageEventManagerCLI.ts index c8edb3d..7838ef3 100644 --- a/src/apps/cli/managers/StorageEventManagerCLI.ts +++ b/src/apps/cli/managers/StorageEventManagerCLI.ts @@ -2,6 +2,7 @@ import { StorageEventManagerBase, type StorageEventManagerBaseDependencies } fro import { CLIStorageEventManagerAdapter } from "./CLIStorageEventManagerAdapter"; import type { IMinimumLiveSyncCommands, LiveSyncBaseCore } from "../../../LiveSyncBaseCore"; import type { ServiceContext } from "@lib/services/base/ServiceBase"; +import type { IgnoreRules } from "../serviceModules/IgnoreRules"; // import type { IMinimumLiveSyncCommands } from "@lib/services/base/IService"; export class StorageEventManagerCLI extends StorageEventManagerBase { @@ -11,9 +12,10 @@ export class StorageEventManagerCLI extends StorageEventManagerBase, dependencies: StorageEventManagerBaseDependencies, + ignoreRules?: IgnoreRules, watchEnabled?: boolean ) { - const adapter = new CLIStorageEventManagerAdapter(basePath, watchEnabled); + const adapter = new CLIStorageEventManagerAdapter(basePath, ignoreRules, watchEnabled); super(adapter, dependencies); this.core = core; } diff --git a/src/apps/cli/serviceModules/CLIServiceModules.ts b/src/apps/cli/serviceModules/CLIServiceModules.ts index b0e67ba..6c4cce5 100644 --- a/src/apps/cli/serviceModules/CLIServiceModules.ts +++ b/src/apps/cli/serviceModules/CLIServiceModules.ts @@ -9,6 +9,7 @@ import { ServiceFileAccessCLI } from "./ServiceFileAccessImpl"; import { ServiceDatabaseFileAccessCLI } from "./DatabaseFileAccess"; import { StorageEventManagerCLI } from "../managers/StorageEventManagerCLI"; import type { ServiceModules } from "@lib/interfaces/ServiceModule"; +import type { IgnoreRules } from "./IgnoreRules"; /** * Initialize service modules for CLI version @@ -23,6 +24,7 @@ export function initialiseServiceModulesCLI( basePath: string, core: LiveSyncBaseCore, services: InjectableServiceHub, + ignoreRules?: IgnoreRules, watchEnabled: boolean = false, ): ServiceModules { const storageAccessManager = new StorageAccessManager(); @@ -43,7 +45,7 @@ export function initialiseServiceModulesCLI( vaultService: services.vault, storageAccessManager: storageAccessManager, APIService: services.API, - }, false); + }, ignoreRules, watchEnabled); // Close the file watcher during graceful shutdown so the process can exit cleanly. services.appLifecycle.onUnload.addHandler(async () => { diff --git a/src/apps/cli/serviceModules/IgnoreRules.ts b/src/apps/cli/serviceModules/IgnoreRules.ts new file mode 100644 index 0000000..9764fd2 --- /dev/null +++ b/src/apps/cli/serviceModules/IgnoreRules.ts @@ -0,0 +1,129 @@ +import * as fs from "fs/promises"; +import * as path from "path"; + +import { minimatch } from "minimatch"; + +/** + * Loads and evaluates ignore rules from `.livesync/ignore` inside the vault. + * + * File format: + * - Lines starting with `#` are comments. + * - Blank lines are ignored. + * - `import: .gitignore` (exactly) — merges patterns from the vault's `.gitignore`. + * - All other lines are minimatch glob patterns relative to the vault root. + * + * Negation patterns (lines starting with `!`) are not supported. Loading a + * ruleset containing them throws an error — use separate include/exclude files + * instead. + * + * Missing files (`.livesync/ignore` or `.gitignore`) are silently skipped. + */ +export class IgnoreRules { + private patterns: string[] = []; + + constructor(private vaultPath: string) {} + + /** + * Reads `.livesync/ignore` (and optionally `.gitignore`) and populates the + * pattern list. Safe to call multiple times — each call replaces the + * previous state. Does not throw if files are absent. + * + * @throws if any pattern line begins with `!` (negation is unsupported). + */ + async load(): Promise { + this.patterns = []; + const ignorePath = path.join(this.vaultPath, ".livesync", "ignore"); + let rawLines: string[]; + try { + const content = await fs.readFile(ignorePath, "utf-8"); + rawLines = content.split(/\r?\n/); + } catch { + // File absent or unreadable — treat as empty ruleset. + return; + } + + for (const line of rawLines) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) { + continue; + } + // NOTE: Only the exact string "import: .gitignore" is recognised. + // Any future generalisation of this directive must validate that + // the resolved path stays within the vault directory. + if (trimmed === "import: .gitignore") { + await this._importGitignore(); + continue; + } + if (trimmed.startsWith("import:")) { + console.error(`[IgnoreRules] Warning: unrecognised directive '${trimmed}' — only 'import: .gitignore' is supported`); + continue; + } + this._addPattern(trimmed); + } + if (this.patterns.length > 0) { + console.error(`[IgnoreRules] Loaded ${this.patterns.length} ignore patterns`); + } + } + + // Normalises a single gitignore-style pattern: + // - Patterns ending with `/` (directory patterns like `build/`) are + // converted to `build/**` so they match all files inside that directory. + // - Patterns without a `/` are prefixed with `**/` to give them matchBase + // semantics (e.g. `*.tmp` → `**/*.tmp`), matching the basename in any + // subdirectory as gitignore does. + // - Patterns that already contain a `/` (but don't end with one) are + // path-specific and used as-is. + private _normalisePattern(pattern: string): string { + if (pattern.endsWith("/")) { + return "**/" + pattern + "**"; + } else if (!pattern.includes("/")) { + return "**/" + pattern; + } + return pattern; + } + + private async _importGitignore(): Promise { + const gitignorePath = path.join(this.vaultPath, ".gitignore"); + let content: string; + try { + content = await fs.readFile(gitignorePath, "utf-8"); + } catch { + return; + } + this._parseLines(content); + } + + private _parseLines(content: string): void { + for (const line of content.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + this._addPattern(trimmed); + } + } + + private _addPattern(raw: string): void { + if (raw.startsWith("!")) { + throw new Error( + `[IgnoreRules] Negation pattern '${raw}' is not supported. ` + + `Remove it from .livesync/ignore or use a separate include/exclude file.` + ); + } + this.patterns.push(this._normalisePattern(raw)); + } + + /** + * Returns `true` if the given vault-relative path matches any loaded + * ignore pattern. + * + * @param relativePath - Path relative to the vault root, using forward + * slashes or the OS separator. + */ + shouldIgnore(relativePath: string): boolean { + if (this.patterns.length === 0) { + return false; + } + // Normalise to forward slashes for minimatch. + const normalised = relativePath.replace(/\\/g, "/"); + return this.patterns.some((p) => minimatch(normalised, p, { dot: true })); + } +} diff --git a/src/apps/cli/serviceModules/IgnoreRules.unit.spec.ts b/src/apps/cli/serviceModules/IgnoreRules.unit.spec.ts new file mode 100644 index 0000000..59bfb12 --- /dev/null +++ b/src/apps/cli/serviceModules/IgnoreRules.unit.spec.ts @@ -0,0 +1,172 @@ +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { IgnoreRules } from "./IgnoreRules"; + +describe("IgnoreRules", () => { + const tempDirs: string[] = []; + + async function createVault(): Promise { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "livesync-ignorerules-")); + tempDirs.push(tempDir); + return tempDir; + } + + async function writeIgnoreFile(vaultPath: string, content: string): Promise { + const ignoreDir = path.join(vaultPath, ".livesync"); + await fs.mkdir(ignoreDir, { recursive: true }); + await fs.writeFile(path.join(ignoreDir, "ignore"), content, "utf-8"); + } + + afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); + }); + + describe("pattern normalisation", () => { + it("adds **/ prefix to basename patterns (no slash)", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "*.tmp\n"); + const rules = new IgnoreRules(vaultPath); + await rules.load(); + expect(rules.shouldIgnore("notes/scratch.tmp")).toBe(true); + expect(rules.shouldIgnore("scratch.tmp")).toBe(true); + expect(rules.shouldIgnore("deep/nested/file.tmp")).toBe(true); + }); + + it("appends ** to directory patterns ending with / and prepends **/", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "build/\n"); + const rules = new IgnoreRules(vaultPath); + await rules.load(); + expect(rules.shouldIgnore("build/output.js")).toBe(true); + expect(rules.shouldIgnore("build/nested/file.js")).toBe(true); + expect(rules.shouldIgnore("subproject/build/output.js")).toBe(true); + }); + + it("leaves patterns containing / as-is", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "docs/private.md\n"); + const rules = new IgnoreRules(vaultPath); + await rules.load(); + expect(rules.shouldIgnore("docs/private.md")).toBe(true); + expect(rules.shouldIgnore("other/docs/private.md")).toBe(false); + }); + }); + + describe("shouldIgnore", () => { + it("matches **/*.tmp against notes/scratch.tmp", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "*.tmp\n"); + const rules = new IgnoreRules(vaultPath); + await rules.load(); + expect(rules.shouldIgnore("notes/scratch.tmp")).toBe(true); + }); + + it("does not match notes/readme.md against **/*.tmp", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "*.tmp\n"); + const rules = new IgnoreRules(vaultPath); + await rules.load(); + expect(rules.shouldIgnore("notes/readme.md")).toBe(false); + }); + + it("returns false when no patterns are loaded", async () => { + const vaultPath = await createVault(); + const rules = new IgnoreRules(vaultPath); + // No load() call — patterns are empty + expect(rules.shouldIgnore("anything.md")).toBe(false); + }); + }); + + describe("negation patterns", () => { + it("throws when a negation pattern is encountered", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "*.tmp\n!important.tmp\n"); + const rules = new IgnoreRules(vaultPath); + await expect(rules.load()).rejects.toThrow(/Negation pattern/); + }); + + it("throws when a .gitignore imported via directive contains negation", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "import: .gitignore\n"); + await fs.writeFile(path.join(vaultPath, ".gitignore"), "*.log\n!keep.log\n", "utf-8"); + const rules = new IgnoreRules(vaultPath); + await expect(rules.load()).rejects.toThrow(/Negation pattern/); + }); + }); + + describe("unrecognised import: directives", () => { + it("warns and skips unrecognised import: forms (does not add as literal pattern)", async () => { + const vaultPath = await createVault(); + // Typo: "import:.gitignore" instead of "import: .gitignore" + await writeIgnoreFile(vaultPath, "*.tmp\nimport:.gitignore\n"); + const rules = new IgnoreRules(vaultPath); + await rules.load(); + // *.tmp still loaded; import:.gitignore is skipped (not treated as a literal pattern) + expect(rules.shouldIgnore("scratch.tmp")).toBe(true); + expect(rules.shouldIgnore("import:.gitignore")).toBe(false); + }); + }); + + describe("load() with missing file", () => { + it("returns without error when .livesync/ignore is absent", async () => { + const vaultPath = await createVault(); + // No ignore file created + const rules = new IgnoreRules(vaultPath); + await expect(rules.load()).resolves.toBeUndefined(); + expect(rules.shouldIgnore("anything.md")).toBe(false); + }); + }); + + describe("load() with comments and blank lines", () => { + it("skips # comment lines and blank lines", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile( + vaultPath, + "# This is a comment\n\n \n*.tmp\n# another comment\nbuild/\n" + ); + const rules = new IgnoreRules(vaultPath); + await rules.load(); + expect(rules.shouldIgnore("scratch.tmp")).toBe(true); + expect(rules.shouldIgnore("build/output.js")).toBe(true); + expect(rules.shouldIgnore("readme.md")).toBe(false); + }); + }); + + describe("import: .gitignore directive", () => { + it("reads and normalises patterns from .gitignore", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "import: .gitignore\n"); + await fs.writeFile(path.join(vaultPath, ".gitignore"), "*.log\nnode_modules/\n", "utf-8"); + const rules = new IgnoreRules(vaultPath); + await rules.load(); + expect(rules.shouldIgnore("app.log")).toBe(true); + expect(rules.shouldIgnore("node_modules/package.json")).toBe(true); + expect(rules.shouldIgnore("src/node_modules/package.json")).toBe(true); + expect(rules.shouldIgnore("src/index.ts")).toBe(false); + }); + + it("merges .gitignore patterns with other patterns", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "*.tmp\nimport: .gitignore\n"); + await fs.writeFile(path.join(vaultPath, ".gitignore"), "*.log\n", "utf-8"); + const rules = new IgnoreRules(vaultPath); + await rules.load(); + expect(rules.shouldIgnore("scratch.tmp")).toBe(true); + expect(rules.shouldIgnore("error.log")).toBe(true); + }); + }); + + describe("import: .gitignore with missing .gitignore", () => { + it("does not throw when .gitignore is absent", async () => { + const vaultPath = await createVault(); + await writeIgnoreFile(vaultPath, "*.tmp\nimport: .gitignore\n"); + // No .gitignore created + const rules = new IgnoreRules(vaultPath); + await expect(rules.load()).resolves.toBeUndefined(); + // The *.tmp pattern from the ignore file still works + expect(rules.shouldIgnore("scratch.tmp")).toBe(true); + }); + }); +}); diff --git a/src/apps/cli/test/test-daemon-linux.sh b/src/apps/cli/test/test-daemon-linux.sh new file mode 100755 index 0000000..96db2c7 --- /dev/null +++ b/src/apps/cli/test/test-daemon-linux.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# Test: daemon-related ignore rules behaviour +# +# Tests that are runnable without a long-running daemon process are exercised +# here using the `mirror` command, which calls the same `isTargetFile` handler +# stack that the daemon uses. +# +# Covered cases: +# 1. .livesync/ignore with *.tmp pattern → ignored file is NOT synced to DB +# 2. .livesync/ignore missing → no error, normal sync continues +# 3. import: .gitignore directive → patterns from .gitignore are merged +# +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +CLI_DIR="$(cd -- "$SCRIPT_DIR/.." && pwd)" +cd "$CLI_DIR" +source "$SCRIPT_DIR/test-helpers.sh" +display_test_info + +RUN_BUILD="${RUN_BUILD:-1}" +cli_test_init_cli_cmd + +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/livesync-cli-daemon-test.XXXXXX")" +trap 'rm -rf "$WORK_DIR"' EXIT + +SETTINGS_FILE="$WORK_DIR/data.json" +VAULT_DIR="$WORK_DIR/vault" +mkdir -p "$VAULT_DIR/notes" + +if [[ "$RUN_BUILD" == "1" ]]; then + echo "[INFO] building CLI..." + npm run build +fi + +echo "[INFO] generating settings -> $SETTINGS_FILE" +cli_test_init_settings_file "$SETTINGS_FILE" +cli_test_mark_settings_configured "$SETTINGS_FILE" + +PASS=0 +FAIL=0 + +assert_pass() { echo "[PASS] $1"; PASS=$((PASS + 1)); } +assert_fail() { echo "[FAIL] $1" >&2; FAIL=$((FAIL + 1)); } + +# ───────────────────────────────────────────────────────────────────────────── +# Case 1: .livesync/ignore with *.tmp → matched file should NOT appear in DB +# ───────────────────────────────────────────────────────────────────────────── +echo "" +echo "=== Case 1: .livesync/ignore *.tmp → ignored file not synced to DB ===" + +mkdir -p "$VAULT_DIR/.livesync" +printf '*.tmp\n' > "$VAULT_DIR/.livesync/ignore" + +# Also write a normal file so we can confirm mirror ran at all. +printf 'normal content\n' > "$VAULT_DIR/notes/normal.md" +# Write the file that should be ignored. +printf 'tmp content\n' > "$VAULT_DIR/notes/scratch.tmp" + +run_cli "$VAULT_DIR" --settings "$SETTINGS_FILE" mirror + +# The normal file should be in the DB. +RESULT_NORMAL="$WORK_DIR/case1-normal.txt" +if run_cli "$VAULT_DIR" --settings "$SETTINGS_FILE" pull notes/normal.md "$RESULT_NORMAL" 2>/dev/null; then + if cmp -s "$VAULT_DIR/notes/normal.md" "$RESULT_NORMAL"; then + assert_pass "normal.md was synced to DB" + else + assert_fail "normal.md content mismatch after mirror" + fi +else + assert_fail "normal.md was not found in DB after mirror" +fi + +# The .tmp file should NOT be in the DB. +DB_LIST="$WORK_DIR/case1-ls.txt" +run_cli "$VAULT_DIR" --settings "$SETTINGS_FILE" ls > "$DB_LIST" +if grep -q "scratch.tmp" "$DB_LIST"; then + assert_fail "scratch.tmp (ignored) was unexpectedly synced to DB" + echo "--- DB listing ---" >&2; cat "$DB_LIST" >&2 +else + assert_pass "scratch.tmp (*.tmp pattern) was NOT synced to DB" +fi + +# ───────────────────────────────────────────────────────────────────────────── +# Case 2: .livesync/ignore absent → no error, normal sync continues +# ───────────────────────────────────────────────────────────────────────────── +echo "" +echo "=== Case 2: .livesync/ignore absent → no error, sync continues ===" + +VAULT_DIR2="$WORK_DIR/vault2" +mkdir -p "$VAULT_DIR2/notes" +SETTINGS_FILE2="$WORK_DIR/data2.json" +cli_test_init_settings_file "$SETTINGS_FILE2" +cli_test_mark_settings_configured "$SETTINGS_FILE2" + +# No .livesync directory at all. +printf 'hello\n' > "$VAULT_DIR2/notes/hello.md" + +# mirror should succeed without error. +set +e +MIRROR_OUTPUT="$WORK_DIR/case2-mirror.txt" +run_cli "$VAULT_DIR2" --settings "$SETTINGS_FILE2" mirror >"$MIRROR_OUTPUT" 2>&1 +MIRROR_EXIT=$? +set -e + +if [[ "$MIRROR_EXIT" -ne 0 ]]; then + assert_fail "mirror exited non-zero ($MIRROR_EXIT) when .livesync/ignore is absent" + cat "$MIRROR_OUTPUT" >&2 +else + assert_pass "mirror succeeded when .livesync/ignore is absent" +fi + +# The normal file should have been synced. +RESULT_HELLO="$WORK_DIR/case2-hello.txt" +if run_cli "$VAULT_DIR2" --settings "$SETTINGS_FILE2" pull notes/hello.md "$RESULT_HELLO" 2>/dev/null; then + assert_pass "file synced normally when .livesync/ignore is absent" +else + assert_fail "file was not synced when .livesync/ignore is absent" +fi + +# ───────────────────────────────────────────────────────────────────────────── +# Case 3: import: .gitignore merges patterns +# ───────────────────────────────────────────────────────────────────────────── +echo "" +echo "=== Case 3: import: .gitignore directive merges patterns ===" + +VAULT_DIR3="$WORK_DIR/vault3" +mkdir -p "$VAULT_DIR3/notes" +SETTINGS_FILE3="$WORK_DIR/data3.json" +cli_test_init_settings_file "$SETTINGS_FILE3" +cli_test_mark_settings_configured "$SETTINGS_FILE3" + +mkdir -p "$VAULT_DIR3/.livesync" +printf 'import: .gitignore\n' > "$VAULT_DIR3/.livesync/ignore" +printf '# gitignore comment\n*.log\nbuild/\n' > "$VAULT_DIR3/.gitignore" + +printf 'regular note\n' > "$VAULT_DIR3/notes/regular.md" +printf 'log content\n' > "$VAULT_DIR3/notes/debug.log" + +run_cli "$VAULT_DIR3" --settings "$SETTINGS_FILE3" mirror + +DB_LIST3="$WORK_DIR/case3-ls.txt" +run_cli "$VAULT_DIR3" --settings "$SETTINGS_FILE3" ls > "$DB_LIST3" + +if grep -q "debug.log" "$DB_LIST3"; then + assert_fail "debug.log (ignored via .gitignore import) was unexpectedly synced to DB" + echo "--- DB listing ---" >&2; cat "$DB_LIST3" >&2 +else + assert_pass "debug.log (*.log from imported .gitignore) was NOT synced to DB" +fi + +# regular.md should still be present. +if grep -q "regular.md" "$DB_LIST3"; then + assert_pass "regular.md was synced normally alongside .gitignore import rules" +else + assert_fail "regular.md was NOT synced — .gitignore import may have been too broad" +fi + +# ───────────────────────────────────────────────────────────────────────────── +# Summary +# ───────────────────────────────────────────────────────────────────────────── +echo "" +echo "Results: PASS=$PASS FAIL=$FAIL" +if [[ "$FAIL" -gt 0 ]]; then + exit 1 +fi diff --git a/src/apps/cli/vite.config.ts b/src/apps/cli/vite.config.ts index 6850a94..74c4ba6 100644 --- a/src/apps/cli/vite.config.ts +++ b/src/apps/cli/vite.config.ts @@ -30,6 +30,10 @@ if (typeof globalThis.FileReader === "undefined") { if (this.onload) this.onload({ target: this }); }).catch((err) => { if (this.onerror) this.onerror({ target: this, error: err }); }); } + readAsArrayBuffer() { throw new Error("FileReader.readAsArrayBuffer is not implemented in this polyfill"); } + readAsBinaryString() { throw new Error("FileReader.readAsBinaryString is not implemented in this polyfill"); } + readAsText() { throw new Error("FileReader.readAsText is not implemented in this polyfill"); } + abort() { throw new Error("FileReader.abort is not implemented in this polyfill"); } }; } `;