public/obsidian-livesync
1
0
Fork 0
mirror of https://github.com/vrtmrz/obsidian-livesync.git synced 2026-08-27 22:07:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge current main into PR 1039

Browse Source
This commit is contained in:
vorotamoroz
2026-08-09 08:45:16 +00:00
parent a3a09df3c8 f2976bc89a
commit 00de35e5d4
1036 changed files with 87150 additions and 45505 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/apps/cli/Dockerfile
+3 -3
View File
@@ -101,9 +101,9 @@ COPY --from=runtime-deps /deps/node_modules ./node_modules
# Copy the built CLI bundle from builder stage
COPY --from=builder /build/src/apps/cli/dist ./dist
# Install entrypoint wrapper
COPY src/apps/cli/docker-entrypoint.sh /usr/local/bin/livesync-cli
RUN chmod +x /usr/local/bin/livesync-cli
# Install the entrypoint wrapper with a deterministic mode, regardless of
# source checkout permissions.
COPY --chmod=755 src/apps/cli/docker-entrypoint.sh /usr/local/bin/livesync-cli
# Mount your vault / local database directory here
VOLUME ["/data"]
src/apps/cli/README.md
+16 -11
View File
@@ -15,9 +15,10 @@ This CLI version is built using the same core as the Obsidian plug-in:
```
CLI Main
└─ LiveSyncBaseCore<ServiceContext, IMinimumLiveSyncCommands>
├─ NodeServiceHub (All services without Obsidian dependencies)
└─ ServiceModules (wired by initialiseServiceModulesCLI)
└─ NodeServiceContext (events, translation, database root, and injected standard I/O)
└─ LiveSyncBaseCore<NodeServiceContext, IMinimumLiveSyncCommands>
├─ NodeServiceHub (All services without Obsidian dependencies)
└─ ServiceModules (wired by initialiseServiceModulesCLI)
├─ FileAccessCLI (Node.js FileSystemAdapter)
├─ StorageEventManagerCLI
├─ ServiceFileAccessCLI
@@ -37,7 +38,9 @@ CLI Main
- All core sync functionality preserved
3. **Service Hub and Settings Services** (`services/`)
- `NodeServiceHub` provides the CLI service context
- `NodeServiceContext` owns the host-selected database root and standard input/output implementation
- `NodeServiceHub` receives that exact Context instead of constructing platform capabilities implicitly
- Internal adapter diagnostics use injected callbacks wired to the service logging API
- Node-specific settings and key-value services are provided without Obsidian dependencies
4. **Main Entry Point** (`main.ts`)
@@ -64,6 +67,10 @@ livesync-cli [database-path] [command] [args...]
- `--vault <path>` / `-V <path>`: (daemon/mirror only) Path to the vault directory containing `.md` files.
- Allows the PouchDB database directory and the actual vault directory to be different locations.
- For `mirror` command, the positional `[vault-path]` argument takes precedence over `--vault`.
- `--write-settings`: Write setting migrations and other lasting changes after the command succeeds.
- `init-settings` writes its target file. `setup`, `remote-add`, `remote-rm`, `remote-set`, and `remote-activate` write their settings changes without this option.
- All remaining commands leave the settings file unchanged by default.
- Temporary values used to suspend synchronisation or select a remote for one command are never written.
### Commands
@@ -108,13 +115,10 @@ livesync-cli ./my-db pull folder/note.md ./note.md
### Build from source
```bash
# Clone with submodules, because the shared core lives in src/lib
git clone --recurse-submodules <repository-url>
# Clone the repository
git clone <repository-url>
cd obsidian-livesync
# If you already cloned without submodules, run this once instead
git submodule update --init --recursive
# Install dependencies from the repository root
npm install
@@ -126,7 +130,7 @@ cd src/apps/cli
npm run build
```
If `src/lib` is missing, the build process stops early with a targeted message instead of a low-level Vite `ENOENT` error.
The shared core is installed as the exact `@vrtmrz/livesync-commonlib` package artefact recorded in the root lockfile.
Run the CLI:
@@ -333,11 +337,12 @@ Options:
--debug, -d Enable debug logging (includes verbose)
--interval <N>, -i <N> (daemon only) Poll CouchDB every N seconds instead of using the _changes feed
--vault <path>, -V <path> (daemon/mirror) Path to vault directory, decoupled from database-path
--write-settings Write setting changes after a successful command
--help, -h Show this help message
Commands:
daemon (default) Run mirror scan then continuously sync CouchDB <-> local filesystem
init-settings [path] Create settings JSON from DEFAULT_SETTINGS
init-settings [path] Create unconfigured settings JSON with the new-Vault recommendations
sync Run one replication cycle and exit
p2p-peers <timeout> Show discovered peers as [peer]<TAB><peer-id><TAB><peer-name>
p2p-sync <peer> <timeout> Synchronise with specified peer-id or peer-name
src/apps/cli/adapters/NodeConversionAdapter.ts
+3 -3
View File
@@ -1,7 +1,7 @@
import type { FilePath, UXFileInfoStub, UXFolderInfo } from "@lib/common/types";
import type { IConversionAdapter } from "@lib/serviceModules/adapters";
import type { FilePath, UXFileInfoStub, UXFolderInfo } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { IConversionAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
import type { NodeFile, NodeFolder } from "./NodeTypes";
import { path } from "@/apps/cli/node-compat";
import { path } from "@vrtmrz/livesync-commonlib/node";
/**
* Conversion adapter implementation for Node.js
src/apps/cli/adapters/NodeFileSystemAdapter.ts
+59 -33
View File
@@ -1,12 +1,13 @@
import type { FilePath, UXStat } from "@lib/common/types";
import type { IFileSystemAdapter } from "@lib/serviceModules/adapters";
import type { FilePath, UXStat } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { IFileSystemAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
import { NodePathAdapter } from "./NodePathAdapter";
import { NodeTypeGuardAdapter } from "./NodeTypeGuardAdapter";
import { NodeConversionAdapter } from "./NodeConversionAdapter";
import { NodeStorageAdapter } from "./NodeStorageAdapter";
import { NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node";
import { NodeVaultAdapter } from "./NodeVaultAdapter";
import type { NodeFile, NodeFolder, NodeStat } from "./NodeTypes";
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
import { path } from "@vrtmrz/livesync-commonlib/node";
import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput";
/**
* Complete file system adapter implementation for Node.js
@@ -20,12 +21,15 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
private fileCache = new Map<string, NodeFile>();
constructor(private basePath: string) {
constructor(
private basePath: string,
private reportDiagnostic: CliDiagnosticReporter = () => undefined
) {
this.path = new NodePathAdapter();
this.typeGuard = new NodeTypeGuardAdapter();
this.conversion = new NodeConversionAdapter();
this.storage = new NodeStorageAdapter(basePath);
this.vault = new NodeVaultAdapter(basePath);
this.vault = new NodeVaultAdapter(this.storage);
}
private resolvePath(p: FilePath | string): string {
@@ -33,11 +37,31 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
}
private normalisePath(p: FilePath | string): string {
return this.path.normalisePath(p as string);
return this.path.normalisePath(p);
}
private async hasExactPathCase(pathStr: string): Promise<boolean> {
try {
const segments = pathStr.split("/").filter((segment) => segment !== "");
let currentPath = "";
for (const segment of segments) {
const entries = await this.storage.list(currentPath);
const candidatePath = currentPath === "" ? segment : `${currentPath}/${segment}`;
if (!entries.files.includes(candidatePath) && !entries.folders.includes(candidatePath)) return false;
currentPath = candidatePath;
}
return segments.length > 0;
} catch {
return false;
}
}
async getAbstractFileByPath(p: FilePath | string): Promise<NodeFile | null> {
const pathStr = this.normalisePath(p);
if (!this.fileCache.has(pathStr) && !(await this.hasExactPathCase(pathStr))) {
this.fileCache.delete(pathStr);
return null;
}
return await this.refreshFile(pathStr);
}
@@ -74,6 +98,15 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
return Array.from(this.fileCache.values());
}
async renameFile(file: NodeFile, newPath: string): Promise<NodeFile> {
const oldPath = file.path;
await this.vault.rename(file, newPath);
this.fileCache.delete(oldPath);
const renamedFile = await this.refreshFile(newPath);
if (!renamedFile) throw new Error(`Could not find renamed file: ${newPath}`);
return renamedFile;
}
async statFromNative(file: NodeFile): Promise<UXStat> {
return file.stat;
}
@@ -86,9 +119,8 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
async refreshFile(p: string): Promise<NodeFile | null> {
const pathStr = this.normalisePath(p);
try {
const fullPath = this.resolvePath(pathStr);
const stat = await fs.stat(fullPath);
if (!stat.isFile()) {
const stat = await this.storage.stat(pathStr);
if (stat?.type !== "file") {
this.fileCache.delete(pathStr);
return null;
}
@@ -97,8 +129,8 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
path: pathStr as FilePath,
stat: {
size: stat.size,
mtime: Math.floor(stat.mtimeMs),
ctime: Math.floor(stat.ctimeMs),
mtime: stat.mtime,
ctime: stat.ctime,
type: "file",
},
};
@@ -117,31 +149,25 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
async scanDirectory(relativePath: string = ""): Promise<void> {
const fullPath = this.resolvePath(relativePath);
try {
const entries = await fs.readdir(fullPath, { withFileTypes: true });
const directoryStat = await this.storage.stat(relativePath);
if (directoryStat?.type !== "folder") throw new Error(`Directory does not exist: ${fullPath}`);
const entries = await this.storage.list(relativePath);
for (const entry of entries) {
const entryRelativePath = path.join(relativePath, entry.name).replace(/\\/g, "/");
if (entry.isDirectory()) {
await this.scanDirectory(entryRelativePath);
} else if (entry.isFile()) {
const entryFullPath = this.resolvePath(entryRelativePath);
const stat = await fs.stat(entryFullPath);
const file: NodeFile = {
path: entryRelativePath as FilePath,
stat: {
size: stat.size,
mtime: Math.floor(stat.mtimeMs),
ctime: Math.floor(stat.ctimeMs),
type: "file",
},
};
this.fileCache.set(entryRelativePath, file);
}
for (const entryPath of entries.files) {
const stat = await this.storage.stat(entryPath);
if (stat?.type !== "file") continue;
const file: NodeFile = {
path: entryPath as FilePath,
stat,
};
this.fileCache.set(entryPath, file);
}
for (const entryPath of entries.folders) {
await this.scanDirectory(entryPath);
}
} catch (error) {
// Directory doesn't exist or is not readable
console.error(`Error scanning directory ${fullPath}:`, error);
this.reportDiagnostic(`Error scanning directory ${fullPath}:`, error);
}
}
}
src/apps/cli/adapters/NodePathAdapter.ts
+3 -3
View File
@@ -1,7 +1,7 @@
import type { FilePath } from "@lib/common/types";
import type { IPathAdapter } from "@lib/serviceModules/adapters";
import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { IPathAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
import type { NodeFile } from "./NodeTypes";
import { path } from "@/apps/cli/node-compat";
import { path } from "@vrtmrz/livesync-commonlib/node";
/**
* Path adapter implementation for Node.js
src/apps/cli/adapters/NodeStorageAdapter.ts
-127
View File
@@ -1,127 +0,0 @@
import type { UXDataWriteOptions } from "@lib/common/types";
import type { IStorageAdapter } from "@lib/serviceModules/adapters";
import type { NodeStat } from "./NodeTypes";
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
import { validateStoragePath } from "@/apps/storagePath";
/**
* Storage adapter implementation for Node.js
*/
export class NodeStorageAdapter implements IStorageAdapter<NodeStat> {
constructor(private readonly basePath: string) {}
private resolvePath(p: string, allowRoot: boolean = true): string {
return path.join(this.basePath, validateStoragePath(p, allowRoot));
}
async exists(p: string): Promise<boolean> {
const fullPath = this.resolvePath(p);
try {
await fs.access(fullPath);
return true;
} catch {
return false;
}
}
async trystat(p: string): Promise<NodeStat | null> {
const fullPath = this.resolvePath(p);
try {
const stat = await fs.stat(fullPath);
return {
size: stat.size,
mtime: Math.floor(stat.mtimeMs),
ctime: Math.floor(stat.ctimeMs),
type: stat.isDirectory() ? "folder" : "file",
};
} catch {
return null;
}
}
async stat(p: string): Promise<NodeStat | null> {
return await this.trystat(p);
}
async mkdir(p: string): Promise<void> {
await fs.mkdir(this.resolvePath(p), { recursive: true });
}
async remove(p: string): Promise<void> {
const fullPath = this.resolvePath(p, false);
const stat = await fs.stat(fullPath);
if (stat.isDirectory()) {
await fs.rm(fullPath, { recursive: true, force: true });
} else {
await fs.unlink(fullPath);
}
}
async read(p: string): Promise<string> {
return await fs.readFile(this.resolvePath(p, false), "utf-8");
}
async readBinary(p: string): Promise<ArrayBuffer> {
const buffer = await fs.readFile(this.resolvePath(p, false));
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion -- required in environments where Buffer.buffer is ArrayBufferLike
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer;
}
async write(p: string, data: string, options?: UXDataWriteOptions): Promise<void> {
const fullPath = this.resolvePath(p, false);
await fs.mkdir(path.dirname(fullPath), { recursive: true });
await fs.writeFile(fullPath, data, "utf-8");
if (options?.mtime || options?.ctime) {
const atime = options.mtime ? new Date(options.mtime) : new Date();
const mtime = options.mtime ? new Date(options.mtime) : new Date();
await fs.utimes(fullPath, atime, mtime);
}
}
async writeBinary(p: string, data: ArrayBuffer, options?: UXDataWriteOptions): Promise<void> {
const fullPath = this.resolvePath(p, false);
await fs.mkdir(path.dirname(fullPath), { recursive: true });
await fs.writeFile(fullPath, new Uint8Array(data));
if (options?.mtime || options?.ctime) {
const atime = options.mtime ? new Date(options.mtime) : new Date();
const mtime = options.mtime ? new Date(options.mtime) : new Date();
await fs.utimes(fullPath, atime, mtime);
}
}
async append(p: string, data: string, options?: UXDataWriteOptions): Promise<void> {
const fullPath = this.resolvePath(p, false);
await fs.mkdir(path.dirname(fullPath), { recursive: true });
await fs.appendFile(fullPath, data, "utf-8");
if (options?.mtime || options?.ctime) {
const atime = options.mtime ? new Date(options.mtime) : new Date();
const mtime = options.mtime ? new Date(options.mtime) : new Date();
await fs.utimes(fullPath, atime, mtime);
}
}
async list(basePath: string): Promise<{ files: string[]; folders: string[] }> {
const fullPath = this.resolvePath(basePath);
try {
const entries = await fs.readdir(fullPath, { withFileTypes: true });
const files: string[] = [];
const folders: string[] = [];
for (const entry of entries) {
const entryPath = path.join(basePath, entry.name).replace(/\\/g, "/");
if (entry.isDirectory()) {
folders.push(entryPath);
} else if (entry.isFile()) {
files.push(entryPath);
}
}
return { files, folders };
} catch {
return { files: [], folders: [] };
}
}
}
src/apps/cli/adapters/NodeStorageAdapter.unit.spec.ts
+1 -4
View File
@@ -1,9 +1,6 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { storageAdapterContractCases } from "@/apps/_test/storageAdapterContract";
import { NodeStorageAdapter } from "./NodeStorageAdapter";
import { fsPromises as fs, os, path, NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node";
describe("NodeStorageAdapter", () => {
const tempDirs: string[] = [];
src/apps/cli/adapters/NodeTypeGuardAdapter.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import type { ITypeGuardAdapter } from "@lib/serviceModules/adapters";
import type { ITypeGuardAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
import type { NodeFile, NodeFolder } from "./NodeTypes";
/**
src/apps/cli/adapters/NodeTypes.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import type { FilePath, UXStat } from "@lib/common/types";
import type { FilePath, UXStat } from "@vrtmrz/livesync-commonlib/compat/common/types";
/**
* Node.js file representation
src/apps/cli/adapters/NodeVaultAdapter.ts
+28 -73
View File
@@ -1,20 +1,21 @@
import type { FilePath, UXDataWriteOptions } from "@lib/common/types";
import type { IVaultAdapter } from "@lib/serviceModules/adapters";
import type { FilePath, UXDataWriteOptions } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { IVaultAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
import type { NodeFile, NodeFolder } from "./NodeTypes";
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
import { NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node";
/**
* Vault adapter implementation for Node.js
*/
export class NodeVaultAdapter implements IVaultAdapter<NodeFile> {
constructor(private basePath: string) {}
private readonly storage: NodeStorageAdapter;
private resolvePath(p: string): string {
return path.join(this.basePath, p);
constructor(rootPathOrStorage: string | NodeStorageAdapter) {
this.storage =
typeof rootPathOrStorage === "string" ? new NodeStorageAdapter(rootPathOrStorage) : rootPathOrStorage;
}
async read(file: NodeFile): Promise<string> {
const content = await fs.readFile(this.resolvePath(file.path), "utf-8");
const content = await this.storage.read(file.path);
// Correct stale stat.size — chokidar stats may be from a poll before the final write.
// The downstream document integrity check compares stat.size to content length, so
// they must agree or other clients reject the file as corrupted.
@@ -28,89 +29,37 @@ export class NodeVaultAdapter implements IVaultAdapter<NodeFile> {
}
async readBinary(file: NodeFile): Promise<ArrayBuffer> {
const buffer = await fs.readFile(this.resolvePath(file.path));
const buffer = await this.storage.readBinary(file.path);
// Same correction as read() — ensure stat.size matches actual byte length.
file.stat.size = buffer.length;
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion -- required in environments where Buffer.buffer is ArrayBufferLike
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer;
file.stat.size = buffer.byteLength;
return buffer;
}
async modify(file: NodeFile, data: string, options?: UXDataWriteOptions): Promise<void> {
const fullPath = this.resolvePath(file.path);
await fs.writeFile(fullPath, data, "utf-8");
if (options?.mtime || options?.ctime) {
const atime = options.mtime ? new Date(options.mtime) : new Date();
const mtime = options.mtime ? new Date(options.mtime) : new Date();
await fs.utimes(fullPath, atime, mtime);
}
await this.storage.write(file.path, data, options);
}
async modifyBinary(file: NodeFile, data: ArrayBuffer, options?: UXDataWriteOptions): Promise<void> {
const fullPath = this.resolvePath(file.path);
await fs.writeFile(fullPath, new Uint8Array(data));
if (options?.mtime || options?.ctime) {
const atime = options.mtime ? new Date(options.mtime) : new Date();
const mtime = options.mtime ? new Date(options.mtime) : new Date();
await fs.utimes(fullPath, atime, mtime);
}
await this.storage.writeBinary(file.path, data, options);
}
async create(p: string, data: string, options?: UXDataWriteOptions): Promise<NodeFile> {
const fullPath = this.resolvePath(p);
await fs.mkdir(path.dirname(fullPath), { recursive: true });
await fs.writeFile(fullPath, data, "utf-8");
if (options?.mtime || options?.ctime) {
const atime = options.mtime ? new Date(options.mtime) : new Date();
const mtime = options.mtime ? new Date(options.mtime) : new Date();
await fs.utimes(fullPath, atime, mtime);
}
const stat = await fs.stat(fullPath);
return {
path: p as FilePath,
stat: {
size: stat.size,
mtime: Math.floor(stat.mtimeMs),
ctime: Math.floor(stat.ctimeMs),
type: "file",
},
};
await this.storage.write(p, data, options);
return await this.toNodeFile(p);
}
async createBinary(p: string, data: ArrayBuffer, options?: UXDataWriteOptions): Promise<NodeFile> {
const fullPath = this.resolvePath(p);
await fs.mkdir(path.dirname(fullPath), { recursive: true });
await fs.writeFile(fullPath, new Uint8Array(data));
await this.storage.writeBinary(p, data, options);
return await this.toNodeFile(p);
}
if (options?.mtime || options?.ctime) {
const atime = options.mtime ? new Date(options.mtime) : new Date();
const mtime = options.mtime ? new Date(options.mtime) : new Date();
await fs.utimes(fullPath, atime, mtime);
}
const stat = await fs.stat(fullPath);
return {
path: p as FilePath,
stat: {
size: stat.size,
mtime: Math.floor(stat.mtimeMs),
ctime: Math.floor(stat.ctimeMs),
type: "file",
},
};
async rename(file: NodeFile, newPath: string): Promise<void> {
await this.storage.rename(file.path, newPath);
file.path = newPath as FilePath;
}
async delete(file: NodeFile | NodeFolder, force = false): Promise<void> {
const fullPath = this.resolvePath(file.path);
const stat = await fs.stat(fullPath);
if (stat.isDirectory()) {
await fs.rm(fullPath, { recursive: true, force });
} else {
await fs.unlink(fullPath);
}
await this.storage.remove(file.path);
}
async trash(file: NodeFile | NodeFolder, force = false): Promise<void> {
@@ -122,4 +71,10 @@ export class NodeVaultAdapter implements IVaultAdapter<NodeFile> {
// No-op in CLI version (no event system)
return undefined;
}
private async toNodeFile(path: string): Promise<NodeFile> {
const stat = await this.storage.stat(path);
if (stat?.type !== "file") throw new Error(`Could not read created file metadata: ${path}`);
return { path: path as FilePath, stat };
}
}
src/apps/cli/adapters/NodeVaultAdapter.unit.spec.ts
+140
View File
@@ -0,0 +1,140 @@
import { fsPromises, os, path } from "@vrtmrz/livesync-commonlib/node";
import { describe, expect, it, vi } from "vitest";
import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { NodeFileSystemAdapter } from "./NodeFileSystemAdapter";
import { NodeVaultAdapter } from "./NodeVaultAdapter";
describe("NodeVaultAdapter.rename", () => {
it("changes the directory entry case without changing the content", async () => {
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-case-rename-"));
try {
await fsPromises.writeFile(path.join(directory, "Calculus.md"), "content", "utf8");
const adapter = new NodeVaultAdapter(directory);
const file = {
path: "Calculus.md" as FilePath,
stat: { ctime: 1, mtime: 2, size: 7, type: "file" as const },
};
await adapter.rename(file, "calculus.md");
expect(await fsPromises.readdir(directory)).toEqual(["calculus.md"]);
expect(await fsPromises.readFile(path.join(directory, "calculus.md"), "utf8")).toBe("content");
expect(file.path).toBe("calculus.md");
} finally {
await fsPromises.rm(directory, { recursive: true, force: true });
}
});
it("does not move a file through a symbolic link outside the vault root", async () => {
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-rename-root-"));
const outsideDirectory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-rename-outside-"));
try {
await fsPromises.writeFile(path.join(directory, "source.md"), "content", "utf8");
await fsPromises.symlink(
outsideDirectory,
path.join(directory, "linked"),
process.platform === "win32" ? "junction" : "dir"
);
const adapter = new NodeVaultAdapter(directory);
const file = {
path: "source.md" as FilePath,
stat: { ctime: 1, mtime: 2, size: 7, type: "file" as const },
};
await expect(adapter.rename(file, "linked/moved.md")).rejects.toThrow(/symbolic link/i);
await expect(fsPromises.readFile(path.join(directory, "source.md"), "utf8")).resolves.toBe("content");
await expect(fsPromises.stat(path.join(outsideDirectory, "moved.md"))).rejects.toMatchObject({
code: "ENOENT",
});
} finally {
await fsPromises.rm(directory, { recursive: true, force: true });
await fsPromises.rm(outsideDirectory, { recursive: true, force: true });
}
});
it("does not modify a file through a symbolic link outside the vault root", async () => {
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-modify-root-"));
const outsideDirectory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-modify-outside-"));
try {
await fsPromises.writeFile(path.join(outsideDirectory, "victim.md"), "before", "utf8");
await fsPromises.symlink(
outsideDirectory,
path.join(directory, "linked"),
process.platform === "win32" ? "junction" : "dir"
);
const adapter = new NodeVaultAdapter(directory);
const file = {
path: "linked/victim.md" as FilePath,
stat: { ctime: 1, mtime: 2, size: 6, type: "file" as const },
};
await expect(adapter.modify(file, "after")).rejects.toThrow(/symbolic link/i);
await expect(fsPromises.readFile(path.join(outsideDirectory, "victim.md"), "utf8")).resolves.toBe("before");
} finally {
await fsPromises.rm(directory, { recursive: true, force: true });
await fsPromises.rm(outsideDirectory, { recursive: true, force: true });
}
});
});
describe("NodeFileSystemAdapter path case", () => {
it("finds the stored case and refreshes the cache after a case-only rename", async () => {
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-case-cache-"));
try {
await fsPromises.writeFile(path.join(directory, "Calculus.md"), "content", "utf8");
const adapter = new NodeFileSystemAdapter(directory);
await expect(adapter.getAbstractFileByPath("calculus.md")).resolves.toBeNull();
const existingFile = await adapter.getAbstractFileByPathInsensitive("calculus.md");
expect(existingFile?.path).toBe("Calculus.md");
if (!existingFile) throw new Error("Expected to find Calculus.md case-insensitively");
const renamedFile = await adapter.renameFile(existingFile, "calculus.md");
expect(renamedFile.path).toBe("calculus.md");
expect((await adapter.getFiles()).map((file) => file.path)).toEqual(["calculus.md"]);
expect(await fsPromises.readdir(directory)).toEqual(["calculus.md"]);
} finally {
await fsPromises.rm(directory, { recursive: true, force: true });
}
});
it("reports directory scan failures through the injected diagnostic callback", async () => {
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-scan-diagnostic-"));
const missingDirectory = path.join(directory, "missing");
const reportDiagnostic = vi.fn();
try {
const adapter = new NodeFileSystemAdapter(missingDirectory, reportDiagnostic);
await adapter.scanDirectory();
expect(reportDiagnostic).toHaveBeenCalledWith(
`Error scanning directory ${missingDirectory}:`,
expect.any(Error)
);
} finally {
await fsPromises.rm(directory, { recursive: true, force: true });
}
});
it("does not discover a file through a symbolic link outside the vault root", async () => {
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-discovery-root-"));
const outsideDirectory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-discovery-outside-"));
try {
await fsPromises.writeFile(path.join(outsideDirectory, "outside.md"), "content", "utf8");
await fsPromises.symlink(
outsideDirectory,
path.join(directory, "linked"),
process.platform === "win32" ? "junction" : "dir"
);
const adapter = new NodeFileSystemAdapter(directory);
await expect(adapter.getAbstractFileByPath("linked/outside.md")).resolves.toBeNull();
await expect(adapter.getFiles()).resolves.toEqual([]);
} finally {
await fsPromises.rm(directory, { recursive: true, force: true });
await fsPromises.rm(outsideDirectory, { recursive: true, force: true });
}
});
});
src/apps/cli/cliOutput.ts
+30
View File
@@ -0,0 +1,30 @@
import type { StandardIo } from "@vrtmrz/livesync-commonlib/context";
/** Report a CLI-owned diagnostic without selecting its final presentation channel. */
export type CliDiagnosticReporter = (message: string, detail?: unknown) => void;
function formatValue(value: unknown): string {
if (typeof value === "string") return value;
if (value instanceof Error) return value.stack ?? value.message;
try {
const encoded = JSON.stringify(value);
if (encoded !== undefined) return encoded;
} catch {
// Fall through to the host-independent string conversion.
}
return String(value);
}
function formatLine(values: readonly unknown[]): string {
return `${values.map(formatValue).join(" ")}\n`;
}
/** Render one user-facing line on standard output. */
export function writeStdoutLine(standardIo: StandardIo, ...values: readonly unknown[]): void {
standardIo.writeStdout(formatLine(values));
}
/** Render one user-facing or diagnostic line on standard error. */
export function writeStderrLine(standardIo: StandardIo, ...values: readonly unknown[]): void {
standardIo.writeStderr(formatLine(values));
}
src/apps/cli/cliSettingsDefaults.ts
+10
View File
@@ -0,0 +1,10 @@
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings";
export function createDefaultCliSettings(): ObsidianLiveSyncSettings {
return {
...createNewVaultSettings(),
useIndexedDBAdapter: false,
isConfigured: false,
};
}
src/apps/cli/cliSettingsDefaults.unit.spec.ts
+16
View File
@@ -0,0 +1,16 @@
import { describe, expect, it } from "vitest";
import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings";
import { createDefaultCliSettings } from "./cliSettingsDefaults.ts";
describe("createDefaultCliSettings", () => {
it("uses the recommended new-Vault settings with the Node database adapter", () => {
const settings = createDefaultCliSettings();
const recommended = createNewVaultSettings();
expect(settings).toEqual({
...recommended,
useIndexedDBAdapter: false,
isConfigured: false,
});
});
});
src/apps/cli/commands/daemonCommand.unit.spec.ts
+20 -14
View File
@@ -1,14 +1,15 @@
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
import { createServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { runCommand } from "./runCommand";
import type { CLIOptions } from "./types";
// Mock performFullScan so daemon tests don't require a real CouchDB connection.
vi.mock("@lib/serviceFeatures/offlineScanner", () => ({
vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", () => ({
performFullScan: vi.fn(async () => true),
}));
// Mock UnresolvedErrorManager to avoid event-hub side effects.
vi.mock("@lib/services/base/UnresolvedErrorManager", () => ({
vi.mock("@vrtmrz/livesync-commonlib/compat/services/base/UnresolvedErrorManager", () => ({
UnresolvedErrorManager: class UnresolvedErrorManager {
showError() {}
clearError() {}
@@ -16,11 +17,18 @@ vi.mock("@lib/services/base/UnresolvedErrorManager", () => ({
},
}));
import * as offlineScanner from "@lib/serviceFeatures/offlineScanner";
import * as offlineScanner from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
function createCoreMock() {
const standardIo = {
readStdin: vi.fn(async () => ""),
prompt: vi.fn(async () => ""),
writeStdout: vi.fn((_chunk: string | Uint8Array) => undefined),
writeStderr: vi.fn((_chunk: string | Uint8Array) => undefined),
};
return {
services: {
context: Object.assign(createServiceContext(), { standardIo }),
control: {
activated: Promise.resolve(),
applySettings: vi.fn(async () => {}),
@@ -155,13 +163,13 @@ describe("daemon command", () => {
syncOnStart: false,
}));
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const result = await runCommand(makeDaemonOptions(), { ...baseContext, core });
expect(result).toBe(true);
const warningCalls = consoleSpy.mock.calls.filter(
(args) => typeof args[0] === "string" && args[0].includes("liveSync and syncOnStart are both disabled")
const warningCalls = core.services.context.standardIo.writeStderr.mock.calls.filter(
([chunk]: [string | Uint8Array]) =>
typeof chunk === "string" && chunk.includes("liveSync and syncOnStart are both disabled")
);
expect(warningCalls.length).toBeGreaterThan(0);
});
@@ -173,12 +181,12 @@ describe("daemon command", () => {
syncOnStart: false,
}));
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
await runCommand(makeDaemonOptions(), { ...baseContext, core });
const warningCalls = consoleSpy.mock.calls.filter(
(args) => typeof args[0] === "string" && args[0].includes("liveSync and syncOnStart are both disabled")
const warningCalls = core.services.context.standardIo.writeStderr.mock.calls.filter(
([chunk]: [string | Uint8Array]) =>
typeof chunk === "string" && chunk.includes("liveSync and syncOnStart are both disabled")
);
expect(warningCalls.length).toBe(0);
});
@@ -231,7 +239,6 @@ describe("daemon command", () => {
it("polling backoff: interval escalates on failure, caps at 300000ms, then halves on recovery", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
vi.spyOn(console, "error").mockImplementation(() => {});
// startup replicate (call 1) succeeds; poll calls 2–7 fail; call 8 succeeds.
let callCount = 0;
@@ -284,10 +291,9 @@ describe("daemon command", () => {
expect(setTimeoutSpy.mock.calls[afterSuccessCallCount - 1][1]).toBe(150_000);
});
it("polling error handling: replicate rejection is caught and console.error is called", async () => {
it("polling error handling: replicate rejection is caught and written to standard error", async () => {
const core = createCoreMock();
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
// Make replicate succeed on the initial call (startup), then fail on the poll.
let callCount = 0;
@@ -304,8 +310,8 @@ describe("daemon command", () => {
await vi.advanceTimersByTimeAsync(intervalMs);
// No unhandled rejection — the error was caught internally.
const errorCalls = consoleSpy.mock.calls.filter(
(args) => typeof args[0] === "string" && args[0].includes("Poll error")
const errorCalls = core.services.context.standardIo.writeStderr.mock.calls.filter(
([chunk]: [string | Uint8Array]) => typeof chunk === "string" && chunk.includes("Poll error")
);
expect(errorCalls.length).toBeGreaterThan(0);
});
src/apps/cli/commands/p2p.ts
+14 -14
View File
@@ -1,11 +1,11 @@
import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
import { P2P_DEFAULT_SETTINGS } from "@lib/common/types";
import type { ServiceContext } from "@lib/services/base/ServiceBase";
import { LiveSyncTrysteroReplicator } from "@lib/replication/trystero/LiveSyncTrysteroReplicator";
import { compatGlobal } from "@lib/common/coreEnvFunctions.ts";
import { LiveSyncError } from "@lib/common/LSError";
import { getPeerConnectionStats } from "@lib/rpc/transports/DiagRTCPeerConnections.utils";
import { appendFile } from "node:fs/promises";
import { P2P_DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { LiveSyncTrysteroReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/LiveSyncTrysteroReplicator";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError";
import { getPeerConnectionStats } from "@vrtmrz/livesync-commonlib/compat/rpc/transports/DiagRTCPeerConnections.utils";
import { fsPromises } from "@vrtmrz/livesync-commonlib/node";
type CLIP2PPeer = {
peerId: string;
@@ -13,10 +13,10 @@ type CLIP2PPeer = {
};
type CandidateSummary = {
id: string | "unknown";
candidateType: string | "unknown";
protocol: string | "unknown";
relayProtocol: string | "unknown";
id: string;
candidateType: string;
protocol: string;
relayProtocol: string;
};
function delay(ms: number): Promise<void> {
@@ -98,7 +98,7 @@ function getReportValue<T extends string | number>(
return typeof value === "string" || typeof value === "number" ? (value as T) : "unknown";
}
function summariseCandidate(reports: unknown[], candidateId: string | "unknown"): CandidateSummary | undefined {
function summariseCandidate(reports: unknown[], candidateId: string): CandidateSummary | undefined {
if (candidateId === "unknown") {
return undefined;
}
@@ -155,7 +155,7 @@ async function writePeerConnectionStatsIfRequested(
localCandidate,
remoteCandidate,
};
await appendFile(outputPath, `${JSON.stringify(payload)}\n`, "utf8");
await fsPromises.appendFile(outputPath, `${JSON.stringify(payload)}\n`, "utf8");
}
export async function syncWithPeer(
@@ -189,7 +189,7 @@ export async function syncWithPeer(
}
const pushResult = await replicator.requestSynchroniseToPeer(targetPeer.peerId);
if (!pushResult || pushResult.ok !== true) {
const err = pushResult?.error;
const err: unknown = pushResult && "error" in pushResult ? pushResult.error : undefined;
throw err instanceof Error
? err
: LiveSyncError.fromError(err ?? "P2P sync failed while requesting remote sync");
src/apps/cli/commands/runCommand.ts
+94 -82
View File
@@ -1,5 +1,5 @@
import { decodeSettingsFromSetupURI } from "@lib/API/processSetting";
import { configURIBase } from "@lib/common/models/shared.const";
import { decodeSettingsFromSetupURI } from "@vrtmrz/livesync-commonlib/compat/API/processSetting";
import { configURIBase } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const";
import {
DEFAULT_SETTINGS,
MILESTONE_DOCID,
@@ -9,19 +9,23 @@ import {
REMOTE_MINIO,
type EntryMilestoneInfo,
type EntryDoc,
} from "@lib/common/types";
import { ConnectionStringParser } from "@lib/common/ConnectionString";
import { activateRemoteConfiguration, createRemoteConfigurationId } from "@lib/serviceFeatures/remoteConfig";
import { stripAllPrefixes } from "@lib/string_and_binary/path";
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
import {
activateRemoteConfiguration,
createRemoteConfigurationId,
} from "@vrtmrz/livesync-commonlib/remote-configurations";
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import type { CLICommandContext, CLIOptions } from "./types";
import { promptForPassphrase, readStdinAsUtf8, toArrayBuffer, toDatabaseRelativePath } from "./utils";
import { toArrayBuffer, toDatabaseRelativePath } from "./utils";
import { collectPeers, openP2PHost, parseTimeoutSeconds, syncWithPeer } from "./p2p";
import { performFullScan } from "@lib/serviceFeatures/offlineScanner";
import { UnresolvedErrorManager } from "@lib/services/base/UnresolvedErrorManager";
import { compatGlobal } from "@lib/common/coreEnvFunctions.ts";
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
import type { LiveSyncCouchDBReplicator } from "@lib/replication/couchdb/LiveSyncReplicator";
import type { LiveSyncJournalReplicator } from "@lib/replication/journal/LiveSyncJournalReplicator";
import { performFullScan } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
import { UnresolvedErrorManager } from "@vrtmrz/livesync-commonlib/compat/services/base/UnresolvedErrorManager";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
import { fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node";
import type { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator";
import type { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator";
import { writeStderrLine, writeStdoutLine } from "@/apps/cli/cliOutput";
function redactConnectionString(uri: string): string {
return uri.replace(/\/\/([^@/]+)@/u, "//***@");
@@ -31,9 +35,10 @@ async function verifyRemoteState(
core: CLICommandContext["core"],
settings: ObsidianLiveSyncSettings
): Promise<boolean> {
const { standardIo } = core.services.context;
const replicator = core.services.replicator.getActiveReplicator();
if (!replicator) {
process.stderr.write("[Verification] No active replicator found\n");
standardIo.writeStderr("[Verification] No active replicator found\n");
return false;
}
@@ -50,7 +55,7 @@ async function verifyRemoteState(
true
);
if (typeof dbRet === "string") {
process.stderr.write(`[Verification] Failed to connect to remote CouchDB: ${dbRet}\n`);
standardIo.writeStderr(`[Verification] Failed to connect to remote CouchDB: ${dbRet}\n`);
return false;
}
milestone = await dbRet.db.get(MILESTONE_DOCID);
@@ -61,29 +66,30 @@ async function verifyRemoteState(
if (milestone) {
const isLocked = !!milestone.locked;
const isAccepted = !!milestone.accepted_nodes?.includes(replicator.nodeid);
process.stderr.write(`[Verification] Remote Database: ${isLocked ? "LOCKED" : "UNLOCKED"}\n`);
process.stderr.write(
standardIo.writeStderr(`[Verification] Remote Database: ${isLocked ? "LOCKED" : "UNLOCKED"}\n`);
standardIo.writeStderr(
`[Verification] Current Device Node ID (${replicator.nodeid}): ${isAccepted ? "ACCEPTED" : "NOT ACCEPTED"}\n`
);
return true;
} else {
process.stderr.write("[Verification] Milestone document not found on remote.\n");
standardIo.writeStderr("[Verification] Milestone document not found on remote.\n");
return false;
}
} catch (e) {
const message = e instanceof Error ? e.message : String(e);
process.stderr.write(`[Verification] Failed to fetch milestone document: ${message}\n`);
standardIo.writeStderr(`[Verification] Failed to fetch milestone document: ${message}\n`);
return false;
}
}
export async function runCommand(options: CLIOptions, context: CLICommandContext): Promise<boolean> {
const { databasePath, core, settingsPath } = context;
const { standardIo } = core.services.context;
const vaultPath = context.vaultPath || databasePath;
await core.services.control.activated;
if (options.command === "daemon") {
const log = (msg: unknown) => console.error(`[Daemon] ${msg}`);
const log = (msg: unknown) => writeStderrLine(standardIo, `[Daemon] ${String(msg)}`);
// Skip the config mismatch dialog — the daemon cannot resolve it interactively
// and the default "Dismiss" action would block replication. The daemon should
@@ -94,17 +100,17 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
log("Replicating from CouchDB...");
const replResult = await core.services.replication.replicate(true);
if (!replResult) {
console.error("[Daemon] Initial CouchDB replication failed, cannot continue");
writeStderrLine(standardIo, "[Daemon] Initial CouchDB replication failed, cannot continue");
return false;
}
log("CouchDB replication complete");
// 2. Mirror scan to reconcile PouchDB ↔ local filesystem.
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle);
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events);
log("Running mirror scan...");
const scanOk = await performFullScan(core, log, errorManager, false, true);
if (!scanOk) {
console.error("[Daemon] Mirror scan failed, cannot continue");
writeStderrLine(standardIo, "[Daemon] Mirror scan failed, cannot continue");
return false;
}
log("Mirror scan complete");
@@ -152,9 +158,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
} catch (err) {
consecutiveFailures++;
currentIntervalMs = Math.min(baseIntervalMs * Math.pow(2, consecutiveFailures), maxIntervalMs);
console.error(`[Daemon] Poll error (${consecutiveFailures} consecutive):`, err);
writeStderrLine(standardIo, `[Daemon] Poll error (${consecutiveFailures} consecutive):`, err);
if (consecutiveFailures >= 5) {
console.error(
writeStderrLine(
standardIo,
`[Daemon] Warning: ${consecutiveFailures} consecutive failures, backing off to ${Math.round(currentIntervalMs / 1000)}s`
);
}
@@ -179,7 +186,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
log("LiveSync active");
const currentSettings = core.services.setting.currentSettings();
if (!currentSettings.liveSync && !currentSettings.syncOnStart) {
console.error(
writeStderrLine(
standardIo,
"[Daemon] Warning: liveSync and syncOnStart are both disabled in settings. " +
"No sync will occur. Set liveSync=true in your settings file for continuous sync, " +
"or use --interval for polling mode."
@@ -191,7 +199,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}
if (options.command === "sync") {
console.log("[Command] sync");
writeStdoutLine(standardIo, "[Command] sync");
const result = await core.services.replication.replicate(true);
if (!result) {
// TODO: Standardise the logic for identifying the cause of replication
@@ -199,7 +207,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
// error, etc.) is surfaced with a CLI-specific actionable message.
const replicator = core.services.replicator.getActiveReplicator();
if (replicator?.remoteLockedAndDeviceNotAccepted) {
console.error(
writeStderrLine(
standardIo,
`[Error] The remote database is locked and this device is not yet accepted.\n` +
`[Error] Please unlock the database from the Obsidian plugin and retry.`
);
@@ -213,10 +222,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
throw new Error("p2p-peers requires one argument: <timeout>");
}
const timeoutSec = parseTimeoutSeconds(options.commandArgs[0], "p2p-peers");
console.error(`[Command] p2p-peers timeout=${timeoutSec}s`);
writeStderrLine(standardIo, `[Command] p2p-peers timeout=${timeoutSec}s`);
const peers = await collectPeers(core, timeoutSec);
if (peers.length > 0) {
process.stdout.write(peers.map((peer) => `[peer]\t${peer.peerId}\t${peer.name}`).join("\n") + "\n");
standardIo.writeStdout(peers.map((peer) => `[peer]\t${peer.peerId}\t${peer.name}`).join("\n") + "\n");
}
return true;
}
@@ -230,16 +239,16 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
throw new Error("p2p-sync requires a non-empty <peer>");
}
const timeoutSec = parseTimeoutSeconds(options.commandArgs[1], "p2p-sync");
console.error(`[Command] p2p-sync peer=${peerToken} timeout=${timeoutSec}s`);
writeStderrLine(standardIo, `[Command] p2p-sync peer=${peerToken} timeout=${timeoutSec}s`);
const peer = await syncWithPeer(core, peerToken, timeoutSec);
console.error(`[Done] P2P sync completed with ${peer.name} (${peer.peerId})`);
writeStderrLine(standardIo, `[Done] P2P sync completed with ${peer.name} (${peer.peerId})`);
return true;
}
if (options.command === "p2p-host") {
console.error("[Command] p2p-host");
writeStderrLine(standardIo, "[Command] p2p-host");
await openP2PHost(core);
console.error("[Ready] P2P host is running. Press Ctrl+C to stop.");
writeStderrLine(standardIo, "[Ready] P2P host is running. Press Ctrl+C to stop.");
await new Promise(() => {});
return true;
}
@@ -252,7 +261,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
const destinationDatabasePath = toDatabaseRelativePath(options.commandArgs[1], vaultPath);
const sourceData = await fs.readFile(sourcePath);
const sourceStat = await fs.stat(sourcePath);
console.log(`[Command] push ${sourcePath} -> ${destinationDatabasePath}`);
writeStdoutLine(standardIo, `[Command] push ${sourcePath} -> ${destinationDatabasePath}`);
await core.serviceModules.storageAccess.writeFileAuto(destinationDatabasePath, toArrayBuffer(sourceData), {
mtime: Math.floor(sourceStat.mtimeMs),
@@ -269,7 +278,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}
const sourceDatabasePath = toDatabaseRelativePath(options.commandArgs[0], vaultPath);
const destinationPath = path.resolve(options.commandArgs[1]);
console.log(`[Command] pull ${sourceDatabasePath} -> ${destinationPath}`);
writeStdoutLine(standardIo, `[Command] pull ${sourceDatabasePath} -> ${destinationPath}`);
const sourcePathWithPrefix = sourceDatabasePath as FilePathWithPrefix;
const restored = await core.serviceModules.fileHandler.dbToStorage(sourcePathWithPrefix, null, true);
@@ -296,7 +305,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
if (!rev) {
throw new Error("pull-rev requires a non-empty revision");
}
console.log(`[Command] pull-rev ${sourceDatabasePath}@${rev} -> ${destinationPath}`);
writeStdoutLine(standardIo, `[Command] pull-rev ${sourceDatabasePath}@${rev} -> ${destinationPath}`);
const source = await core.serviceModules.databaseFileAccess.fetch(
sourceDatabasePath as FilePathWithPrefix,
@@ -325,7 +334,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
if (!setupURI.startsWith(configURIBase)) {
throw new Error(`setup URI must start with ${configURIBase}`);
}
const passphrase = await promptForPassphrase();
const passphrase = await standardIo.prompt("Enter setup URI passphrase: ");
if (!passphrase) {
throw new Error("Passphrase is required");
}
const decoded = await decodeSettingsFromSetupURI(setupURI, passphrase);
if (!decoded) {
throw new Error("Failed to decode settings from setup URI");
@@ -337,7 +349,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
isConfigured: true,
} as ObsidianLiveSyncSettings;
console.log(`[Command] setup -> ${settingsPath}`);
writeStdoutLine(standardIo, `[Command] setup -> ${settingsPath}`);
await core.services.setting.applyExternalSettings(nextSettings, true);
await core.services.control.applySettings();
return true;
@@ -348,8 +360,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
throw new Error("put requires one argument: <dst>");
}
const destinationDatabasePath = toDatabaseRelativePath(options.commandArgs[0], vaultPath);
const content = await readStdinAsUtf8();
console.log(`[Command] put stdin -> ${destinationDatabasePath}`);
const content = await standardIo.readStdin();
writeStdoutLine(standardIo, `[Command] put stdin -> ${destinationDatabasePath}`);
return await core.serviceModules.databaseFileAccess.storeContent(
destinationDatabasePath as FilePathWithPrefix,
content
@@ -361,7 +373,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
throw new Error("cat requires one argument: <src>");
}
const sourceDatabasePath = toDatabaseRelativePath(options.commandArgs[0], vaultPath);
console.error(`[Command] cat ${sourceDatabasePath}`);
writeStderrLine(standardIo, `[Command] cat ${sourceDatabasePath}`);
const source = await core.serviceModules.databaseFileAccess.fetch(
sourceDatabasePath as FilePathWithPrefix,
undefined,
@@ -372,10 +384,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}
const body = source.body;
if (body.type === "text/plain") {
process.stdout.write(await body.text());
standardIo.writeStdout(await body.text());
} else {
const buffer = Buffer.from(await body.arrayBuffer());
process.stdout.write(new Uint8Array(buffer));
standardIo.writeStdout(new Uint8Array(buffer));
}
return true;
}
@@ -389,7 +401,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
if (!rev) {
throw new Error("cat-rev requires a non-empty revision");
}
console.error(`[Command] cat-rev ${sourceDatabasePath} @ ${rev}`);
writeStderrLine(standardIo, `[Command] cat-rev ${sourceDatabasePath} @ ${rev}`);
const source = await core.serviceModules.databaseFileAccess.fetch(
sourceDatabasePath as FilePathWithPrefix,
rev,
@@ -400,10 +412,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}
const body = source.body;
if (body.type === "text/plain") {
process.stdout.write(await body.text());
standardIo.writeStdout(await body.text());
} else {
const buffer = Buffer.from(await body.arrayBuffer());
process.stdout.write(new Uint8Array(buffer));
standardIo.writeStdout(new Uint8Array(buffer));
}
return true;
}
@@ -432,9 +444,9 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
rows.sort((a, b) => a.path.localeCompare(b.path));
if (rows.length > 0) {
process.stdout.write(rows.map((e) => e.line).join("\n") + "\n");
standardIo.writeStdout(rows.map((e) => e.line).join("\n") + "\n");
} else {
process.stderr.write("[Info] No documents found in the local database.\n");
standardIo.writeStderr("[Info] No documents found in the local database.\n");
}
return true;
}
@@ -475,11 +487,11 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
chunks: children.length,
children: children,
};
process.stdout.write(JSON.stringify(out, null, 2) + "\n");
standardIo.writeStdout(JSON.stringify(out, null, 2) + "\n");
return true;
}
process.stderr.write(`[Info] File not found: ${targetPath}\n`);
standardIo.writeStderr(`[Info] File not found: ${targetPath}\n`);
return false;
}
@@ -488,7 +500,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
throw new Error("rm requires one argument: <path>");
}
const targetPath = toDatabaseRelativePath(options.commandArgs[0], vaultPath);
console.error(`[Command] rm ${targetPath}`);
writeStderrLine(standardIo, `[Command] rm ${targetPath}`);
return await core.serviceModules.databaseFileAccess.delete(targetPath as FilePathWithPrefix);
}
@@ -504,30 +516,30 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
const currentMeta = await core.serviceModules.databaseFileAccess.fetchEntryMeta(targetPath, undefined, true);
if (currentMeta === false || currentMeta._deleted || currentMeta.deleted) {
process.stderr.write(`[Info] File not found: ${targetPath}\n`);
standardIo.writeStderr(`[Info] File not found: ${targetPath}\n`);
return false;
}
const conflicts = await core.serviceModules.databaseFileAccess.getConflictedRevs(targetPath);
const candidateRevisions = [currentMeta._rev, ...conflicts];
if (!candidateRevisions.includes(revisionToKeep)) {
process.stderr.write(`[Info] Revision not found for ${targetPath}: ${revisionToKeep}\n`);
standardIo.writeStderr(`[Info] Revision not found for ${targetPath}: ${revisionToKeep}\n`);
return false;
}
if (conflicts.length === 0 && currentMeta._rev === revisionToKeep) {
console.error(`[Command] resolve ${targetPath} keep ${revisionToKeep} (already resolved)`);
writeStderrLine(standardIo, `[Command] resolve ${targetPath} keep ${revisionToKeep} (already resolved)`);
return true;
}
console.error(`[Command] resolve ${targetPath} keep ${revisionToKeep}`);
writeStderrLine(standardIo, `[Command] resolve ${targetPath} keep ${revisionToKeep}`);
for (const revision of candidateRevisions) {
if (revision === revisionToKeep) {
continue;
}
const resolved = await core.services.conflict.resolveByDeletingRevision(targetPath, revision ?? "", "CLI");
if (!resolved) {
process.stderr.write(`[Info] Failed to delete revision ${revision} for ${targetPath}\n`);
standardIo.writeStderr(`[Info] Failed to delete revision ${revision} for ${targetPath}\n`);
return false;
}
}
@@ -535,9 +547,9 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}
if (options.command === "mirror") {
console.error("[Command] mirror");
const log = (msg: unknown) => console.error(`[Mirror] ${msg}`);
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle);
writeStderrLine(standardIo, "[Command] mirror");
const log = (msg: unknown) => writeStderrLine(standardIo, `[Mirror] ${String(msg)}`);
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events);
return await performFullScan(core, log, errorManager, false, true);
}
@@ -579,7 +591,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
await core.services.control.applySettings();
}
process.stdout.write(`${id}\t${name}\t${redactConnectionString(canonicalUri)}\n`);
standardIo.writeStdout(`${id}\t${name}\t${redactConnectionString(canonicalUri)}\n`);
return true;
}
@@ -594,7 +606,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
const current = core.services.setting.currentSettings();
if (!current.remoteConfigurations?.[id]) {
process.stderr.write(`[Info] Remote configuration not found: ${id}\n`);
standardIo.writeStderr(`[Info] Remote configuration not found: ${id}\n`);
return false;
}
@@ -624,7 +636,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
await core.services.control.applySettings();
}
console.error(`[Command] remote-rm ${id}`);
writeStderrLine(standardIo, `[Command] remote-rm ${id}`);
return true;
}
@@ -634,7 +646,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
configs.sort((a, b) => a.name.localeCompare(b.name));
if (configs.length === 0) {
process.stderr.write("[Info] No remote configurations found.\n");
standardIo.writeStderr("[Info] No remote configurations found.\n");
return true;
}
@@ -642,7 +654,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
const status = config.id === settings.activeConfigurationId ? "active" : "inactive";
return `${config.id}\t${config.name}\t${status}\t${redactConnectionString(config.uri)}`;
});
process.stdout.write(lines.join("\n") + "\n");
standardIo.writeStdout(lines.join("\n") + "\n");
return true;
}
@@ -657,11 +669,11 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
const config = core.services.setting.currentSettings().remoteConfigurations?.[id];
if (!config) {
process.stderr.write(`[Info] Remote configuration not found: ${id}\n`);
standardIo.writeStderr(`[Info] Remote configuration not found: ${id}\n`);
return false;
}
process.stdout.write(`${config.uri}\n`);
standardIo.writeStdout(`${config.uri}\n`);
return true;
}
@@ -701,7 +713,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
const updated = core.services.setting.currentSettings().remoteConfigurations?.[id];
if (!updated) {
process.stderr.write(`[Info] Remote configuration not found: ${id}\n`);
standardIo.writeStderr(`[Info] Remote configuration not found: ${id}\n`);
return false;
}
@@ -709,7 +721,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
await core.services.control.applySettings();
}
console.error(`[Command] remote-set ${id}`);
writeStderrLine(standardIo, `[Command] remote-set ${id}`);
return true;
}
if (options.command === "remote-activate") {
@@ -732,12 +744,12 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}, true);
if (!switched) {
process.stderr.write(`[Info] Failed to activate remote configuration: ${id}\n`);
standardIo.writeStderr(`[Info] Failed to activate remote configuration: ${id}\n`);
return false;
}
await core.services.control.applySettings();
console.error(`[Command] remote-activate ${id}`);
writeStderrLine(standardIo, `[Command] remote-activate ${id}`);
return true;
}
@@ -755,14 +767,14 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}, false);
if (!switched) {
process.stderr.write(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
standardIo.writeStderr(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
return false;
}
await core.services.control.applySettings();
}
console.error(`[Command] mark-resolved${id ? ` ${id}` : ""}`);
writeStderrLine(standardIo, `[Command] mark-resolved${id ? ` ${id}` : ""}`);
await core.services.replication.markResolved();
const settings = core.services.setting.currentSettings();
await verifyRemoteState(core, settings);
@@ -783,14 +795,14 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}, false);
if (!switched) {
process.stderr.write(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
standardIo.writeStderr(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
return false;
}
await core.services.control.applySettings();
}
console.error(`[Command] unlock-remote${id ? ` ${id}` : ""}`);
writeStderrLine(standardIo, `[Command] unlock-remote${id ? ` ${id}` : ""}`);
await core.services.replication.markUnlocked();
const settings = core.services.setting.currentSettings();
await verifyRemoteState(core, settings);
@@ -811,14 +823,14 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}, false);
if (!switched) {
process.stderr.write(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
standardIo.writeStderr(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
return false;
}
await core.services.control.applySettings();
}
console.error(`[Command] lock-remote${id ? ` ${id}` : ""}`);
writeStderrLine(standardIo, `[Command] lock-remote${id ? ` ${id}` : ""}`);
await core.services.replication.markLocked();
const settings = core.services.setting.currentSettings();
await verifyRemoteState(core, settings);
@@ -839,26 +851,26 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
}, false);
if (!switched) {
process.stderr.write(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
standardIo.writeStderr(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
return false;
}
await core.services.control.applySettings();
}
console.error(`[Command] remote-status${id ? ` ${id}` : ""}`);
writeStderrLine(standardIo, `[Command] remote-status${id ? ` ${id}` : ""}`);
const replicator = core.services.replicator.getActiveReplicator();
if (!replicator) {
process.stderr.write("[Error] No active replicator found\n");
standardIo.writeStderr("[Error] No active replicator found\n");
return false;
}
const settings = core.services.setting.currentSettings();
const status = await replicator.getRemoteStatus(settings);
if (status === false) {
process.stderr.write("[Error] Failed to fetch remote status\n");
standardIo.writeStderr("[Error] Failed to fetch remote status\n");
return false;
}
process.stdout.write(JSON.stringify(status, null, 2) + "\n");
standardIo.writeStdout(JSON.stringify(status, null, 2) + "\n");
return true;
}
src/apps/cli/commands/runCommand.unit.spec.ts
+91 -29
View File
@@ -1,14 +1,20 @@
import * as path from "path";
import * as fs from "fs/promises";
import * as os from "os";
import * as processSetting from "@lib/API/processSetting";
import { ConnectionStringParser } from "@lib/common/ConnectionString";
import { configURIBase } from "@lib/common/models/shared.const";
import { DEFAULT_SETTINGS, REMOTE_COUCHDB, REMOTE_MINIO, REMOTE_P2P } from "@lib/common/types";
import { fsPromises as fs, os, path } from "@vrtmrz/livesync-commonlib/node";
import * as processSetting from "@vrtmrz/livesync-commonlib/compat/API/processSetting";
import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
import { configURIBase } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const";
import { DEFAULT_SETTINGS, REMOTE_COUCHDB, REMOTE_MINIO, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
import { runCommand } from "./runCommand";
import type { CLIOptions } from "./types";
import * as commandUtils from "./utils";
function createStandardIoMock() {
return {
readStdin: vi.fn(async () => ""),
prompt: vi.fn(async () => ""),
writeStdout: vi.fn((_chunk: string | Uint8Array) => undefined),
writeStderr: vi.fn((_chunk: string | Uint8Array) => undefined),
};
}
function createCoreMock() {
const liveSettings = {
@@ -19,6 +25,9 @@ function createCoreMock() {
} as any;
return {
services: {
context: {
standardIo: createStandardIoMock(),
},
control: {
activated: Promise.resolve(),
applySettings: vi.fn(async () => {}),
@@ -71,6 +80,7 @@ function createCoreMock() {
},
databaseFileAccess: {
fetch: vi.fn(async () => undefined),
storeContent: vi.fn(async () => true),
},
},
} as any;
@@ -98,20 +108,20 @@ async function createSetupURI(passphrase: string): Promise<string> {
return await processSetting.encodeSettingsToSetupURI(settings, passphrase);
}
function captureStdout() {
const writes: string[] = [];
const spy = vi.spyOn(process.stdout, "write").mockImplementation((chunk: any) => {
writes.push(typeof chunk === "string" ? chunk : String(chunk));
return true;
});
function captureStdout(core: ReturnType<typeof createCoreMock>) {
const spy = core.services.context.standardIo.writeStdout;
spy.mockClear();
return {
spy,
lines: () =>
writes
spy.mock.calls
.map(([chunk]: [string | Uint8Array]) =>
typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk)
)
.join("")
.split("\n")
.map((e) => e.trim())
.filter((e) => e.length > 0),
.map((entry: string) => entry.trim())
.filter((entry: string) => entry.length > 0),
};
}
@@ -306,7 +316,7 @@ describe("runCommand abnormal cases", () => {
it("setup rejects empty passphrase", async () => {
const core = createCoreMock();
vi.spyOn(commandUtils, "promptForPassphrase").mockRejectedValue(new Error("Passphrase is required"));
core.services.context.standardIo.prompt.mockResolvedValue("");
await expect(
runCommand(makeOptions("setup", [`${configURIBase}dummy`]), {
@@ -320,7 +330,7 @@ describe("runCommand abnormal cases", () => {
const core = createCoreMock();
const passphrase = "correct-passphrase";
const setupURI = await createSetupURI(passphrase);
vi.spyOn(commandUtils, "promptForPassphrase").mockResolvedValue(passphrase);
core.services.context.standardIo.prompt.mockResolvedValue(passphrase);
const result = await runCommand(makeOptions("setup", [setupURI]), {
...context,
@@ -341,7 +351,7 @@ describe("runCommand abnormal cases", () => {
it("setup rejects encoded URI when passphrase is wrong", async () => {
const core = createCoreMock();
const setupURI = await createSetupURI("correct-passphrase");
vi.spyOn(commandUtils, "promptForPassphrase").mockResolvedValue("wrong-passphrase");
core.services.context.standardIo.prompt.mockResolvedValue("wrong-passphrase");
await expect(
runCommand(makeOptions("setup", [setupURI]), {
@@ -354,9 +364,61 @@ describe("runCommand abnormal cases", () => {
expect(core.services.control.applySettings).not.toHaveBeenCalled();
});
it("put reads content from the injected standard input", async () => {
const core = createCoreMock();
core.services.context.standardIo.readStdin.mockResolvedValue("content from stdin");
const result = await runCommand(makeOptions("put", ["notes/input.md"]), {
...context,
core,
});
expect(result).toBe(true);
expect(core.services.context.standardIo.readStdin).toHaveBeenCalledOnce();
expect(core.serviceModules.databaseFileAccess.storeContent).toHaveBeenCalledWith(
"notes/input.md",
"content from stdin"
);
});
it("cat writes text to the injected standard output without adding a delimiter", async () => {
const core = createCoreMock();
core.serviceModules.databaseFileAccess.fetch.mockResolvedValue({
deleted: false,
body: new Blob(["exact text"], { type: "text/plain" }),
});
const result = await runCommand(makeOptions("cat", ["notes/output.md"]), {
...context,
core,
});
expect(result).toBe(true);
expect(core.services.context.standardIo.writeStdout).toHaveBeenCalledWith("exact text");
});
it("cat preserves binary bytes through the injected standard output", async () => {
const core = createCoreMock();
const expected = Uint8Array.from([0x00, 0x7f, 0x80, 0xff]);
core.serviceModules.databaseFileAccess.fetch.mockResolvedValue({
deleted: false,
body: new Blob([expected], { type: "application/octet-stream" }),
});
const result = await runCommand(makeOptions("cat", ["binary/output.bin"]), {
...context,
core,
});
expect(result).toBe(true);
const chunk = core.services.context.standardIo.writeStdout.mock.calls.at(-1)?.[0];
expect(chunk).toBeInstanceOf(Uint8Array);
expect([...(chunk as Uint8Array)]).toEqual([...expected]);
});
it("remote-add stores canonical URI and prints the created id", async () => {
const core = createCoreMock();
const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
const stdout = core.services.context.standardIo.writeStdout;
const result = await runCommand(makeOptions("remote-add", ["my-remote", "sls+https://example.com/db"]), {
...context,
@@ -437,7 +499,7 @@ describe("runCommand abnormal cases", () => {
uri: "sls+https://example.com/db?db=vault",
isEncrypted: false,
};
const stdout = captureStdout();
const stdout = captureStdout(core);
const result = await runCommand(makeOptions("remote-export", ["r1"]), {
...context,
@@ -567,7 +629,7 @@ describe("runCommand abnormal cases", () => {
])("remote command round-trip works for %s", async (_protocol, initialConnStr) => {
const core = createCoreMock();
const addOut = captureStdout();
const addOut = captureStdout(core);
const addResult = await runCommand(makeOptions("remote-add", ["rt", initialConnStr]), {
...context,
core,
@@ -576,7 +638,7 @@ describe("runCommand abnormal cases", () => {
const remoteId = parseAddedRemoteIdFromLines(addOut.lines());
expect(remoteId).not.toBe("");
const export1Out = captureStdout();
const export1Out = captureStdout(core);
const export1Result = await runCommand(makeOptions("remote-export", [remoteId]), {
...context,
core,
@@ -593,7 +655,7 @@ describe("runCommand abnormal cases", () => {
});
expect(setResult).toBe(true);
const export2Out = captureStdout();
const export2Out = captureStdout(core);
const export2Result = await runCommand(makeOptions("remote-export", [remoteId]), {
...context,
core,
@@ -742,13 +804,13 @@ describe("runCommand abnormal cases", () => {
it("remote-status without args outputs status of active remote configuration", async () => {
const core = createCoreMock();
const stdout = captureStdout();
const stdout = captureStdout(core);
const result = await runCommand(makeOptions("remote-status", []), {
...context,
core,
});
expect(result).toBe(true);
const fullOutput = stdout.spy.mock.calls.map((c) => c[0]).join("");
const fullOutput = stdout.spy.mock.calls.map((call: [string | Uint8Array]) => call[0]).join("");
const parsedStatus = JSON.parse(fullOutput);
expect(parsedStatus.db_name).toBe("test-db");
expect(parsedStatus.doc_count).toBe(42);
@@ -763,13 +825,13 @@ describe("runCommand abnormal cases", () => {
uri: "sls+https://example.com/db1",
isEncrypted: false,
};
const stdout = captureStdout();
const stdout = captureStdout(core);
const result = await runCommand(makeOptions("remote-status", ["r1"]), {
...context,
core,
});
expect(result).toBe(true);
const fullOutput = stdout.spy.mock.calls.map((c) => c[0]).join("");
const fullOutput = stdout.spy.mock.calls.map((call: [string | Uint8Array]) => call[0]).join("");
const parsedStatus = JSON.parse(fullOutput);
expect(parsedStatus.db_name).toBe("test-db");
expect(parsedStatus.doc_count).toBe(42);
src/apps/cli/commands/types.ts
+11 -3
View File
@@ -1,6 +1,7 @@
import { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
import { ServiceContext } from "@lib/services/base/ServiceBase";
import type { ObsidianLiveSyncSettings } from "@lib/common/types";
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { NodeServiceContext } from "@/apps/cli/services/NodeServiceContext";
import type { UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/UseP2PReplicatorResult";
export type CLICommand =
| "daemon"
@@ -39,6 +40,7 @@ export interface CLIOptions {
verbose?: boolean;
debug?: boolean;
force?: boolean;
writeSettings?: boolean;
command: CLICommand;
commandArgs: string[];
interval?: number;
@@ -47,7 +49,9 @@ export interface CLIOptions {
export interface CLICommandContext {
databasePath: string;
vaultPath: string;
core: LiveSyncBaseCore<ServiceContext, never>;
core: LiveSyncBaseCore<NodeServiceContext, never>;
/** Current-result contract owned by the P2P service feature. */
p2pReplicator?: UseP2PReplicatorResult;
settingsPath: string;
originalSyncSettings: Pick<
ObsidianLiveSyncSettings,
@@ -91,3 +95,7 @@ export const VALID_COMMANDS = new Set([
"remote-status",
"init-settings",
] as const);
export function isCLICommand(value: string): value is CLICommand {
return (VALID_COMMANDS as ReadonlySet<string>).has(value);
}
src/apps/cli/commands/utils.ts
+1 -27
View File
@@ -1,7 +1,6 @@
import { path, readline } from "@/apps/cli/node-compat";
import { path } from "@vrtmrz/livesync-commonlib/node";
export function toArrayBuffer(data: Buffer): ArrayBuffer {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion -- required in environments where Buffer.buffer is ArrayBufferLike
return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
}
@@ -23,28 +22,3 @@ export function toDatabaseRelativePath(inputPath: string, databasePath: string):
}
return rel.replace(/\\/g, "/");
}
export async function readStdinAsUtf8(): Promise<string> {
const chunks = [];
for await (const chunk of process.stdin) {
if (typeof chunk === "string") {
chunks.push(Buffer.from(chunk, "utf-8"));
} else {
chunks.push(chunk as Buffer);
}
}
return Buffer.concat(chunks as Uint8Array[]).toString("utf-8");
}
export async function promptForPassphrase(prompt = "Enter setup URI passphrase: "): Promise<string> {
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
try {
const passphrase = await rl.question(prompt);
if (!passphrase) {
throw new Error("Passphrase is required");
}
return passphrase;
} finally {
rl.close();
}
}
src/apps/cli/commands/utils.unit.spec.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import * as path from "path";
import { path } from "@vrtmrz/livesync-commonlib/node";
import { describe, expect, it } from "vitest";
import { toDatabaseRelativePath } from "./utils";
src/apps/cli/docker-image.unit.spec.ts
+11
View File
@@ -0,0 +1,11 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
const dockerfile = readFileSync(new URL("./Dockerfile", import.meta.url), "utf8");
describe("CLI Docker image", () => {
it("sets a deterministic readable and executable entrypoint mode", () => {
expect(dockerfile).toContain("COPY --chmod=755 src/apps/cli/docker-entrypoint.sh /usr/local/bin/livesync-cli");
expect(dockerfile).not.toContain("RUN chmod +x /usr/local/bin/livesync-cli");
});
});
src/apps/cli/entrypoint.ts
+10 -9
View File
@@ -1,20 +1,21 @@
#!/usr/bin/env node
// eslint-disable -- This is the entry point for the CLI application.
import * as polyfill from "werift";
import { RTCPeerConnection } from "werift";
import { main } from "./main";
import { compatGlobal } from "@lib/common/coreEnvFunctions";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
import { createNodeStandardIo } from "@vrtmrz/livesync-commonlib/node";
import { writeStderrLine } from "./cliOutput";
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Polyfill
const rtcPolyfillCtor = (polyfill as any).RTCPeerConnection;
if (
typeof (compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection === "undefined" &&
typeof rtcPolyfillCtor === "function"
typeof RTCPeerConnection === "function"
) {
// Fill only the standard WebRTC global in Node CLI runtime.
(compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection = rtcPolyfillCtor;
(compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection = RTCPeerConnection;
}
main().catch((error) => {
console.error(`[Fatal Error]`, error);
const standardIo = createNodeStandardIo();
main(standardIo).catch((error) => {
writeStderrLine(standardIo, `[Fatal Error]`, error);
process.exit(1);
});
src/apps/cli/lib/pouchdb-node.ts
+100 -71
View File
@@ -8,11 +8,8 @@ import LevelDBAdapter from "pouchdb-adapter-leveldb";
import find from "pouchdb-find";
import transform from "transform-pouch";
//@ts-ignore
import { findPathToLeaf } from "pouchdb-merge";
//@ts-ignore
import { findPathToLeaf, type RevisionTreeNode } from "pouchdb-merge";
import { adapterFun } from "pouchdb-utils";
//@ts-ignore
import { createError, MISSING_DOC, UNKNOWN_ERROR } from "pouchdb-errors";
import { mapAllTasksWithConcurrencyLimit, unwrapTaskResult } from "octagonal-wheels/concurrency/task";
@@ -24,113 +21,145 @@ type PurgeMultiResult = {
documentWasRemovedCompletely: boolean;
};
type PurgeMultiParam = [docId: string, rev$$1: string];
function appendPurgeSeqs(db: PouchDB.Database, docs: PurgeMultiParam[]) {
return (
db
.get("_local/purges")
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Internal method patching.
.then(function (doc: any) {
for (const [docId, rev$$1] of docs) {
const purgeSeq = doc.purgeSeq + 1;
doc.purges.push({
docId,
rev: rev$$1,
purgeSeq,
});
//@ts-ignore : missing type def
if (doc.purges.length > db.purged_infos_limit) {
//@ts-ignore : missing type def
doc.purges.splice(0, doc.purges.length - db.purged_infos_limit);
}
doc.purgeSeq = purgeSeq;
type PurgeLogDocument = {
purgeSeq: number;
purges: Array<{ docId: string; rev: string; purgeSeq: number }>;
};
type PurgeMultiResultMap = Record<string, unknown>;
interface PouchDBPrivateDatabase extends PouchDB.Database {
adapter: string;
purged_infos_limit: number;
_getRevisionTree(
documentId: string,
callback: (error: Error | undefined, revisions?: RevisionTreeNode[]) => void
): void;
_purge(
documentId: string,
revisionPath: string[],
callback: (error: Error | undefined, result?: PurgeMultiResult) => void
): void;
purgeMulti(documents: PurgeMultiParam[]): Promise<PurgeMultiResultMap>;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
function isSuccessfulPurge(value: unknown): value is PurgeMultiResult {
return isRecord(value) && value.ok === true;
}
function appendPurgeSeqs(db: PouchDBPrivateDatabase, docs: PurgeMultiParam[]) {
return db
.get<PurgeLogDocument>("_local/purges")
.then(function (doc) {
for (const [docId, rev$$1] of docs) {
const purgeSeq = doc.purgeSeq + 1;
doc.purges.push({
docId,
rev: rev$$1,
purgeSeq,
});
if (doc.purges.length > db.purged_infos_limit) {
doc.purges.splice(0, doc.purges.length - db.purged_infos_limit);
}
return doc;
})
.catch(function (err) {
if (err.status !== 404) {
throw err;
}
return {
_id: "_local/purges",
purges: docs.map(([docId, rev$$1], idx) => ({
docId,
rev: rev$$1,
purgeSeq: idx,
})),
purgeSeq: docs.length,
};
})
.then(function (doc) {
return db.put(doc);
})
);
doc.purgeSeq = purgeSeq;
}
return doc;
})
.catch(function (error: unknown) {
if (!isRecord(error) || error.status !== 404) {
throw error;
}
return {
_id: "_local/purges",
purges: docs.map(([docId, rev$$1], idx) => ({
docId,
rev: rev$$1,
purgeSeq: idx,
})),
purgeSeq: docs.length,
};
})
.then(function (doc) {
return db.put(doc);
});
}
/**
* purge multiple documents at once.
*/
PouchDB.prototype.purgeMulti = adapterFun(
const pouchDBPrototype = (PouchDB as typeof PouchDB & { prototype: PouchDBPrivateDatabase }).prototype;
pouchDBPrototype.purgeMulti = adapterFun<PouchDBPrivateDatabase, [documents: PurgeMultiParam[]], PurgeMultiResultMap>(
"_purgeMulti",
function (
this: PouchDBPrivateDatabase,
docs: PurgeMultiParam[],
callback: (
error: Error,
result?: {
[x: string]: PurgeMultiResult | Error;
}
) => void
callback: (error?: Error, result?: PurgeMultiResultMap) => void
) {
//@ts-ignore
if (typeof this._purge === "undefined") {
return callback(
//@ts-ignore: this ts-ignore might be hiding a `this` bug where we don't have "this" conext.
createError(UNKNOWN_ERROR, "Purge is not implemented in the " + this.adapter + " adapter.")
);
}
//@ts-ignore
// eslint-disable-next-line @typescript-eslint/no-this-alias
// eslint-disable-next-line @typescript-eslint/no-this-alias -- The adapter task callbacks must retain this PouchDB instance.
const self = this;
const tasks = docs.map(
(param) => () =>
new Promise<[PurgeMultiParam, PurgeMultiResult | Error]>((res, rej) => {
new Promise<[PurgeMultiParam, unknown]>((res) => {
const [docId, rev$$1] = param;
self._getRevisionTree(docId, (error: Error, revs: string[]) => {
self._getRevisionTree(docId, (error, revs) => {
if (error) {
return res([param, error]);
}
if (!revs) {
return res([param, createError(MISSING_DOC)]);
}
let path;
let path: string[];
try {
path = findPathToLeaf(revs, rev$$1);
} catch (error) {
//@ts-ignore
return res([param, error.message || error]);
} catch (caught: unknown) {
const failure = caught instanceof Error && caught.message ? caught.message : caught;
return res([param, failure]);
}
self._purge(docId, path, (error: Error, result: PurgeMultiResult) => {
self._purge(docId, path, (error, result) => {
if (error) {
return res([param, error]);
} else {
return res([param, result]);
}
return res([param, result]);
});
});
})
);
(async () => {
const ret = await mapAllTasksWithConcurrencyLimit(1, tasks);
const retAll = ret.map((e) => unwrapTaskResult(e)) as [PurgeMultiParam, PurgeMultiResult | Error][];
await appendPurgeSeqs(
self,
retAll.filter((e) => "ok" in e[1]).map((e) => e[0])
);
const result = Object.fromEntries(retAll.map((e) => [e[0][0], e[1]]));
const retAll: Array<[PurgeMultiParam, unknown]> = [];
for (const entry of ret) {
const outcome = unwrapTaskResult(entry);
if (outcome instanceof Error) {
throw outcome;
}
retAll.push(outcome);
}
const successfullyPurged: PurgeMultiParam[] = [];
const resultEntries: Array<[string, unknown]> = [];
for (const [document, outcome] of retAll) {
if (isSuccessfulPurge(outcome)) {
successfullyPurged.push(document);
}
resultEntries.push([document[0], outcome]);
}
await appendPurgeSeqs(self, successfullyPurged);
const result: PurgeMultiResultMap = Object.fromEntries(resultEntries);
return result;
})()
//@ts-ignore
.then((result) => callback(undefined, result))
.catch((error) => callback(error));
.catch((caught: unknown) => {
const error = caught instanceof Error ? caught : new Error(String(caught));
callback(error);
});
}
);
src/apps/cli/main.log-handler.unit.spec.ts
+55
View File
@@ -0,0 +1,55 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
const mocks = vi.hoisted(() => ({
setLogHandler: vi.fn(),
}));
vi.mock("./services/NodeServiceHub", () => ({
NodeServiceContext: class {},
NodeServiceHub: class {
API = {
addLog: {
setHandler: mocks.setLogHandler,
},
};
},
}));
import { main } from "./main";
function createStandardIoMock() {
return {
readStdin: vi.fn(async () => ""),
prompt: vi.fn(async () => ""),
writeStdout: vi.fn(),
writeStderr: vi.fn(),
};
}
describe("CLI log handler", () => {
const originalArgv = process.argv.slice();
let databasePath: string;
beforeEach(async () => {
databasePath = await mkdtemp(join(tmpdir(), "livesync-cli-log-handler-"));
mocks.setLogHandler.mockReset();
mocks.setLogHandler.mockImplementation(() => {
throw new Error("__LOG_HANDLER_CONFIGURED__");
});
});
afterEach(async () => {
process.argv = originalArgv.slice();
await rm(databasePath, { recursive: true, force: true });
});
it("replaces the default Headless API log handler", async () => {
process.argv = ["node", "livesync-cli", databasePath, "remote-ls"];
await expect(main(createStandardIoMock())).rejects.toThrow("__LOG_HANDLER_CONFIGURED__");
expect(mocks.setLogHandler).toHaveBeenCalledWith(expect.any(Function), true);
});
});
src/apps/cli/main.ts
+210 -97
View File
@@ -2,9 +2,13 @@ import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub";
import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage";
import { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
import { initialiseServiceModulesCLI } from "./serviceModules/CLIServiceModules";
import { DEFAULT_SETTINGS, LOG_LEVEL_VERBOSE, type LOG_LEVEL, type ObsidianLiveSyncSettings } from "@lib/common/types";
import type { InjectableServiceHub } from "@lib/services/implements/injectable/InjectableServiceHub";
import type { InjectableSettingService } from "@lib/services/implements/injectable/InjectableSettingService";
import {
LOG_LEVEL_VERBOSE,
type LOG_LEVEL,
type ObsidianLiveSyncSettings,
} from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub";
import type { InjectableSettingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableSettingService";
import {
LOG_LEVEL_DEBUG,
setGlobalLogFunction,
@@ -14,20 +18,39 @@ import {
LOG_LEVEL_NOTICE,
} from "octagonal-wheels/common/logger";
import { runCommand } from "./commands/runCommand";
import { VALID_COMMANDS } from "./commands/types";
import type { CLICommand, CLIOptions } from "./commands/types";
import { getPathFromUXFileInfo } from "@lib/common/typeUtils";
import { stripAllPrefixes } from "@lib/string_and_binary/path";
import { isCLICommand } from "./commands/types";
import type { CLICommand, CLICommandContext, CLIOptions } from "./commands/types";
import { getPathFromUXFileInfo } from "@vrtmrz/livesync-commonlib/compat/common/typeUtils";
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
import { IgnoreRules } from "./serviceModules/IgnoreRules";
import { useP2PReplicatorFeature } from "@lib/replication/trystero/useP2PReplicatorFeature";
import { fsPromises as fs, path, fs as fsSync } from "./node-compat";
import { useP2PReplicatorFeature } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/useP2PReplicatorFeature";
import type { UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/UseP2PReplicatorResult";
import { createNodeStandardIo, fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node";
import type { StandardIo } from "@vrtmrz/livesync-commonlib/context";
import { writeStderrLine, writeStdoutLine } from "./cliOutput";
import { createDefaultCliSettings } from "./cliSettingsDefaults";
import {
applyStoredSetting,
changedSettingKeys,
CLI_RUNTIME_ONLY_SETTING_KEYS,
cloneSettings,
isSettingsWriteCommand,
preserveStoredSetting,
reconcileDurableSettings,
settingsEqual,
} from "./settingsPersistence";
const SETTINGS_FILE = ".livesync/settings.json";
ensureGlobalNodeLocalStorage();
defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG;
function printHelp(): void {
console.log(`
/** Injectable command boundary used by CLI integration probes. */
export type CliCommandRunner = (options: CLIOptions, context: CLICommandContext) => Promise<boolean>;
function printHelp(standardIo: StandardIo): void {
writeStdoutLine(
standardIo,
`
Self-hosted LiveSync CLI
Usage:
@@ -79,6 +102,7 @@ Options:
--vault <path>, -V <path> (daemon/mirror) Path to the vault directory containing .md files
(defaults to database-path; allows separate PouchDB and vault dirs)
--interval <N>, -i <N> (daemon only) Poll CouchDB every N seconds instead of using the _changes feed
--write-settings Write setting changes after a successful command
Examples:
livesync-cli ./my-database Run daemon (LiveSync mode)
@@ -110,14 +134,15 @@ Examples:
livesync-cli ./my-database remote-status remote-abc123
livesync-cli init-settings ./data.json
livesync-cli ./my-database --verbose
`);
`
);
}
export function parseArgs(): CLIOptions {
export function parseArgs(standardIo: StandardIo = createNodeStandardIo()): CLIOptions {
const args = process.argv.slice(2);
if (args.length === 0 || args.includes("--help") || args.includes("-h")) {
printHelp();
printHelp(standardIo);
process.exit(0);
}
@@ -127,6 +152,7 @@ export function parseArgs(): CLIOptions {
let verbose = false;
let debug = false;
let force = false;
let writeSettings = false;
let interval: number | undefined;
let command: CLICommand = "daemon";
const commandArgs: string[] = [];
@@ -138,7 +164,7 @@ export function parseArgs(): CLIOptions {
case "-V": {
i++;
if (!args[i]) {
console.error(`Error: Missing value for ${token}`);
writeStderrLine(standardIo, `Error: Missing value for ${token}`);
process.exit(1);
}
vaultPath = args[i];
@@ -148,7 +174,7 @@ export function parseArgs(): CLIOptions {
case "-s": {
i++;
if (!args[i]) {
console.error(`Error: Missing value for ${token}`);
writeStderrLine(standardIo, `Error: Missing value for ${token}`);
process.exit(1);
}
settingsPath = args[i];
@@ -158,12 +184,12 @@ export function parseArgs(): CLIOptions {
case "-i": {
i++;
if (!args[i]) {
console.error(`Error: Missing value for ${token}`);
writeStderrLine(standardIo, `Error: Missing value for ${token}`);
process.exit(1);
}
const n = parseInt(args[i], 10);
if (!Number.isInteger(n) || n <= 0) {
console.error(`Error: --interval requires a positive integer, got '${args[i]}'`);
writeStderrLine(standardIo, `Error: --interval requires a positive integer, got '${args[i]}'`);
process.exit(1);
}
interval = n;
@@ -173,7 +199,8 @@ export function parseArgs(): CLIOptions {
case "-d":
// debugging automatically enables verbose logging, as it is intended for debugging issues.
debug = true;
// falls through
verbose = true;
break;
case "--verbose":
case "-v":
verbose = true;
@@ -182,11 +209,13 @@ export function parseArgs(): CLIOptions {
case "-f":
force = true;
break;
case "--write-settings":
writeSettings = true;
break;
default: {
if (!databasePath) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Set checking
if (command === "daemon" && VALID_COMMANDS.has(token as any)) {
command = token as CLICommand;
if (command === "daemon" && isCLICommand(token)) {
command = token;
break;
}
if (command === "init-settings") {
@@ -196,9 +225,8 @@ export function parseArgs(): CLIOptions {
databasePath = token;
break;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Set checking
if (command === "daemon" && VALID_COMMANDS.has(token as any)) {
command = token as CLICommand;
if (command === "daemon" && isCLICommand(token)) {
command = token;
break;
}
commandArgs.push(token);
@@ -208,12 +236,12 @@ export function parseArgs(): CLIOptions {
}
if (!databasePath && command !== "init-settings") {
console.error("Error: database-path is required");
writeStderrLine(standardIo, "Error: database-path is required");
process.exit(1);
}
if (command === "daemon" && commandArgs.length > 0) {
console.error(`Error: Unknown command '${commandArgs[0]}'`);
writeStderrLine(standardIo, `Error: Unknown command '${commandArgs[0]}'`);
process.exit(1);
}
@@ -224,13 +252,14 @@ export function parseArgs(): CLIOptions {
verbose,
debug,
force,
writeSettings,
command,
commandArgs,
interval,
};
}
async function createDefaultSettingsFile(options: CLIOptions) {
async function createDefaultSettingsFile(options: CLIOptions, standardIo: StandardIo) {
const targetPath = options.settingsPath
? path.resolve(options.settingsPath)
: options.commandArgs[0]
@@ -248,20 +277,20 @@ async function createDefaultSettingsFile(options: CLIOptions) {
}
}
const settings = {
...DEFAULT_SETTINGS,
useIndexedDBAdapter: false,
} as ObsidianLiveSyncSettings;
const settings = createDefaultCliSettings();
await fs.mkdir(path.dirname(targetPath), { recursive: true });
await fs.writeFile(targetPath, JSON.stringify(settings, null, 2), "utf-8");
console.log(`[Done] Created settings file: ${targetPath}`);
writeStdoutLine(standardIo, `[Done] Created settings file: ${targetPath}`);
}
export async function main() {
const options = parseArgs();
export async function main(
standardIo: StandardIo = createNodeStandardIo(),
commandRunner: CliCommandRunner = runCommand
) {
const options = parseArgs(standardIo);
if (options.interval && options.command !== "daemon") {
console.error(`Warning: --interval is only used in daemon mode, ignored for '${options.command}'`);
writeStderrLine(standardIo, `Warning: --interval is only used in daemon mode, ignored for '${options.command}'`);
}
const avoidStdoutNoise =
options.command === "cat" ||
@@ -278,13 +307,13 @@ export async function main() {
options.command === "unlock-remote" ||
options.command === "lock-remote" ||
options.command === "remote-status";
const infoLog = avoidStdoutNoise ? console.error : console.log;
const infoLog = (...values: readonly unknown[]) => {
const writeLine = avoidStdoutNoise ? writeStderrLine : writeStdoutLine;
writeLine(standardIo, ...values);
};
if (options.debug) {
setGlobalLogFunction((msg, level) => {
console.error(`[${level}] ${typeof msg === "string" ? msg : JSON.stringify(msg)}`);
if (msg instanceof Error) {
console.error(msg);
}
writeStderrLine(standardIo, `[${level}]`, msg);
});
} else {
setGlobalLogFunction((msg, level) => {
@@ -292,7 +321,7 @@ export async function main() {
});
}
if (options.command === "init-settings") {
await createDefaultSettingsFile(options);
await createDefaultSettingsFile(options, standardIo);
return;
}
@@ -302,11 +331,11 @@ export async function main() {
try {
const stat = await fs.stat(databasePath);
if (!stat.isDirectory()) {
console.error(`Error: ${databasePath} is not a directory`);
writeStderrLine(standardIo, `Error: ${databasePath} is not a directory`);
process.exit(1);
}
} catch {
console.error(`Error: Database directory ${databasePath} does not exist`);
writeStderrLine(standardIo, `Error: Database directory ${databasePath} does not exist`);
process.exit(1);
}
@@ -332,11 +361,11 @@ export async function main() {
try {
const stat = await fs.stat(vaultPath);
if (!stat.isDirectory()) {
console.error(`Error: Vault path ${vaultPath} is not a directory`);
writeStderrLine(standardIo, `Error: Vault path ${vaultPath} is not a directory`);
process.exit(1);
}
} catch {
console.error(`Error: Vault directory ${vaultPath} does not exist`);
writeStderrLine(standardIo, `Error: Vault directory ${vaultPath} does not exist`);
process.exit(1);
}
@@ -347,14 +376,20 @@ export async function main() {
infoLog("");
let ignoreRules: IgnoreRules | undefined;
if (options.command === "daemon" || options.command === "mirror") {
ignoreRules = new IgnoreRules(vaultPath);
ignoreRules = new IgnoreRules(vaultPath, (message, detail) => {
if (detail === undefined) {
writeStderrLine(standardIo, message);
} else {
writeStderrLine(standardIo, message, detail);
}
});
await ignoreRules.load();
}
// Create service context and hub
const context = new NodeServiceContext(databasePath);
const context = new NodeServiceContext(databasePath, standardIo);
const serviceHubInstance = new NodeServiceHub<NodeServiceContext>(databasePath, context);
serviceHubInstance.API.addLog.setHandler((message: string, level: LOG_LEVEL) => {
serviceHubInstance.API.addLog.setHandler((message: unknown, level: LOG_LEVEL) => {
let levelStr = "";
switch (level) {
case LOG_LEVEL_DEBUG:
@@ -373,35 +408,45 @@ export async function main() {
levelStr = "Urgent";
break;
default:
levelStr = `${level}`;
levelStr = String(level);
}
const prefix = `(${levelStr})`;
if (level <= LOG_LEVEL_INFO) {
if (!options.verbose) return;
}
console.error(`${prefix} ${message}`);
});
writeStderrLine(standardIo, prefix, message);
}, true);
// Prevent replication result from being processed automatically in non-daemon commands.
// In daemon mode the default handler must run so changes are applied to the filesystem.
if (options.command !== "daemon") {
serviceHubInstance.replication.processSynchroniseResult.addHandler(async () => {
console.error(`[Info] Replication result received, but not processed automatically in CLI mode.`);
writeStderrLine(standardIo, `[Info] Replication result received, but not processed automatically in CLI mode.`);
return await Promise.resolve(true);
}, -100);
}
// Setup settings handlers
const settingService = serviceHubInstance.setting;
const originalSettingsText = await fs.readFile(settingsPath, "utf-8").catch(() => undefined);
let latestPreparedSettingsText: string | undefined;
let preparedSettingsRevision = 0;
let commandIsRunning = false;
let commandPreparedSettingsTexts: string[] = [];
(settingService as InjectableSettingService<NodeServiceContext>).saveData.setHandler(
async (data: ObsidianLiveSyncSettings) => {
try {
await fs.writeFile(settingsPath, JSON.stringify(data, null, 2), "utf-8");
latestPreparedSettingsText = JSON.stringify(data, null, 2);
preparedSettingsRevision++;
if (commandIsRunning) {
commandPreparedSettingsTexts.push(latestPreparedSettingsText);
}
if (options.verbose) {
console.error(`[Settings] Saved to ${settingsPath}`);
writeStderrLine(standardIo, `[Settings] Prepared an update for ${settingsPath}`);
}
} catch (error) {
console.error(`[Settings] Failed to save:`, error);
writeStderrLine(standardIo, `[Settings] Failed to prepare an update:`, error);
throw error;
}
}
);
@@ -410,16 +455,15 @@ export async function main() {
async (): Promise<ObsidianLiveSyncSettings | undefined> => {
try {
const content = await fs.readFile(settingsPath, "utf-8");
const data = JSON.parse(content);
const data = JSON.parse(content) as ObsidianLiveSyncSettings;
if (options.verbose) {
console.error(`[Settings] Loaded from ${settingsPath}`);
writeStderrLine(standardIo, `[Settings] Loaded from ${settingsPath}`);
}
// Force disable IndexedDB adapter in CLI environment
data.useIndexedDBAdapter = false;
return data;
// Force disable IndexedDB adapter in CLI environment without mutating the loaded settings object.
return { ...data, useIndexedDBAdapter: false };
} catch {
if (options.verbose) {
console.error(`[Settings] File not found, using defaults`);
writeStderrLine(standardIo, `[Settings] File not found, using defaults`);
}
return undefined;
}
@@ -427,6 +471,7 @@ export async function main() {
);
// Create LiveSync core
let p2pReplicator: UseP2PReplicatorResult | undefined;
const core = new LiveSyncBaseCore(
serviceHubInstance,
(core: LiveSyncBaseCore<NodeServiceContext, never>, serviceHub: InjectableServiceHub<NodeServiceContext>) => {
@@ -436,7 +481,7 @@ export async function main() {
() => [], // No add-ons
(core) => {
// Register P2P replicator feature.
useP2PReplicatorFeature(core);
p2pReplicator = useP2PReplicatorFeature(core);
// Add target filter to prevent internal files are handled
core.services.vault.isTargetFile.addHandler(async (target) => {
const targetPath = stripAllPrefixes(getPathFromUXFileInfo(target));
@@ -469,14 +514,14 @@ export async function main() {
// Setup signal handlers for graceful shutdown
const shutdown = async (signal: string) => {
console.log();
console.log(`[Shutdown] Received ${signal}, shutting down gracefully...`);
writeStdoutLine(standardIo);
writeStdoutLine(standardIo, `[Shutdown] Received ${signal}, shutting down gracefully...`);
try {
await core.services.control.onUnload();
console.log(`[Shutdown] Complete`);
writeStdoutLine(standardIo, `[Shutdown] Complete`);
process.exit(0);
} catch (error) {
console.error(`[Shutdown] Error:`, error);
writeStderrLine(standardIo, `[Shutdown] Error:`, error);
process.exit(1);
}
};
@@ -484,24 +529,21 @@ export async function main() {
process.on("SIGINT", () => void shutdown("SIGINT"));
process.on("SIGTERM", () => void shutdown("SIGTERM"));
// Save the settings file before any lifecycle events can mutate and persist them.
// suspendAllSync and other lifecycle hooks clobber sync settings in memory, and
// various code paths persist the clobbered state to disk. We restore on shutdown.
const settingsBackup = await fs.readFile(settingsPath, "utf-8").catch(() => null!);
// Restore settings file on any exit to undo lifecycle mutations.
// Write to a temp path first so a crash mid-write doesn't leave a truncated file.
process.on("exit", () => {
if (settingsBackup) {
const tmpPath = settingsPath + ".tmp";
try {
fsSync.writeFileSync(tmpPath, settingsBackup, "utf-8");
fsSync.renameSync(tmpPath, settingsPath);
} catch (err) {
console.error("[Settings] Failed to restore settings on exit:", err);
const writeSettingsAtomically = async (content: string | undefined): Promise<void> => {
if (content === undefined || content === originalSettingsText) return;
const temporaryPath = `${settingsPath}.${process.pid}.tmp`;
await fs.mkdir(path.dirname(settingsPath), { recursive: true });
try {
await fs.writeFile(temporaryPath, content, "utf-8");
await fs.rename(temporaryPath, settingsPath);
if (options.verbose) {
writeStderrLine(standardIo, `[Settings] Saved to ${settingsPath}`);
}
} catch (error) {
await fs.unlink(temporaryPath).catch(() => {});
throw error;
}
});
};
// Start the core
try {
@@ -509,12 +551,18 @@ export async function main() {
const loadResult = await core.services.control.onLoad();
if (!loadResult) {
console.error(`[Error] Failed to initialize LiveSync`);
writeStderrLine(standardIo, `[Error] Failed to initialize LiveSync`);
process.exit(1);
}
const settingsAfterLoadText = latestPreparedSettingsText
? preserveStoredSetting(latestPreparedSettingsText, originalSettingsText, "useIndexedDBAdapter")
: originalSettingsText;
// Capture sync settings before suspendAllSync() clobbers them.
// Used by daemon mode to restore the correct sync behaviour after the mirror scan.
const settingsBeforeSuspend = core.services.setting.currentSettings();
const settingsBeforeSuspend = cloneSettings(core.services.setting.currentSettings());
const durableSettingsBeforeSuspend = cloneSettings(settingsBeforeSuspend);
applyStoredSetting(durableSettingsBeforeSuspend, settingsAfterLoadText, "useIndexedDBAdapter");
const originalSyncSettings = {
liveSync: settingsBeforeSuspend.liveSync,
syncOnStart: settingsBeforeSuspend.syncOnStart,
@@ -525,7 +573,19 @@ export async function main() {
syncAfterMerge: settingsBeforeSuspend.syncAfterMerge,
};
await core.services.setting.suspendAllSync();
const settingsAfterSuspend = cloneSettings(core.services.setting.currentSettings());
await core.services.control.onReady();
const settingsBeforeCommand = cloneSettings(core.services.setting.currentSettings());
const transientSettingKeys = changedSettingKeys(settingsBeforeSuspend, settingsAfterSuspend);
for (const key of CLI_RUNTIME_ONLY_SETTING_KEYS) {
transientSettingKeys.add(key);
}
const durableSettingsBeforeCommand = reconcileDurableSettings({
durableBase: durableSettingsBeforeSuspend,
runtimeBaseline: settingsAfterSuspend,
runtimeCurrent: settingsBeforeCommand,
preserveKeys: transientSettingKeys,
});
infoLog(`[Ready] LiveSync is running`);
infoLog(`[Ready] Press Ctrl+C to stop`);
@@ -534,37 +594,90 @@ export async function main() {
// Check if configured
const settings = core.services.setting.currentSettings();
if (!settings.isConfigured) {
console.warn(`[Warning] LiveSync is not configured yet`);
console.warn(`[Warning] Please edit ${settingsPath} to configure CouchDB connection`);
console.warn();
console.warn(`Required settings:`);
console.warn(` - couchDB_URI: CouchDB server URL`);
console.warn(` - couchDB_USER: CouchDB username`);
console.warn(` - couchDB_PASSWORD: CouchDB password`);
console.warn(` - couchDB_DBNAME: Database name`);
console.warn();
writeStderrLine(standardIo, `[Warning] LiveSync is not configured yet`);
writeStderrLine(standardIo, `[Warning] Please edit ${settingsPath} to configure CouchDB connection`);
writeStderrLine(standardIo);
writeStderrLine(standardIo, `Required settings:`);
writeStderrLine(standardIo, ` - couchDB_URI: CouchDB server URL`);
writeStderrLine(standardIo, ` - couchDB_USER: CouchDB username`);
writeStderrLine(standardIo, ` - couchDB_PASSWORD: CouchDB password`);
writeStderrLine(standardIo, ` - couchDB_DBNAME: Database name`);
writeStderrLine(standardIo);
} else {
infoLog(`[Info] LiveSync is configured and ready`);
infoLog(`[Info] Database: ${settings.couchDB_URI}/${settings.couchDB_DBNAME}`);
infoLog("");
}
const result = await runCommand(options, { databasePath, vaultPath, core, settingsPath, originalSyncSettings });
commandPreparedSettingsTexts = [];
let result: boolean;
try {
commandIsRunning = true;
result = await commandRunner(options, {
databasePath,
vaultPath,
core,
p2pReplicator,
settingsPath,
originalSyncSettings,
});
} finally {
commandIsRunning = false;
}
let settingsTextToCommit: string | undefined;
if (result && options.command === "setup") {
settingsTextToCommit = commandPreparedSettingsTexts[0];
if (settingsTextToCommit === undefined) {
throw new Error("The setup command completed without preparing its settings update.");
}
} else if (result && (isSettingsWriteCommand(options.command) || options.writeSettings)) {
const runtimeSettingsAfterCommand = cloneSettings(core.services.setting.currentSettings());
const durableSettingsAfterCommand = reconcileDurableSettings({
durableBase: durableSettingsBeforeCommand,
runtimeBaseline: settingsBeforeCommand,
runtimeCurrent: runtimeSettingsAfterCommand,
preserveKeys: transientSettingKeys,
command: options.command,
});
if (
isSettingsWriteCommand(options.command) ||
!settingsEqual(durableSettingsAfterCommand, durableSettingsBeforeSuspend)
) {
const runtimeSettings = cloneSettings(core.services.setting.currentSettings());
const revisionBeforeSave = preparedSettingsRevision;
try {
await core.services.setting.updateSettings(() => cloneSettings(durableSettingsAfterCommand), true);
if (preparedSettingsRevision === revisionBeforeSave || latestPreparedSettingsText === undefined) {
throw new Error("The setting service did not prepare the requested settings update.");
}
settingsTextToCommit = latestPreparedSettingsText;
} finally {
await core.services.setting.updateSettings(() => runtimeSettings, false);
}
} else {
settingsTextToCommit = settingsAfterLoadText;
}
}
if (!result) {
console.error(`[Error] Command '${options.command}' failed`);
writeStderrLine(standardIo, `[Error] Command '${options.command}' failed`);
process.exitCode = 1;
} else if (options.command !== "daemon") {
infoLog(`[Done] Command '${options.command}' completed`);
}
if (options.command === "daemon" && result) {
await writeSettingsAtomically(settingsTextToCommit);
// Keep the process running
await new Promise(() => {});
} else {
await core.services.control.onUnload();
await writeSettingsAtomically(settingsTextToCommit);
}
} catch (error) {
console.error(`[Error] Failed to start:`, error);
writeStderrLine(standardIo, `[Error] Failed to start:`, error);
process.exit(1);
}
// To prevent unexpected hanging in webRTC connections.
src/apps/cli/main.unit.spec.ts
+34 -17
View File
@@ -1,5 +1,14 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { parseArgs } from "./main";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { parseArgs as parseCliArgs } from "./main";
function createStandardIoMock() {
return {
readStdin: vi.fn(async () => ""),
prompt: vi.fn(async () => ""),
writeStdout: vi.fn(),
writeStderr: vi.fn(),
};
}
function mockProcessExit() {
const exitMock = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
@@ -10,6 +19,13 @@ function mockProcessExit() {
describe("CLI parseArgs", () => {
const originalArgv = process.argv.slice();
let standardIo: ReturnType<typeof createStandardIoMock>;
beforeEach(() => {
standardIo = createStandardIoMock();
});
const parseArgs = () => parseCliArgs(standardIo);
afterEach(() => {
process.argv = originalArgv.slice();
@@ -19,42 +35,38 @@ describe("CLI parseArgs", () => {
it("exits 1 when --settings has no value", () => {
process.argv = ["node", "livesync-cli", "./databasePath", "--settings"];
const exitMock = mockProcessExit();
const stderr = vi.spyOn(console, "error").mockImplementation(() => {});
expect(() => parseArgs()).toThrowError("__EXIT__:1");
expect(exitMock).toHaveBeenCalledWith(1);
expect(stderr).toHaveBeenCalledWith("Error: Missing value for --settings");
expect(standardIo.writeStderr).toHaveBeenCalledWith("Error: Missing value for --settings\n");
});
it("exits 1 when database-path is missing", () => {
process.argv = ["node", "livesync-cli", "sync"];
const exitMock = mockProcessExit();
const stderr = vi.spyOn(console, "error").mockImplementation(() => {});
expect(() => parseArgs()).toThrowError("__EXIT__:1");
expect(exitMock).toHaveBeenCalledWith(1);
expect(stderr).toHaveBeenCalledWith("Error: database-path is required");
expect(standardIo.writeStderr).toHaveBeenCalledWith("Error: database-path is required\n");
});
it("exits 1 for unknown command after database-path", () => {
process.argv = ["node", "livesync-cli", "./databasePath", "unknown-cmd"];
const exitMock = mockProcessExit();
const stderr = vi.spyOn(console, "error").mockImplementation(() => {});
expect(() => parseArgs()).toThrowError("__EXIT__:1");
expect(exitMock).toHaveBeenCalledWith(1);
expect(stderr).toHaveBeenCalledWith("Error: Unknown command 'unknown-cmd'");
expect(standardIo.writeStderr).toHaveBeenCalledWith("Error: Unknown command 'unknown-cmd'\n");
});
it("exits 0 and prints help for --help", () => {
process.argv = ["node", "livesync-cli", "--help"];
const exitMock = mockProcessExit();
const stdout = vi.spyOn(console, "log").mockImplementation(() => {});
expect(() => parseArgs()).toThrowError("__EXIT__:0");
expect(exitMock).toHaveBeenCalledWith(0);
expect(stdout).toHaveBeenCalled();
const combined = stdout.mock.calls.flat().join("\n");
expect(standardIo.writeStdout).toHaveBeenCalled();
const combined = standardIo.writeStdout.mock.calls.flat().join("");
expect(combined).toContain("Usage:");
expect(combined).toContain("livesync-cli <database-path> [options] <command> [command-args]");
});
@@ -152,7 +164,6 @@ describe("CLI parseArgs", () => {
it("exits 1 when --interval has no value", () => {
process.argv = ["node", "livesync-cli", "./vault", "--interval"];
const exitMock = mockProcessExit();
vi.spyOn(console, "error").mockImplementation(() => {});
expect(() => parseArgs()).toThrowError("__EXIT__:1");
expect(exitMock).toHaveBeenCalledWith(1);
});
@@ -160,22 +171,19 @@ describe("CLI parseArgs", () => {
it("exits 1 when --interval is not a positive integer", () => {
process.argv = ["node", "livesync-cli", "./vault", "--interval", "0"];
const exitMock = mockProcessExit();
vi.spyOn(console, "error").mockImplementation(() => {});
expect(() => parseArgs()).toThrowError("__EXIT__:1");
expect(exitMock).toHaveBeenCalledWith(1);
});
it("exits 1 when --interval is negative", () => {
process.argv = ["node", "livesync-cli", "./vault", "--interval", "-5"];
const exitMock = mockProcessExit();
vi.spyOn(console, "error").mockImplementation(() => {});
mockProcessExit();
expect(() => parseArgs()).toThrowError("__EXIT__:1");
});
it("exits 1 when --interval is not numeric", () => {
process.argv = ["node", "livesync-cli", "./vault", "--interval", "abc"];
const exitMock = mockProcessExit();
vi.spyOn(console, "error").mockImplementation(() => {});
mockProcessExit();
expect(() => parseArgs()).toThrowError("__EXIT__:1");
});
@@ -198,4 +206,13 @@ describe("CLI parseArgs", () => {
expect(parsed.command).toBe("daemon");
expect(parsed.interval).toBe(30);
});
it("parses --write-settings as a global option", () => {
process.argv = ["node", "livesync-cli", "./vault", "--write-settings", "ls"];
const parsed = parseArgs();
expect(parsed.command).toBe("ls");
expect(parsed.writeSettings).toBe(true);
expect(parsed.commandArgs).toEqual([]);
});
});
src/apps/cli/managers/CLIStorageEventManagerAdapter.ts
+25 -15
View File
@@ -1,6 +1,6 @@
import type { FilePath, UXFileInfoStub, UXInternalFileInfoStub } from "@lib/common/types";
import type { FileEventItem } from "@lib/common/types";
import type { IStorageEventManagerAdapter } from "@lib/managers/adapters";
import type { FilePath, UXFileInfoStub, UXInternalFileInfoStub } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { FileEventItem } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { IStorageEventManagerAdapter } from "@vrtmrz/livesync-commonlib/compat/managers/adapters";
import type {
IStorageEventTypeGuardAdapter,
IStorageEventPersistenceAdapter,
@@ -8,12 +8,13 @@ import type {
IStorageEventStatusAdapter,
IStorageEventConverterAdapter,
IStorageEventWatchHandlers,
} from "@lib/managers/adapters";
import type { FileEventItemSentinel } from "@lib/managers/StorageEventManager";
} from "@vrtmrz/livesync-commonlib/compat/managers/adapters";
import type { FileEventItemSentinel } from "@vrtmrz/livesync-commonlib/compat/managers/StorageEventManager";
import type { NodeFile, NodeFolder } from "@/apps/cli/adapters/NodeTypes";
import { watch as chokidarWatch, type FSWatcher } from "chokidar";
import type { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules";
import { fsPromises as fs, path, type Stats } from "@/apps/cli/node-compat";
import { fsPromises as fs, path, type Stats } from "@vrtmrz/livesync-commonlib/node";
import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput";
/**
* CLI-specific type guard adapter
@@ -45,7 +46,10 @@ class CLITypeGuardAdapter implements IStorageEventTypeGuardAdapter<NodeFile, Nod
class CLIPersistenceAdapter implements IStorageEventPersistenceAdapter {
private snapshotPath: string;
constructor(basePath: string) {
constructor(
basePath: string,
private reportDiagnostic: CliDiagnosticReporter = () => undefined
) {
this.snapshotPath = path.join(basePath, ".livesync-snapshot.json");
}
@@ -53,14 +57,14 @@ class CLIPersistenceAdapter implements IStorageEventPersistenceAdapter {
try {
await fs.writeFile(this.snapshotPath, JSON.stringify(snapshot, null, 2), "utf-8");
} catch (error) {
console.error("Failed to save snapshot:", error);
this.reportDiagnostic("Failed to save snapshot:", error);
}
}
async loadSnapshot(): Promise<(FileEventItem | FileEventItemSentinel)[] | null> {
try {
const content = await fs.readFile(this.snapshotPath, "utf-8");
return JSON.parse(content);
return JSON.parse(content) as (FileEventItem | FileEventItemSentinel)[];
} catch {
return null;
}
@@ -109,7 +113,8 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter {
constructor(
private basePath: string,
private ignoreRules?: IgnoreRules,
private watchEnabled: boolean = false
private watchEnabled: boolean = false,
private reportDiagnostic: CliDiagnosticReporter = () => undefined
) {}
private _toNodeFile(filePath: string, stats: Stats | undefined): NodeFile {
@@ -182,8 +187,8 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter {
});
watcher.on("error", (err) => {
console.error("[CLIWatchAdapter] Fatal watcher error — file watching stopped:", err);
console.error("[CLIWatchAdapter] Exiting for systemd restart.");
this.reportDiagnostic("[CLIWatchAdapter] Fatal watcher error — file watching stopped:", err);
this.reportDiagnostic("[CLIWatchAdapter] Exiting for systemd restart.");
void watcher.close();
this._watcher = undefined;
// Use exit(1) rather than SIGTERM so systemd Restart=on-failure engages.
@@ -212,10 +217,15 @@ export class CLIStorageEventManagerAdapter implements IStorageEventManagerAdapte
readonly status: CLIStatusAdapter;
readonly converter: CLIConverterAdapter;
constructor(basePath: string, ignoreRules?: IgnoreRules, watchEnabled: boolean = false) {
constructor(
basePath: string,
ignoreRules?: IgnoreRules,
watchEnabled: boolean = false,
reportDiagnostic: CliDiagnosticReporter = () => undefined
) {
this.typeGuard = new CLITypeGuardAdapter();
this.persistence = new CLIPersistenceAdapter(basePath);
this.watch = new CLIWatchAdapter(basePath, ignoreRules, watchEnabled);
this.persistence = new CLIPersistenceAdapter(basePath, reportDiagnostic);
this.watch = new CLIWatchAdapter(basePath, ignoreRules, watchEnabled, reportDiagnostic);
this.status = new CLIStatusAdapter();
this.converter = new CLIConverterAdapter();
}
src/apps/cli/managers/CLIStorageEventManagerAdapter.unit.spec.ts
+8 -2
View File
@@ -1,5 +1,5 @@
import { describe, expect, it, vi, beforeEach } from "vitest";
import type { IStorageEventWatchHandlers } from "@lib/managers/adapters";
import type { IStorageEventWatchHandlers } from "@vrtmrz/livesync-commonlib/compat/managers/adapters";
import type { NodeFile } from "@/apps/cli/adapters/NodeTypes";
// ── chokidar mock ──────────────────────────────────────────────────────────────
@@ -124,7 +124,8 @@ describe("CLIStorageEventManagerAdapter", () => {
});
it("error event triggers process.exit(1)", async () => {
const adapter = new CLIStorageEventManagerAdapter("/base", undefined, true);
const reportDiagnostic = vi.fn();
const adapter = new CLIStorageEventManagerAdapter("/base", undefined, true, reportDiagnostic);
const handlers = makeHandlers();
await adapter.watch.beginWatch(handlers);
@@ -138,6 +139,11 @@ describe("CLIStorageEventManagerAdapter", () => {
errorCallback(new Error("disk failure"));
expect(processExitSpy).toHaveBeenCalledWith(1);
expect(reportDiagnostic).toHaveBeenCalledWith(
"[CLIWatchAdapter] Fatal watcher error — file watching stopped:",
expect.any(Error)
);
expect(reportDiagnostic).toHaveBeenCalledWith("[CLIWatchAdapter] Exiting for systemd restart.");
processExitSpy.mockRestore();
});
src/apps/cli/managers/StorageEventManagerCLI.ts
+10 -4
View File
@@ -1,9 +1,10 @@
import { StorageEventManagerBase, type StorageEventManagerBaseDependencies } from "@lib/managers/StorageEventManager";
import { StorageEventManagerBase, type StorageEventManagerBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/managers/StorageEventManager";
import { CLIStorageEventManagerAdapter } from "./CLIStorageEventManagerAdapter";
import type { IMinimumLiveSyncCommands, LiveSyncBaseCore } from "@/LiveSyncBaseCore";
import type { ServiceContext } from "@lib/services/base/ServiceBase";
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import type { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules";
// import type { IMinimumLiveSyncCommands } from "@lib/services/base/IService";
import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
// import type { IMinimumLiveSyncCommands } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
export class StorageEventManagerCLI extends StorageEventManagerBase<CLIStorageEventManagerAdapter> {
core: LiveSyncBaseCore<ServiceContext, IMinimumLiveSyncCommands>;
@@ -15,7 +16,12 @@ export class StorageEventManagerCLI extends StorageEventManagerBase<CLIStorageEv
ignoreRules?: IgnoreRules,
watchEnabled?: boolean
) {
const adapter = new CLIStorageEventManagerAdapter(basePath, ignoreRules, watchEnabled);
const adapter = new CLIStorageEventManagerAdapter(basePath, ignoreRules, watchEnabled, (message, detail) => {
dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE);
if (detail !== undefined) {
dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE);
}
});
super(adapter, dependencies);
this.core = core;
}
src/apps/cli/node-compat.ts
-11
View File
@@ -1,11 +0,0 @@
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
import * as nodeFs from "node:fs";
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
import * as nodeFsPromises from "node:fs/promises";
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
import * as nodePath from "node:path";
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
import * as nodeReadlinePromises from "node:readline/promises";
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
import type { Stats } from "node:fs";
export { nodeFs as fs, nodeFsPromises as fsPromises, nodePath as path, nodeReadlinePromises as readline, type Stats };
src/apps/cli/package.json
+8 -13
View File
@@ -1,47 +1,43 @@
{
"name": "self-hosted-livesync-cli",
"private": true,
"version": "0.25.82-cli",
"version": "1.0.10-cli",
"main": "dist/index.cjs",
"type": "module",
"scripts": {
"dev": "vite",
"prebuild": "node scripts/check-submodule.mjs",
"build": "vite build",
"preview": "vite preview",
"cli": "node dist/index.cjs",
"buildRun": "npm run build && npm run cli --",
"build:docker": "docker build -f Dockerfile -t livesync-cli ../../..",
"check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json",
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts",
"check": "tsc -p tsconfig.json",
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/settingsPersistence.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts",
"test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh",
"test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh",
"test:e2e:two-vaults:matrix": "bash test/test-e2e-two-vaults-matrix.sh",
"test:e2e:push-pull": "bash test/test-push-pull-linux.sh",
"test:e2e:setup-put-cat": "bash test/test-setup-put-cat-linux.sh",
"test:e2e:sync-two-local": "bash test/test-sync-two-local-databases-linux.sh",
"test:e2e:p2p": "bash test/test-p2p-three-nodes-conflict-linux.sh",
"test:e2e:p2p-upload-download-repro": "bash test/test-p2p-upload-download-repro-linux.sh",
"test:e2e:p2p-host": "bash test/test-p2p-host-linux.sh",
"test:e2e:p2p-sync": "bash test/test-p2p-sync-linux.sh",
"pretest:e2e:ci": "npm run build",
"test:e2e:ci": "deno task --cwd testdeno test:ci",
"test:e2e:p2p": "deno task --cwd testdeno test:p2p:compose",
"test:e2e:mirror": "bash test/test-mirror-linux.sh",
"test:e2e:remote-commands": "bash test/test-remote-commands-linux.sh",
"pretest:e2e:all": "npm run build",
"test:e2e:all": " export RUN_BUILD=0 && npm run test:e2e:setup-put-cat && npm run test:e2e:push-pull && npm run test:e2e:sync-two-local && npm run test:e2e:p2p && npm run test:e2e:mirror && npm run test:e2e:two-vaults && npm run test:e2e:remote-commands",
"test:e2e:all": "deno task --cwd testdeno test:ci && deno task --cwd testdeno test:p2p:compose",
"pretest:e2e:docker:all": "npm run build:docker",
"test:e2e:docker:push-pull": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-push-pull-linux.sh",
"test:e2e:docker:setup-put-cat": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-setup-put-cat-linux.sh",
"test:e2e:docker:mirror": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-mirror-linux.sh",
"test:e2e:docker:remote-commands": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-remote-commands-linux.sh",
"test:e2e:docker:sync-two-local": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-sync-two-local-databases-linux.sh",
"test:e2e:docker:p2p": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-p2p-three-nodes-conflict-linux.sh",
"test:e2e:docker:p2p-sync": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-p2p-sync-linux.sh",
"test:e2e:docker:all": "export RUN_BUILD=0 && npm run test:e2e:docker:setup-put-cat && npm run test:e2e:docker:push-pull && npm run test:e2e:docker:sync-two-local && npm run test:e2e:docker:mirror && npm run test:e2e:docker:remote-commands"
},
"dependencies": {
"chokidar": "^4.0.0",
"minimatch": "^10.2.5",
"octagonal-wheels": "^0.1.51",
"octagonal-wheels": "^0.1.52",
"pouchdb-adapter-http": "^9.0.0",
"pouchdb-adapter-leveldb": "^9.0.0",
"pouchdb-core": "^9.0.0",
@@ -55,7 +51,6 @@
"werift": "^0.23.0"
},
"devDependencies": {
"@sveltejs/vite-plugin-svelte": "^7.1.2",
"typescript": "5.9.3",
"vite": "^8.0.16",
"vitest": "^4.1.8"
src/apps/cli/pouchdb-internals.d.ts
Vendored
+21
View File
@@ -0,0 +1,21 @@
declare module "pouchdb-merge" {
export interface RevisionTreeNode {
pos: number;
ids: [revision: string, metadata: Record<string, unknown>, branches: RevisionTreeNode["ids"][]];
}
export function findPathToLeaf(revisions: RevisionTreeNode[], targetRevision: string): string[];
}
declare module "pouchdb-utils" {
export function adapterFun<TThis, TArguments extends unknown[], TResult>(
name: string,
callback: (this: TThis, ...args: [...TArguments, callback: (error?: Error, result?: TResult) => void]) => void
): (this: TThis, ...args: TArguments) => Promise<TResult>;
}
declare module "pouchdb-errors" {
export const MISSING_DOC: unknown;
export const UNKNOWN_ERROR: unknown;
export function createError(error: unknown, reason?: string): Error;
}
src/apps/cli/scripts/check-submodule.mjs
-36
View File
@@ -1,36 +0,0 @@
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
const cliDir = process.cwd();
const repoRoot = path.resolve(cliDir, "../../..");
const requiredFiles = [
path.join(repoRoot, "src/lib/src/common/types.ts"),
];
const missingFiles = requiredFiles.filter((filePath) => !fs.existsSync(filePath));
if (missingFiles.length === 0) {
process.exit(0);
}
console.error("[CLI Build Error] Required shared sources were not found.");
console.error("This repository uses Git submodules, and the CLI depends on src/lib.");
console.error("");
console.error("Missing file(s):");
for (const filePath of missingFiles) {
console.error(` - ${path.relative(repoRoot, filePath)}`);
}
console.error("");
console.error("Initialize submodules, then retry the CLI build:");
console.error(" git submodule update --init --recursive");
console.error("");
console.error("For a fresh clone, prefer:");
console.error(" git clone --recurse-submodules <repository-url>");
console.error("");
console.error("Then run:");
console.error(" npm install");
console.error(" cd src/apps/cli");
console.error(" npm run build");
process.exit(1);
src/apps/cli/serviceModules/CLIServiceModules.ts
+10 -5
View File
@@ -1,15 +1,16 @@
import type { InjectableServiceHub } from "@lib/services/implements/injectable/InjectableServiceHub";
import { ServiceRebuilder } from "@lib/serviceModules/Rebuilder";
import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub";
import { ServiceRebuilder } from "@vrtmrz/livesync-commonlib/compat/serviceModules/Rebuilder";
import { ServiceFileHandler } from "@/serviceModules/FileHandler";
import { StorageAccessManager } from "@lib/managers/StorageProcessingManager";
import { StorageAccessManager } from "@vrtmrz/livesync-commonlib/compat/managers/StorageProcessingManager";
import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
import type { ServiceContext } from "@lib/services/base/ServiceBase";
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { FileAccessCLI } from "./FileAccessCLI";
import { ServiceFileAccessCLI } from "./ServiceFileAccessImpl";
import { ServiceDatabaseFileAccessCLI } from "./DatabaseFileAccess";
import { StorageEventManagerCLI } from "@/apps/cli/managers/StorageEventManagerCLI";
import type { ServiceModules } from "@lib/interfaces/ServiceModule";
import type { ServiceModules } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule";
import type { IgnoreRules } from "./IgnoreRules";
import { createFileReflectionProvenance } from "@/serviceModules/FileReflectionProvenance";
/**
* Initialize service modules for CLI version
@@ -73,6 +74,7 @@ export function initialiseServiceModulesCLI(
// Database file access (platform-independent)
const databaseFileAccess = new ServiceDatabaseFileAccessCLI({
events: services.context.events,
API: services.API,
database: services.database,
path: services.path,
@@ -82,6 +84,7 @@ export function initialiseServiceModulesCLI(
// File handler (platform-independent)
const fileHandler = new ServiceFileHandler({
events: services.context.events,
API: services.API,
databaseFileAccess: databaseFileAccess,
conflict: services.conflict,
@@ -91,10 +94,12 @@ export function initialiseServiceModulesCLI(
path: services.path,
replication: services.replication,
storageAccess: storageAccess,
fileReflectionProvenance: createFileReflectionProvenance(services.keyValueDB),
});
// Rebuilder (platform-independent)
const rebuilder = new ServiceRebuilder({
events: services.context.events,
API: services.API,
database: services.database,
appLifecycle: services.appLifecycle,
src/apps/cli/serviceModules/DatabaseFileAccess.ts
+2 -2
View File
@@ -1,8 +1,8 @@
import {
ServiceDatabaseFileAccessBase,
type ServiceDatabaseFileAccessDependencies,
} from "@lib/serviceModules/ServiceDatabaseFileAccessBase";
import type { DatabaseFileAccess } from "@lib/interfaces/DatabaseFileAccess";
} from "@vrtmrz/livesync-commonlib/compat/serviceModules/ServiceDatabaseFileAccessBase";
import type { DatabaseFileAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/DatabaseFileAccess";
/**
* CLI-specific implementation of ServiceDatabaseFileAccess
src/apps/cli/serviceModules/FileAccessCLI.ts
+8 -2
View File
@@ -1,5 +1,6 @@
import { FileAccessBase, type FileAccessBaseDependencies } from "@lib/serviceModules/FileAccessBase";
import { FileAccessBase, type FileAccessBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/serviceModules/FileAccessBase";
import { NodeFileSystemAdapter } from "@/apps/cli/adapters/NodeFileSystemAdapter";
import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
/**
* CLI-specific implementation of FileAccessBase
@@ -7,7 +8,12 @@ import { NodeFileSystemAdapter } from "@/apps/cli/adapters/NodeFileSystemAdapter
*/
export class FileAccessCLI extends FileAccessBase<NodeFileSystemAdapter> {
constructor(basePath: string, dependencies: FileAccessBaseDependencies) {
const adapter = new NodeFileSystemAdapter(basePath);
const adapter = new NodeFileSystemAdapter(basePath, (message, detail) => {
dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE);
if (detail !== undefined) {
dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE);
}
});
super(adapter, dependencies);
}
src/apps/cli/serviceModules/IgnoreRules.ts
+8 -4
View File
@@ -1,5 +1,6 @@
import { Minimatch } from "minimatch";
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
import { fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node";
import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput";
/**
* Loads and evaluates ignore rules from `.livesync/ignore` inside the vault.
@@ -19,7 +20,10 @@ import { fsPromises as fs, path } from "@/apps/cli/node-compat";
export class IgnoreRules {
private patterns: Minimatch[] = [];
constructor(private vaultPath: string) {}
constructor(
private vaultPath: string,
private reportDiagnostic: CliDiagnosticReporter = () => undefined
) {}
/**
* Reads `.livesync/ignore` (and optionally `.gitignore`) and populates the
@@ -53,7 +57,7 @@ export class IgnoreRules {
continue;
}
if (trimmed.startsWith("import:")) {
console.error(
this.reportDiagnostic(
`[IgnoreRules] Warning: unrecognised directive '${trimmed}' — only 'import: .gitignore' is supported`
);
continue;
@@ -61,7 +65,7 @@ export class IgnoreRules {
this._addPattern(trimmed);
}
if (this.patterns.length > 0) {
console.error(`[IgnoreRules] Loaded ${this.patterns.length} ignore patterns`);
this.reportDiagnostic(`[IgnoreRules] Loaded ${this.patterns.length} ignore patterns`);
}
}
src/apps/cli/serviceModules/IgnoreRules.unit.spec.ts
+4 -4
View File
@@ -1,6 +1,4 @@
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import { fsPromises as fs, os, path } from "@vrtmrz/livesync-commonlib/node";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const minimatchStats = vi.hoisted(() => ({ constructions: 0 }));
@@ -140,11 +138,13 @@ describe("IgnoreRules", () => {
const vaultPath = await createVault();
// Typo: "import:.gitignore" instead of "import: .gitignore"
await writeIgnoreFile(vaultPath, "*.tmp\nimport:.gitignore\n");
const rules = new IgnoreRules(vaultPath);
const reportDiagnostic = vi.fn();
const rules = new IgnoreRules(vaultPath, reportDiagnostic);
await rules.load();
// *.tmp still loaded; import:.gitignore is skipped (not treated as a literal pattern)
expect(rules.shouldIgnore("scratch.tmp")).toBe(true);
expect(rules.shouldIgnore("import:.gitignore")).toBe(false);
expect(reportDiagnostic).toHaveBeenCalledWith(expect.stringContaining("unrecognised directive"));
});
});
src/apps/cli/serviceModules/ServiceFileAccessImpl.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { ServiceFileAccessBase, type StorageAccessBaseDependencies } from "@lib/serviceModules/ServiceFileAccessBase";
import { ServiceFileAccessBase, type StorageAccessBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/serviceModules/ServiceFileAccessBase";
import { NodeFileSystemAdapter } from "@/apps/cli/adapters/NodeFileSystemAdapter";
/**
src/apps/cli/services/NodeKeyValueDBService.ts
+48 -24
View File
@@ -1,13 +1,14 @@
import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@lib/common/logger";
import type { KeyValueDatabase } from "@lib/interfaces/KeyValueDatabase";
import type { IKeyValueDBService } from "@lib/services/base/IService";
import { ServiceBase, type ServiceContext } from "@lib/services/base/ServiceBase";
import type { InjectableAppLifecycleService } from "@lib/services/implements/injectable/InjectableAppLifecycleService";
import type { InjectableDatabaseEventService } from "@lib/services/implements/injectable/InjectableDatabaseEventService";
import type { IVaultService } from "@lib/services/base/IService";
import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/logger";
import type { KeyValueDatabase } from "@vrtmrz/livesync-commonlib/compat/interfaces/KeyValueDatabase";
import type { IKeyValueDBService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { ServiceBase } from "@vrtmrz/livesync-commonlib/compat/services/base/ServiceBase";
import type { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAppLifecycleService";
import type { InjectableDatabaseEventService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableDatabaseEventService";
import type { IVaultService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase";
import { createInstanceLogFunction } from "@lib/services/lib/logUtils";
import { fs as nodeFs, path as nodePath } from "@/apps/cli/node-compat";
import { createInstanceLogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils";
import { fs as nodeFs, path as nodePath } from "@vrtmrz/livesync-commonlib/node";
const NODE_KV_TYPED_KEY = "__nodeKvType";
const NODE_KV_VALUES_KEY = "values";
@@ -81,6 +82,17 @@ function deserializeFromNodeKV(value: unknown): unknown {
return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, deserializeFromNodeKV(v)]));
}
function asKeyString(key: unknown): string {
if (typeof key === "string") {
return key;
}
const serialised = JSON.stringify(key);
if (typeof serialised !== "string") {
throw new TypeError("The IndexedDB key could not be serialised");
}
return serialised;
}
class NodeFileKeyValueDatabase implements KeyValueDatabase {
private filePath: string;
private data = new Map<string, unknown>();
@@ -90,13 +102,6 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase {
this.load();
}
private asKeyString(key: IDBValidKey): string {
if (typeof key === "string") {
return key;
}
return JSON.stringify(key);
}
private load() {
try {
const loaded = JSON.parse(nodeFs.readFileSync(this.filePath, "utf-8")) as Record<string, unknown>;
@@ -115,17 +120,17 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase {
}
async get<T>(key: IDBValidKey): Promise<T> {
return this.data.get(this.asKeyString(key)) as T;
return this.data.get(asKeyString(key)) as T;
}
async set<T>(key: IDBValidKey, value: T): Promise<IDBValidKey> {
this.data.set(this.asKeyString(key), value);
this.data.set(asKeyString(key), value);
this.flush();
return key;
}
async del(key: IDBValidKey): Promise<void> {
this.data.delete(this.asKeyString(key));
this.data.delete(asKeyString(key));
this.flush();
}
@@ -143,11 +148,12 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase {
let filtered = allKeys;
if (typeof query !== "undefined") {
if (this.isIDBKeyRangeLike(query)) {
const lower = query.lower?.toString() ?? "";
const upper = query.upper?.toString() ?? "\uffff";
const lower = query.lower === undefined ? "" : String(query.lower);
const upper = query.upper === undefined ? "\uffff" : String(query.upper);
filtered = filtered.filter((key) => key >= lower && key <= upper);
} else {
const exact = query.toString();
const exactValue: unknown = query;
const exact = String(exactValue);
filtered = filtered.filter((key) => key === exact);
}
}
@@ -253,6 +259,14 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
}
openSimpleStore<T>(kind: string): SimpleStore<T> {
// Service modules are composed before onSettingLoaded opens the file-
// backed database, so handle creation must not touch it. Actual store
// operations are deliberately fail-fast: the sequential lifecycle opens
// the database before scans, watchers, or replication start. Waiting here
// could hang forever after failed initialisation, or deadlock if a future
// initialisation handler tried to use the store it was waiting to open.
// Reset is likewise a transient unavailable boundary, not a wait state;
// callers must avoid store work there because an operation may fail.
const getDB = () => {
if (!this._kvDB) {
throw new Error("KeyValueDB is not initialized yet");
@@ -271,7 +285,15 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
await getDB().del(`${prefix}${key}`);
},
keys: async (from: string | undefined, to: string | undefined, count?: number): Promise<string[]> => {
const allKeys = (await getDB().keys(undefined, count)).map((e) => e.toString());
const rawKeys: unknown = await getDB().keys(undefined, count);
if (!Array.isArray(rawKeys)) {
throw new TypeError("The key-value database returned an invalid key list");
}
const keyList: unknown[] = rawKeys;
const allKeys: string[] = [];
for (const key of keyList) {
allKeys.push(String(key));
}
const lower = `${prefix}${from ?? ""}`;
const upper = `${prefix}${to ?? "\uffff"}`;
return allKeys
@@ -279,7 +301,9 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
.filter((key) => key >= lower && key <= upper)
.map((key) => key.substring(prefix.length));
},
db: Promise.resolve(getDB()),
get db() {
return Promise.resolve(getDB());
},
} satisfies SimpleStore<T>;
}
}
src/apps/cli/services/NodeKeyValueDBService.unit.spec.ts
+47
View File
@@ -0,0 +1,47 @@
import { describe, expect, it, vi } from "vitest";
import { createServiceContext } from "@vrtmrz/livesync-commonlib/compat/services/base/ServiceBase";
import type { NodeKeyValueDBDependencies } from "./NodeKeyValueDBService";
import { NodeKeyValueDBService } from "./NodeKeyValueDBService";
describe("NodeKeyValueDBService.openSimpleStore", () => {
it("creates a namespaced store handle before the backing database is initialised", () => {
const dependencies = {
appLifecycle: { onSettingLoaded: { addHandler: vi.fn() } },
databaseEvents: {
onResetDatabase: { addHandler: vi.fn() },
onDatabaseInitialisation: { addHandler: vi.fn() },
onUnloadDatabase: { addHandler: vi.fn() },
onCloseDatabase: { addHandler: vi.fn() },
},
vault: {},
} as unknown as NodeKeyValueDBDependencies;
const service = new NodeKeyValueDBService(
createServiceContext(),
dependencies,
"/tmp/obsidian-livesync-node-kv-handle-test.json"
);
expect(() => service.openSimpleStore("early-composition")).not.toThrow();
});
it("fails store operations promptly instead of waiting for lifecycle initialisation", async () => {
const dependencies = {
appLifecycle: { onSettingLoaded: { addHandler: vi.fn() } },
databaseEvents: {
onResetDatabase: { addHandler: vi.fn() },
onDatabaseInitialisation: { addHandler: vi.fn() },
onUnloadDatabase: { addHandler: vi.fn() },
onCloseDatabase: { addHandler: vi.fn() },
},
vault: {},
} as unknown as NodeKeyValueDBDependencies;
const service = new NodeKeyValueDBService(
createServiceContext(),
dependencies,
"/tmp/obsidian-livesync-node-kv-uninitialised-test.json"
);
const store = service.openSimpleStore("early-composition");
await expect(store.get("key")).rejects.toThrow("KeyValueDB is not initialized yet");
});
});
src/apps/cli/services/NodeLocalStorage.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import { fs as nodeFs, path as nodePath } from "@/apps/cli/node-compat";
import { compatGlobal } from "@lib/common/coreEnvFunctions";
import { fs as nodeFs, path as nodePath } from "@vrtmrz/livesync-commonlib/node";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
type LocalStorageShape = {
getItem(key: string): string | null;
src/apps/cli/services/NodeLocalStorage.unit.spec.ts
+1 -3
View File
@@ -1,6 +1,4 @@
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { fs, os, path } from "@vrtmrz/livesync-commonlib/node";
import { afterEach, describe, expect, it } from "vitest";
import {
clearNodeLocalStorage,
src/apps/cli/services/NodeServiceContext.ts
+13
View File
@@ -0,0 +1,13 @@
import { eventHub } from "@/common/events";
import { translateLiveSyncMessage } from "@/common/translation";
import { ServiceContext, type StandardIo } from "@vrtmrz/livesync-commonlib/context";
/** Host capabilities owned by one Self-hosted LiveSync CLI composition. */
export class NodeServiceContext extends ServiceContext {
constructor(
readonly databasePath: string,
readonly standardIo: StandardIo
) {
super({ events: eventHub, translate: translateLiveSyncMessage });
}
}
src/apps/cli/services/NodeServiceContext.unit.spec.ts
+41
View File
@@ -0,0 +1,41 @@
import { describe, expect, it } from "vitest";
import { eventHub } from "@/common/events";
import { translateLiveSyncMessage } from "@/common/translation";
import {
observeServiceComposition,
observeServiceContext,
SERVICE_CONTEXT_MEMBERS,
} from "../../../../test/contracts/serviceContext";
import { NodeServiceContext } from "./NodeServiceContext";
import { NodeServiceHub } from "./NodeServiceHub";
import type { StandardIo } from "@vrtmrz/livesync-commonlib/context";
const TRANSLATION_KEY = "Replicator.Message.InitialiseFatalError";
describe("NodeServiceContext contract", () => {
it("preserves the CLI capabilities and host-neutral API results", () => {
const standardIo: StandardIo = {
readStdin: async () => "input",
prompt: async () => "answer",
writeStdout: () => undefined,
writeStderr: () => undefined,
};
const context = new NodeServiceContext("/tmp/livesync-context-contract", standardIo);
expect(observeServiceContext(context, TRANSLATION_KEY)).toEqual({
translation: translateLiveSyncMessage(TRANSLATION_KEY),
receivedEvents: ["context-contract-event"],
});
expect(context.events).toBe(eventHub);
expect(context.databasePath).toBe("/tmp/livesync-context-contract");
expect(context.standardIo).toBe(standardIo);
const hub = new NodeServiceHub(context.databasePath, context);
const composition = observeServiceComposition(hub, context);
expect(composition.hubUsesExpectedContext).toBe(true);
expect(SERVICE_CONTEXT_MEMBERS.filter((member) => !composition.servicesUsingExpectedContext[member])).toEqual(
[]
);
});
});
src/apps/cli/services/NodeServiceHub.ts
+53 -60
View File
@@ -1,39 +1,36 @@
import type { AppLifecycleService, AppLifecycleServiceDependencies } from "@lib/services/base/AppLifecycleService";
import { ServiceContext } from "@lib/services/base/ServiceBase";
import { ConfigServiceBrowserCompat } from "@lib/services/implements/browser/ConfigServiceBrowserCompat";
import { SvelteDialogManagerBase, type ComponentHasResult } from "@lib/services/implements/base/SvelteDialog";
import { UIService } from "@lib/services/implements/base/UIService";
import { InjectableServiceHub } from "@lib/services/implements/injectable/InjectableServiceHub";
import { InjectableAppLifecycleService } from "@lib/services/implements/injectable/InjectableAppLifecycleService";
import { InjectableConflictService } from "@lib/services/implements/injectable/InjectableConflictService";
import { InjectableDatabaseEventService } from "@lib/services/implements/injectable/InjectableDatabaseEventService";
import { InjectableFileProcessingService } from "@lib/services/implements/injectable/InjectableFileProcessingService";
import { PathServiceCompat } from "@lib/services/implements/injectable/InjectablePathService";
import { InjectableRemoteService } from "@lib/services/implements/injectable/InjectableRemoteService";
import { InjectableReplicationService } from "@lib/services/implements/injectable/InjectableReplicationService";
import { InjectableReplicatorService } from "@lib/services/implements/injectable/InjectableReplicatorService";
import { InjectableTestService } from "@lib/services/implements/injectable/InjectableTestService";
import { InjectableTweakValueService } from "@lib/services/implements/injectable/InjectableTweakValueService";
import { InjectableVaultServiceCompat } from "@lib/services/implements/injectable/InjectableVaultService";
import { ControlService } from "@lib/services/base/ControlService";
import type { IControlService } from "@lib/services/base/IService";
import { HeadlessAPIService } from "@lib/services/implements/headless/HeadlessAPIService";
// import { HeadlessDatabaseService } from "@lib/services/implements/headless/HeadlessDatabaseService";
import type { ServiceInstances } from "@lib/services/ServiceHub";
import type { AppLifecycleServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService";
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { ConfigServiceBrowserCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/browser/ConfigServiceBrowserCompat";
import type {
ComponentHasResult,
SvelteDialogManager,
} from "@vrtmrz/livesync-commonlib/compat/services/implements/base/SvelteDialog";
import { UIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/base/UIService";
import { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub";
import { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAppLifecycleService";
import { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService";
import { InjectableDatabaseEventService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableDatabaseEventService";
import { InjectableFileProcessingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableFileProcessingService";
import { PathServiceCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectablePathService";
import { InjectableRemoteService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableRemoteService";
import { InjectableReplicationService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableReplicationService";
import { InjectableReplicatorService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableReplicatorService";
import { InjectableTestService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableTestService";
import { InjectableTweakValueService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableTweakValueService";
import { InjectableVaultServiceCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableVaultService";
import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService";
import { HeadlessAPIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/headless/HeadlessAPIService";
import { NodeKeyValueDBService } from "./NodeKeyValueDBService";
import { NodeSettingService } from "./NodeSettingService";
import { DatabaseService } from "@lib/services/base/DatabaseService";
import type { ObsidianLiveSyncSettings } from "@lib/common/types";
import { path as nodePath } from "@/apps/cli/node-compat";
import type { KeyValueDBService } from "@lib/services/base/KeyValueDBService";
import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService";
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { path as nodePath } from "@vrtmrz/livesync-commonlib/node";
import type { KeyValueDBService } from "@vrtmrz/livesync-commonlib/compat/services/base/KeyValueDBService";
import { PouchDB } from "@/apps/cli/lib/pouchdb-node";
import { NodeServiceContext } from "./NodeServiceContext";
import { setLang } from "@/common/translation";
export class NodeServiceContext extends ServiceContext {
databasePath: string;
constructor(databasePath: string) {
super();
this.databasePath = databasePath;
}
}
export { NodeServiceContext } from "./NodeServiceContext";
class NodeAppLifecycleService<T extends ServiceContext> extends InjectableAppLifecycleService<T> {
constructor(context: T, dependencies: AppLifecycleServiceDependencies) {
@@ -41,21 +38,24 @@ class NodeAppLifecycleService<T extends ServiceContext> extends InjectableAppLif
}
}
class NodeSvelteDialogManager<T extends ServiceContext> extends SvelteDialogManagerBase<T> {
openSvelteDialog<TValue, UInitial>(
component: ComponentHasResult<TValue, UInitial>,
initialData?: UInitial
class NodeDialogManager<T extends ServiceContext> implements SvelteDialogManager<T> {
open<TValue, UInitial>(
_component: ComponentHasResult<TValue, UInitial>,
_initialData?: UInitial
): Promise<TValue | undefined> {
throw new Error("Method not implemented.");
return Promise.reject(new Error("Interactive dialogues are not available in the CLI."));
}
openWithExplicitCancel<TValue, UInitial>(
_component: ComponentHasResult<TValue, UInitial>,
_initialData?: UInitial
): Promise<TValue> {
return Promise.reject(new Error("Interactive dialogues are not available in the CLI."));
}
}
type NodeUIServiceDependencies<T extends ServiceContext = ServiceContext> = {
appLifecycle: AppLifecycleService<T>;
config: ConfigServiceBrowserCompat<T>;
replicator: InjectableReplicatorService<T>;
APIService: HeadlessAPIService<T>;
control: IControlService;
};
class NodeDatabaseService<T extends NodeServiceContext> extends DatabaseService<T> {
protected override modifyDatabaseOptions(
@@ -77,17 +77,9 @@ class NodeUIService<T extends ServiceContext> extends UIService<T> {
}
constructor(context: T, dependencies: NodeUIServiceDependencies<T>) {
const headlessConfirm = dependencies.APIService.confirm;
const dialogManager = new NodeSvelteDialogManager<T>(context, {
confirm: headlessConfirm,
appLifecycle: dependencies.appLifecycle,
config: dependencies.config,
replicator: dependencies.replicator,
control: dependencies.control,
});
const dialogManager = new NodeDialogManager<T>();
super(context, {
appLifecycle: dependencies.appLifecycle,
dialogManager,
APIService: dependencies.APIService,
});
@@ -95,7 +87,7 @@ class NodeUIService<T extends ServiceContext> extends UIService<T> {
}
export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServiceHub<T> {
constructor(basePath: string, context: T = new NodeServiceContext(basePath) as T) {
constructor(basePath: string, context: T) {
const runtimeDir = nodePath.join(basePath, ".livesync", "runtime");
const localStoragePath = nodePath.join(runtimeDir, "local-storage.json");
const keyValueDBPath = nodePath.join(runtimeDir, "keyvalue-db.json");
@@ -104,13 +96,18 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
const conflict = new InjectableConflictService(context);
const fileProcessing = new InjectableFileProcessingService(context);
const setting = new NodeSettingService(context, { APIService: API }, localStoragePath);
const setting = new NodeSettingService(
context,
{ APIService: API, onDisplayLanguageChanged: setLang },
localStoragePath
);
const appLifecycle = new NodeAppLifecycleService<T>(context, {
settingService: setting,
});
const remote = new InjectableRemoteService(context, {
pouchDB: PouchDB,
APIService: API,
appLifecycle,
setting,
@@ -128,6 +125,7 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
});
const database = new NodeDatabaseService<T>(context, {
pouchDB: PouchDB,
API: API,
path,
vault,
@@ -174,14 +172,10 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
});
const ui = new NodeUIService<T>(context, {
appLifecycle,
config,
replicator,
APIService: API,
control,
});
const serviceInstancesToInit: Required<ServiceInstances<T>> = {
const serviceInstancesToInit = {
appLifecycle,
conflict,
database,
@@ -201,7 +195,6 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
keyValueDB: keyValueDB as unknown as KeyValueDBService<T>,
control,
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- (Forcibly )
super(context, serviceInstancesToInit as any);
super(context, serviceInstancesToInit);
}
}
src/apps/cli/services/NodeSettingService.ts
+7 -8
View File
@@ -1,10 +1,9 @@
import { EVENT_SETTING_SAVED } from "@lib/events/coreEvents";
import { EVENT_SETTING_SAVED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents";
import { EVENT_REQUEST_RELOAD_SETTING_TAB } from "@/common/events";
import { eventHub } from "@lib/hub/hub";
import { handlers } from "@lib/services/lib/HandlerUtils";
import type { ObsidianLiveSyncSettings } from "@lib/common/types";
import type { ServiceContext } from "@lib/services/base/ServiceBase";
import { SettingService, type SettingServiceDependencies } from "@lib/services/base/SettingService";
import { handlers } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils";
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
import { SettingService, type SettingServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/SettingService";
import {
configureNodeLocalStorage,
deleteNodeLocalStorageItem,
@@ -17,11 +16,11 @@ export class NodeSettingService<T extends ServiceContext> extends SettingService
super(context, dependencies);
configureNodeLocalStorage(storagePath);
this.onSettingSaved.addHandler((settings) => {
eventHub.emitEvent(EVENT_SETTING_SAVED, settings);
this.context.events.emitEvent(EVENT_SETTING_SAVED, settings);
return Promise.resolve(true);
});
this.onSettingLoaded.addHandler((settings) => {
eventHub.emitEvent(EVENT_REQUEST_RELOAD_SETTING_TAB);
this.context.events.emitEvent(EVENT_REQUEST_RELOAD_SETTING_TAB);
return Promise.resolve(true);
});
}
src/apps/cli/settingsPersistence.ts
+173
View File
@@ -0,0 +1,173 @@
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
import { activateRemoteConfiguration } from "@vrtmrz/livesync-commonlib/remote-configurations";
import type { CLICommand } from "./commands/types";
const SETTINGS_WRITE_COMMANDS = new Set<CLICommand>([
"setup",
"remote-add",
"remote-rm",
"remote-set",
"remote-activate",
]);
const REMOTE_SETTINGS_WRITE_COMMANDS = new Set<CLICommand>([
"remote-add",
"remote-rm",
"remote-set",
"remote-activate",
]);
export const CLI_RUNTIME_ONLY_SETTING_KEYS = new Set<keyof ObsidianLiveSyncSettings>([
"disableCheckingConfigMismatch",
"suspendFileWatching",
"suspendParseReplicationResult",
]);
function cloneJsonValue<T>(value: T): T {
if (value === undefined) return value;
return JSON.parse(JSON.stringify(value)) as T;
}
export function cloneSettings(settings: ObsidianLiveSyncSettings): ObsidianLiveSyncSettings {
return cloneJsonValue(settings);
}
function settingValuesEqual(left: unknown, right: unknown): boolean {
return JSON.stringify(left) === JSON.stringify(right);
}
export function settingsEqual(left: ObsidianLiveSyncSettings, right: ObsidianLiveSyncSettings): boolean {
return settingValuesEqual(left, right);
}
function settingsKeys(...settings: ObsidianLiveSyncSettings[]): Set<keyof ObsidianLiveSyncSettings> {
return new Set(settings.flatMap((value) => Object.keys(value) as Array<keyof ObsidianLiveSyncSettings>));
}
function copySetting(
target: ObsidianLiveSyncSettings,
source: ObsidianLiveSyncSettings,
key: keyof ObsidianLiveSyncSettings
): void {
const targetRecord = target as unknown as Record<string, unknown>;
const sourceRecord = source as unknown as Record<string, unknown>;
if (Object.prototype.hasOwnProperty.call(sourceRecord, key)) {
targetRecord[key] = cloneJsonValue(sourceRecord[key]);
} else {
delete targetRecord[key];
}
}
function remoteSettingKeys(...settings: ObsidianLiveSyncSettings[]): Set<keyof ObsidianLiveSyncSettings> {
const keys = new Set<keyof ObsidianLiveSyncSettings>([
"remoteConfigurations",
"activeConfigurationId",
"P2P_ActiveRemoteConfigurationId",
"remoteType",
]);
for (const current of settings) {
for (const configuration of Object.values(current.remoteConfigurations ?? {})) {
try {
const parsed = ConnectionStringParser.parse(configuration.uri);
for (const key of Object.keys(parsed.settings) as Array<keyof ObsidianLiveSyncSettings>) {
keys.add(key);
}
} catch {
// The setting service reports invalid remote configurations when loading them.
}
}
}
return keys;
}
export function changedSettingKeys(
before: ObsidianLiveSyncSettings,
after: ObsidianLiveSyncSettings
): Set<keyof ObsidianLiveSyncSettings> {
const changed = new Set<keyof ObsidianLiveSyncSettings>();
for (const key of settingsKeys(before, after)) {
if (!settingValuesEqual(before[key], after[key])) {
changed.add(key);
}
}
return changed;
}
export function isSettingsWriteCommand(command: CLICommand): boolean {
return SETTINGS_WRITE_COMMANDS.has(command);
}
export function reconcileDurableSettings(options: {
durableBase: ObsidianLiveSyncSettings;
runtimeBaseline: ObsidianLiveSyncSettings;
runtimeCurrent: ObsidianLiveSyncSettings;
preserveKeys: ReadonlySet<keyof ObsidianLiveSyncSettings>;
command?: CLICommand;
}): ObsidianLiveSyncSettings {
const durable = cloneSettings(options.durableBase);
for (const key of settingsKeys(options.runtimeBaseline, options.runtimeCurrent)) {
if (options.preserveKeys.has(key)) continue;
if (!settingValuesEqual(options.runtimeBaseline[key], options.runtimeCurrent[key])) {
copySetting(durable, options.runtimeCurrent, key);
}
}
if (options.command) {
if (REMOTE_SETTINGS_WRITE_COMMANDS.has(options.command)) {
copySetting(durable, options.runtimeCurrent, "remoteConfigurations");
copySetting(durable, options.runtimeCurrent, "activeConfigurationId");
copySetting(durable, options.runtimeCurrent, "P2P_ActiveRemoteConfigurationId");
if (durable.activeConfigurationId) {
activateRemoteConfiguration(durable, durable.activeConfigurationId);
}
} else {
// Commands such as remote-status may activate a profile temporarily. Only
// the dedicated remote settings commands are allowed to retain that switch.
for (const key of remoteSettingKeys(options.durableBase, options.runtimeBaseline, options.runtimeCurrent)) {
copySetting(durable, options.durableBase, key);
}
}
}
return durable;
}
export function preserveStoredSetting(
candidateText: string,
originalText: string | undefined,
key: keyof ObsidianLiveSyncSettings
): string {
if (originalText === undefined) return candidateText;
try {
const candidate = JSON.parse(candidateText) as ObsidianLiveSyncSettings;
const original = JSON.parse(originalText) as ObsidianLiveSyncSettings;
if (Object.prototype.hasOwnProperty.call(original, key)) {
copySetting(candidate, original, key);
} else {
delete (candidate as unknown as Record<string, unknown>)[key];
}
return JSON.stringify(candidate, null, 2);
} catch {
return candidateText;
}
}
export function applyStoredSetting(
target: ObsidianLiveSyncSettings,
storedText: string | undefined,
key: keyof ObsidianLiveSyncSettings
): void {
if (storedText === undefined) return;
try {
const stored = JSON.parse(storedText) as ObsidianLiveSyncSettings;
if (Object.prototype.hasOwnProperty.call(stored, key)) {
copySetting(target, stored, key);
} else {
delete (target as unknown as Record<string, unknown>)[key];
}
} catch {
// The setting service owns validation and recovery of malformed files.
}
}
src/apps/cli/settingsPersistence.unit.spec.ts
+162
View File
@@ -0,0 +1,162 @@
import { describe, expect, it } from "vitest";
import { DEFAULT_SETTINGS, type ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { activateRemoteConfiguration } from "@vrtmrz/livesync-commonlib/remote-configurations";
import {
applyStoredSetting,
changedSettingKeys,
cloneSettings,
isSettingsWriteCommand,
preserveStoredSetting,
reconcileDurableSettings,
} from "./settingsPersistence";
function settings(overrides: Partial<ObsidianLiveSyncSettings> = {}): ObsidianLiveSyncSettings {
return Object.assign(cloneSettings(DEFAULT_SETTINGS), overrides);
}
describe("CLI settings persistence", () => {
it("identifies commands which change the settings file automatically", () => {
expect(isSettingsWriteCommand("setup")).toBe(true);
expect(isSettingsWriteCommand("remote-add")).toBe(true);
expect(isSettingsWriteCommand("remote-rm")).toBe(true);
expect(isSettingsWriteCommand("remote-set")).toBe(true);
expect(isSettingsWriteCommand("remote-activate")).toBe(true);
expect(isSettingsWriteCommand("ls")).toBe(false);
expect(isSettingsWriteCommand("remote-status")).toBe(false);
});
it("retains lasting changes without retaining CLI suspension values", () => {
const durableBase = settings({
liveSync: true,
syncOnStart: true,
periodicReplication: true,
P2P_AutoStart: true,
settingVersion: 9,
customChunkSize: 40,
});
const runtimeBaseline = settings({
...durableBase,
liveSync: false,
syncOnStart: false,
periodicReplication: false,
P2P_AutoStart: false,
});
const runtimeCurrent = settings({
...runtimeBaseline,
settingVersion: 10,
customChunkSize: 60,
});
const reconciled = reconcileDurableSettings({
durableBase,
runtimeBaseline,
runtimeCurrent,
preserveKeys: changedSettingKeys(durableBase, runtimeBaseline),
command: "ls",
});
expect(reconciled.liveSync).toBe(true);
expect(reconciled.syncOnStart).toBe(true);
expect(reconciled.periodicReplication).toBe(true);
expect(reconciled.P2P_AutoStart).toBe(true);
expect(reconciled.settingVersion).toBe(10);
expect(reconciled.customChunkSize).toBe(60);
});
it("retains a remote profile change and restores the durable sync values", () => {
const durableBase = settings({
liveSync: true,
remoteConfigurations: {},
activeConfigurationId: "",
});
const runtimeBaseline = settings({ ...durableBase, liveSync: false });
const runtimeCurrent = settings({
...runtimeBaseline,
remoteConfigurations: {
main: {
id: "main",
name: "Main",
uri: "sls+https://user:pass@example.com/?db=notes",
isEncrypted: false,
},
},
activeConfigurationId: "main",
});
activateRemoteConfiguration(runtimeCurrent, "main");
const reconciled = reconcileDurableSettings({
durableBase,
runtimeBaseline,
runtimeCurrent,
preserveKeys: changedSettingKeys(durableBase, runtimeBaseline),
command: "remote-add",
});
expect(reconciled.liveSync).toBe(true);
expect(reconciled.activeConfigurationId).toBe("main");
expect(reconciled.remoteConfigurations.main?.name).toBe("Main");
expect(reconciled.couchDB_URI).toBe("https://example.com");
expect(reconciled.couchDB_DBNAME).toBe("notes");
});
it("does not retain a remote profile selected temporarily by an operational command", () => {
const durableBase = settings({
remoteConfigurations: {
first: {
id: "first",
name: "First",
uri: "sls+https://first:pass@example.com/?db=first",
isEncrypted: false,
},
second: {
id: "second",
name: "Second",
uri: "sls+https://second:pass@example.net/?db=second",
isEncrypted: false,
},
},
activeConfigurationId: "first",
});
activateRemoteConfiguration(durableBase, "first");
const runtimeBaseline = cloneSettings(durableBase);
const runtimeCurrent = cloneSettings(durableBase);
activateRemoteConfiguration(runtimeCurrent, "second");
const reconciled = reconcileDurableSettings({
durableBase,
runtimeBaseline,
runtimeCurrent,
preserveKeys: new Set(),
command: "remote-status",
});
expect(reconciled.activeConfigurationId).toBe("first");
expect(reconciled.couchDB_URI).toBe("https://example.com");
expect(reconciled.couchDB_USER).toBe("first");
expect(reconciled.couchDB_DBNAME).toBe("first");
});
it("preserves the stored adapter choice while the CLI uses its Node.js adapter", () => {
const original = JSON.stringify({ useIndexedDBAdapter: true, settingVersion: 9 });
const prepared = JSON.stringify({ useIndexedDBAdapter: false, settingVersion: 10 });
const preserved = preserveStoredSetting(prepared, original, "useIndexedDBAdapter");
const target = settings({ useIndexedDBAdapter: false });
applyStoredSetting(target, preserved, "useIndexedDBAdapter");
expect(JSON.parse(preserved).useIndexedDBAdapter).toBe(true);
expect(target.useIndexedDBAdapter).toBe(true);
});
it("does not add the CLI adapter override to an older settings file", () => {
const original = JSON.stringify({ settingVersion: 9 });
const prepared = JSON.stringify({ useIndexedDBAdapter: false, settingVersion: 10 });
const preserved = preserveStoredSetting(prepared, original, "useIndexedDBAdapter");
const target = settings({ useIndexedDBAdapter: false });
applyStoredSetting(target, preserved, "useIndexedDBAdapter");
expect(JSON.parse(preserved)).not.toHaveProperty("useIndexedDBAdapter");
expect(target).not.toHaveProperty("useIndexedDBAdapter");
});
});
src/apps/cli/setup-uri-e2e-helper.unit.spec.ts
+11
View File
@@ -0,0 +1,11 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
const setupPutCatHelper = readFileSync(new URL("./test/test-setup-put-cat-linux.sh", import.meta.url), "utf8");
describe("CLI setup URI E2E helper", () => {
it("evaluates Commonlib package imports as ESM", () => {
expect(setupPutCatHelper).toContain("node --input-type=module -e");
expect(setupPutCatHelper).not.toContain("npx tsx -e");
});
});
src/apps/cli/test-support/p2p-lifecycle-entrypoint.ts
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env node
import { RTCPeerConnection } from "werift";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
import { createNodeStandardIo } from "@vrtmrz/livesync-commonlib/node";
import { writeStderrLine } from "@/apps/cli/cliOutput";
import { main, type CliCommandRunner } from "@/apps/cli/main";
import { parseTimeoutSeconds } from "@/apps/cli/commands/p2p";
import { runP2PReplicatorReplacementProbe } from "./p2p-replicator-replacement";
if (
typeof (compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection === "undefined" &&
typeof RTCPeerConnection === "function"
) {
(compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection = RTCPeerConnection;
}
const standardIo = createNodeStandardIo();
const runLifecycleProbe: CliCommandRunner = async (options, context) => {
if (options.command !== "p2p-sync" || options.commandArgs.length < 2) {
throw new Error("The P2P lifecycle test entry requires: p2p-sync <peer> <timeout> [note-path] [note-content]");
}
const peerToken = options.commandArgs[0].trim();
if (!peerToken) {
throw new Error("The P2P lifecycle test entry requires a non-empty peer");
}
const timeoutSec = parseTimeoutSeconds(options.commandArgs[1], "P2P lifecycle test entry");
return await runP2PReplicatorReplacementProbe(
context,
peerToken,
timeoutSec * 1000,
options.commandArgs[2],
options.commandArgs[3]
);
};
main(standardIo, runLifecycleProbe).catch((error) => {
writeStderrLine(standardIo, "[Fatal Error]", error);
process.exit(1);
});
src/apps/cli/test-support/p2p-replicator-replacement.ts
+132
View File
@@ -0,0 +1,132 @@
import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
import { LiveSyncTrysteroReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/LiveSyncTrysteroReplicator";
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
import type { CLICommandContext } from "@/apps/cli/commands/types";
import { openP2PHost } from "@/apps/cli/commands/p2p";
const DEFAULT_NOTE_PATH = "p2p-replicator-replacement.md";
const DEFAULT_NOTE_CONTENT = "Replicated after replacing the active P2P replicator.";
function delay(ms: number): Promise<void> {
return new Promise((resolve) => compatGlobal.setTimeout(resolve, ms));
}
function describeError(value: unknown): string {
return value instanceof Error ? (value.stack ?? value.message) : String(value);
}
async function waitForServing(replicator: LiveSyncTrysteroReplicator, timeoutMs: number): Promise<void> {
const started = Date.now();
while (Date.now() - started <= timeoutMs) {
if (replicator.server?.isServing) return;
await delay(200);
}
throw new Error("The replacement P2P replicator did not start serving within the timeout");
}
async function waitForPeer(
replicator: LiveSyncTrysteroReplicator,
targetPeer: string,
timeoutMs: number
): Promise<{ peerId: string; name: string }> {
const started = Date.now();
while (Date.now() - started <= timeoutMs) {
const peer = replicator.knownAdvertisements.find(
(candidate) => candidate.name === targetPeer || candidate.peerId === targetPeer
);
if (peer) return peer;
await delay(200);
}
const knownPeers = replicator.knownAdvertisements.map((peer) => `${peer.name} (${peer.peerId})`).join(", ");
throw new Error(
`Peer '${targetPeer}' was not discovered within the timeout. Known peers: ${knownPeers || "none"}`
);
}
function assertPullSucceeded(result: unknown): void {
if (result && typeof result === "object" && "error" in result && result.error) {
throw new Error(`P2P pull failed: ${describeError(result.error)}`);
}
}
async function communicateWithPeer(
replicator: LiveSyncTrysteroReplicator,
targetPeer: string,
timeoutMs: number
): Promise<{ peerId: string; name: string }> {
await replicator.open();
await waitForServing(replicator, timeoutMs);
const peer = await waitForPeer(replicator, targetPeer, timeoutMs);
assertPullSucceeded(await replicator.replicateFrom(peer.peerId, false));
const pushResult = await replicator.requestSynchroniseToPeer(peer.peerId);
if (!pushResult || pushResult.ok !== true) {
throw new Error(`P2P push failed: ${describeError(pushResult?.error)}`);
}
return peer;
}
/** Runs the real-transport lifecycle probe used by the Deno and Compose P2P suites. */
export async function runP2PReplicatorReplacementProbe(
context: CLICommandContext,
targetPeer: string,
timeoutMs: number,
notePath = DEFAULT_NOTE_PATH,
noteContent = DEFAULT_NOTE_CONTENT
): Promise<boolean> {
const { core, p2pReplicator } = context;
if (!p2pReplicator) {
throw new Error("The CLI did not expose its P2P service-feature result to the integration probe");
}
const firstReplicator = await openP2PHost(core);
if (p2pReplicator.replicator !== firstReplicator) {
throw new Error("The P2P service feature did not expose the newly created replicator");
}
const firstPeer = await communicateWithPeer(firstReplicator, targetPeer, timeoutMs);
const initialised = await core.services.databaseEvents.initialiseDatabase(false, true, false);
if (!initialised) {
throw new Error("Database reinitialisation failed during the P2P replacement probe");
}
const replacementReplicator = p2pReplicator.replicator;
if (core.services.replicator.getActiveReplicator() !== replacementReplicator) {
throw new Error("ReplicatorService did not activate the P2P service feature's replacement replicator");
}
if (replacementReplicator === firstReplicator) {
throw new Error("Database reinitialisation retained the previous P2P replicator instance");
}
if (firstReplicator.server !== undefined) {
throw new Error("The previous P2P replicator remained open after replacement");
}
const settings = core.services.setting.currentSettings();
settings.P2P_AutoStart = true;
await core.services.control.applySettings();
const resumedReplicator = p2pReplicator.replicator;
await waitForServing(resumedReplicator, timeoutMs);
if (firstReplicator.server !== undefined) {
throw new Error("A setting event reopened the previous P2P replicator");
}
const encoded = new TextEncoder().encode(noteContent);
const noteBody = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength);
const timestamp = Date.now();
await core.serviceModules.storageAccess.writeFileAuto(notePath, noteBody, {
ctime: timestamp,
mtime: timestamp,
});
await core.serviceModules.fileHandler.storeFileToDB(notePath as FilePathWithPrefix, true);
const replacementPeer = await communicateWithPeer(resumedReplicator, targetPeer, timeoutMs);
if (replacementPeer.name !== firstPeer.name) {
throw new Error(
`The replacement replicator reached '${replacementPeer.name}' instead of the original peer '${firstPeer.name}'`
);
}
core.services.context.standardIo.writeStdout(
`[Probe] P2P replicator replaced, old transport stayed closed, and ${notePath} was sent through the replacement.\n`
);
return true;
}
src/apps/cli/test/test-p2p-upload-download-repro-linux.sh
+1 -2
View File
@@ -93,9 +93,8 @@ data.encrypt = true;
data.passphrase = process.env.PASSPHRASE_VAL;
data.usePathObfuscation = true;
data.handleFilenameCaseSensitive = false;
data.customChunkSize = 50;
data.customChunkSize = 60;
data.usePluginSyncV2 = true;
data.doNotUseFixedRevisionForChunks = false;
data.P2P_DevicePeerName = process.env.DEVICE_NAME;
data.isConfigured = true;
src/apps/cli/test/test-setup-put-cat-linux.sh
+3 -4
View File
@@ -3,7 +3,6 @@ set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
CLI_DIR="$(cd -- "$SCRIPT_DIR/.." && pwd)"
REPO_ROOT="$(cd -- "$CLI_DIR/../../.." && pwd)"
cd "$CLI_DIR"
source "$SCRIPT_DIR/test-helpers.sh"
display_test_info
@@ -28,10 +27,10 @@ cli_test_init_settings_file "$SETTINGS_FILE"
echo "[INFO] creating setup URI from settings"
SETUP_URI="$(
REPO_ROOT="$REPO_ROOT" SETTINGS_FILE="$SETTINGS_FILE" SETUP_PASSPHRASE="$SETUP_PASSPHRASE" npx tsx -e '
import fs from "node:fs";
SETTINGS_FILE="$SETTINGS_FILE" SETUP_PASSPHRASE="$SETUP_PASSPHRASE" node --input-type=module -e '
import { fs } from "@vrtmrz/livesync-commonlib/node";
import { encodeSettingsToSetupURI } from "@vrtmrz/livesync-commonlib/compat/API/processSetting";
(async () => {
const { encodeSettingsToSetupURI } = await import(process.env.REPO_ROOT + "/src/lib/src/API/processSetting.ts");
const settingsPath = process.env.SETTINGS_FILE;
const setupPassphrase = process.env.SETUP_PASSPHRASE;
const settings = JSON.parse(fs.readFileSync(settingsPath, "utf-8"));
src/apps/cli/testdeno/CONTRIBUTING_TESTS.md
+1
View File
@@ -107,6 +107,7 @@ Deno.test("feature: behaviour", async () => {
- Re-run sync operations where the protocol is eventually consistent.
- For network-sensitive commands, use `LIVESYNC_CLI_RETRY` during debugging.
- Keep Docker container reuse disabled by default unless debugging.
- Use `npm run test:e2e:cli:p2p` for canonical P2P validation. It runs the Deno scenario in Compose because host networking and WebRTC candidate selection are not reproducible across environments. Individual `deno task test:p2p-*` tasks remain available when explicitly invoked for cross-platform diagnostics, but are not selected by the default suite or CI.
## Environment variables
src/apps/cli/testdeno/bench-compression.ts
+603
View File
@@ -0,0 +1,603 @@
import { join } from "@std/path";
import { TempDir } from "./helpers/temp.ts";
import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
import { assertFilesEqual } from "./helpers/cli.ts";
import { runMeasuredCliOrFail, type CliProcessMeasurement } from "./helpers/measuredCli.ts";
import { createCouchdbDatabase, startCouchdb, stopCouchdb } from "./helpers/docker.ts";
import {
createCompressionBenchmarkDataset,
type CompressionDataset,
type CompressionDatasetEntry,
} from "./helpers/compressionDataset.ts";
import { computeDatasetDigestSha256 } from "./helpers/benchmarkVerification.ts";
import { startCouchdbProxy, type CouchdbProxyCounters } from "./bench-couchdb.ts";
import type { DatasetKind } from "./helpers/dataset.ts";
type CompressionCondition = {
name: string;
encrypt: boolean;
enableCompression: boolean;
};
type CouchDbSizes = {
file: number;
external: number;
active: number;
};
type PerKindRemoteMeasurement = {
sourceFiles: number;
sourceBytes: number;
mappedFiles: number;
uniqueReferencedChunks: number;
storedChunkDataBytes: number;
storedChunkJsonBytes: number;
};
type RemoteMeasurement = {
couchdbSizes: CouchDbSizes;
documentCount: number;
chunkDocumentCount: number;
metadataDocumentCount: number;
compressedMarkerCount: number;
encryptedChunkCount: number;
storedChunkDataBytes: number;
storedChunkJsonBytes: number;
perKind: Record<DatasetKind, PerKindRemoteMeasurement>;
};
type RunResult = {
condition: CompressionCondition;
repeatIndex: number;
executionOrder: number;
databaseName: string;
datasetDigestSha256: string;
dataset: {
totalFiles: number;
totalBytes: number;
filesByKind: CompressionDataset["filesByKind"];
bytesByKind: CompressionDataset["bytesByKind"];
jpegGenerator: string;
};
effectiveSettings: Record<string, unknown>;
mirror: CliProcessMeasurement;
upload: CliProcessMeasurement & { http: CouchdbProxyCounters };
download: CliProcessMeasurement & { http: CouchdbProxyCounters };
materialisation: CliProcessMeasurement & { http: CouchdbProxyCounters };
verification: {
verifiedFiles: number;
complete: boolean;
};
remote: RemoteMeasurement;
};
const CONDITIONS: CompressionCondition[] = [
{ name: "plain", encrypt: false, enableCompression: false },
{ name: "plain-compressed", encrypt: false, enableCompression: true },
{ name: "e2ee", encrypt: true, enableCompression: false },
{ name: "e2ee-compressed", encrypt: true, enableCompression: true },
];
const DATASET_KINDS: DatasetKind[] = ["md", "jpg", "png", "json", "ts", "gz", "bin"];
const COMPRESSED_MARKER = "\u000eLZ\u001d";
function readEnvString(name: string, fallback: string): string {
const value = Deno.env.get(name)?.trim();
return value ? value : fallback;
}
function readEnvPositiveInteger(name: string, fallback: number): number {
const raw = Deno.env.get(name)?.trim();
if (!raw) return fallback;
const value = Number(raw);
if (!Number.isInteger(value) || value <= 0) {
throw new Error(`${name} must be a positive integer, got '${raw}'`);
}
return value;
}
function readEnvPositiveNumber(name: string, fallback: number): number {
const raw = Deno.env.get(name)?.trim();
if (!raw) return fallback;
const value = Number(raw);
if (!Number.isFinite(value) || value <= 0) {
throw new Error(`${name} must be positive, got '${raw}'`);
}
return value;
}
function readEnvBoolean(name: string, fallback: boolean): boolean {
const raw = Deno.env.get(name)?.trim();
if (!raw) return fallback;
return /^(1|true|yes|on)$/i.test(raw);
}
function byteLength(value: string): number {
return new TextEncoder().encode(value).byteLength;
}
function median(values: number[]): number {
if (values.length === 0) return 0;
const sorted = [...values].sort((a, b) => a - b);
const middle = Math.floor(sorted.length / 2);
if (sorted.length % 2 === 1) return sorted[middle];
return (sorted[middle - 1] + sorted[middle]) / 2;
}
function rounded(value: number): number {
return Number(value.toFixed(4));
}
function deltaPercent(enabled: number, disabled: number): number | null {
if (disabled === 0) return null;
return rounded(((enabled - disabled) / disabled) * 100);
}
function reductionPercent(enabled: number, disabled: number): number | null {
if (disabled === 0) return null;
return rounded((1 - enabled / disabled) * 100);
}
function basicAuth(user: string, password: string): string {
return `Basic ${btoa(`${user}:${password}`)}`;
}
async function couchRequest(
baseUri: string,
user: string,
password: string,
path: string,
init: RequestInit = {},
allowedStatuses: number[] = []
): Promise<Response> {
const response = await fetch(`${baseUri.replace(/\/$/, "")}${path}`, {
...init,
headers: {
Authorization: basicAuth(user, password),
...(init.method === "POST" || init.body ? { "Content-Type": "application/json" } : {}),
...init.headers,
},
});
if (!response.ok && !allowedStatuses.includes(response.status)) {
throw new Error(`${init.method ?? "GET"} ${path}: ${response.status} ${await response.text()}`);
}
return response;
}
async function deleteDatabase(baseUri: string, user: string, password: string, databaseName: string): Promise<void> {
const response = await couchRequest(
baseUri,
user,
password,
`/${encodeURIComponent(databaseName)}`,
{ method: "DELETE" },
[404]
);
await response.body?.cancel().catch(() => {});
}
function blankPerKind(dataset: CompressionDataset): Record<DatasetKind, PerKindRemoteMeasurement> {
return Object.fromEntries(
DATASET_KINDS.map((kind) => [
kind,
{
sourceFiles: dataset.filesByKind[kind],
sourceBytes: dataset.bytesByKind[kind],
mappedFiles: 0,
uniqueReferencedChunks: 0,
storedChunkDataBytes: 0,
storedChunkJsonBytes: 0,
},
])
) as Record<DatasetKind, PerKindRemoteMeasurement>;
}
function findDatasetEntry(path: unknown, entries: CompressionDatasetEntry[]): CompressionDatasetEntry | undefined {
if (typeof path !== "string") return undefined;
return entries.find((entry) => path === entry.relativePath || path.endsWith(`/${entry.relativePath}`));
}
async function inspectRemoteDatabase(options: {
baseUri: string;
user: string;
password: string;
databaseName: string;
dataset: CompressionDataset;
}): Promise<RemoteMeasurement> {
const dbPath = `/${encodeURIComponent(options.databaseName)}`;
await couchRequest(options.baseUri, options.user, options.password, `${dbPath}/_ensure_full_commit`, {
method: "POST",
body: "{}",
});
const [info, allDocs] = await Promise.all([
couchRequest(options.baseUri, options.user, options.password, dbPath).then((response) => response.json()),
couchRequest(options.baseUri, options.user, options.password, `${dbPath}/_all_docs?include_docs=true`).then(
(response) => response.json()
),
]);
const rows = (allDocs as { rows?: Array<{ doc?: Record<string, unknown> }> }).rows ?? [];
const docs = rows.flatMap((row) => (row.doc ? [row.doc] : []));
const docsById = new Map(docs.flatMap((doc) => (typeof doc._id === "string" ? [[doc._id, doc] as const] : [])));
const metadataDocs = docs.filter((doc) => Array.isArray(doc.children) && typeof doc.path === "string");
const chunkDocs = docs.filter(
(doc) =>
(doc.type === "leaf" || doc.type === "chunkpack") &&
typeof doc.data === "string" &&
typeof doc._id === "string"
);
const perKind = blankPerKind(options.dataset);
const chunkIdsByKind = new Map(DATASET_KINDS.map((kind) => [kind, new Set<string>()] as const));
for (const doc of metadataDocs) {
const entry = findDatasetEntry(doc.path, options.dataset.entries);
if (!entry) continue;
perKind[entry.kind].mappedFiles += 1;
for (const child of doc.children as unknown[]) {
if (typeof child === "string") chunkIdsByKind.get(entry.kind)!.add(child);
}
}
for (const kind of DATASET_KINDS) {
const chunkIds = chunkIdsByKind.get(kind)!;
perKind[kind].uniqueReferencedChunks = chunkIds.size;
for (const chunkId of chunkIds) {
const chunk = docsById.get(chunkId);
if (!chunk || typeof chunk.data !== "string") continue;
perKind[kind].storedChunkDataBytes += byteLength(chunk.data);
perKind[kind].storedChunkJsonBytes += byteLength(JSON.stringify(chunk));
}
}
const sizeInfo = (info as { sizes?: Partial<CouchDbSizes>; doc_count?: number }).sizes ?? {};
return {
couchdbSizes: {
file: sizeInfo.file ?? 0,
external: sizeInfo.external ?? 0,
active: sizeInfo.active ?? 0,
},
documentCount: (info as { doc_count?: number }).doc_count ?? docs.length,
chunkDocumentCount: chunkDocs.length,
metadataDocumentCount: metadataDocs.length,
compressedMarkerCount: chunkDocs.filter(
(doc) => typeof doc.data === "string" && doc.data.startsWith(COMPRESSED_MARKER)
).length,
encryptedChunkCount: chunkDocs.filter((doc) => doc.e_ === true).length,
storedChunkDataBytes: chunkDocs.reduce(
(sum, doc) => sum + (typeof doc.data === "string" ? byteLength(doc.data) : 0),
0
),
storedChunkJsonBytes: chunkDocs.reduce((sum, doc) => sum + byteLength(JSON.stringify(doc)), 0),
perKind,
};
}
async function verifyDataset(
workDir: TempDir,
vaultB: string,
settingsB: string,
entries: CompressionDatasetEntry[]
): Promise<CliProcessMeasurement> {
const started = performance.now();
const measurements: CliProcessMeasurement[] = [];
for (const entry of entries) {
const pulledPath = workDir.join(`verify-${entry.kind}-${entry.relativePath.split("/").at(-1)}`);
measurements.push(
await runMeasuredCliOrFail(vaultB, "--settings", settingsB, "pull", entry.relativePath, pulledPath)
);
await assertFilesEqual(entry.absolutePath, pulledPath, `compression benchmark mismatch: ${entry.relativePath}`);
}
const elapsedMs = performance.now() - started;
const userCpuMs = measurements.reduce((sum, measurement) => sum + measurement.userCpuMs, 0);
const systemCpuMs = measurements.reduce((sum, measurement) => sum + measurement.systemCpuMs, 0);
const totalCpuMs = userCpuMs + systemCpuMs;
return {
elapsedMs: rounded(elapsedMs),
userCpuMs: rounded(userCpuMs),
systemCpuMs: rounded(systemCpuMs),
totalCpuMs: rounded(totalCpuMs),
cpuToWallRatio: rounded(totalCpuMs / elapsedMs),
maxResidentSetKiB: Math.max(...measurements.map((measurement) => measurement.maxResidentSetKiB)),
};
}
function summariseResults(results: RunResult[]) {
const byCondition = Object.fromEntries(
CONDITIONS.map((condition) => {
const runs = results.filter((result) => result.condition.name === condition.name);
return [
condition.name,
{
repeats: runs.length,
remoteStoredChunkDataBytesMedian: median(runs.map((run) => run.remote.storedChunkDataBytes)),
couchdbExternalBytesMedian: median(runs.map((run) => run.remote.couchdbSizes.external)),
couchdbFileBytesMedian: median(runs.map((run) => run.remote.couchdbSizes.file)),
uploadRequestBodyBytesMedian: median(runs.map((run) => run.upload.http.requestBodyBytes)),
uploadResponseBodyBytesMedian: median(runs.map((run) => run.upload.http.responseBodyBytes)),
downloadRequestBodyBytesMedian: median(runs.map((run) => run.download.http.requestBodyBytes)),
downloadResponseBodyBytesMedian: median(runs.map((run) => run.download.http.responseBodyBytes)),
uploadElapsedMsMedian: median(runs.map((run) => run.upload.elapsedMs)),
uploadCpuMsMedian: median(runs.map((run) => run.upload.totalCpuMs)),
downloadElapsedMsMedian: median(runs.map((run) => run.download.elapsedMs)),
downloadCpuMsMedian: median(runs.map((run) => run.download.totalCpuMs)),
materialisationElapsedMsMedian: median(runs.map((run) => run.materialisation.elapsedMs)),
materialisationCpuMsMedian: median(runs.map((run) => run.materialisation.totalCpuMs)),
materialisationResponseBodyBytesMedian: median(
runs.map((run) => run.materialisation.http.responseBodyBytes)
),
completeDownloadResponseBodyBytesMedian: median(
runs.map(
(run) => run.download.http.responseBodyBytes + run.materialisation.http.responseBodyBytes
)
),
completeDownloadElapsedMsMedian: median(
runs.map((run) => run.download.elapsedMs + run.materialisation.elapsedMs)
),
completeDownloadCpuMsMedian: median(
runs.map((run) => run.download.totalCpuMs + run.materialisation.totalCpuMs)
),
maxResidentSetKiBMedian: median(
runs.map((run) =>
Math.max(
run.upload.maxResidentSetKiB,
run.download.maxResidentSetKiB,
run.materialisation.maxResidentSetKiB
)
)
),
perKindStoredChunkDataBytesMedian: Object.fromEntries(
DATASET_KINDS.map((kind) => [
kind,
median(runs.map((run) => run.remote.perKind[kind].storedChunkDataBytes)),
])
),
},
];
})
);
const comparisons = [false, true].map((encrypt) => {
const disabledName = encrypt ? "e2ee" : "plain";
const enabledName = encrypt ? "e2ee-compressed" : "plain-compressed";
const disabled = byCondition[disabledName] as Record<string, unknown>;
const enabled = byCondition[enabledName] as Record<string, unknown>;
const disabledPerKind = disabled.perKindStoredChunkDataBytesMedian as Record<DatasetKind, number>;
const enabledPerKind = enabled.perKindStoredChunkDataBytesMedian as Record<DatasetKind, number>;
return {
encrypt,
disabledCondition: disabledName,
enabledCondition: enabledName,
storedChunkDataReductionPercent: reductionPercent(
enabled.remoteStoredChunkDataBytesMedian as number,
disabled.remoteStoredChunkDataBytesMedian as number
),
couchdbExternalReductionPercent: reductionPercent(
enabled.couchdbExternalBytesMedian as number,
disabled.couchdbExternalBytesMedian as number
),
couchdbFileReductionPercent: reductionPercent(
enabled.couchdbFileBytesMedian as number,
disabled.couchdbFileBytesMedian as number
),
uploadRequestBodyReductionPercent: reductionPercent(
enabled.uploadRequestBodyBytesMedian as number,
disabled.uploadRequestBodyBytesMedian as number
),
completeDownloadResponseBodyReductionPercent: reductionPercent(
enabled.completeDownloadResponseBodyBytesMedian as number,
disabled.completeDownloadResponseBodyBytesMedian as number
),
uploadElapsedDeltaPercent: deltaPercent(
enabled.uploadElapsedMsMedian as number,
disabled.uploadElapsedMsMedian as number
),
uploadCpuDeltaPercent: deltaPercent(
enabled.uploadCpuMsMedian as number,
disabled.uploadCpuMsMedian as number
),
downloadElapsedDeltaPercent: deltaPercent(
enabled.downloadElapsedMsMedian as number,
disabled.downloadElapsedMsMedian as number
),
downloadCpuDeltaPercent: deltaPercent(
enabled.downloadCpuMsMedian as number,
disabled.downloadCpuMsMedian as number
),
completeDownloadElapsedDeltaPercent: deltaPercent(
enabled.completeDownloadElapsedMsMedian as number,
disabled.completeDownloadElapsedMsMedian as number
),
completeDownloadCpuDeltaPercent: deltaPercent(
enabled.completeDownloadCpuMsMedian as number,
disabled.completeDownloadCpuMsMedian as number
),
perKindStoredChunkDataReductionPercent: Object.fromEntries(
DATASET_KINDS.map((kind) => [kind, reductionPercent(enabledPerKind[kind], disabledPerKind[kind])])
),
};
});
return { byCondition, comparisons };
}
async function main(): Promise<void> {
const backendUri = readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989");
const proxyUri = readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989");
const user = readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin"));
const password = readEnvString("BENCH_COUCHDB_PASSWORD", readEnvString("password", "password"));
const databasePrefix = readEnvString("BENCH_COUCHDB_DBNAME", `compression-bench-${Date.now()}`);
const repeatCount = readEnvPositiveInteger("BENCH_COMPRESSION_REPEAT_COUNT", 1);
const requestedRttMs = readEnvPositiveNumber("BENCH_COUCHDB_RTT_MS", 1);
const managedCouchdb = readEnvBoolean("BENCH_COUCHDB_MANAGED", true);
const passphrase = readEnvString("BENCH_PASSPHRASE", "compression-benchmark-passphrase");
const resultRoot = readEnvString("BENCH_COMPRESSION_RESULT_ROOT", "bench-results");
const resultPath =
Deno.env.get("BENCH_COMPRESSION_RESULT_JSON")?.trim() ||
join(resultRoot, `compression-${new Date().toISOString().replaceAll(":", "-")}.json`);
const createdDatabases = new Set<string>();
const results: RunResult[] = [];
let managedStarted = false;
await Deno.mkdir(resultRoot, { recursive: true });
const proxy = startCouchdbProxy({ backendUri, proxyUri, requestedRttMs });
try {
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
const rotation = (repeatIndex - 1) % CONDITIONS.length;
const orderedConditions = [...CONDITIONS.slice(rotation), ...CONDITIONS.slice(0, rotation)];
for (const [executionOffset, condition] of orderedConditions.entries()) {
const databaseName = `${databasePrefix}-${repeatIndex}-${condition.name}`.toLowerCase();
if (managedCouchdb && !managedStarted) {
await startCouchdb(backendUri, user, password, databaseName);
managedStarted = true;
} else {
await createCouchdbDatabase(backendUri, user, password, databaseName);
}
createdDatabases.add(databaseName);
await using workDir = await TempDir.create(`livesync-compression-${condition.name}`);
const vaultA = workDir.join("vault-a");
const vaultB = workDir.join("vault-b");
const settingsA = workDir.join("settings-a.json");
const settingsB = workDir.join("settings-b.json");
await Deno.mkdir(vaultA, { recursive: true });
await Deno.mkdir(vaultB, { recursive: true });
await initSettingsFile(settingsA);
await initSettingsFile(settingsB);
await Promise.all(
[settingsA, settingsB].map((settingsFile) =>
applyRemoteSyncSettings(settingsFile, {
remoteType: "COUCHDB",
couchdbUri: proxyUri,
couchdbUser: user,
couchdbPassword: password,
couchdbDbname: databaseName,
encrypt: condition.encrypt,
passphrase,
enableCompression: condition.enableCompression,
usePathObfuscation: false,
})
)
);
const dataset = await createCompressionBenchmarkDataset({ rootDir: vaultA });
const datasetDigestSha256 = await computeDatasetDigestSha256(dataset.entries);
const mirror = await runMeasuredCliOrFail(vaultA, "--settings", settingsA, "mirror");
proxy.resetCounters();
const uploadMeasurement = await runMeasuredCliOrFail(vaultA, "--settings", settingsA, "sync");
const upload = { ...uploadMeasurement, http: proxy.snapshotCounters() };
proxy.resetCounters();
const downloadMeasurement = await runMeasuredCliOrFail(vaultB, "--settings", settingsB, "sync");
const download = { ...downloadMeasurement, http: proxy.snapshotCounters() };
proxy.resetCounters();
const materialisationMeasurement = await verifyDataset(workDir, vaultB, settingsB, dataset.entries);
const materialisation = {
...materialisationMeasurement,
http: proxy.snapshotCounters(),
};
const remote = await inspectRemoteDatabase({
baseUri: backendUri,
user,
password,
databaseName,
dataset,
});
const settings = JSON.parse(await Deno.readTextFile(settingsA)) as Record<string, unknown>;
const effectiveSettings = Object.fromEntries(
[
"encrypt",
"enableCompression",
"E2EEAlgorithm",
"usePathObfuscation",
"chunkSplitterVersion",
"customChunkSize",
"minimumChunkSize",
"hashAlg",
].map((key) => [key, settings[key]])
);
results.push({
condition,
repeatIndex,
executionOrder: executionOffset + 1,
databaseName,
datasetDigestSha256,
dataset: {
totalFiles: dataset.totalFiles,
totalBytes: dataset.totalBytes,
filesByKind: dataset.filesByKind,
bytesByKind: dataset.bytesByKind,
jpegGenerator: dataset.jpegGenerator,
},
effectiveSettings,
mirror,
upload,
download,
materialisation,
verification: { verifiedFiles: dataset.entries.length, complete: true },
remote,
});
await deleteDatabase(backendUri, user, password, databaseName);
createdDatabases.delete(databaseName);
console.error(
`[Compression benchmark] repeat ${repeatIndex}/${repeatCount} ${condition.name}: ` +
`${dataset.totalFiles} files, ${remote.chunkDocumentCount} chunks, ` +
`${remote.storedChunkDataBytes} stored chunk-data bytes`
);
}
}
const output = {
schemaVersion: 1,
mode: "couchdb-cli-compression-benchmark",
generatedAt: new Date().toISOString(),
commonlibVersion: (
JSON.parse(
await Deno.readTextFile(
join(import.meta.dirname!, "../../../../node_modules/@vrtmrz/livesync-commonlib/package.json")
)
) as { version: string }
).version,
couchdbVersion: (
(await couchRequest(backendUri, user, password, "/").then((response) => response.json())) as {
version?: string;
}
).version,
compressionImplementation: "Commonlib replicationFilter using fflate level 8 before E2EE V2",
chunkingImplementation: "LiveSync CLI mirror using the effective settings recorded for each run",
requestedRttMs,
httpByteScope:
"Decoded HTTP request and response body bytes observed by the local proxy; headers are excluded.",
limitations: [
"Synthetic JPEGs exercise a deterministic image-like fixture but are not a photographic corpus.",
"PNG, Markdown, JSON, and TypeScript inputs are current repository files and therefore change with the source tree.",
"Wall and CPU times include CLI process start-up; compare repeated medians rather than treating one run as a universal result.",
"Full materialisation starts one CLI process per file and can repeat lazy chunk fetches; treat it as a CLI workflow measurement rather than a raw download lower bound.",
"The benchmark uses a local CouchDB and a fixed latency proxy, not a contended production server or a real WAN.",
"Path obfuscation is explicitly disabled so raw metadata can be mapped back to file kinds.",
],
repeatCount,
conditions: CONDITIONS,
summary: summariseResults(results),
runs: results,
};
await Deno.writeTextFile(resultPath, JSON.stringify(output, null, 2));
console.log(JSON.stringify(output, null, 2));
console.error(`[Compression benchmark] wrote ${resultPath}`);
} finally {
await proxy.stop();
for (const databaseName of createdDatabases) {
await deleteDatabase(backendUri, user, password, databaseName).catch((error) => console.error(error));
}
if (managedCouchdb && managedStarted) {
await stopCouchdb().catch(() => {});
}
}
}
if (import.meta.main) {
main().catch((error) => {
console.error("[Compression benchmark fatal]", error);
Deno.exit(1);
});
}
src/apps/cli/testdeno/bench-couchdb.ts
+22
View File
@@ -188,11 +188,19 @@ function readOptionalResultPath(): string | undefined {
export type CouchdbProxyHandle = {
stop: () => Promise<void>;
resetCounters: () => void;
snapshotCounters: () => CouchdbProxyCounters;
applied: boolean;
note: string;
directionalDelayMs: number;
};
export type CouchdbProxyCounters = {
requestCount: number;
requestBodyBytes: number;
responseBodyBytes: number;
};
export function startCouchdbProxy(
options: {
backendUri: string;
@@ -208,6 +216,11 @@ export function startCouchdbProxy(
((milliseconds: number) =>
new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
const controller = new AbortController();
const counters: CouchdbProxyCounters = {
requestCount: 0,
requestBodyBytes: 0,
responseBodyBytes: 0,
};
const listener = Deno.serve(
{
@@ -238,6 +251,8 @@ export function startCouchdbProxy(
requestBody = undefined;
}
}
counters.requestCount += 1;
counters.requestBodyBytes += requestBody?.byteLength ?? 0;
const upstream = await fetch(targetUrl, {
method: request.method,
@@ -249,6 +264,7 @@ export function startCouchdbProxy(
const responseHeaders = new Headers(upstream.headers);
responseHeaders.delete("content-length");
const responseBody = await upstream.arrayBuffer();
counters.responseBodyBytes += responseBody.byteLength;
await delay(halfDelayMs);
return new Response(responseBody, {
@@ -264,6 +280,12 @@ export function startCouchdbProxy(
directionalDelayMs: halfDelayMs,
note:
`local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms request-path and ${halfDelayMs}ms response-path delay`,
resetCounters: () => {
counters.requestCount = 0;
counters.requestBodyBytes = 0;
counters.responseBodyBytes = 0;
},
snapshotCounters: () => ({ ...counters }),
stop: async () => {
controller.abort();
await listener.finished.catch(() => {});
src/apps/cli/testdeno/deno.json
+8 -1
View File
@@ -1,10 +1,13 @@
{
"tasks": {
"test": "deno test --env-file=.test.env -A --no-check test-*.ts",
"test": "deno task test:ci",
"test:ci": "deno run -A --no-check run-ci-suite.ts",
"test:p2p:compose": "deno run -A --no-check run-compose-p2p.ts",
"test:local": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts test-mirror.ts test-daemon.ts",
"test:daemon": "deno test --env-file=.test.env -A --no-check test-daemon.ts",
"test:decoupled-vault": "deno test --env-file=.test.env -A --no-check test-decoupled-vault.ts",
"test:remote-commands": "deno test --env-file=.test.env -A --no-check test-remote-commands.ts",
"test:settings-writeback": "deno test -A --no-check test-settings-writeback.ts",
"test:push-pull": "deno test --env-file=.test.env -A --no-check test-push-pull.ts",
"test:setup-put-cat": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts",
"test:mirror": "deno test --env-file=.test.env -A --no-check test-mirror.ts",
@@ -13,11 +16,15 @@
"test:p2p-host": "deno test --env-file=.test.env -A --no-check test-p2p-host.ts",
"test:p2p-peers": "deno test --env-file=.test.env -A --no-check test-p2p-peers-local-relay.ts",
"test:p2p-sync": "deno test --env-file=.test.env -A --no-check test-p2p-sync.ts",
"test:p2p-replacement": "deno test --env-file=.test.env -A --no-check test-p2p-replicator-replacement.ts",
"test:p2p-relay-disconnect": "deno test --env-file=.test.env -A --no-check test-p2p-relay-disconnect.ts",
"test:p2p:ci": "deno test --env-file=.test.env -A --no-check test-p2p-sync.ts test-p2p-replicator-replacement.ts test-p2p-relay-disconnect.ts",
"test:p2p-three-nodes": "deno test --env-file=.test.env -A --no-check test-p2p-three-nodes-conflict.ts",
"test:p2p-upload-download": "deno test --env-file=.test.env -A --no-check test-p2p-upload-download-repro.ts",
"test:benchmark-contract": "deno test --env-file=.test.env -A --no-check test-benchmark-contract.ts",
"bench:p2p": "deno run --env-file=.test.env -A --no-check bench-p2p.ts",
"bench:couchdb": "deno run --env-file=.test.env -A --no-check bench-couchdb.ts",
"bench:compression": "deno run --env-file=.test.env -A --no-check bench-compression.ts",
"bench:cases": "deno run --env-file=.test.env -A --no-check bench-network-cases.ts",
"bench:latency-sweep": "deno run --env-file=.test.env -A --no-check bench-latency-sweep.ts",
"bench:p2p-split-node": "deno run --env-file=.test.env -A --no-check bench-p2p-split-node.ts",
src/apps/cli/testdeno/helpers/cli.ts
+1 -1
View File
@@ -7,7 +7,7 @@ import { join } from "@std/path";
// CLI root (src/apps/cli/) is two levels up.
// import.meta.dirname is available in Deno 1.40+ as an OS-native path string.
export const CLI_DIR: string = join(import.meta.dirname!, "..", "..");
const CLI_DIST = join(CLI_DIR, "dist", "index.cjs");
export const CLI_DIST = join(CLI_DIR, "dist", "index.cjs");
// ---------------------------------------------------------------------------
// Result type
src/apps/cli/testdeno/helpers/compressionDataset.ts
+188
View File
@@ -0,0 +1,188 @@
import { join } from "@std/path";
import type { DatasetEntry, DatasetKind } from "./dataset.ts";
export type CompressionDatasetEntry = DatasetEntry & {
source: string;
};
export type CompressionDataset = {
entries: CompressionDatasetEntry[];
totalFiles: number;
totalBytes: number;
bytesByKind: Record<DatasetKind, number>;
filesByKind: Record<DatasetKind, number>;
jpegGenerator: string;
};
export type JpegEncoder = (inputPpm: string, outputJpeg: string) => Promise<string>;
const ALL_KINDS: DatasetKind[] = ["md", "jpg", "png", "json", "ts", "gz", "bin"];
const REPOSITORY_ROOT = join(import.meta.dirname!, "..", "..", "..", "..", "..");
function fnv1a32(input: string): number {
let hash = 0x811c9dc5;
for (let i = 0; i < input.length; i++) {
hash ^= input.charCodeAt(i) & 0xff;
hash = Math.imul(hash, 0x01000193);
}
return hash >>> 0;
}
function createXorshift32(seed: number): () => number {
let state = seed || 0x9e3779b9;
return () => {
state ^= state << 13;
state ^= state >>> 17;
state ^= state << 5;
return state >>> 0;
};
}
function createSyntheticPpm(width: number, height: number, seed: string, textured: boolean): Uint8Array {
const header = new TextEncoder().encode(`P6\n${width} ${height}\n255\n`);
const pixels = new Uint8Array(width * height * 3);
const nextRandom = createXorshift32(fnv1a32(seed));
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const offset = (y * width + x) * 3;
const noise = textured ? (nextRandom() & 0x3f) - 32 : 0;
pixels[offset] = Math.max(0, Math.min(255, Math.floor((x * 255) / (width - 1)) + noise));
pixels[offset + 1] = Math.max(0, Math.min(255, Math.floor((y * 255) / (height - 1)) + noise));
pixels[offset + 2] = Math.max(0, Math.min(255, Math.floor(((x + y) * 255) / (width + height - 2)) - noise));
}
}
const result = new Uint8Array(header.length + pixels.length);
result.set(header);
result.set(pixels, header.length);
return result;
}
async function gzip(input: Uint8Array): Promise<Uint8Array> {
const copied = new Uint8Array(input.byteLength);
copied.set(input);
const stream = new Blob([copied.buffer]).stream().pipeThrough(new CompressionStream("gzip"));
return new Uint8Array(await new Response(stream).arrayBuffer());
}
export async function encodeJpegWithCjpeg(inputPpm: string, outputJpeg: string): Promise<string> {
const command = new Deno.Command("cjpeg", {
args: ["-quality", "85", "-optimize", "-outfile", outputJpeg, inputPpm],
stdin: "null",
stdout: "piped",
stderr: "piped",
});
let result: Deno.CommandOutput;
try {
result = await command.output();
} catch (error) {
if (error instanceof Deno.errors.NotFound) {
throw new Error(
"cjpeg is required for the compression benchmark. Use the Compose runner or install libjpeg tools."
);
}
throw error;
}
if (!result.success) {
throw new Error(`cjpeg failed: ${new TextDecoder().decode(result.stderr)}`);
}
return "cjpeg quality=85, optimise=true, synthetic PPM 640x480";
}
async function writeRandomBinary(path: string, size: number, seed: string): Promise<void> {
const bytes = new Uint8Array(size);
const nextRandom = createXorshift32(fnv1a32(seed));
for (let index = 0; index < bytes.length; index++) {
bytes[index] = nextRandom() & 0xff;
}
await Deno.writeFile(path, bytes);
}
export async function createCompressionBenchmarkDataset(options: {
rootDir: string;
datasetDirName?: string;
repositoryRoot?: string;
seed?: string;
jpegEncoder?: JpegEncoder;
}): Promise<CompressionDataset> {
const datasetDirName = options.datasetDirName ?? "compression-benchmark";
const repositoryRoot = options.repositoryRoot ?? REPOSITORY_ROOT;
const seed = options.seed ?? "livesync-compression-benchmark";
const jpegEncoder = options.jpegEncoder ?? encodeJpegWithCjpeg;
const datasetRoot = join(options.rootDir, datasetDirName);
const entries: CompressionDatasetEntry[] = [];
let jpegGenerator = "";
for (const kind of ALL_KINDS) {
await Deno.mkdir(join(datasetRoot, kind), { recursive: true });
}
const addFile = async (kind: DatasetKind, absolutePath: string, source: string) => {
const relativePath = absolutePath
.slice(options.rootDir.length + 1)
.split("\\")
.join("/");
const size = (await Deno.stat(absolutePath)).size;
entries.push({ kind, relativePath, absolutePath, size, source });
};
const copyRepositoryFile = async (kind: DatasetKind, sourcePath: string, targetName: string) => {
const destination = join(datasetRoot, kind, targetName);
await Deno.copyFile(join(repositoryRoot, sourcePath), destination);
await addFile(kind, destination, sourcePath);
};
await copyRepositoryFile("md", "docs/settings.md", "settings.md");
await copyRepositoryFile("md", "docs/quick_setup.md", "quick-setup.md");
await copyRepositoryFile("md", "updates.md", "updates.md");
await copyRepositoryFile("png", "instruction_images/cloudant_1.png", "cloudant-1.png");
await copyRepositoryFile(
"png",
"images/quick-setup/guide-quick-setup-first-setup-uri.png",
"quick-setup-first-setup-uri.png"
);
await copyRepositoryFile("json", "package.json", "package.json");
await copyRepositoryFile("json", "manifest.json", "manifest.json");
await copyRepositoryFile("ts", "src/modules/core/ModuleReplicator.ts", "ModuleReplicator.ts");
await copyRepositoryFile("ts", "src/modules/core/ReplicateResultProcessor.ts", "ReplicateResultProcessor.ts");
const markdownBytes = await Deno.readFile(join(repositoryRoot, "docs/settings.md"));
const gzipPath = join(datasetRoot, "gz", "settings.md.gz");
await Deno.writeFile(gzipPath, await gzip(markdownBytes));
await addFile("gz", gzipPath, "generated gzip of docs/settings.md");
const randomPath = join(datasetRoot, "bin", "deterministic-random.bin");
await writeRandomBinary(randomPath, 256 * 1024, seed);
await addFile("bin", randomPath, `deterministic xorshift32 seed=${seed}`);
for (const [name, textured] of [
["smooth-gradient.jpg", false],
["textured-gradient.jpg", true],
] as const) {
const ppmPath = await Deno.makeTempFile({ dir: options.rootDir, prefix: "compression-jpeg-", suffix: ".ppm" });
const jpegPath = join(datasetRoot, "jpg", name);
try {
await Deno.writeFile(ppmPath, createSyntheticPpm(640, 480, `${seed}-${name}`, textured));
jpegGenerator = await jpegEncoder(ppmPath, jpegPath);
} finally {
await Deno.remove(ppmPath).catch(() => {});
}
await addFile("jpg", jpegPath, `${jpegGenerator}; ${textured ? "textured" : "smooth"}`);
}
const bytesByKind = Object.fromEntries(ALL_KINDS.map((kind) => [kind, 0])) as Record<DatasetKind, number>;
const filesByKind = Object.fromEntries(ALL_KINDS.map((kind) => [kind, 0])) as Record<DatasetKind, number>;
for (const entry of entries) {
bytesByKind[entry.kind] += entry.size;
filesByKind[entry.kind] += 1;
}
return {
entries,
totalFiles: entries.length,
totalBytes: entries.reduce((sum, entry) => sum + entry.size, 0),
bytesByKind,
filesByKind,
jpegGenerator,
};
}
src/apps/cli/testdeno/helpers/dataset.ts
+3 -1
View File
@@ -9,8 +9,10 @@ export type DeterministicDatasetConfig = {
binSizeBytes: number;
};
export type DatasetKind = "md" | "jpg" | "png" | "json" | "ts" | "gz" | "bin";
export type DatasetEntry = {
kind: "md" | "bin";
kind: DatasetKind;
relativePath: string;
absolutePath: string;
size: number;
src/apps/cli/testdeno/helpers/measuredCli.ts
+61
View File
@@ -0,0 +1,61 @@
import { CLI_DIR, CLI_DIST } from "./cli.ts";
export type CliProcessMeasurement = {
elapsedMs: number;
userCpuMs: number;
systemCpuMs: number;
totalCpuMs: number;
cpuToWallRatio: number;
maxResidentSetKiB: number;
};
const MARKER = "__LIVESYNC_GNU_TIME__";
export async function runMeasuredCliOrFail(...args: string[]): Promise<CliProcessMeasurement> {
const started = performance.now();
let output: Deno.CommandOutput;
try {
output = await new Deno.Command("/usr/bin/time", {
args: ["-f", `${MARKER}%U\t%S\t%M`, "node", CLI_DIST, ...args],
cwd: CLI_DIR,
stdin: "null",
stdout: "piped",
stderr: "piped",
}).output();
} catch (error) {
if (error instanceof Deno.errors.NotFound) {
throw new Error(
"GNU /usr/bin/time is required for the compression benchmark. Use the Compose runner or install GNU time."
);
}
throw error;
}
const elapsedMs = performance.now() - started;
const stderr = new TextDecoder().decode(output.stderr);
const stdout = new TextDecoder().decode(output.stdout);
const measurementLine = stderr.split(/\r?\n/).find((line) => line.startsWith(MARKER));
if (!output.success) {
throw new Error(`CLI exited with code ${output.code}\nstdout: ${stdout}\nstderr: ${stderr}`);
}
if (!measurementLine) {
throw new Error(`GNU time did not emit the expected measurement marker\nstderr: ${stderr}`);
}
const [userSeconds, systemSeconds, maxResidentSetKiB] = measurementLine
.slice(MARKER.length)
.split("\t")
.map(Number);
if (![userSeconds, systemSeconds, maxResidentSetKiB].every(Number.isFinite)) {
throw new Error(`Could not parse GNU time measurement: ${measurementLine}`);
}
const userCpuMs = userSeconds * 1000;
const systemCpuMs = systemSeconds * 1000;
const totalCpuMs = userCpuMs + systemCpuMs;
return {
elapsedMs: Number(elapsedMs.toFixed(1)),
userCpuMs: Number(userCpuMs.toFixed(1)),
systemCpuMs: Number(systemCpuMs.toFixed(1)),
totalCpuMs: Number(totalCpuMs.toFixed(1)),
cpuToWallRatio: Number((totalCpuMs / elapsedMs).toFixed(4)),
maxResidentSetKiB,
};
}
src/apps/cli/testdeno/helpers/settings.ts
+13 -11
View File
@@ -1,4 +1,3 @@
import { join } from "@std/path";
import { CLI_DIR, runCliOrFail } from "./cli.ts";
// ---------------------------------------------------------------------------
@@ -11,18 +10,14 @@ export async function initSettingsFile(settingsFile: string): Promise<void> {
}
/**
* Generate a full setup URI from a settings file via src/lib API.
* Generate a full setup URI from a settings file via the Commonlib package API.
* Mirrors the bash flow in test-setup-put-cat-linux.sh.
*/
export async function generateSetupUriFromSettings(settingsFile: string, setupPassphrase: string): Promise<string> {
const repoRoot = join(CLI_DIR, "..", "..", "..");
const script = [
"import fs from 'node:fs';",
"import { pathToFileURL } from 'node:url';",
"import { fs } from '@vrtmrz/livesync-commonlib/node';",
"import { encodeSettingsToSetupURI } from '@vrtmrz/livesync-commonlib/compat/API/processSetting';",
"(async () => {",
" const modulePath = process.env.REPO_ROOT + '/src/lib/src/API/processSetting.ts';",
" const moduleUrl = pathToFileURL(modulePath).href;",
" const { encodeSettingsToSetupURI } = await import(moduleUrl);",
" const settingsPath = process.env.SETTINGS_FILE;",
" const passphrase = process.env.SETUP_PASSPHRASE;",
" const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf-8'));",
@@ -39,6 +34,7 @@ export async function generateSetupUriFromSettings(settingsFile: string, setupPa
].join("\n");
const scriptPath = await Deno.makeTempFile({
dir: CLI_DIR,
prefix: "livesync-setup-uri-",
suffix: ".mts",
});
@@ -49,7 +45,6 @@ export async function generateSetupUriFromSettings(settingsFile: string, setupPa
args: ["tsx", scriptPath],
cwd: CLI_DIR,
env: {
REPO_ROOT: repoRoot,
SETTINGS_FILE: settingsFile,
SETUP_PASSPHRASE: setupPassphrase,
},
@@ -128,6 +123,8 @@ export async function applyRemoteSyncSettings(
minioSecretKey?: string;
encrypt?: boolean;
passphrase?: string;
enableCompression?: boolean;
usePathObfuscation?: boolean;
}
): Promise<void> {
const data = JSON.parse(await Deno.readTextFile(settingsFile));
@@ -154,6 +151,12 @@ export async function applyRemoteSyncSettings(
data.usePluginSync = false;
data.encrypt = options.encrypt === true;
data.passphrase = options.encrypt ? (options.passphrase ?? "") : "";
if (options.enableCompression !== undefined) {
data.enableCompression = options.enableCompression;
}
if (options.usePathObfuscation !== undefined) {
data.usePathObfuscation = options.usePathObfuscation;
}
data.isConfigured = true;
await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2));
}
@@ -200,9 +203,8 @@ export async function applyP2pTestTweaks(settingsFile: string, deviceName: strin
data.passphrase = passphrase;
data.usePathObfuscation = true;
data.handleFilenameCaseSensitive = false;
data.customChunkSize = 50;
data.customChunkSize = 60;
data.usePluginSyncV2 = true;
data.doNotUseFixedRevisionForChunks = false;
data.P2P_DevicePeerName = deviceName;
data.isConfigured = true;
await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2));
src/apps/cli/testdeno/relay-disconnect-probe.mjs
+129
View File
@@ -0,0 +1,129 @@
import { createRequire } from "node:module";
import { pathToFileURL } from "node:url";
import { RTCPeerConnection } from "werift";
import { TrysteroReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/TrysteroReplicator";
const requireFromProbe = createRequire(import.meta.url);
const commonlibEntry = requireFromProbe.resolve("@vrtmrz/livesync-commonlib/context");
const requireFromCommonlib = createRequire(commonlibEntry);
const nostrEntry = requireFromCommonlib.resolve("@trystero-p2p/nostr");
const { getRelaySockets, joinRoom, pauseRelayReconnection } = await import(pathToFileURL(nostrEntry).href);
const relayUrl = process.env.RELAY ?? "ws://nostr-relay:7777/";
const timeoutMs = Number(process.env.RELAY_TIMEOUT_MS ?? 15_000);
function delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function waitFor(description, predicate) {
const deadline = Date.now() + timeoutMs;
while (Date.now() <= deadline) {
if (predicate()) return;
await delay(50);
}
throw new Error(`Timed out waiting for ${description}`);
}
async function waitForSocketOpen(socket, description) {
if (socket.readyState === WebSocket.OPEN) {
// Node can expose OPEN before it dispatches the event. Yield once so
// Trystero's previously registered onopen handler has completed before
// this probe starts the close handshake.
await delay(0);
if (socket.readyState === WebSocket.OPEN) return;
}
await new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
cleanup();
reject(new Error(`Timed out waiting for ${description}; readyState=${socket.readyState}`));
}, timeoutMs);
const cleanup = () => {
clearTimeout(timeout);
socket.removeEventListener("open", onOpen);
socket.removeEventListener("close", onClose);
socket.removeEventListener("error", onError);
};
const onOpen = () => {
cleanup();
resolve();
};
const onClose = () => {
cleanup();
reject(new Error(`${description} closed before opening`));
};
const onError = () => {
cleanup();
reject(new Error(`${description} failed before opening`));
};
// Trystero installs its onopen handler while constructing the socket,
// before this observer is registered. Waiting for the actual event
// therefore establishes transport readiness without a fixed delay.
socket.addEventListener("open", onOpen, { once: true });
socket.addEventListener("close", onClose, { once: true });
socket.addEventListener("error", onError, { once: true });
});
}
const room = joinRoom(
{
appId: `livesync-relay-disconnect-probe-${Date.now()}`,
password: "local-test-only",
relayConfig: {
urls: [relayUrl],
manualReconnection: true,
},
rtcPolyfill: RTCPeerConnection,
},
"disconnect-probe"
);
try {
await waitFor("the relay WebSocket to be registered", () => Object.values(getRelaySockets()).length > 0);
const originalSockets = Object.values(getRelaySockets());
await Promise.all(
originalSockets.map((socket, index) => waitForSocketOpen(socket, `relay WebSocket ${index + 1} to open`))
);
const replicator = new TrysteroReplicator(
{},
{
close: async () => undefined,
dispatchConnectionStatus: async () => undefined,
}
);
replicator.disconnectFromServer();
await waitFor("all relay WebSockets to close", () =>
originalSockets.every((socket) => socket.readyState === WebSocket.CLOSED)
);
await delay(4_000);
if (!originalSockets.every((socket) => socket.readyState === WebSocket.CLOSED)) {
throw new Error("A relay WebSocket reconnected while reconnection was paused");
}
replicator.allowReconnection();
await waitFor("a replacement relay WebSocket to be registered", () =>
Object.values(getRelaySockets()).some((socket) => !originalSockets.includes(socket))
);
const replacementSockets = Object.values(getRelaySockets()).filter((socket) => !originalSockets.includes(socket));
await Promise.all(
replacementSockets.map((socket, index) =>
waitForSocketOpen(socket, `replacement relay WebSocket ${index + 1} to open`)
)
);
console.log(
JSON.stringify({
socketsClosed: originalSockets.length,
stayedDisconnectedWhilePaused: true,
reconnectedAfterResume: true,
})
);
} finally {
await room.leave();
pauseRelayReconnection();
for (const socket of Object.values(getRelaySockets())) socket.close();
}
src/apps/cli/testdeno/run-ci-suite.ts
+33
View File
@@ -0,0 +1,33 @@
const TASKS = [
"test:settings-writeback",
"test:setup-put-cat",
"test:mirror",
"test:daemon",
"test:push-pull",
"test:decoupled-vault",
"test:sync-two-local",
"test:sync-locked-remote",
"test:remote-commands",
"test:e2e-matrix:couchdb-enc0",
"test:e2e-matrix:couchdb-enc1",
"test:e2e-matrix:minio-enc0",
"test:e2e-matrix:minio-enc1",
] as const;
for (const [index, task] of TASKS.entries()) {
console.log(`\n[CLI E2E ${index + 1}/${TASKS.length}] ${task}`);
const child = new Deno.Command(Deno.execPath(), {
args: ["task", task],
cwd: import.meta.dirname,
stdin: "inherit",
stdout: "inherit",
stderr: "inherit",
}).spawn();
const status = await child.status;
if (!status.success) {
console.error(`[CLI E2E] ${task} failed with exit code ${status.code}.`);
Deno.exit(status.code);
}
}
console.log(`\n[CLI E2E] CI suite passed (${TASKS.length} tasks).`);
src/apps/cli/testdeno/run-cli-e2e.sh
+3 -3
View File
@@ -1,15 +1,15 @@
#!/usr/bin/env sh
set -eu
TASK="${CLI_E2E_TASK:-test:p2p-sync}"
TASK="${CLI_E2E_TASK:-test:p2p:ci}"
case "$TASK" in
test:p2p-host|test:p2p-peers|test:p2p-sync|test:p2p-three-nodes|test:p2p-upload-download)
test:p2p-host|test:p2p-peers|test:p2p-sync|test:p2p-replacement|test:p2p-relay-disconnect|test:p2p:ci|test:p2p-three-nodes|test:p2p-upload-download)
exec deno task "$TASK"
;;
*)
echo "Unknown CLI_E2E_TASK: $TASK" >&2
echo "Expected one of: test:p2p-host, test:p2p-peers, test:p2p-sync, test:p2p-three-nodes, test:p2p-upload-download" >&2
echo "Expected one of: test:p2p-host, test:p2p-peers, test:p2p-sync, test:p2p-replacement, test:p2p-relay-disconnect, test:p2p:ci, test:p2p-three-nodes, test:p2p-upload-download" >&2
exit 2
;;
esac
src/apps/cli/testdeno/run-compose-p2p.ts
+50
View File
@@ -0,0 +1,50 @@
const repositoryRoot = await Deno.realPath(new URL("../../../../", import.meta.url));
const composeArgs = ["compose", "-f", "test/bench-network/compose.yml"];
const p2pEnvironment = {
CLI_E2E_TASK: Deno.env.get("CLI_E2E_TASK") ?? "test:p2p:ci",
RELAY: Deno.env.get("RELAY") ?? "ws://nostr-relay:7777/",
PEERS_TIMEOUT: Deno.env.get("PEERS_TIMEOUT") ?? "20",
SYNC_TIMEOUT: Deno.env.get("SYNC_TIMEOUT") ?? "60",
LIVESYNC_USE_COTURN: Deno.env.get("LIVESYNC_USE_COTURN") ?? "0",
TURN_SERVERS: Deno.env.get("TURN_SERVERS") ?? "none",
LIVESYNC_P2P_PEERS_RETRY: Deno.env.get("LIVESYNC_P2P_PEERS_RETRY") ?? "1",
LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "60000",
BENCH_LIVESYNC_TEST_TEE: Deno.env.get("BENCH_LIVESYNC_TEST_TEE") ?? "0",
LIVESYNC_CLI_DEBUG: Deno.env.get("LIVESYNC_CLI_DEBUG") ?? "0",
LIVESYNC_CLI_VERBOSE: Deno.env.get("LIVESYNC_CLI_VERBOSE") ?? "0",
};
async function runDocker(args: string[], env?: Record<string, string>): Promise<Deno.CommandStatus> {
return await new Deno.Command("docker", {
args,
cwd: repositoryRoot,
env,
stdin: "inherit",
stdout: "inherit",
stderr: "inherit",
}).spawn().status;
}
let testStatus: Deno.CommandStatus | undefined;
try {
testStatus = await runDocker(
[...composeArgs, "run", "--build", "--rm", "bench-runner", "run-livesync-cli-e2e"],
p2pEnvironment
);
} finally {
const cleanupStatus = await runDocker([...composeArgs, "down", "-v", "--remove-orphans"]);
if (!cleanupStatus.success) {
console.error(`[CLI E2E] Compose cleanup failed with exit code ${cleanupStatus.code}.`);
if (testStatus?.success) {
Deno.exit(cleanupStatus.code);
}
}
}
if (!testStatus?.success) {
const code = testStatus?.code ?? 1;
console.error(`[CLI E2E] Compose P2P suite failed with exit code ${code}.`);
Deno.exit(code);
}
console.log("\n[CLI E2E] Compose P2P suite passed.");
src/apps/cli/testdeno/test-benchmark-contract.ts
+111
View File
@@ -6,6 +6,7 @@ import {
selectVerificationEntries,
} from "./helpers/benchmarkVerification.ts";
import type { DatasetEntry } from "./helpers/dataset.ts";
import { createCompressionBenchmarkDataset } from "./helpers/compressionDataset.ts";
function getFreePort(): number {
const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 });
@@ -106,6 +107,22 @@ Deno.test("CouchDB latency proxy applies half the requested RTT in each directio
assertEquals(await response.text(), "ok");
assertEquals(proxy.directionalDelayMs, 10);
assertEquals(delays, [10, 10]);
assertEquals(proxy.snapshotCounters(), {
requestCount: 1,
requestBodyBytes: 0,
responseBodyBytes: 2,
});
proxy.resetCounters();
const posted = await fetch(`http://127.0.0.1:${proxyPort}/probe`, {
method: "POST",
body: "abc",
});
assertEquals(await posted.text(), "ok");
assertEquals(proxy.snapshotCounters(), {
requestCount: 1,
requestBodyBytes: 3,
responseBodyBytes: 2,
});
} finally {
await proxy.stop();
await backend.shutdown();
@@ -124,6 +141,100 @@ Deno.test("CouchDB latency proxy applies half the requested RTT in each directio
}
});
Deno.test("compression benchmark dataset covers representative file kinds deterministically", async () => {
const fixtureRoot = await Deno.makeTempDir({
prefix: "livesync-compression-contract-",
});
const repositoryRoot = `${fixtureRoot}/repository`;
const vaultA = `${fixtureRoot}/vault-a`;
const vaultB = `${fixtureRoot}/vault-b`;
const repositoryFiles = [
"docs/settings.md",
"docs/quick_setup.md",
"updates.md",
"instruction_images/cloudant_1.png",
"images/quick-setup/guide-quick-setup-first-setup-uri.png",
"package.json",
"manifest.json",
"src/modules/core/ModuleReplicator.ts",
"src/modules/core/ReplicateResultProcessor.ts",
];
try {
for (const [index, relativePath] of repositoryFiles.entries()) {
const absolutePath = `${repositoryRoot}/${relativePath}`;
await Deno.mkdir(
absolutePath.slice(0, absolutePath.lastIndexOf("/")),
{ recursive: true },
);
await Deno.writeFile(
absolutePath,
new TextEncoder().encode(
`fixture-${index}-${relativePath}\n`.repeat(20),
),
);
}
await Deno.mkdir(vaultA, { recursive: true });
await Deno.mkdir(vaultB, { recursive: true });
const jpegEncoder = async (_input: string, output: string) => {
await Deno.writeFile(
output,
new Uint8Array([
0xff,
0xd8,
0xff,
0xdb,
0,
1,
2,
3,
0xff,
0xd9,
]),
);
return "contract JPEG encoder";
};
const first = await createCompressionBenchmarkDataset({
rootDir: vaultA,
repositoryRoot,
seed: "contract-seed",
jpegEncoder,
});
const second = await createCompressionBenchmarkDataset({
rootDir: vaultB,
repositoryRoot,
seed: "contract-seed",
jpegEncoder,
});
assertEquals(first.filesByKind, {
md: 3,
jpg: 2,
png: 2,
json: 2,
ts: 2,
gz: 1,
bin: 1,
});
assertEquals(first.totalFiles, 13);
assertEquals(first.jpegGenerator, "contract JPEG encoder");
assertEquals(
first.entries.map((entry) => [
entry.kind,
entry.relativePath,
entry.size,
]),
second.entries.map((entry) => [
entry.kind,
entry.relativePath,
entry.size,
]),
);
assert(first.entries.every((entry) => entry.size > 0));
} finally {
await Deno.remove(fixtureRoot, { recursive: true }).catch(() => {});
}
});
Deno.test("benchmark verification mode selects either all files or a labelled sample", () => {
const entries: DatasetEntry[] = [
{ kind: "md", relativePath: "a.md", absolutePath: "/a", size: 1 },
src/apps/cli/testdeno/test-p2p-relay-disconnect.ts
+26
View File
@@ -0,0 +1,26 @@
import { assert, assertEquals } from "@std/assert";
Deno.test("p2p lifecycle: explicit disconnect closes and pauses relay WebSockets", async () => {
const command = new Deno.Command("node", {
args: [new URL("./relay-disconnect-probe.mjs", import.meta.url).pathname],
env: {
RELAY: Deno.env.get("RELAY") ?? "ws://nostr-relay:7777/",
RELAY_TIMEOUT_MS: Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000",
},
stdout: "piped",
stderr: "piped",
});
const result = await command.output();
const stdout = new TextDecoder().decode(result.stdout).trim();
const stderr = new TextDecoder().decode(result.stderr).trim();
assert(result.success, `Relay disconnect probe failed\nstdout: ${stdout}\nstderr: ${stderr}`);
const report = JSON.parse(stdout.split("\n").at(-1) ?? "{}") as {
socketsClosed?: number;
stayedDisconnectedWhilePaused?: boolean;
reconnectedAfterResume?: boolean;
};
assert((report.socketsClosed ?? 0) > 0, "The probe did not observe an open relay WebSocket");
assertEquals(report.stayedDisconnectedWhilePaused, true);
assertEquals(report.reconnectedAfterResume, true);
});
src/apps/cli/testdeno/test-p2p-replicator-replacement.ts
+117
View File
@@ -0,0 +1,117 @@
import { assert, assertEquals, assertStringIncludes } from "@std/assert";
import { join } from "@std/path";
import { TempDir } from "./helpers/temp.ts";
import { initSettingsFile, applyP2pSettings, applyP2pTestTweaks } from "./helpers/settings.ts";
import { startCliInBackground } from "./helpers/backgroundCli.ts";
import { maybeStartLocalRelay, stopLocalRelayIfStarted, maybeStartCoturn, stopCoturnIfStarted } from "./helpers/p2p.ts";
import { CLI_DIR, runCli, sanitiseCatStdout } from "./helpers/cli.ts";
const NOTE_PATH = "p2p-replicator-replacement.md";
const NOTE_CONTENT = "Replicated after replacing the active P2P replicator.";
async function runReplacementProbe(
vaultPath: string,
settingsPath: string,
targetPeer: string,
timeoutMs: number
): Promise<{ code: number; stdout: string; stderr: string }> {
const command = new Deno.Command("node", {
args: [
join(CLI_DIR, "dist", "p2p-lifecycle-test.cjs"),
vaultPath,
"--settings",
settingsPath,
"p2p-sync",
targetPeer,
String(timeoutMs / 1000),
NOTE_PATH,
NOTE_CONTENT,
],
cwd: CLI_DIR,
stdout: "piped",
stderr: "piped",
});
const result = await command.output();
return {
code: result.code,
stdout: new TextDecoder().decode(result.stdout),
stderr: new TextDecoder().decode(result.stderr),
};
}
Deno.test("p2p lifecycle: replacement keeps real CLI communication on the current replicator", async () => {
const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/";
const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "20");
const syncTimeout = Number(Deno.env.get("SYNC_TIMEOUT") ?? "60");
const probeTimeoutMs = Math.max(peersTimeout, syncTimeout) * 1000;
const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`;
const roomId = Deno.env.get("ROOM_ID") ?? `replacement-room-${nonce}`;
const passphrase = Deno.env.get("PASSPHRASE") ?? `replacement-pass-${nonce}`;
const appId = "self-hosted-livesync-cli-replacement-test";
const hostPeerName = `p2p-replacement-host-${nonce}`;
const probePeerName = `p2p-replacement-probe-${nonce}`;
const verifierPeerName = `p2p-replacement-verifier-${nonce}`;
const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0";
const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? "turn:127.0.0.1:3478" : "none");
await using workDir = await TempDir.create("livesync-cli-p2p-replacement");
const hostVault = workDir.join("vault-host");
const probeVault = workDir.join("vault-probe");
const verifierVault = workDir.join("vault-verifier");
const hostSettings = workDir.join("settings-host.json");
const probeSettings = workDir.join("settings-probe.json");
const verifierSettings = workDir.join("settings-verifier.json");
await Promise.all([
Deno.mkdir(hostVault, { recursive: true }),
Deno.mkdir(probeVault, { recursive: true }),
Deno.mkdir(verifierVault, { recursive: true }),
]);
const relayStarted = await maybeStartLocalRelay(relay);
const coturnStarted = await maybeStartCoturn(turnServers);
try {
for (const settingsPath of [hostSettings, probeSettings, verifierSettings]) {
await initSettingsFile(settingsPath);
await applyP2pSettings(settingsPath, roomId, passphrase, appId, relay, "~.*", turnServers);
}
await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase);
await applyP2pTestTweaks(probeSettings, probePeerName, passphrase);
await applyP2pTestTweaks(verifierSettings, verifierPeerName, passphrase);
const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
try {
await host.waitUntilContains("P2P host is running", 20000);
const probe = await runReplacementProbe(probeVault, probeSettings, hostPeerName, probeTimeoutMs);
assert(
probe.code === 0,
`P2P replacement probe failed\nstdout: ${probe.stdout}\nstderr: ${probe.stderr}`
);
assertStringIncludes(probe.stdout, "[Probe] P2P replicator replaced");
const syncResult = await runCli(
verifierVault,
"--settings",
verifierSettings,
"p2p-sync",
hostPeerName,
String(syncTimeout)
);
assert(
syncResult.code === 0,
`Verifier P2P sync failed\nstdout: ${syncResult.stdout}\nstderr: ${syncResult.stderr}`
);
const catResult = await runCli(verifierVault, "--settings", verifierSettings, "cat", NOTE_PATH);
assert(
catResult.code === 0,
`Verifier could not read ${NOTE_PATH}\nstdout: ${catResult.stdout}\nstderr: ${catResult.stderr}`
);
assertEquals(sanitiseCatStdout(catResult.stdout).trim(), NOTE_CONTENT);
} finally {
await host.stop();
}
} finally {
await stopLocalRelayIfStarted(relayStarted);
await stopCoturnIfStarted(coturnStarted);
}
});
src/apps/cli/testdeno/test-settings-writeback.ts
+134
View File
@@ -0,0 +1,134 @@
import { assert, assertEquals } from "@std/assert";
import { TempDir } from "./helpers/temp.ts";
import { runCli } from "./helpers/cli.ts";
import { initSettingsFile } from "./helpers/settings.ts";
async function prepareSettingsFixture(prefix: string) {
const workDir = await TempDir.create(prefix);
const settingsFile = workDir.join("settings.json");
const databaseDir = workDir.join("database");
await Deno.mkdir(databaseDir, { recursive: true });
await initSettingsFile(settingsFile);
return { workDir, settingsFile, databaseDir };
}
Deno.test("settings-changing commands persist durable settings without CLI runtime suspension", async () => {
const fixture = await prepareSettingsFixture("livesync-cli-settings-command");
await using workDir = fixture.workDir;
const { settingsFile, databaseDir } = fixture;
const settings = JSON.parse(await Deno.readTextFile(settingsFile));
settings.liveSync = true;
settings.syncOnStart = true;
settings.periodicReplication = true;
settings.P2P_Enabled = true;
settings.P2P_AutoStart = true;
settings.P2P_AutoBroadcast = true;
await Deno.writeTextFile(settingsFile, JSON.stringify(settings, null, 2));
const result = await runCli(
databaseDir,
"--settings",
settingsFile,
"remote-add",
"test-remote",
"sls+https://user:pass@example.com/database"
);
assertEquals(result.code, 0, result.combined);
const firstRemoteId = result.stdout.trim().split("\t")[0];
assert(firstRemoteId, `remote-add did not return an ID: ${result.combined}`);
let persisted = JSON.parse(await Deno.readTextFile(settingsFile));
let remotes = Object.values(persisted.remoteConfigurations ?? {}) as Array<{ name?: string }>;
assert(
remotes.some((remote) => remote.name === "test-remote"),
"remote-add did not persist the new profile"
);
assertEquals(persisted.liveSync, true);
assertEquals(persisted.syncOnStart, true);
assertEquals(persisted.periodicReplication, true);
assertEquals(persisted.P2P_Enabled, true);
assertEquals(persisted.P2P_AutoStart, true);
assertEquals(persisted.P2P_AutoBroadcast, true);
const secondAdd = await runCli(
databaseDir,
"--settings",
settingsFile,
"remote-add",
"second-remote",
"sls+https://other:secret@example.net/second"
);
assertEquals(secondAdd.code, 0, secondAdd.combined);
const secondRemoteId = secondAdd.stdout.trim().split("\t")[0];
assert(secondRemoteId, `second remote-add did not return an ID: ${secondAdd.combined}`);
const activate = await runCli(databaseDir, "--settings", settingsFile, "remote-activate", secondRemoteId);
assertEquals(activate.code, 0, activate.combined);
persisted = JSON.parse(await Deno.readTextFile(settingsFile));
assertEquals(persisted.activeConfigurationId, secondRemoteId);
const set = await runCli(
databaseDir,
"--settings",
settingsFile,
"remote-set",
secondRemoteId,
"sls+https://replacement:secret@example.org/replaced"
);
assertEquals(set.code, 0, set.combined);
const exported = await runCli(databaseDir, "--settings", settingsFile, "remote-export", secondRemoteId);
assertEquals(exported.code, 0, exported.combined);
assert(exported.stdout.includes("replacement"), "remote-set did not persist the replacement URI");
const remove = await runCli(databaseDir, "--settings", settingsFile, "remote-rm", secondRemoteId);
assertEquals(remove.code, 0, remove.combined);
persisted = JSON.parse(await Deno.readTextFile(settingsFile));
remotes = Object.values(persisted.remoteConfigurations ?? {}) as Array<{ id?: string }>;
assert(!remotes.some((remote) => remote.id === secondRemoteId), "remote-rm did not persist the removal");
assertEquals(persisted.activeConfigurationId, firstRemoteId);
});
Deno.test("ordinary commands keep the settings file unchanged by default", async () => {
const fixture = await prepareSettingsFixture("livesync-cli-settings-readonly");
await using workDir = fixture.workDir;
const { settingsFile, databaseDir } = fixture;
const settings = JSON.parse(await Deno.readTextFile(settingsFile));
settings.settingVersion = 9;
const original = JSON.stringify(settings, null, 2);
await Deno.writeTextFile(settingsFile, original);
const result = await runCli(databaseDir, "--settings", settingsFile, "ls");
assertEquals(result.code, 0, result.combined);
assertEquals(await Deno.readTextFile(settingsFile), original);
});
Deno.test("--write-settings persists durable start-up setting changes", async () => {
const fixture = await prepareSettingsFixture("livesync-cli-settings-explicit");
await using workDir = fixture.workDir;
const { settingsFile, databaseDir } = fixture;
const settings = JSON.parse(await Deno.readTextFile(settingsFile));
settings.settingVersion = 9;
delete settings.useIndexedDBAdapter;
await Deno.writeTextFile(settingsFile, JSON.stringify(settings, null, 2));
const result = await runCli(databaseDir, "--settings", settingsFile, "--write-settings", "ls");
assertEquals(result.code, 0, result.combined);
const persisted = JSON.parse(await Deno.readTextFile(settingsFile));
assertEquals(persisted.settingVersion, 10);
assert(!("useIndexedDBAdapter" in persisted), "the CLI-only adapter override was written to the settings file");
});
Deno.test("failed settings-changing commands leave the settings file unchanged", async () => {
const fixture = await prepareSettingsFixture("livesync-cli-settings-failure");
await using workDir = fixture.workDir;
const { settingsFile, databaseDir } = fixture;
const original = await Deno.readTextFile(settingsFile);
const result = await runCli(databaseDir, "--settings", settingsFile, "remote-rm", "missing-remote");
assert(result.code !== 0, "remote-rm unexpectedly succeeded");
assertEquals(await Deno.readTextFile(settingsFile), original);
});
src/apps/cli/testdeno/test-setup-put-cat.ts
+7
View File
@@ -41,6 +41,13 @@ Deno.test("CLI file operations: push / cat / ls / info / rm / resolve / cat-rev
setupResult.combined.includes("[Command] setup ->"),
`setup command did not execute expected code path\n${setupResult.combined}`
);
const persistedSetup = JSON.parse(await Deno.readTextFile(settingsFile));
assertEquals(persistedSetup.isConfigured, true, "setup did not persist the configured state");
assert(
typeof persistedSetup.encryptedCouchDBConnection === "string" &&
persistedSetup.encryptedCouchDBConnection.length > 0,
"setup did not persist the encrypted connection settings"
);
const run = (...args: string[]) => runCliOrFail(vaultDir, "--settings", settingsFile, ...args);
src/apps/cli/testdeno/test_dev_deno.md
+22 -3
View File
@@ -1,13 +1,13 @@
# CLI Deno Test Development Notes
This document provides an overview of the Deno-based compatibility tests under `src/apps/cli/testdeno/`.
The existing bash tests under `src/apps/cli/test/` are preserved, while a Windows-friendly suite is maintained in parallel.
The Deno suite is the canonical CLI E2E entry point. P2P scenarios run through the repository Compose entry point so that networking, signalling, and the runner environment are reproducible. Existing Bash tests under `src/apps/cli/test/` remain as legacy implementation references, but are not exposed as supported P2P entry points.
---
## Goals
- Keep existing bash tests intact.
- Keep the existing Bash tests as migration references while using Deno and Compose for supported execution.
- Provide direct execution from Windows PowerShell.
- Establish a TypeScript (Deno) foundation for core end-to-end and integration scenarios.
@@ -18,6 +18,8 @@ The existing bash tests under `src/apps/cli/test/` are preserved, while a Window
```
src/apps/cli/testdeno/
deno.json
run-ci-suite.ts
run-compose-p2p.ts
CONTRIBUTING_TESTS.md
helpers/
backgroundCli.ts
@@ -56,6 +58,8 @@ src/apps/cli/testdeno/
Main tasks:
- `deno task test`
- `deno task test:ci`
- `deno task test:p2p:compose`
- `deno task test:local`
- `deno task test:daemon`
- `deno task test:decoupled-vault`
@@ -73,6 +77,8 @@ Main tasks:
- `deno task test:e2e-couchdb`
- `deno task test:e2e-matrix`
`deno task test` is an alias for the non-P2P `test:ci` suite. The individual P2P tasks are explicit host-direct entry points for cross-platform diagnostics; they are never selected by the default suite or CI. Use `test:p2p:compose` for canonical P2P verification.
### `helpers/cli.ts`
- CLI execution wrappers.
@@ -206,11 +212,23 @@ Both CouchDB and P2P relay flows are bash-independent.
## Running tests (PowerShell)
From the repository root, use the canonical package scripts. `test:e2e:cli` runs the same non-P2P task set selected by the default CLI CI workflow. P2P validation runs in Compose so peer discovery does not depend on host loopback, firewall, or WebRTC candidate behaviour.
```powershell
npm run test:e2e:cli
npm run test:e2e:cli:p2p
npm run test:e2e:cli:all
```
From `src/apps/cli/testdeno`:
```powershell
cd src/apps/cli/testdeno
# Canonical suites
deno task test:ci
deno task test:p2p:compose
# Local-only set
deno task test:local
@@ -227,7 +245,8 @@ deno task test:decoupled-vault
deno task test:remote-commands
deno task test:e2e-couchdb
# P2P-based tests
# Explicit host-direct P2P diagnostics for cross-platform investigations.
# These are not part of the default suite or release evidence.
deno task test:p2p-host
deno task test:p2p-peers
deno task test:p2p-sync
src/apps/cli/tsconfig.json
+1 -2
View File
@@ -23,8 +23,7 @@
// "rootDir": "../../../",
/* Path mapping */
"paths": {
"@/*": ["../../*"],
"@lib/*": ["../../lib/src/*", "../../../_types/src/lib/src/*"]
"@/*": ["../../*"]
}
},
"include": ["*.ts", "**/*.ts", "**/*.tsx"],
src/apps/cli/vite.config.ts
+31 -21
View File
@@ -1,13 +1,20 @@
import { defineConfig } from "vite";
import { defaultServerConditions, defaultServerMainFields, defineConfig } from "vite";
import { svelte } from "@sveltejs/vite-plugin-svelte";
import path from "node:path";
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { fileURLToPath, fs, isBuiltin, path } from "@vrtmrz/livesync-commonlib/node";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const resolve = (...args: string[]) => path.resolve(...args).replace(/\\/g, "/");
const repoRoot = path.resolve(__dirname, "../../..");
const packageJson = JSON.parse(readFileSync(path.resolve(repoRoot, "package.json"), "utf-8"));
const manifestJson = JSON.parse(readFileSync(path.resolve(repoRoot, "manifest.json"), "utf-8"));
function readVersion(filePath: string): string | undefined {
const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8"));
if (typeof parsed !== "object" || parsed === null || !("version" in parsed)) {
return undefined;
}
return typeof parsed.version === "string" ? parsed.version : undefined;
}
const packageVersion = readVersion(path.resolve(repoRoot, "package.json"));
const manifestVersion = readVersion(path.resolve(repoRoot, "manifest.json"));
// https://vite.dev/config/
const defaultExternal = [
"obsidian",
@@ -47,7 +54,7 @@ function injectBanner(): import("vite").Plugin {
name: "inject-banner",
generateBundle(_options, bundle) {
for (const chunk of Object.values(bundle)) {
if (chunk.type === "chunk" && chunk.fileName.startsWith("entrypoint")) {
if (chunk.type === "chunk" && chunk.isEntry) {
// Insert after the shebang line if present, otherwise at the top.
if (chunk.code.startsWith("#!")) {
const newline = chunk.code.indexOf("\n");
@@ -62,20 +69,27 @@ function injectBanner(): import("vite").Plugin {
};
}
const buildInputs: Record<string, string> = {
index: resolve(__dirname, "entrypoint.ts"),
};
if (process.env.LIVESYNC_CLI_TEST_SUPPORT === "1") {
buildInputs["p2p-lifecycle-test"] = resolve(__dirname, "test-support/p2p-lifecycle-entrypoint.ts");
}
export default defineConfig({
plugins: [svelte(), injectBanner()],
resolve: {
// This bundle runs in Node. Vite's client defaults include the `browser`
// export condition, which would select Commonlib's inline Web Worker.
conditions: [...defaultServerConditions],
mainFields: [...defaultServerMainFields],
alias: {
"@lib/worker/bgWorker.ts": "../../lib/src/worker/bgWorker.mock.ts",
"@lib/pouchdb/pouchdb-browser.ts": resolve(__dirname, "lib/pouchdb-node.ts"),
// The CLI runs on Node.js; force AWS XML builder to its CJS Node entry
// so Vite does not resolve the browser DOMParser-based XML parser.
"@aws-sdk/xml-builder": resolve(__dirname, "../../../node_modules/@aws-sdk/xml-builder/dist-cjs/index.js"),
// Force fflate to the Node CJS entry; browser entry expects Web Worker globals.
fflate: resolve(__dirname, "../../../node_modules/fflate/lib/node.cjs"),
"@": resolve(__dirname, "../../"),
"@lib": resolve(__dirname, "../../lib/src"),
"../../src/worker/bgWorker.ts": "../../src/worker/bgWorker.mock.ts",
},
},
@@ -85,26 +99,22 @@ export default defineConfig({
emptyOutDir: true,
minify: false,
rollupOptions: {
input: {
index: resolve(__dirname, "entrypoint.ts"),
},
input: buildInputs,
external: (id) => {
if (isBuiltin(id)) return true;
if (defaultExternal.includes(id)) return true;
if (id.startsWith(".") || id.startsWith("/")) return false;
if (id.startsWith("@/") || id.startsWith("@lib/")) return false;
if (id.startsWith("@/")) return false;
if (id.endsWith(".ts") || id.endsWith(".js")) return false;
if (id === "fs" || id === "fs/promises" || id === "path" || id === "crypto" || id === "worker_threads")
return true;
if (id.startsWith("pouchdb-")) return true;
if (id.startsWith("werift")) return true;
if (id.startsWith("node:")) return true;
return false;
},
},
lib: {
entry: resolve(__dirname, "entrypoint.ts"),
formats: ["cjs"],
fileName: "index",
fileName: (_format, entryName) => `${entryName}.cjs`,
},
},
define: {
@@ -112,7 +122,7 @@ export default defineConfig({
global: "globalThis",
nonInteractive: "true",
// localStorage: "undefined", // Prevent usage of localStorage in the CLI environment
MANIFEST_VERSION: JSON.stringify(process.env.MANIFEST_VERSION || manifestJson.version || "0.0.0"),
PACKAGE_VERSION: JSON.stringify(process.env.PACKAGE_VERSION || packageJson.version || "0.0.0"),
MANIFEST_VERSION: JSON.stringify(process.env.MANIFEST_VERSION || manifestVersion || "0.0.0"),
PACKAGE_VERSION: JSON.stringify(process.env.PACKAGE_VERSION || packageVersion || "0.0.0"),
},
});