mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-08-28 14:27:08 +00:00
chore: merge upstream main into history revision branch
This commit is contained in:
+19
-19
@@ -1,22 +1,22 @@
|
||||
import { LOG_LEVEL_INFO } from "octagonal-wheels/common/logger";
|
||||
import type PouchDB from "pouchdb-core";
|
||||
import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase";
|
||||
import type { HasSettings, ObsidianLiveSyncSettings, EntryDoc } from "@lib/common/types";
|
||||
import { __$checkInstanceBinding } from "@lib/dev/checks";
|
||||
import type { Confirm } from "@lib/interfaces/Confirm";
|
||||
import type { DatabaseFileAccess } from "@lib/interfaces/DatabaseFileAccess";
|
||||
import type { Rebuilder } from "@lib/interfaces/DatabaseRebuilder";
|
||||
import type { IFileHandler } from "@lib/interfaces/FileHandler";
|
||||
import type { StorageAccess } from "@lib/interfaces/StorageAccess";
|
||||
import type { LiveSyncLocalDBEnv } from "@lib/pouchdb/LiveSyncLocalDB";
|
||||
import type { LiveSyncCouchDBReplicatorEnv } from "@lib/replication/couchdb/LiveSyncReplicator";
|
||||
import type { CheckPointInfo } from "@lib/replication/journal/JournalSyncTypes";
|
||||
import type { LiveSyncJournalReplicatorEnv } from "@lib/replication/journal/LiveSyncJournalReplicatorEnv";
|
||||
import type { LiveSyncReplicatorEnv } from "@lib/replication/LiveSyncAbstractReplicator";
|
||||
import { useTargetFilters } from "@lib/serviceFeatures/targetFilter";
|
||||
import { useRemoteConfigurationMigration } from "@lib/serviceFeatures/remoteConfig";
|
||||
import type { ServiceContext } from "@lib/services/base/ServiceBase";
|
||||
import type { InjectableServiceHub } from "@lib/services/InjectableServices";
|
||||
import type { HasSettings, ObsidianLiveSyncSettings, EntryDoc } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { __$checkInstanceBinding } from "@vrtmrz/livesync-commonlib/compat/dev/checks";
|
||||
import type { Confirm } from "@vrtmrz/livesync-commonlib/compat/interfaces/Confirm";
|
||||
import type { DatabaseFileAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/DatabaseFileAccess";
|
||||
import type { Rebuilder } from "@vrtmrz/livesync-commonlib/compat/interfaces/DatabaseRebuilder";
|
||||
import type { IFileHandler } from "@vrtmrz/livesync-commonlib/compat/interfaces/FileHandler";
|
||||
import type { StorageAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/StorageAccess";
|
||||
import type { LiveSyncLocalDBEnv } from "@vrtmrz/livesync-commonlib/compat/pouchdb/LiveSyncLocalDB";
|
||||
import type { LiveSyncCouchDBReplicatorEnv } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator";
|
||||
import type { CheckPointInfo } from "@vrtmrz/livesync-commonlib/compat/replication/journal/JournalSyncTypes";
|
||||
import type { LiveSyncJournalReplicatorEnv } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicatorEnv";
|
||||
import type { LiveSyncReplicatorEnv } from "@vrtmrz/livesync-commonlib/compat/replication/LiveSyncAbstractReplicator";
|
||||
import { useTargetFilters } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/targetFilter";
|
||||
import { useRemoteConfigurationMigration } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/remoteConfig";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub";
|
||||
import { AbstractModule } from "./modules/AbstractModule";
|
||||
import { ModulePeriodicProcess } from "./modules/core/ModulePeriodicProcess";
|
||||
import { ModuleReplicator } from "./modules/core/ModuleReplicator";
|
||||
@@ -26,10 +26,10 @@ import { ModuleConflictChecker } from "./modules/coreFeatures/ModuleConflictChec
|
||||
import { ModuleConflictResolver } from "./modules/coreFeatures/ModuleConflictResolver";
|
||||
import { ModuleResolvingMismatchedTweaks } from "./modules/coreFeatures/ModuleResolveMismatchedTweaks";
|
||||
import { ModuleLiveSyncMain } from "./modules/main/ModuleLiveSyncMain";
|
||||
import type { ServiceModules } from "@lib/interfaces/ServiceModule";
|
||||
import type { ServiceModules } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule";
|
||||
import { ModuleBasicMenu } from "./modules/essential/ModuleBasicMenu";
|
||||
import { usePrepareDatabaseForUse } from "@lib/serviceFeatures/prepareDatabaseForUse";
|
||||
import type { Constructor } from "@lib/common/utils.type";
|
||||
import { usePrepareDatabaseForUse } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/prepareDatabaseForUse";
|
||||
import type { Constructor } from "@vrtmrz/livesync-commonlib/compat/common/utils.type";
|
||||
|
||||
export class LiveSyncBaseCore<
|
||||
T extends ServiceContext = ServiceContext,
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import type { IStorageAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
|
||||
|
||||
/** One platform-neutral storage adapter contract case. */
|
||||
export interface StorageAdapterContractCase {
|
||||
readonly name: string;
|
||||
run(adapter: IStorageAdapter): Promise<void>;
|
||||
}
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) throw new Error(message);
|
||||
}
|
||||
|
||||
function assertEqual(actual: unknown, expected: unknown, message: string): void {
|
||||
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
|
||||
throw new Error(`${message}\nactual=${JSON.stringify(actual)}\nexpected=${JSON.stringify(expected)}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertRejects(operation: () => Promise<unknown>, message: string): Promise<void> {
|
||||
try {
|
||||
await operation();
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
/** Passing baseline shared by Node, FSAPI, and future storage adapters. */
|
||||
export const storageAdapterContractCases: readonly StorageAdapterContractCase[] = [
|
||||
{
|
||||
name: "reports missing paths consistently",
|
||||
async run(adapter) {
|
||||
assertEqual(await adapter.exists("missing.txt"), false, "missing path should not exist");
|
||||
assertEqual(await adapter.stat("missing.txt"), null, "missing stat should be null");
|
||||
assertEqual(await adapter.trystat("missing.txt"), null, "missing trystat should be null");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "creates parent directories for nested text writes",
|
||||
async run(adapter) {
|
||||
await adapter.write("notes/nested/note.md", "hello");
|
||||
assertEqual(await adapter.read("notes/nested/note.md"), "hello", "text should round-trip");
|
||||
assert(await adapter.exists("notes/nested/note.md"), "written text path should exist");
|
||||
assertEqual((await adapter.stat("notes/nested/note.md"))?.type, "file", "written path should be a file");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "round-trips exact binary bytes",
|
||||
async run(adapter) {
|
||||
const expected = Uint8Array.from([0x00, 0x7f, 0x80, 0xff, 0x42]);
|
||||
await adapter.writeBinary("binary/blob.bin", expected.buffer.slice(0));
|
||||
const result = await adapter.readBinary("binary/blob.bin");
|
||||
assertEqual([...new Uint8Array(result)], [...expected], "binary data should round-trip exactly");
|
||||
assertEqual(result.byteLength, expected.byteLength, "binary result should have the exact visible length");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "creates and extends text through append",
|
||||
async run(adapter) {
|
||||
await adapter.append("logs/events.log", "first");
|
||||
await adapter.append("logs/events.log", ":second");
|
||||
assertEqual(await adapter.read("logs/events.log"), "first:second", "append should create then extend text");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "lists direct files and folders",
|
||||
async run(adapter) {
|
||||
await adapter.mkdir("listing/folder");
|
||||
await adapter.write("listing/file.txt", "content");
|
||||
const listed = await adapter.list("listing");
|
||||
assertEqual([...listed.files].sort(), ["listing/file.txt"], "list should contain the direct file");
|
||||
assertEqual([...listed.folders].sort(), ["listing/folder"], "list should contain the direct folder");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "removes files and directory trees",
|
||||
async run(adapter) {
|
||||
await adapter.write("remove/file.txt", "content");
|
||||
await adapter.write("remove/folder/nested.txt", "content");
|
||||
await adapter.remove("remove/file.txt");
|
||||
assertEqual(await adapter.exists("remove/file.txt"), false, "file should be removed");
|
||||
await adapter.remove("remove/folder");
|
||||
assertEqual(await adapter.exists("remove/folder"), false, "directory tree should be removed");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "keeps operations inside the configured root",
|
||||
async run(adapter) {
|
||||
await assertRejects(() => adapter.exists("../outside"), "parent traversal should be rejected");
|
||||
await assertRejects(() => adapter.write("nested/../outside", "content"), "nested traversal should be rejected");
|
||||
await assertRejects(() => adapter.read("/absolute"), "absolute paths should be rejected");
|
||||
await assertRejects(() => adapter.read("C:\\absolute"), "drive-qualified paths should be rejected");
|
||||
await assertRejects(() => adapter.read("nested\\outside"), "backslash-separated paths should be rejected");
|
||||
await assertRejects(() => adapter.remove(""), "removing the configured root should be rejected");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "uses the empty path only for root-safe operations",
|
||||
async run(adapter) {
|
||||
await adapter.mkdir("");
|
||||
assertEqual(await adapter.exists(""), true, "the configured root should exist");
|
||||
assertEqual((await adapter.stat(""))?.type, "folder", "the configured root should be a folder");
|
||||
assertEqual(await adapter.list(""), { files: [], folders: [] }, "the configured root should be listable");
|
||||
await assertRejects(() => adapter.write("", "content"), "writing over the configured root should be rejected");
|
||||
await assertRejects(() => adapter.append("", "content"), "appending to the configured root should be rejected");
|
||||
},
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,136 @@
|
||||
import type { Confirm, ConfirmActionLayout } from "@vrtmrz/livesync-commonlib/compat/interfaces/Confirm";
|
||||
import { createNativeElement } from "@/apps/browserDom";
|
||||
|
||||
import MessageBox from "./ui/MessageBox.svelte";
|
||||
import TextInputBox from "./ui/TextInputBox.svelte";
|
||||
|
||||
import { mount } from "svelte";
|
||||
import { promiseWithResolvers } from "octagonal-wheels/promises";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { _activeDocument, compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
|
||||
function displayMessageBox<T, U extends string[]>(
|
||||
message: string,
|
||||
buttons: U,
|
||||
title: string,
|
||||
commit: (ret: U[number]) => T,
|
||||
actionLayout: ConfirmActionLayout = "vertical"
|
||||
): Promise<T> {
|
||||
const el = createNativeElement(_activeDocument, "div");
|
||||
const p = promiseWithResolvers<T>();
|
||||
mount(MessageBox, {
|
||||
target: el,
|
||||
props: {
|
||||
message,
|
||||
buttons: buttons as string[],
|
||||
title: title,
|
||||
actionLayout,
|
||||
commit: (action: U[number]) => {
|
||||
const ret = commit(action);
|
||||
p.resolve(ret);
|
||||
},
|
||||
},
|
||||
});
|
||||
_activeDocument.body.appendChild(el);
|
||||
void p.promise.finally(() => {
|
||||
el.remove();
|
||||
});
|
||||
return p.promise;
|
||||
}
|
||||
function promptForInput(
|
||||
title: string,
|
||||
key: string,
|
||||
placeholder: string,
|
||||
isPassword?: boolean
|
||||
): Promise<string | false> {
|
||||
const el = createNativeElement(_activeDocument, "div");
|
||||
const p = promiseWithResolvers<string | false>();
|
||||
mount(TextInputBox, {
|
||||
target: el,
|
||||
props: {
|
||||
title,
|
||||
message: key,
|
||||
placeholder,
|
||||
isPassword,
|
||||
commit: (text: string | false) => {
|
||||
p.resolve(text);
|
||||
},
|
||||
},
|
||||
});
|
||||
_activeDocument.body.appendChild(el);
|
||||
void p.promise.finally(() => {
|
||||
el.remove();
|
||||
});
|
||||
return p.promise;
|
||||
}
|
||||
|
||||
export class BrowserConfirm<T extends ServiceContext> implements Confirm {
|
||||
_context: T;
|
||||
constructor(context: T) {
|
||||
this._context = context;
|
||||
}
|
||||
askYesNo(message: string): Promise<"yes" | "no"> {
|
||||
return displayMessageBox(message, ["Yes", "No"] as const, "Confirm", (action) =>
|
||||
action == "Yes" ? "yes" : "no"
|
||||
);
|
||||
}
|
||||
askString(title: string, key: string, placeholder: string, isPassword?: boolean): Promise<string | false> {
|
||||
return promptForInput(title, key, placeholder, isPassword);
|
||||
}
|
||||
askYesNoDialog(
|
||||
message: string,
|
||||
opt: { title?: string; defaultOption?: "Yes" | "No"; timeout?: number }
|
||||
): Promise<"yes" | "no"> {
|
||||
return displayMessageBox(message, ["Yes", "No"] as const, opt.title ?? "Confirm", (action) =>
|
||||
action == "Yes" ? "yes" : "no"
|
||||
);
|
||||
}
|
||||
askSelectString(message: string, items: string[]): Promise<string> {
|
||||
return displayMessageBox(message, [...items] as const, "Confirm", (action) => action);
|
||||
}
|
||||
askSelectStringDialogue<T extends readonly string[]>(
|
||||
message: string,
|
||||
buttons: T,
|
||||
opt: { title?: string; defaultAction: T[number]; timeout?: number }
|
||||
): Promise<T[number] | false> {
|
||||
return displayMessageBox(message, [...buttons] as const, opt.title ?? "Confirm", (action) => action);
|
||||
}
|
||||
askInPopup(
|
||||
key: string,
|
||||
dialogText: string,
|
||||
anchorCallback: (anchor: HTMLAnchorElement) => void,
|
||||
durationMs: number = 20000
|
||||
): void {
|
||||
const existing = _activeDocument.querySelector(`[data-livesync-popup="${CSS.escape(key)}"]`);
|
||||
existing?.remove();
|
||||
|
||||
const notice = createNativeElement(_activeDocument, "div");
|
||||
notice.className = "livesync-browser-notice";
|
||||
notice.dataset.livesyncPopup = key;
|
||||
const [beforeText, afterText] = dialogText.split("{HERE}", 2);
|
||||
notice.append(beforeText);
|
||||
const anchor = createNativeElement(_activeDocument, "a");
|
||||
anchor.href = "#";
|
||||
anchorCallback(anchor);
|
||||
anchor.addEventListener("click", () => notice.remove());
|
||||
notice.append(anchor, afterText ?? "");
|
||||
_activeDocument.body.appendChild(notice);
|
||||
compatGlobal.setTimeout(() => notice.remove(), durationMs);
|
||||
}
|
||||
confirmWithMessage(
|
||||
title: string,
|
||||
contentMd: string,
|
||||
buttons: string[],
|
||||
defaultAction: (typeof buttons)[number],
|
||||
timeout?: number,
|
||||
actionLayout?: ConfirmActionLayout
|
||||
): Promise<(typeof buttons)[number] | false> {
|
||||
return displayMessageBox(
|
||||
contentMd,
|
||||
[...buttons] as const,
|
||||
title ?? "Confirm",
|
||||
(action) => action,
|
||||
actionLayout
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { promiseWithResolvers, type PromiseWithResolvers } from "octagonal-wheels/promises";
|
||||
import { createNativeElement } from "@/apps/browserDom";
|
||||
import { mount } from "svelte";
|
||||
import MenuView from "./ui/MenuView.svelte";
|
||||
import { _activeDocument } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
|
||||
export class MenuItem {
|
||||
type = "item";
|
||||
title = "";
|
||||
handler?: () => void | Promise<void>;
|
||||
icon: string = "";
|
||||
setTitle(title: string) {
|
||||
this.title = title;
|
||||
return this;
|
||||
}
|
||||
onClick(callback: () => void | Promise<void>) {
|
||||
this.handler = callback;
|
||||
return this;
|
||||
}
|
||||
setIcon(icon: string | null) {
|
||||
this.icon = icon || "";
|
||||
return this;
|
||||
}
|
||||
}
|
||||
export class MenuSeparator {
|
||||
type = "separator";
|
||||
}
|
||||
export class Menu {
|
||||
type = "menu";
|
||||
items: (MenuItem | MenuSeparator)[] = [];
|
||||
|
||||
constructor() {}
|
||||
addItem(callback: (item: MenuItem) => void) {
|
||||
const item = new MenuItem();
|
||||
callback(item);
|
||||
this.items.push(item);
|
||||
return this;
|
||||
}
|
||||
addSeparator() {
|
||||
this.items.push(new MenuSeparator());
|
||||
return this;
|
||||
}
|
||||
waitingForClose?: PromiseWithResolvers<void>;
|
||||
showAtPosition(pos: { x: number; y: number }) {
|
||||
const el = createNativeElement(_activeDocument, "div");
|
||||
if (this.waitingForClose) {
|
||||
this.waitingForClose.resolve();
|
||||
}
|
||||
this.waitingForClose = promiseWithResolvers<void>();
|
||||
mount(MenuView, {
|
||||
target: el,
|
||||
props: {
|
||||
items: this.items,
|
||||
closeMenu: () => {
|
||||
this.waitingForClose?.resolve();
|
||||
this.waitingForClose = undefined;
|
||||
},
|
||||
x: pos.x,
|
||||
y: pos.y,
|
||||
},
|
||||
});
|
||||
_activeDocument.body.appendChild(el);
|
||||
void this.waitingForClose.promise.finally(() => {
|
||||
el.remove();
|
||||
});
|
||||
return this.waitingForClose.promise;
|
||||
}
|
||||
hide() {
|
||||
this.waitingForClose?.resolve();
|
||||
this.waitingForClose = undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import {
|
||||
type ComponentHasResult,
|
||||
SvelteDialogManagerBase,
|
||||
SvelteDialogMixIn,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/services/implements/base/SvelteDialog";
|
||||
import { createNativeElement } from "@/apps/browserDom";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import type { SvelteDialogManagerDependencies } from "@vrtmrz/livesync-commonlib/compat/services/implements/base/SvelteDialog";
|
||||
import { _activeDocument } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
import DialogHost from "@/modules/services/LiveSyncUI/DialogHost.svelte";
|
||||
|
||||
export class ShimModal {
|
||||
contentEl: HTMLElement;
|
||||
titleEl: HTMLElement;
|
||||
modalEl: HTMLElement;
|
||||
isOpen: boolean = false;
|
||||
baseEl: HTMLElement;
|
||||
constructor() {
|
||||
const baseEl = createNativeElement(_activeDocument, "popup");
|
||||
this.baseEl = baseEl;
|
||||
this.contentEl = createNativeElement(_activeDocument, "div");
|
||||
this.contentEl.className = "modal-content";
|
||||
this.titleEl = createNativeElement(_activeDocument, "div");
|
||||
this.titleEl.className = "modal-title";
|
||||
this.modalEl = createNativeElement(_activeDocument, "div");
|
||||
this.modalEl.className = "modal";
|
||||
this.modalEl.hidden = true;
|
||||
this.modalEl.appendChild(this.titleEl);
|
||||
this.modalEl.appendChild(this.contentEl);
|
||||
this.baseEl.appendChild(this.modalEl);
|
||||
}
|
||||
open() {
|
||||
this.isOpen = true;
|
||||
this.modalEl.hidden = false;
|
||||
if (!this.baseEl.parentElement) {
|
||||
_activeDocument.body.appendChild(this.baseEl);
|
||||
}
|
||||
this.onOpen();
|
||||
}
|
||||
close() {
|
||||
this.isOpen = false;
|
||||
this.modalEl.hidden = true;
|
||||
this.baseEl.remove();
|
||||
this.onClose();
|
||||
}
|
||||
onOpen() {}
|
||||
onClose() {}
|
||||
setPlaceholder(p: string) {}
|
||||
setTitle(t: string) {
|
||||
this.titleEl.textContent = t;
|
||||
}
|
||||
}
|
||||
|
||||
const BrowserSvelteDialogBase = SvelteDialogMixIn(ShimModal, DialogHost);
|
||||
|
||||
export class LiveSyncBrowserDialog<T, U, C extends ServiceContext = ServiceContext> extends BrowserSvelteDialogBase<
|
||||
T,
|
||||
U,
|
||||
C
|
||||
> {
|
||||
constructor(
|
||||
context: C,
|
||||
dependents: SvelteDialogManagerDependencies<C>,
|
||||
component: ComponentHasResult<T, U>,
|
||||
initialData?: U
|
||||
) {
|
||||
super();
|
||||
this.initDialog(context, dependents, component, initialData);
|
||||
}
|
||||
}
|
||||
export class BrowserSvelteDialogManager<T extends ServiceContext> extends SvelteDialogManagerBase<T> {
|
||||
override async openSvelteDialog<TT, TU>(
|
||||
component: ComponentHasResult<TT, TU>,
|
||||
initialData?: TU
|
||||
): Promise<TT | undefined> {
|
||||
const dialog = new LiveSyncBrowserDialog<TT, TU, T>(this.context, this.dependents, component, initialData);
|
||||
dialog.open();
|
||||
return await dialog.waitForClose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import type { BrowserServiceHostDependencies } from "@vrtmrz/livesync-commonlib/compat/services/BrowserServices";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { UIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/base/UIService";
|
||||
import DialogToCopy from "@/modules/services/LiveSyncUI/dialogues/DialogueToCopy.svelte";
|
||||
import { BrowserSvelteDialogManager } from "./BrowserSvelteDialogManager";
|
||||
|
||||
export class LiveSyncBrowserUIService<T extends ServiceContext> extends UIService<T> {
|
||||
override get dialogToCopy() {
|
||||
return DialogToCopy;
|
||||
}
|
||||
constructor(context: T, dependents: BrowserServiceHostDependencies<T>) {
|
||||
const browserConfirm = dependents.API.confirm;
|
||||
const obsidianSvelteDialogManager = new BrowserSvelteDialogManager<T>(context, {
|
||||
appLifecycle: dependents.appLifecycle,
|
||||
config: dependents.config,
|
||||
replicator: dependents.replicator,
|
||||
confirm: browserConfirm,
|
||||
control: dependents.control,
|
||||
});
|
||||
super(context, {
|
||||
dialogManager: obsidianSvelteDialogManager,
|
||||
APIService: dependents.API,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { BrowserServiceHub, type BrowserServiceHost } from "@vrtmrz/livesync-commonlib/compat/services/BrowserServices";
|
||||
import type { KeyValueDatabaseFactory } from "@vrtmrz/livesync-commonlib/compat/interfaces/KeyValueDatabase";
|
||||
import { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { BrowserAPIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/browser/BrowserAPIService";
|
||||
import { BrowserConfirm } from "./BrowserConfirm";
|
||||
import { LiveSyncBrowserUIService } from "./LiveSyncBrowserUIService";
|
||||
import { setLang, translateLiveSyncMessage } from "@/common/translation";
|
||||
|
||||
export type LiveSyncBrowserServiceHubOptions<T extends ServiceContext> = {
|
||||
context?: T;
|
||||
openKeyValueDatabase?: KeyValueDatabaseFactory;
|
||||
};
|
||||
|
||||
function createLiveSyncBrowserHost<T extends ServiceContext>(): BrowserServiceHost<T> {
|
||||
return {
|
||||
createAPI(context) {
|
||||
return new BrowserAPIService(context, {
|
||||
confirm: new BrowserConfirm(context),
|
||||
});
|
||||
},
|
||||
createUI(context, dependencies) {
|
||||
return new LiveSyncBrowserUIService(context, dependencies);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createLiveSyncBrowserServiceHub<T extends ServiceContext>(
|
||||
options: LiveSyncBrowserServiceHubOptions<T> = {}
|
||||
): BrowserServiceHub<T> {
|
||||
const context = options.context ?? (new ServiceContext({ translate: translateLiveSyncMessage }) as T);
|
||||
return new BrowserServiceHub<T>({
|
||||
...options,
|
||||
context,
|
||||
onDisplayLanguageChanged: setLang,
|
||||
host: createLiveSyncBrowserHost<T>(),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { createServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
observeServiceComposition,
|
||||
observeServiceContext,
|
||||
SERVICE_CONTEXT_MEMBERS,
|
||||
} from "../../../test/contracts/serviceContext";
|
||||
import { createLiveSyncBrowserServiceHub } from "./createLiveSyncBrowserServiceHub";
|
||||
|
||||
describe("LiveSync browser service context contract", () => {
|
||||
it("preserves one injected context and its API results throughout the Webapp composition", () => {
|
||||
const context = createServiceContext({
|
||||
translate: (key) => `webapp:${key}`,
|
||||
});
|
||||
const hub = createLiveSyncBrowserServiceHub({ context });
|
||||
|
||||
expect(observeServiceContext(context, "moduleLocalDatabase.logWaitingForReady")).toEqual({
|
||||
translation: "webapp:moduleLocalDatabase.logWaitingForReady",
|
||||
receivedEvents: ["context-contract-event"],
|
||||
});
|
||||
const composition = observeServiceComposition(hub, context);
|
||||
expect(composition.hubUsesExpectedContext).toBe(true);
|
||||
expect(SERVICE_CONTEXT_MEMBERS.filter((member) => !composition.servicesUsingExpectedContext[member])).toEqual(
|
||||
[]
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"extends": "../../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": true,
|
||||
"paths": {
|
||||
"@/*": ["../../*"]
|
||||
}
|
||||
},
|
||||
"include": ["**/*.ts", "**/*.svelte"],
|
||||
"exclude": ["**/*.unit.spec.ts"]
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
<script lang="ts">
|
||||
import type { MenuItem } from "@/apps/browser/BrowserMenu";
|
||||
import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
|
||||
|
||||
type Props = {
|
||||
item: MenuItem;
|
||||
closeMenu: () => void;
|
||||
};
|
||||
const { item = $bindable(), closeMenu }: Props = $props();
|
||||
function handleCommit(event: MouseEvent | KeyboardEvent) {
|
||||
event.preventDefault();
|
||||
try {
|
||||
if (item.handler) {
|
||||
item.handler?.();
|
||||
}
|
||||
} catch (ex) {
|
||||
Logger(ex, LOG_LEVEL_VERBOSE, "browser-menu");
|
||||
}
|
||||
closeMenu();
|
||||
}
|
||||
const icons = {
|
||||
checkmark: "✓",
|
||||
} as { [key: string]: string };
|
||||
function renderIcon(item: MenuItem) {
|
||||
if (item.icon && item.icon in icons) {
|
||||
return icons[item.icon] ?? item.icon;
|
||||
} else if (item.icon !== undefined) {
|
||||
return "";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
</script>
|
||||
|
||||
<li>
|
||||
<span class="icon">{renderIcon(item)}</span>
|
||||
<label for=""
|
||||
><!-- svelte-ignore a11y_invalid_attribute -->
|
||||
<a onclick={handleCommit} onkeydown={handleCommit} role="button" tabindex="0" href="#">{item.title}</a></label
|
||||
>
|
||||
</li>
|
||||
|
||||
<style>
|
||||
span.icon {
|
||||
display: inline-block;
|
||||
min-width: 1.5em;
|
||||
text-align: center;
|
||||
}
|
||||
li {
|
||||
list-style: none;
|
||||
padding: 0.5em 1em;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,10 @@
|
||||
<script lang="ts">
|
||||
import type { MenuSeparator } from "@/apps/browser/BrowserMenu";
|
||||
|
||||
type Props = {
|
||||
item: MenuSeparator;
|
||||
};
|
||||
const { item = $bindable() }: Props = $props();
|
||||
</script>
|
||||
|
||||
<hr />
|
||||
@@ -0,0 +1,89 @@
|
||||
<script lang="ts">
|
||||
import type { Menu, MenuItem, MenuSeparator } from "@/apps/browser/BrowserMenu";
|
||||
import MenuItemView from "./MenuItemView.svelte";
|
||||
import MenuSeparatorView from "./MenuSeparatorView.svelte";
|
||||
|
||||
type Props = {
|
||||
items: (MenuItem | MenuSeparator)[];
|
||||
closeMenu: () => void;
|
||||
};
|
||||
const { items = $bindable(), closeMenu }: Props = $props();
|
||||
function handleKey(event: KeyboardEvent) {
|
||||
if (event.key === "Escape") {
|
||||
event.preventDefault();
|
||||
closeMenu();
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<popup>
|
||||
<nav>
|
||||
{#each items as item}
|
||||
{#if item.type === "separator"}
|
||||
<MenuSeparatorView {item} />
|
||||
{:else if item.type === "item"}
|
||||
<MenuItemView item={item as MenuItem} {closeMenu} />
|
||||
{/if}
|
||||
{/each}
|
||||
</nav>
|
||||
</popup>
|
||||
<div class="background" onclick={() => closeMenu()} onkeydown={handleKey} role="none"></div>
|
||||
|
||||
<style>
|
||||
popup {
|
||||
z-index: 1000;
|
||||
position: fixed;
|
||||
background: rgba(255, 255, 255, 0.8);
|
||||
max-width: 70vw;
|
||||
max-height: 80vh;
|
||||
margin: auto;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
backdrop-filter: blur(5px);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
border-radius: 5px;
|
||||
box-shadow: 0 0 10px rgba(0, 0, 0, 0.5);
|
||||
border: 1px solid var(--background-primary-alt);
|
||||
justify-content: space-between;
|
||||
width: fit-content;
|
||||
height: fit-content;
|
||||
}
|
||||
/* popup header {
|
||||
background: var(--background-primary);
|
||||
color: var(--text-normal);
|
||||
padding: 1em;
|
||||
border-bottom: 1px solid var(--background-primary-alt);
|
||||
font: size 1.4em;
|
||||
}
|
||||
popup article {
|
||||
padding: 1em;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
overflow-y: auto;
|
||||
}
|
||||
popup article .msg {
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
popup .buttons {
|
||||
border-top: 1px solid var(--background-primary-alt);
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
padding: 1em;
|
||||
}
|
||||
popup .buttons button {
|
||||
margin: 0 0.5em;
|
||||
background-color: var(--background-primary-alt);
|
||||
} */
|
||||
popup ~ .background {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: rgba(0, 0, 0, 0.125);
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,137 @@
|
||||
<script lang="ts">
|
||||
import { renderMessageMarkdown } from "./renderMessageMarkdown";
|
||||
|
||||
type Props = {
|
||||
title: string;
|
||||
message: string;
|
||||
buttons: string[];
|
||||
actionLayout?: ConfirmActionLayout;
|
||||
commit: (button: string) => void;
|
||||
};
|
||||
type ConfirmActionLayout = "auto" | "vertical";
|
||||
let { title, message, buttons, actionLayout, commit }: Props = $props();
|
||||
const renderedMessage = $derived(renderMessageMarkdown(message));
|
||||
|
||||
function handleEsc(event: KeyboardEvent) {
|
||||
if (event.key === "Escape") {
|
||||
commit("");
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<popup>
|
||||
<header>{title}</header>
|
||||
<article><div class="msg">{@html renderedMessage}</div></article>
|
||||
<div class:vertical={actionLayout === "vertical"} class="buttons">
|
||||
{#each buttons as button}
|
||||
<button onclick={() => commit(button)}>{button}</button>
|
||||
{/each}
|
||||
</div>
|
||||
</popup>
|
||||
<div class="background" onclick={() => commit("")} onkeydown={handleEsc} role="none"></div>
|
||||
|
||||
<style>
|
||||
popup {
|
||||
z-index: 1000;
|
||||
position: fixed;
|
||||
background: rgba(255, 255, 255, 0.8);
|
||||
max-width: 70vw;
|
||||
max-height: 80vh;
|
||||
margin: auto;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
min-width: 50vw;
|
||||
min-height: 50vh;
|
||||
backdrop-filter: blur(5px);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
border-radius: 5px;
|
||||
box-shadow: 0 0 10px rgba(0, 0, 0, 0.5);
|
||||
border: 1px solid var(--background-primary-alt);
|
||||
justify-content: space-between;
|
||||
}
|
||||
popup header {
|
||||
background: var(--background-primary);
|
||||
color: var(--text-normal);
|
||||
padding: 1em;
|
||||
border-bottom: 1px solid var(--background-primary-alt);
|
||||
font: size 1.4em;
|
||||
}
|
||||
popup article {
|
||||
padding: 1em;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
overflow-y: auto;
|
||||
}
|
||||
popup article .msg {
|
||||
width: 100%;
|
||||
line-height: 1.5;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
popup article .msg :global(:first-child) {
|
||||
margin-top: 0;
|
||||
}
|
||||
popup article .msg :global(:last-child) {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
popup article .msg :global(pre) {
|
||||
overflow-x: auto;
|
||||
padding: 0.75em;
|
||||
border-radius: 4px;
|
||||
background: var(--background-secondary);
|
||||
}
|
||||
popup article .msg :global(code) {
|
||||
font-family: var(--font-monospace);
|
||||
}
|
||||
popup article .msg :global(blockquote) {
|
||||
margin: 0;
|
||||
padding-left: 1em;
|
||||
border-left: 3px solid var(--background-modifier-border);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
popup article .msg :global(ul),
|
||||
popup article .msg :global(ol) {
|
||||
padding-left: 1.5em;
|
||||
}
|
||||
popup article .msg :global(table) {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
popup article .msg :global(th),
|
||||
popup article .msg :global(td) {
|
||||
padding: 0.4em 0.6em;
|
||||
border: 1px solid var(--background-modifier-border);
|
||||
}
|
||||
popup article .msg :global(a) {
|
||||
color: var(--text-accent);
|
||||
}
|
||||
popup .buttons {
|
||||
border-top: 1px solid var(--background-primary-alt);
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
padding: 1em;
|
||||
}
|
||||
popup .buttons button {
|
||||
margin: 0 0.5em;
|
||||
background-color: var(--background-primary-alt);
|
||||
}
|
||||
popup .buttons.vertical {
|
||||
align-items: stretch;
|
||||
flex-direction: column;
|
||||
}
|
||||
popup .buttons.vertical button {
|
||||
margin: 0.25em 0;
|
||||
width: 100%;
|
||||
}
|
||||
popup ~ .background {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: rgba(0, 0, 0, 0.125);
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,126 @@
|
||||
<script lang="ts">
|
||||
type Props = {
|
||||
title: string;
|
||||
message: string;
|
||||
initialText?: string;
|
||||
placeholder?: string;
|
||||
isPassword?: boolean;
|
||||
commit: (text: string | false) => void;
|
||||
};
|
||||
const { title, message, commit, initialText, placeholder, isPassword }: Props = $props();
|
||||
|
||||
function initialTextSeed(): string {
|
||||
return initialText ?? "";
|
||||
}
|
||||
|
||||
let text = $state(initialTextSeed());
|
||||
const type = $derived(isPassword ? "password" : "text");
|
||||
function cancel() {
|
||||
commit(false);
|
||||
}
|
||||
|
||||
function handleKey(event: KeyboardEvent) {
|
||||
if (event.key === "Escape") {
|
||||
handleCancel(event);
|
||||
} else if (event.key === "Enter") {
|
||||
handleCommit(event);
|
||||
}
|
||||
}
|
||||
function handleCancel(event: KeyboardEvent | MouseEvent) {
|
||||
cancel();
|
||||
event.preventDefault();
|
||||
}
|
||||
|
||||
function handleCommit(event: KeyboardEvent | MouseEvent) {
|
||||
commit(text);
|
||||
event.preventDefault();
|
||||
}
|
||||
let textEl: HTMLInputElement;
|
||||
$effect(() => {
|
||||
textEl.focus();
|
||||
});
|
||||
</script>
|
||||
|
||||
<popup>
|
||||
<header>{title}</header>
|
||||
<article>
|
||||
<div class="msg">{message}</div>
|
||||
<div class="input">
|
||||
<input
|
||||
bind:this={textEl}
|
||||
{type}
|
||||
bind:value={text}
|
||||
{placeholder}
|
||||
onkeydown={handleKey}
|
||||
onkeyup={handleKey}
|
||||
/>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<div class="buttons">
|
||||
<button onclick={handleCommit}>OK</button>
|
||||
<button onclick={handleCancel}>Cancel</button>
|
||||
</div>
|
||||
</popup>
|
||||
<div class="background" onclick={handleCancel} onkeydown={handleKey} role="none"></div>
|
||||
|
||||
<style>
|
||||
popup {
|
||||
z-index: 1000;
|
||||
position: fixed;
|
||||
background: rgba(255, 255, 255, 0.8);
|
||||
max-width: 70vw;
|
||||
max-height: 80vh;
|
||||
margin: auto;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
min-width: 50vw;
|
||||
min-height: 50vh;
|
||||
backdrop-filter: blur(5px);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
border-radius: 5px;
|
||||
box-shadow: 0 0 10px rgba(0, 0, 0, 0.5);
|
||||
border: 1px solid var(--background-primary-alt);
|
||||
justify-content: space-between;
|
||||
}
|
||||
popup header {
|
||||
background: var(--background-primary);
|
||||
color: var(--text-normal);
|
||||
padding: 1em;
|
||||
border-bottom: 1px solid var(--background-primary-alt);
|
||||
font: size 1.4em;
|
||||
}
|
||||
popup article {
|
||||
align-items: center;
|
||||
padding: 1em;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
flex-direction: column;
|
||||
}
|
||||
popup article .msg {
|
||||
overflow-y: auto;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
popup .buttons {
|
||||
border-top: 1px solid var(--background-primary-alt);
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
padding: 1em;
|
||||
}
|
||||
popup .buttons button {
|
||||
margin: 0 0.5em;
|
||||
background-color: var(--background-primary-alt);
|
||||
}
|
||||
popup ~ .background {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background: rgba(0, 0, 0, 0.125);
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,21 @@
|
||||
import MarkdownIt from "markdown-it";
|
||||
|
||||
const markdownRenderer = new MarkdownIt({
|
||||
html: false,
|
||||
breaks: true,
|
||||
linkify: true,
|
||||
});
|
||||
|
||||
const defaultLinkOpenRenderer =
|
||||
markdownRenderer.renderer.rules.link_open ??
|
||||
((tokens, idx, options, _env, self) => self.renderToken(tokens, idx, options));
|
||||
|
||||
markdownRenderer.renderer.rules.link_open = (tokens, idx, options, env, self) => {
|
||||
tokens[idx].attrSet("target", "_blank");
|
||||
tokens[idx].attrSet("rel", "noopener noreferrer");
|
||||
return defaultLinkOpenRenderer(tokens, idx, options, env, self);
|
||||
};
|
||||
|
||||
export function renderMessageMarkdown(message: string): string {
|
||||
return markdownRenderer.render(message);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { renderMessageMarkdown } from "./renderMessageMarkdown";
|
||||
|
||||
describe("renderMessageMarkdown", () => {
|
||||
it("renders basic markdown features used by browser dialogues", () => {
|
||||
const html = renderMessageMarkdown("# Title\n\n| left | right |\n| --- | --- |\n| a | b |\n");
|
||||
|
||||
expect(html).toContain("<h1>Title</h1>");
|
||||
expect(html).toContain("<table>");
|
||||
expect(html).toContain("<td>a</td>");
|
||||
});
|
||||
|
||||
it("escapes inline HTML instead of rendering it", () => {
|
||||
const html = renderMessageMarkdown("Before<script>alert('xss')</script>After");
|
||||
|
||||
expect(html).not.toContain("<script>");
|
||||
expect(html).toContain("<script>alert('xss')</script>");
|
||||
});
|
||||
|
||||
it("opens Markdown links safely in a new tab", () => {
|
||||
const html = renderMessageMarkdown("[docs](https://example.com)");
|
||||
|
||||
expect(html).toContain('target="_blank"');
|
||||
expect(html).toContain('rel="noopener noreferrer"');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Native DOM creation for browser applications which run outside Obsidian.
|
||||
*
|
||||
* Obsidian adds creation helpers to its own DOM environment. The standalone
|
||||
* Webapp and WebPeer hosts do not own those prototype extensions, and the
|
||||
* Webapp compatibility layer implements them on top of this native boundary.
|
||||
*/
|
||||
type NativeDocumentCreation = Pick<Document, "createElement" | "createDocumentFragment">;
|
||||
|
||||
export function createNativeElement<K extends keyof HTMLElementTagNameMap>(
|
||||
document: NativeDocumentCreation,
|
||||
tag: K
|
||||
): HTMLElementTagNameMap[K];
|
||||
export function createNativeElement(document: NativeDocumentCreation, tag: string): HTMLElement;
|
||||
export function createNativeElement(document: NativeDocumentCreation, tag: string): HTMLElement {
|
||||
return document.createElement(tag);
|
||||
}
|
||||
|
||||
export function createNativeFragment(document: NativeDocumentCreation): DocumentFragment {
|
||||
return document.createDocumentFragment();
|
||||
}
|
||||
@@ -101,9 +101,9 @@ COPY --from=runtime-deps /deps/node_modules ./node_modules
|
||||
# Copy the built CLI bundle from builder stage
|
||||
COPY --from=builder /build/src/apps/cli/dist ./dist
|
||||
|
||||
# Install entrypoint wrapper
|
||||
COPY src/apps/cli/docker-entrypoint.sh /usr/local/bin/livesync-cli
|
||||
RUN chmod +x /usr/local/bin/livesync-cli
|
||||
# Install the entrypoint wrapper with a deterministic mode, regardless of
|
||||
# source checkout permissions.
|
||||
COPY --chmod=755 src/apps/cli/docker-entrypoint.sh /usr/local/bin/livesync-cli
|
||||
|
||||
# Mount your vault / local database directory here
|
||||
VOLUME ["/data"]
|
||||
|
||||
+11
-11
@@ -15,9 +15,10 @@ This CLI version is built using the same core as the Obsidian plug-in:
|
||||
|
||||
```
|
||||
CLI Main
|
||||
└─ LiveSyncBaseCore<ServiceContext, IMinimumLiveSyncCommands>
|
||||
├─ NodeServiceHub (All services without Obsidian dependencies)
|
||||
└─ ServiceModules (wired by initialiseServiceModulesCLI)
|
||||
└─ NodeServiceContext (events, translation, database root, and injected standard I/O)
|
||||
└─ LiveSyncBaseCore<NodeServiceContext, IMinimumLiveSyncCommands>
|
||||
├─ NodeServiceHub (All services without Obsidian dependencies)
|
||||
└─ ServiceModules (wired by initialiseServiceModulesCLI)
|
||||
├─ FileAccessCLI (Node.js FileSystemAdapter)
|
||||
├─ StorageEventManagerCLI
|
||||
├─ ServiceFileAccessCLI
|
||||
@@ -37,7 +38,9 @@ CLI Main
|
||||
- All core sync functionality preserved
|
||||
|
||||
3. **Service Hub and Settings Services** (`services/`)
|
||||
- `NodeServiceHub` provides the CLI service context
|
||||
- `NodeServiceContext` owns the host-selected database root and standard input/output implementation
|
||||
- `NodeServiceHub` receives that exact Context instead of constructing platform capabilities implicitly
|
||||
- Internal adapter diagnostics use injected callbacks wired to the service logging API
|
||||
- Node-specific settings and key-value services are provided without Obsidian dependencies
|
||||
|
||||
4. **Main Entry Point** (`main.ts`)
|
||||
@@ -108,13 +111,10 @@ livesync-cli ./my-db pull folder/note.md ./note.md
|
||||
### Build from source
|
||||
|
||||
```bash
|
||||
# Clone with submodules, because the shared core lives in src/lib
|
||||
git clone --recurse-submodules <repository-url>
|
||||
# Clone the repository
|
||||
git clone <repository-url>
|
||||
cd obsidian-livesync
|
||||
|
||||
# If you already cloned without submodules, run this once instead
|
||||
git submodule update --init --recursive
|
||||
|
||||
# Install dependencies from the repository root
|
||||
npm install
|
||||
|
||||
@@ -126,7 +126,7 @@ cd src/apps/cli
|
||||
npm run build
|
||||
```
|
||||
|
||||
If `src/lib` is missing, the build process stops early with a targeted message instead of a low-level Vite `ENOENT` error.
|
||||
The shared core is installed as the exact `@vrtmrz/livesync-commonlib` package artefact recorded in the root lockfile.
|
||||
|
||||
Run the CLI:
|
||||
|
||||
@@ -337,7 +337,7 @@ Options:
|
||||
|
||||
Commands:
|
||||
daemon (default) Run mirror scan then continuously sync CouchDB <-> local filesystem
|
||||
init-settings [path] Create settings JSON from DEFAULT_SETTINGS
|
||||
init-settings [path] Create unconfigured settings JSON with the new-Vault recommendations
|
||||
sync Run one replication cycle and exit
|
||||
p2p-peers <timeout> Show discovered peers as [peer]<TAB><peer-id><TAB><peer-name>
|
||||
p2p-sync <peer> <timeout> Synchronise with specified peer-id or peer-name
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { FilePath, UXFileInfoStub, UXFolderInfo } from "@lib/common/types";
|
||||
import type { IConversionAdapter } from "@lib/serviceModules/adapters";
|
||||
import type { FilePath, UXFileInfoStub, UXFolderInfo } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { IConversionAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
|
||||
import type { NodeFile, NodeFolder } from "./NodeTypes";
|
||||
import { path } from "@/apps/cli/node-compat";
|
||||
import { path } from "@vrtmrz/livesync-commonlib/node";
|
||||
|
||||
/**
|
||||
* Conversion adapter implementation for Node.js
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import type { FilePath, UXStat } from "@lib/common/types";
|
||||
import type { IFileSystemAdapter } from "@lib/serviceModules/adapters";
|
||||
import type { FilePath, UXStat } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { IFileSystemAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
|
||||
import { NodePathAdapter } from "./NodePathAdapter";
|
||||
import { NodeTypeGuardAdapter } from "./NodeTypeGuardAdapter";
|
||||
import { NodeConversionAdapter } from "./NodeConversionAdapter";
|
||||
import { NodeStorageAdapter } from "./NodeStorageAdapter";
|
||||
import { NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { NodeVaultAdapter } from "./NodeVaultAdapter";
|
||||
import type { NodeFile, NodeFolder, NodeStat } from "./NodeTypes";
|
||||
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
|
||||
import { path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput";
|
||||
|
||||
/**
|
||||
* Complete file system adapter implementation for Node.js
|
||||
@@ -20,12 +21,15 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
|
||||
|
||||
private fileCache = new Map<string, NodeFile>();
|
||||
|
||||
constructor(private basePath: string) {
|
||||
constructor(
|
||||
private basePath: string,
|
||||
private reportDiagnostic: CliDiagnosticReporter = () => undefined
|
||||
) {
|
||||
this.path = new NodePathAdapter();
|
||||
this.typeGuard = new NodeTypeGuardAdapter();
|
||||
this.conversion = new NodeConversionAdapter();
|
||||
this.storage = new NodeStorageAdapter(basePath);
|
||||
this.vault = new NodeVaultAdapter(basePath);
|
||||
this.vault = new NodeVaultAdapter(this.storage);
|
||||
}
|
||||
|
||||
private resolvePath(p: FilePath | string): string {
|
||||
@@ -33,11 +37,31 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
|
||||
}
|
||||
|
||||
private normalisePath(p: FilePath | string): string {
|
||||
return this.path.normalisePath(p as string);
|
||||
return this.path.normalisePath(p);
|
||||
}
|
||||
|
||||
private async hasExactPathCase(pathStr: string): Promise<boolean> {
|
||||
try {
|
||||
const segments = pathStr.split("/").filter((segment) => segment !== "");
|
||||
let currentPath = "";
|
||||
for (const segment of segments) {
|
||||
const entries = await this.storage.list(currentPath);
|
||||
const candidatePath = currentPath === "" ? segment : `${currentPath}/${segment}`;
|
||||
if (!entries.files.includes(candidatePath) && !entries.folders.includes(candidatePath)) return false;
|
||||
currentPath = candidatePath;
|
||||
}
|
||||
return segments.length > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async getAbstractFileByPath(p: FilePath | string): Promise<NodeFile | null> {
|
||||
const pathStr = this.normalisePath(p);
|
||||
if (!this.fileCache.has(pathStr) && !(await this.hasExactPathCase(pathStr))) {
|
||||
this.fileCache.delete(pathStr);
|
||||
return null;
|
||||
}
|
||||
return await this.refreshFile(pathStr);
|
||||
}
|
||||
|
||||
@@ -74,6 +98,15 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
|
||||
return Array.from(this.fileCache.values());
|
||||
}
|
||||
|
||||
async renameFile(file: NodeFile, newPath: string): Promise<NodeFile> {
|
||||
const oldPath = file.path;
|
||||
await this.vault.rename(file, newPath);
|
||||
this.fileCache.delete(oldPath);
|
||||
const renamedFile = await this.refreshFile(newPath);
|
||||
if (!renamedFile) throw new Error(`Could not find renamed file: ${newPath}`);
|
||||
return renamedFile;
|
||||
}
|
||||
|
||||
async statFromNative(file: NodeFile): Promise<UXStat> {
|
||||
return file.stat;
|
||||
}
|
||||
@@ -86,9 +119,8 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
|
||||
async refreshFile(p: string): Promise<NodeFile | null> {
|
||||
const pathStr = this.normalisePath(p);
|
||||
try {
|
||||
const fullPath = this.resolvePath(pathStr);
|
||||
const stat = await fs.stat(fullPath);
|
||||
if (!stat.isFile()) {
|
||||
const stat = await this.storage.stat(pathStr);
|
||||
if (stat?.type !== "file") {
|
||||
this.fileCache.delete(pathStr);
|
||||
return null;
|
||||
}
|
||||
@@ -97,8 +129,8 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
|
||||
path: pathStr as FilePath,
|
||||
stat: {
|
||||
size: stat.size,
|
||||
mtime: Math.floor(stat.mtimeMs),
|
||||
ctime: Math.floor(stat.ctimeMs),
|
||||
mtime: stat.mtime,
|
||||
ctime: stat.ctime,
|
||||
type: "file",
|
||||
},
|
||||
};
|
||||
@@ -117,31 +149,25 @@ export class NodeFileSystemAdapter implements IFileSystemAdapter<NodeFile, NodeF
|
||||
async scanDirectory(relativePath: string = ""): Promise<void> {
|
||||
const fullPath = this.resolvePath(relativePath);
|
||||
try {
|
||||
const entries = await fs.readdir(fullPath, { withFileTypes: true });
|
||||
const directoryStat = await this.storage.stat(relativePath);
|
||||
if (directoryStat?.type !== "folder") throw new Error(`Directory does not exist: ${fullPath}`);
|
||||
const entries = await this.storage.list(relativePath);
|
||||
|
||||
for (const entry of entries) {
|
||||
const entryRelativePath = path.join(relativePath, entry.name).replace(/\\/g, "/");
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await this.scanDirectory(entryRelativePath);
|
||||
} else if (entry.isFile()) {
|
||||
const entryFullPath = this.resolvePath(entryRelativePath);
|
||||
const stat = await fs.stat(entryFullPath);
|
||||
const file: NodeFile = {
|
||||
path: entryRelativePath as FilePath,
|
||||
stat: {
|
||||
size: stat.size,
|
||||
mtime: Math.floor(stat.mtimeMs),
|
||||
ctime: Math.floor(stat.ctimeMs),
|
||||
type: "file",
|
||||
},
|
||||
};
|
||||
this.fileCache.set(entryRelativePath, file);
|
||||
}
|
||||
for (const entryPath of entries.files) {
|
||||
const stat = await this.storage.stat(entryPath);
|
||||
if (stat?.type !== "file") continue;
|
||||
const file: NodeFile = {
|
||||
path: entryPath as FilePath,
|
||||
stat,
|
||||
};
|
||||
this.fileCache.set(entryPath, file);
|
||||
}
|
||||
for (const entryPath of entries.folders) {
|
||||
await this.scanDirectory(entryPath);
|
||||
}
|
||||
} catch (error) {
|
||||
// Directory doesn't exist or is not readable
|
||||
console.error(`Error scanning directory ${fullPath}:`, error);
|
||||
this.reportDiagnostic(`Error scanning directory ${fullPath}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { FilePath } from "@lib/common/types";
|
||||
import type { IPathAdapter } from "@lib/serviceModules/adapters";
|
||||
import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { IPathAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
|
||||
import type { NodeFile } from "./NodeTypes";
|
||||
import { path } from "@/apps/cli/node-compat";
|
||||
import { path } from "@vrtmrz/livesync-commonlib/node";
|
||||
|
||||
/**
|
||||
* Path adapter implementation for Node.js
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
import type { UXDataWriteOptions } from "@lib/common/types";
|
||||
import type { IStorageAdapter } from "@lib/serviceModules/adapters";
|
||||
import type { NodeStat } from "./NodeTypes";
|
||||
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
|
||||
|
||||
/**
|
||||
* Storage adapter implementation for Node.js
|
||||
*/
|
||||
export class NodeStorageAdapter implements IStorageAdapter<NodeStat> {
|
||||
constructor(private basePath: string) {}
|
||||
|
||||
private resolvePath(p: string): string {
|
||||
return path.join(this.basePath, p);
|
||||
}
|
||||
|
||||
async exists(p: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(this.resolvePath(p));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async trystat(p: string): Promise<NodeStat | null> {
|
||||
try {
|
||||
const stat = await fs.stat(this.resolvePath(p));
|
||||
return {
|
||||
size: stat.size,
|
||||
mtime: Math.floor(stat.mtimeMs),
|
||||
ctime: Math.floor(stat.ctimeMs),
|
||||
type: stat.isDirectory() ? "folder" : "file",
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async stat(p: string): Promise<NodeStat | null> {
|
||||
return await this.trystat(p);
|
||||
}
|
||||
|
||||
async mkdir(p: string): Promise<void> {
|
||||
await fs.mkdir(this.resolvePath(p), { recursive: true });
|
||||
}
|
||||
|
||||
async remove(p: string): Promise<void> {
|
||||
const fullPath = this.resolvePath(p);
|
||||
const stat = await fs.stat(fullPath);
|
||||
if (stat.isDirectory()) {
|
||||
await fs.rm(fullPath, { recursive: true, force: true });
|
||||
} else {
|
||||
await fs.unlink(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
async read(p: string): Promise<string> {
|
||||
return await fs.readFile(this.resolvePath(p), "utf-8");
|
||||
}
|
||||
|
||||
async readBinary(p: string): Promise<ArrayBuffer> {
|
||||
const buffer = await fs.readFile(this.resolvePath(p));
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion -- required in environments where Buffer.buffer is ArrayBufferLike
|
||||
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer;
|
||||
}
|
||||
|
||||
async write(p: string, data: string, options?: UXDataWriteOptions): Promise<void> {
|
||||
const fullPath = this.resolvePath(p);
|
||||
await fs.mkdir(path.dirname(fullPath), { recursive: true });
|
||||
await fs.writeFile(fullPath, data, "utf-8");
|
||||
|
||||
if (options?.mtime || options?.ctime) {
|
||||
const atime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
const mtime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
await fs.utimes(fullPath, atime, mtime);
|
||||
}
|
||||
}
|
||||
|
||||
async writeBinary(p: string, data: ArrayBuffer, options?: UXDataWriteOptions): Promise<void> {
|
||||
const fullPath = this.resolvePath(p);
|
||||
await fs.mkdir(path.dirname(fullPath), { recursive: true });
|
||||
await fs.writeFile(fullPath, new Uint8Array(data));
|
||||
|
||||
if (options?.mtime || options?.ctime) {
|
||||
const atime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
const mtime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
await fs.utimes(fullPath, atime, mtime);
|
||||
}
|
||||
}
|
||||
|
||||
async append(p: string, data: string, options?: UXDataWriteOptions): Promise<void> {
|
||||
const fullPath = this.resolvePath(p);
|
||||
await fs.mkdir(path.dirname(fullPath), { recursive: true });
|
||||
await fs.appendFile(fullPath, data, "utf-8");
|
||||
|
||||
if (options?.mtime || options?.ctime) {
|
||||
const atime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
const mtime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
await fs.utimes(fullPath, atime, mtime);
|
||||
}
|
||||
}
|
||||
|
||||
async list(basePath: string): Promise<{ files: string[]; folders: string[] }> {
|
||||
const fullPath = this.resolvePath(basePath);
|
||||
try {
|
||||
const entries = await fs.readdir(fullPath, { withFileTypes: true });
|
||||
const files: string[] = [];
|
||||
const folders: string[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(basePath, entry.name).replace(/\\/g, "/");
|
||||
if (entry.isDirectory()) {
|
||||
folders.push(entryPath);
|
||||
} else if (entry.isFile()) {
|
||||
files.push(entryPath);
|
||||
}
|
||||
}
|
||||
|
||||
return { files, folders };
|
||||
} catch {
|
||||
return { files: [], folders: [] };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,8 @@
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { NodeStorageAdapter } from "./NodeStorageAdapter";
|
||||
import { storageAdapterContractCases } from "@/apps/_test/storageAdapterContract";
|
||||
import { fsPromises as fs, os, path, NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node";
|
||||
|
||||
describe("NodeStorageAdapter binary I/O", () => {
|
||||
describe("NodeStorageAdapter", () => {
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
async function createAdapter() {
|
||||
@@ -17,6 +15,12 @@ describe("NodeStorageAdapter binary I/O", () => {
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
for (const contractCase of storageAdapterContractCases) {
|
||||
it(contractCase.name, async () => {
|
||||
await contractCase.run(await createAdapter());
|
||||
});
|
||||
}
|
||||
|
||||
it("writes and reads binary data without corruption", async () => {
|
||||
const adapter = await createAdapter();
|
||||
const expected = Uint8Array.from([0x00, 0x7f, 0x80, 0xff, 0x42]);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { ITypeGuardAdapter } from "@lib/serviceModules/adapters";
|
||||
import type { ITypeGuardAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
|
||||
import type { NodeFile, NodeFolder } from "./NodeTypes";
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { FilePath, UXStat } from "@lib/common/types";
|
||||
import type { FilePath, UXStat } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
|
||||
/**
|
||||
* Node.js file representation
|
||||
|
||||
@@ -1,20 +1,21 @@
|
||||
import type { FilePath, UXDataWriteOptions } from "@lib/common/types";
|
||||
import type { IVaultAdapter } from "@lib/serviceModules/adapters";
|
||||
import type { FilePath, UXDataWriteOptions } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { IVaultAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
|
||||
import type { NodeFile, NodeFolder } from "./NodeTypes";
|
||||
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
|
||||
import { NodeStorageAdapter } from "@vrtmrz/livesync-commonlib/node";
|
||||
|
||||
/**
|
||||
* Vault adapter implementation for Node.js
|
||||
*/
|
||||
export class NodeVaultAdapter implements IVaultAdapter<NodeFile> {
|
||||
constructor(private basePath: string) {}
|
||||
private readonly storage: NodeStorageAdapter;
|
||||
|
||||
private resolvePath(p: string): string {
|
||||
return path.join(this.basePath, p);
|
||||
constructor(rootPathOrStorage: string | NodeStorageAdapter) {
|
||||
this.storage =
|
||||
typeof rootPathOrStorage === "string" ? new NodeStorageAdapter(rootPathOrStorage) : rootPathOrStorage;
|
||||
}
|
||||
|
||||
async read(file: NodeFile): Promise<string> {
|
||||
const content = await fs.readFile(this.resolvePath(file.path), "utf-8");
|
||||
const content = await this.storage.read(file.path);
|
||||
// Correct stale stat.size — chokidar stats may be from a poll before the final write.
|
||||
// The downstream document integrity check compares stat.size to content length, so
|
||||
// they must agree or other clients reject the file as corrupted.
|
||||
@@ -28,89 +29,37 @@ export class NodeVaultAdapter implements IVaultAdapter<NodeFile> {
|
||||
}
|
||||
|
||||
async readBinary(file: NodeFile): Promise<ArrayBuffer> {
|
||||
const buffer = await fs.readFile(this.resolvePath(file.path));
|
||||
const buffer = await this.storage.readBinary(file.path);
|
||||
// Same correction as read() — ensure stat.size matches actual byte length.
|
||||
file.stat.size = buffer.length;
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion -- required in environments where Buffer.buffer is ArrayBufferLike
|
||||
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer;
|
||||
file.stat.size = buffer.byteLength;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
async modify(file: NodeFile, data: string, options?: UXDataWriteOptions): Promise<void> {
|
||||
const fullPath = this.resolvePath(file.path);
|
||||
await fs.writeFile(fullPath, data, "utf-8");
|
||||
|
||||
if (options?.mtime || options?.ctime) {
|
||||
const atime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
const mtime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
await fs.utimes(fullPath, atime, mtime);
|
||||
}
|
||||
await this.storage.write(file.path, data, options);
|
||||
}
|
||||
|
||||
async modifyBinary(file: NodeFile, data: ArrayBuffer, options?: UXDataWriteOptions): Promise<void> {
|
||||
const fullPath = this.resolvePath(file.path);
|
||||
await fs.writeFile(fullPath, new Uint8Array(data));
|
||||
|
||||
if (options?.mtime || options?.ctime) {
|
||||
const atime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
const mtime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
await fs.utimes(fullPath, atime, mtime);
|
||||
}
|
||||
await this.storage.writeBinary(file.path, data, options);
|
||||
}
|
||||
|
||||
async create(p: string, data: string, options?: UXDataWriteOptions): Promise<NodeFile> {
|
||||
const fullPath = this.resolvePath(p);
|
||||
await fs.mkdir(path.dirname(fullPath), { recursive: true });
|
||||
await fs.writeFile(fullPath, data, "utf-8");
|
||||
|
||||
if (options?.mtime || options?.ctime) {
|
||||
const atime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
const mtime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
await fs.utimes(fullPath, atime, mtime);
|
||||
}
|
||||
|
||||
const stat = await fs.stat(fullPath);
|
||||
return {
|
||||
path: p as FilePath,
|
||||
stat: {
|
||||
size: stat.size,
|
||||
mtime: Math.floor(stat.mtimeMs),
|
||||
ctime: Math.floor(stat.ctimeMs),
|
||||
type: "file",
|
||||
},
|
||||
};
|
||||
await this.storage.write(p, data, options);
|
||||
return await this.toNodeFile(p);
|
||||
}
|
||||
|
||||
async createBinary(p: string, data: ArrayBuffer, options?: UXDataWriteOptions): Promise<NodeFile> {
|
||||
const fullPath = this.resolvePath(p);
|
||||
await fs.mkdir(path.dirname(fullPath), { recursive: true });
|
||||
await fs.writeFile(fullPath, new Uint8Array(data));
|
||||
await this.storage.writeBinary(p, data, options);
|
||||
return await this.toNodeFile(p);
|
||||
}
|
||||
|
||||
if (options?.mtime || options?.ctime) {
|
||||
const atime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
const mtime = options.mtime ? new Date(options.mtime) : new Date();
|
||||
await fs.utimes(fullPath, atime, mtime);
|
||||
}
|
||||
|
||||
const stat = await fs.stat(fullPath);
|
||||
return {
|
||||
path: p as FilePath,
|
||||
stat: {
|
||||
size: stat.size,
|
||||
mtime: Math.floor(stat.mtimeMs),
|
||||
ctime: Math.floor(stat.ctimeMs),
|
||||
type: "file",
|
||||
},
|
||||
};
|
||||
async rename(file: NodeFile, newPath: string): Promise<void> {
|
||||
await this.storage.rename(file.path, newPath);
|
||||
file.path = newPath as FilePath;
|
||||
}
|
||||
|
||||
async delete(file: NodeFile | NodeFolder, force = false): Promise<void> {
|
||||
const fullPath = this.resolvePath(file.path);
|
||||
const stat = await fs.stat(fullPath);
|
||||
if (stat.isDirectory()) {
|
||||
await fs.rm(fullPath, { recursive: true, force });
|
||||
} else {
|
||||
await fs.unlink(fullPath);
|
||||
}
|
||||
await this.storage.remove(file.path);
|
||||
}
|
||||
|
||||
async trash(file: NodeFile | NodeFolder, force = false): Promise<void> {
|
||||
@@ -122,4 +71,10 @@ export class NodeVaultAdapter implements IVaultAdapter<NodeFile> {
|
||||
// No-op in CLI version (no event system)
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private async toNodeFile(path: string): Promise<NodeFile> {
|
||||
const stat = await this.storage.stat(path);
|
||||
if (stat?.type !== "file") throw new Error(`Could not read created file metadata: ${path}`);
|
||||
return { path: path as FilePath, stat };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { fsPromises, os, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { NodeFileSystemAdapter } from "./NodeFileSystemAdapter";
|
||||
import { NodeVaultAdapter } from "./NodeVaultAdapter";
|
||||
|
||||
describe("NodeVaultAdapter.rename", () => {
|
||||
it("changes the directory entry case without changing the content", async () => {
|
||||
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-case-rename-"));
|
||||
try {
|
||||
await fsPromises.writeFile(path.join(directory, "Calculus.md"), "content", "utf8");
|
||||
const adapter = new NodeVaultAdapter(directory);
|
||||
const file = {
|
||||
path: "Calculus.md" as FilePath,
|
||||
stat: { ctime: 1, mtime: 2, size: 7, type: "file" as const },
|
||||
};
|
||||
|
||||
await adapter.rename(file, "calculus.md");
|
||||
|
||||
expect(await fsPromises.readdir(directory)).toEqual(["calculus.md"]);
|
||||
expect(await fsPromises.readFile(path.join(directory, "calculus.md"), "utf8")).toBe("content");
|
||||
expect(file.path).toBe("calculus.md");
|
||||
} finally {
|
||||
await fsPromises.rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not move a file through a symbolic link outside the vault root", async () => {
|
||||
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-rename-root-"));
|
||||
const outsideDirectory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-rename-outside-"));
|
||||
try {
|
||||
await fsPromises.writeFile(path.join(directory, "source.md"), "content", "utf8");
|
||||
await fsPromises.symlink(
|
||||
outsideDirectory,
|
||||
path.join(directory, "linked"),
|
||||
process.platform === "win32" ? "junction" : "dir"
|
||||
);
|
||||
const adapter = new NodeVaultAdapter(directory);
|
||||
const file = {
|
||||
path: "source.md" as FilePath,
|
||||
stat: { ctime: 1, mtime: 2, size: 7, type: "file" as const },
|
||||
};
|
||||
|
||||
await expect(adapter.rename(file, "linked/moved.md")).rejects.toThrow(/symbolic link/i);
|
||||
|
||||
await expect(fsPromises.readFile(path.join(directory, "source.md"), "utf8")).resolves.toBe("content");
|
||||
await expect(fsPromises.stat(path.join(outsideDirectory, "moved.md"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
} finally {
|
||||
await fsPromises.rm(directory, { recursive: true, force: true });
|
||||
await fsPromises.rm(outsideDirectory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not modify a file through a symbolic link outside the vault root", async () => {
|
||||
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-modify-root-"));
|
||||
const outsideDirectory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-modify-outside-"));
|
||||
try {
|
||||
await fsPromises.writeFile(path.join(outsideDirectory, "victim.md"), "before", "utf8");
|
||||
await fsPromises.symlink(
|
||||
outsideDirectory,
|
||||
path.join(directory, "linked"),
|
||||
process.platform === "win32" ? "junction" : "dir"
|
||||
);
|
||||
const adapter = new NodeVaultAdapter(directory);
|
||||
const file = {
|
||||
path: "linked/victim.md" as FilePath,
|
||||
stat: { ctime: 1, mtime: 2, size: 6, type: "file" as const },
|
||||
};
|
||||
|
||||
await expect(adapter.modify(file, "after")).rejects.toThrow(/symbolic link/i);
|
||||
|
||||
await expect(fsPromises.readFile(path.join(outsideDirectory, "victim.md"), "utf8")).resolves.toBe("before");
|
||||
} finally {
|
||||
await fsPromises.rm(directory, { recursive: true, force: true });
|
||||
await fsPromises.rm(outsideDirectory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("NodeFileSystemAdapter path case", () => {
|
||||
it("finds the stored case and refreshes the cache after a case-only rename", async () => {
|
||||
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-case-cache-"));
|
||||
try {
|
||||
await fsPromises.writeFile(path.join(directory, "Calculus.md"), "content", "utf8");
|
||||
const adapter = new NodeFileSystemAdapter(directory);
|
||||
|
||||
await expect(adapter.getAbstractFileByPath("calculus.md")).resolves.toBeNull();
|
||||
const existingFile = await adapter.getAbstractFileByPathInsensitive("calculus.md");
|
||||
expect(existingFile?.path).toBe("Calculus.md");
|
||||
if (!existingFile) throw new Error("Expected to find Calculus.md case-insensitively");
|
||||
const renamedFile = await adapter.renameFile(existingFile, "calculus.md");
|
||||
|
||||
expect(renamedFile.path).toBe("calculus.md");
|
||||
expect((await adapter.getFiles()).map((file) => file.path)).toEqual(["calculus.md"]);
|
||||
expect(await fsPromises.readdir(directory)).toEqual(["calculus.md"]);
|
||||
} finally {
|
||||
await fsPromises.rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reports directory scan failures through the injected diagnostic callback", async () => {
|
||||
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-scan-diagnostic-"));
|
||||
const missingDirectory = path.join(directory, "missing");
|
||||
const reportDiagnostic = vi.fn();
|
||||
try {
|
||||
const adapter = new NodeFileSystemAdapter(missingDirectory, reportDiagnostic);
|
||||
|
||||
await adapter.scanDirectory();
|
||||
|
||||
expect(reportDiagnostic).toHaveBeenCalledWith(
|
||||
`Error scanning directory ${missingDirectory}:`,
|
||||
expect.any(Error)
|
||||
);
|
||||
} finally {
|
||||
await fsPromises.rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("does not discover a file through a symbolic link outside the vault root", async () => {
|
||||
const directory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-discovery-root-"));
|
||||
const outsideDirectory = await fsPromises.mkdtemp(path.join(os.tmpdir(), "livesync-discovery-outside-"));
|
||||
try {
|
||||
await fsPromises.writeFile(path.join(outsideDirectory, "outside.md"), "content", "utf8");
|
||||
await fsPromises.symlink(
|
||||
outsideDirectory,
|
||||
path.join(directory, "linked"),
|
||||
process.platform === "win32" ? "junction" : "dir"
|
||||
);
|
||||
const adapter = new NodeFileSystemAdapter(directory);
|
||||
|
||||
await expect(adapter.getAbstractFileByPath("linked/outside.md")).resolves.toBeNull();
|
||||
await expect(adapter.getFiles()).resolves.toEqual([]);
|
||||
} finally {
|
||||
await fsPromises.rm(directory, { recursive: true, force: true });
|
||||
await fsPromises.rm(outsideDirectory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import type { StandardIo } from "@vrtmrz/livesync-commonlib/context";
|
||||
|
||||
/** Report a CLI-owned diagnostic without selecting its final presentation channel. */
|
||||
export type CliDiagnosticReporter = (message: string, detail?: unknown) => void;
|
||||
|
||||
function formatValue(value: unknown): string {
|
||||
if (typeof value === "string") return value;
|
||||
if (value instanceof Error) return value.stack ?? value.message;
|
||||
try {
|
||||
const encoded = JSON.stringify(value);
|
||||
if (encoded !== undefined) return encoded;
|
||||
} catch {
|
||||
// Fall through to the host-independent string conversion.
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function formatLine(values: readonly unknown[]): string {
|
||||
return `${values.map(formatValue).join(" ")}\n`;
|
||||
}
|
||||
|
||||
/** Render one user-facing line on standard output. */
|
||||
export function writeStdoutLine(standardIo: StandardIo, ...values: readonly unknown[]): void {
|
||||
standardIo.writeStdout(formatLine(values));
|
||||
}
|
||||
|
||||
/** Render one user-facing or diagnostic line on standard error. */
|
||||
export function writeStderrLine(standardIo: StandardIo, ...values: readonly unknown[]): void {
|
||||
standardIo.writeStderr(formatLine(values));
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings";
|
||||
|
||||
export function createDefaultCliSettings(): ObsidianLiveSyncSettings {
|
||||
return {
|
||||
...createNewVaultSettings(),
|
||||
useIndexedDBAdapter: false,
|
||||
isConfigured: false,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createNewVaultSettings } from "@vrtmrz/livesync-commonlib/settings";
|
||||
import { createDefaultCliSettings } from "./cliSettingsDefaults.ts";
|
||||
|
||||
describe("createDefaultCliSettings", () => {
|
||||
it("uses the recommended new-Vault settings with the Node database adapter", () => {
|
||||
const settings = createDefaultCliSettings();
|
||||
const recommended = createNewVaultSettings();
|
||||
|
||||
expect(settings).toEqual({
|
||||
...recommended,
|
||||
useIndexedDBAdapter: false,
|
||||
isConfigured: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,14 +1,15 @@
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
|
||||
import { createServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { runCommand } from "./runCommand";
|
||||
import type { CLIOptions } from "./types";
|
||||
|
||||
// Mock performFullScan so daemon tests don't require a real CouchDB connection.
|
||||
vi.mock("@lib/serviceFeatures/offlineScanner", () => ({
|
||||
vi.mock("@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner", () => ({
|
||||
performFullScan: vi.fn(async () => true),
|
||||
}));
|
||||
|
||||
// Mock UnresolvedErrorManager to avoid event-hub side effects.
|
||||
vi.mock("@lib/services/base/UnresolvedErrorManager", () => ({
|
||||
vi.mock("@vrtmrz/livesync-commonlib/compat/services/base/UnresolvedErrorManager", () => ({
|
||||
UnresolvedErrorManager: class UnresolvedErrorManager {
|
||||
showError() {}
|
||||
clearError() {}
|
||||
@@ -16,11 +17,18 @@ vi.mock("@lib/services/base/UnresolvedErrorManager", () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
import * as offlineScanner from "@lib/serviceFeatures/offlineScanner";
|
||||
import * as offlineScanner from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
|
||||
|
||||
function createCoreMock() {
|
||||
const standardIo = {
|
||||
readStdin: vi.fn(async () => ""),
|
||||
prompt: vi.fn(async () => ""),
|
||||
writeStdout: vi.fn((_chunk: string | Uint8Array) => undefined),
|
||||
writeStderr: vi.fn((_chunk: string | Uint8Array) => undefined),
|
||||
};
|
||||
return {
|
||||
services: {
|
||||
context: Object.assign(createServiceContext(), { standardIo }),
|
||||
control: {
|
||||
activated: Promise.resolve(),
|
||||
applySettings: vi.fn(async () => {}),
|
||||
@@ -155,13 +163,13 @@ describe("daemon command", () => {
|
||||
syncOnStart: false,
|
||||
}));
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
const result = await runCommand(makeDaemonOptions(), { ...baseContext, core });
|
||||
|
||||
expect(result).toBe(true);
|
||||
const warningCalls = consoleSpy.mock.calls.filter(
|
||||
(args) => typeof args[0] === "string" && args[0].includes("liveSync and syncOnStart are both disabled")
|
||||
const warningCalls = core.services.context.standardIo.writeStderr.mock.calls.filter(
|
||||
([chunk]: [string | Uint8Array]) =>
|
||||
typeof chunk === "string" && chunk.includes("liveSync and syncOnStart are both disabled")
|
||||
);
|
||||
expect(warningCalls.length).toBeGreaterThan(0);
|
||||
});
|
||||
@@ -173,12 +181,12 @@ describe("daemon command", () => {
|
||||
syncOnStart: false,
|
||||
}));
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
await runCommand(makeDaemonOptions(), { ...baseContext, core });
|
||||
|
||||
const warningCalls = consoleSpy.mock.calls.filter(
|
||||
(args) => typeof args[0] === "string" && args[0].includes("liveSync and syncOnStart are both disabled")
|
||||
const warningCalls = core.services.context.standardIo.writeStderr.mock.calls.filter(
|
||||
([chunk]: [string | Uint8Array]) =>
|
||||
typeof chunk === "string" && chunk.includes("liveSync and syncOnStart are both disabled")
|
||||
);
|
||||
expect(warningCalls.length).toBe(0);
|
||||
});
|
||||
@@ -231,7 +239,6 @@ describe("daemon command", () => {
|
||||
it("polling backoff: interval escalates on failure, caps at 300000ms, then halves on recovery", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
// startup replicate (call 1) succeeds; poll calls 2–7 fail; call 8 succeeds.
|
||||
let callCount = 0;
|
||||
@@ -284,10 +291,9 @@ describe("daemon command", () => {
|
||||
expect(setTimeoutSpy.mock.calls[afterSuccessCallCount - 1][1]).toBe(150_000);
|
||||
});
|
||||
|
||||
it("polling error handling: replicate rejection is caught and console.error is called", async () => {
|
||||
it("polling error handling: replicate rejection is caught and written to standard error", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.mocked(offlineScanner.performFullScan).mockResolvedValue(true);
|
||||
const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
// Make replicate succeed on the initial call (startup), then fail on the poll.
|
||||
let callCount = 0;
|
||||
@@ -304,8 +310,8 @@ describe("daemon command", () => {
|
||||
await vi.advanceTimersByTimeAsync(intervalMs);
|
||||
|
||||
// No unhandled rejection — the error was caught internally.
|
||||
const errorCalls = consoleSpy.mock.calls.filter(
|
||||
(args) => typeof args[0] === "string" && args[0].includes("Poll error")
|
||||
const errorCalls = core.services.context.standardIo.writeStderr.mock.calls.filter(
|
||||
([chunk]: [string | Uint8Array]) => typeof chunk === "string" && chunk.includes("Poll error")
|
||||
);
|
||||
expect(errorCalls.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
@@ -1,15 +1,24 @@
|
||||
import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
|
||||
import { P2P_DEFAULT_SETTINGS } from "@lib/common/types";
|
||||
import type { ServiceContext } from "@lib/services/base/ServiceBase";
|
||||
import { LiveSyncTrysteroReplicator } from "@lib/replication/trystero/LiveSyncTrysteroReplicator";
|
||||
import { compatGlobal } from "@lib/common/coreEnvFunctions.ts";
|
||||
import { LiveSyncError } from "@lib/common/LSError";
|
||||
import { P2P_DEFAULT_SETTINGS } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { LiveSyncTrysteroReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/LiveSyncTrysteroReplicator";
|
||||
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
import { LiveSyncError } from "@vrtmrz/livesync-commonlib/compat/common/LSError";
|
||||
import { getPeerConnectionStats } from "@vrtmrz/livesync-commonlib/compat/rpc/transports/DiagRTCPeerConnections.utils";
|
||||
import { fsPromises } from "@vrtmrz/livesync-commonlib/node";
|
||||
|
||||
type CLIP2PPeer = {
|
||||
peerId: string;
|
||||
name: string;
|
||||
};
|
||||
|
||||
type CandidateSummary = {
|
||||
id: string;
|
||||
candidateType: string;
|
||||
protocol: string;
|
||||
relayProtocol: string;
|
||||
};
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => compatGlobal.setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -81,6 +90,74 @@ function resolvePeer(peers: CLIP2PPeer[], peerToken: string): CLIP2PPeer | undef
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function getReportValue<T extends string | number>(
|
||||
report: Record<string, unknown> | undefined,
|
||||
key: string
|
||||
): T | "unknown" {
|
||||
const value = report?.[key];
|
||||
return typeof value === "string" || typeof value === "number" ? (value as T) : "unknown";
|
||||
}
|
||||
|
||||
function summariseCandidate(reports: unknown[], candidateId: string): CandidateSummary | undefined {
|
||||
if (candidateId === "unknown") {
|
||||
return undefined;
|
||||
}
|
||||
const report = reports.map((r) => r as Record<string, unknown>).find((r) => r.id === candidateId);
|
||||
if (!report) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
id: candidateId,
|
||||
candidateType: getReportValue<string>(report, "candidateType"),
|
||||
protocol: getReportValue<string>(report, "protocol"),
|
||||
relayProtocol: getReportValue<string>(report, "relayProtocol"),
|
||||
};
|
||||
}
|
||||
|
||||
async function writePeerConnectionStatsIfRequested(
|
||||
replicator: LiveSyncTrysteroReplicator,
|
||||
peer: CLIP2PPeer
|
||||
): Promise<void> {
|
||||
const outputPath = process.env.LIVESYNC_P2P_STATS_JSONL?.trim();
|
||||
if (!outputPath) {
|
||||
return;
|
||||
}
|
||||
|
||||
const peerConnection = replicator.rawHost?.room?.getPeers()[peer.peerId];
|
||||
const stats = peerConnection ? await getPeerConnectionStats(`cli-p2p-${peer.peerId}`, peerConnection) : undefined;
|
||||
const localCandidate = summariseCandidate(stats?.reports ?? [], stats?.localCandidateId ?? "unknown");
|
||||
const remoteCandidate = summariseCandidate(stats?.reports ?? [], stats?.remoteCandidateId ?? "unknown");
|
||||
const selectedPath =
|
||||
localCandidate && remoteCandidate
|
||||
? `${localCandidate.candidateType}<->${remoteCandidate.candidateType}`
|
||||
: "unknown";
|
||||
|
||||
const payload = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
command: "p2p-sync",
|
||||
peerId: peer.peerId,
|
||||
peerName: peer.name,
|
||||
candidatePathCollected: !!stats?.selectedPair,
|
||||
selectedPath,
|
||||
selectedPair: stats
|
||||
? {
|
||||
id: stats.selectedPairId,
|
||||
state: stats.state,
|
||||
currentRoundTripTime: stats.currentRoundTripTime,
|
||||
totalRoundTripTime: stats.totalRoundTripTime,
|
||||
requestsSent: stats.requestsSent,
|
||||
responsesReceived: stats.responsesReceived,
|
||||
packetsDiscardedOnSend: stats.packetsDiscardedOnSend,
|
||||
bytesSent: stats.bytesSent,
|
||||
bytesReceived: stats.bytesReceived,
|
||||
}
|
||||
: undefined,
|
||||
localCandidate,
|
||||
remoteCandidate,
|
||||
};
|
||||
await fsPromises.appendFile(outputPath, `${JSON.stringify(payload)}\n`, "utf8");
|
||||
}
|
||||
|
||||
export async function syncWithPeer(
|
||||
core: LiveSyncBaseCore<ServiceContext, never>,
|
||||
peerToken: string,
|
||||
@@ -112,12 +189,13 @@ export async function syncWithPeer(
|
||||
}
|
||||
const pushResult = await replicator.requestSynchroniseToPeer(targetPeer.peerId);
|
||||
if (!pushResult || pushResult.ok !== true) {
|
||||
const err = pushResult?.error;
|
||||
const err: unknown = pushResult && "error" in pushResult ? pushResult.error : undefined;
|
||||
throw err instanceof Error
|
||||
? err
|
||||
: LiveSyncError.fromError(err ?? "P2P sync failed while requesting remote sync");
|
||||
}
|
||||
|
||||
await writePeerConnectionStatsIfRequested(replicator, targetPeer);
|
||||
return targetPeer;
|
||||
} finally {
|
||||
await replicator.close();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { decodeSettingsFromSetupURI } from "@lib/API/processSetting";
|
||||
import { configURIBase } from "@lib/common/models/shared.const";
|
||||
import { decodeSettingsFromSetupURI } from "@vrtmrz/livesync-commonlib/compat/API/processSetting";
|
||||
import { configURIBase } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const";
|
||||
import {
|
||||
DEFAULT_SETTINGS,
|
||||
MILESTONE_DOCID,
|
||||
@@ -9,19 +9,23 @@ import {
|
||||
REMOTE_MINIO,
|
||||
type EntryMilestoneInfo,
|
||||
type EntryDoc,
|
||||
} from "@lib/common/types";
|
||||
import { ConnectionStringParser } from "@lib/common/ConnectionString";
|
||||
import { activateRemoteConfiguration, createRemoteConfigurationId } from "@lib/serviceFeatures/remoteConfig";
|
||||
import { stripAllPrefixes } from "@lib/string_and_binary/path";
|
||||
} from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
|
||||
import {
|
||||
activateRemoteConfiguration,
|
||||
createRemoteConfigurationId,
|
||||
} from "@vrtmrz/livesync-commonlib/remote-configurations";
|
||||
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
|
||||
import type { CLICommandContext, CLIOptions } from "./types";
|
||||
import { promptForPassphrase, readStdinAsUtf8, toArrayBuffer, toDatabaseRelativePath } from "./utils";
|
||||
import { toArrayBuffer, toDatabaseRelativePath } from "./utils";
|
||||
import { collectPeers, openP2PHost, parseTimeoutSeconds, syncWithPeer } from "./p2p";
|
||||
import { performFullScan } from "@lib/serviceFeatures/offlineScanner";
|
||||
import { UnresolvedErrorManager } from "@lib/services/base/UnresolvedErrorManager";
|
||||
import { compatGlobal } from "@lib/common/coreEnvFunctions.ts";
|
||||
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
|
||||
import type { LiveSyncCouchDBReplicator } from "@lib/replication/couchdb/LiveSyncReplicator";
|
||||
import type { LiveSyncJournalReplicator } from "@lib/replication/journal/LiveSyncJournalReplicator";
|
||||
import { performFullScan } from "@vrtmrz/livesync-commonlib/compat/serviceFeatures/offlineScanner";
|
||||
import { UnresolvedErrorManager } from "@vrtmrz/livesync-commonlib/compat/services/base/UnresolvedErrorManager";
|
||||
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
import { fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { LiveSyncCouchDBReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/couchdb/LiveSyncReplicator";
|
||||
import type { LiveSyncJournalReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/journal/LiveSyncJournalReplicator";
|
||||
import { writeStderrLine, writeStdoutLine } from "@/apps/cli/cliOutput";
|
||||
|
||||
function redactConnectionString(uri: string): string {
|
||||
return uri.replace(/\/\/([^@/]+)@/u, "//***@");
|
||||
@@ -31,9 +35,10 @@ async function verifyRemoteState(
|
||||
core: CLICommandContext["core"],
|
||||
settings: ObsidianLiveSyncSettings
|
||||
): Promise<boolean> {
|
||||
const { standardIo } = core.services.context;
|
||||
const replicator = core.services.replicator.getActiveReplicator();
|
||||
if (!replicator) {
|
||||
process.stderr.write("[Verification] No active replicator found\n");
|
||||
standardIo.writeStderr("[Verification] No active replicator found\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -50,7 +55,7 @@ async function verifyRemoteState(
|
||||
true
|
||||
);
|
||||
if (typeof dbRet === "string") {
|
||||
process.stderr.write(`[Verification] Failed to connect to remote CouchDB: ${dbRet}\n`);
|
||||
standardIo.writeStderr(`[Verification] Failed to connect to remote CouchDB: ${dbRet}\n`);
|
||||
return false;
|
||||
}
|
||||
milestone = await dbRet.db.get(MILESTONE_DOCID);
|
||||
@@ -61,29 +66,30 @@ async function verifyRemoteState(
|
||||
if (milestone) {
|
||||
const isLocked = !!milestone.locked;
|
||||
const isAccepted = !!milestone.accepted_nodes?.includes(replicator.nodeid);
|
||||
process.stderr.write(`[Verification] Remote Database: ${isLocked ? "LOCKED" : "UNLOCKED"}\n`);
|
||||
process.stderr.write(
|
||||
standardIo.writeStderr(`[Verification] Remote Database: ${isLocked ? "LOCKED" : "UNLOCKED"}\n`);
|
||||
standardIo.writeStderr(
|
||||
`[Verification] Current Device Node ID (${replicator.nodeid}): ${isAccepted ? "ACCEPTED" : "NOT ACCEPTED"}\n`
|
||||
);
|
||||
return true;
|
||||
} else {
|
||||
process.stderr.write("[Verification] Milestone document not found on remote.\n");
|
||||
standardIo.writeStderr("[Verification] Milestone document not found on remote.\n");
|
||||
return false;
|
||||
}
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
process.stderr.write(`[Verification] Failed to fetch milestone document: ${message}\n`);
|
||||
standardIo.writeStderr(`[Verification] Failed to fetch milestone document: ${message}\n`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runCommand(options: CLIOptions, context: CLICommandContext): Promise<boolean> {
|
||||
const { databasePath, core, settingsPath } = context;
|
||||
const { standardIo } = core.services.context;
|
||||
const vaultPath = context.vaultPath || databasePath;
|
||||
|
||||
await core.services.control.activated;
|
||||
if (options.command === "daemon") {
|
||||
const log = (msg: unknown) => console.error(`[Daemon] ${msg}`);
|
||||
const log = (msg: unknown) => writeStderrLine(standardIo, `[Daemon] ${String(msg)}`);
|
||||
|
||||
// Skip the config mismatch dialog — the daemon cannot resolve it interactively
|
||||
// and the default "Dismiss" action would block replication. The daemon should
|
||||
@@ -94,17 +100,17 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
log("Replicating from CouchDB...");
|
||||
const replResult = await core.services.replication.replicate(true);
|
||||
if (!replResult) {
|
||||
console.error("[Daemon] Initial CouchDB replication failed, cannot continue");
|
||||
writeStderrLine(standardIo, "[Daemon] Initial CouchDB replication failed, cannot continue");
|
||||
return false;
|
||||
}
|
||||
log("CouchDB replication complete");
|
||||
|
||||
// 2. Mirror scan to reconcile PouchDB ↔ local filesystem.
|
||||
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle);
|
||||
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events);
|
||||
log("Running mirror scan...");
|
||||
const scanOk = await performFullScan(core, log, errorManager, false, true);
|
||||
if (!scanOk) {
|
||||
console.error("[Daemon] Mirror scan failed, cannot continue");
|
||||
writeStderrLine(standardIo, "[Daemon] Mirror scan failed, cannot continue");
|
||||
return false;
|
||||
}
|
||||
log("Mirror scan complete");
|
||||
@@ -152,9 +158,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
} catch (err) {
|
||||
consecutiveFailures++;
|
||||
currentIntervalMs = Math.min(baseIntervalMs * Math.pow(2, consecutiveFailures), maxIntervalMs);
|
||||
console.error(`[Daemon] Poll error (${consecutiveFailures} consecutive):`, err);
|
||||
writeStderrLine(standardIo, `[Daemon] Poll error (${consecutiveFailures} consecutive):`, err);
|
||||
if (consecutiveFailures >= 5) {
|
||||
console.error(
|
||||
writeStderrLine(
|
||||
standardIo,
|
||||
`[Daemon] Warning: ${consecutiveFailures} consecutive failures, backing off to ${Math.round(currentIntervalMs / 1000)}s`
|
||||
);
|
||||
}
|
||||
@@ -179,7 +186,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
log("LiveSync active");
|
||||
const currentSettings = core.services.setting.currentSettings();
|
||||
if (!currentSettings.liveSync && !currentSettings.syncOnStart) {
|
||||
console.error(
|
||||
writeStderrLine(
|
||||
standardIo,
|
||||
"[Daemon] Warning: liveSync and syncOnStart are both disabled in settings. " +
|
||||
"No sync will occur. Set liveSync=true in your settings file for continuous sync, " +
|
||||
"or use --interval for polling mode."
|
||||
@@ -191,7 +199,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}
|
||||
|
||||
if (options.command === "sync") {
|
||||
console.log("[Command] sync");
|
||||
writeStdoutLine(standardIo, "[Command] sync");
|
||||
const result = await core.services.replication.replicate(true);
|
||||
if (!result) {
|
||||
// TODO: Standardise the logic for identifying the cause of replication
|
||||
@@ -199,7 +207,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
// error, etc.) is surfaced with a CLI-specific actionable message.
|
||||
const replicator = core.services.replicator.getActiveReplicator();
|
||||
if (replicator?.remoteLockedAndDeviceNotAccepted) {
|
||||
console.error(
|
||||
writeStderrLine(
|
||||
standardIo,
|
||||
`[Error] The remote database is locked and this device is not yet accepted.\n` +
|
||||
`[Error] Please unlock the database from the Obsidian plugin and retry.`
|
||||
);
|
||||
@@ -213,10 +222,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
throw new Error("p2p-peers requires one argument: <timeout>");
|
||||
}
|
||||
const timeoutSec = parseTimeoutSeconds(options.commandArgs[0], "p2p-peers");
|
||||
console.error(`[Command] p2p-peers timeout=${timeoutSec}s`);
|
||||
writeStderrLine(standardIo, `[Command] p2p-peers timeout=${timeoutSec}s`);
|
||||
const peers = await collectPeers(core, timeoutSec);
|
||||
if (peers.length > 0) {
|
||||
process.stdout.write(peers.map((peer) => `[peer]\t${peer.peerId}\t${peer.name}`).join("\n") + "\n");
|
||||
standardIo.writeStdout(peers.map((peer) => `[peer]\t${peer.peerId}\t${peer.name}`).join("\n") + "\n");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -230,16 +239,16 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
throw new Error("p2p-sync requires a non-empty <peer>");
|
||||
}
|
||||
const timeoutSec = parseTimeoutSeconds(options.commandArgs[1], "p2p-sync");
|
||||
console.error(`[Command] p2p-sync peer=${peerToken} timeout=${timeoutSec}s`);
|
||||
writeStderrLine(standardIo, `[Command] p2p-sync peer=${peerToken} timeout=${timeoutSec}s`);
|
||||
const peer = await syncWithPeer(core, peerToken, timeoutSec);
|
||||
console.error(`[Done] P2P sync completed with ${peer.name} (${peer.peerId})`);
|
||||
writeStderrLine(standardIo, `[Done] P2P sync completed with ${peer.name} (${peer.peerId})`);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (options.command === "p2p-host") {
|
||||
console.error("[Command] p2p-host");
|
||||
writeStderrLine(standardIo, "[Command] p2p-host");
|
||||
await openP2PHost(core);
|
||||
console.error("[Ready] P2P host is running. Press Ctrl+C to stop.");
|
||||
writeStderrLine(standardIo, "[Ready] P2P host is running. Press Ctrl+C to stop.");
|
||||
await new Promise(() => {});
|
||||
return true;
|
||||
}
|
||||
@@ -252,7 +261,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
const destinationDatabasePath = toDatabaseRelativePath(options.commandArgs[1], vaultPath);
|
||||
const sourceData = await fs.readFile(sourcePath);
|
||||
const sourceStat = await fs.stat(sourcePath);
|
||||
console.log(`[Command] push ${sourcePath} -> ${destinationDatabasePath}`);
|
||||
writeStdoutLine(standardIo, `[Command] push ${sourcePath} -> ${destinationDatabasePath}`);
|
||||
|
||||
await core.serviceModules.storageAccess.writeFileAuto(destinationDatabasePath, toArrayBuffer(sourceData), {
|
||||
mtime: Math.floor(sourceStat.mtimeMs),
|
||||
@@ -269,7 +278,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}
|
||||
const sourceDatabasePath = toDatabaseRelativePath(options.commandArgs[0], vaultPath);
|
||||
const destinationPath = path.resolve(options.commandArgs[1]);
|
||||
console.log(`[Command] pull ${sourceDatabasePath} -> ${destinationPath}`);
|
||||
writeStdoutLine(standardIo, `[Command] pull ${sourceDatabasePath} -> ${destinationPath}`);
|
||||
|
||||
const sourcePathWithPrefix = sourceDatabasePath as FilePathWithPrefix;
|
||||
const restored = await core.serviceModules.fileHandler.dbToStorage(sourcePathWithPrefix, null, true);
|
||||
@@ -296,7 +305,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
if (!rev) {
|
||||
throw new Error("pull-rev requires a non-empty revision");
|
||||
}
|
||||
console.log(`[Command] pull-rev ${sourceDatabasePath}@${rev} -> ${destinationPath}`);
|
||||
writeStdoutLine(standardIo, `[Command] pull-rev ${sourceDatabasePath}@${rev} -> ${destinationPath}`);
|
||||
|
||||
const source = await core.serviceModules.databaseFileAccess.fetch(
|
||||
sourceDatabasePath as FilePathWithPrefix,
|
||||
@@ -325,7 +334,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
if (!setupURI.startsWith(configURIBase)) {
|
||||
throw new Error(`setup URI must start with ${configURIBase}`);
|
||||
}
|
||||
const passphrase = await promptForPassphrase();
|
||||
const passphrase = await standardIo.prompt("Enter setup URI passphrase: ");
|
||||
if (!passphrase) {
|
||||
throw new Error("Passphrase is required");
|
||||
}
|
||||
const decoded = await decodeSettingsFromSetupURI(setupURI, passphrase);
|
||||
if (!decoded) {
|
||||
throw new Error("Failed to decode settings from setup URI");
|
||||
@@ -337,7 +349,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
isConfigured: true,
|
||||
} as ObsidianLiveSyncSettings;
|
||||
|
||||
console.log(`[Command] setup -> ${settingsPath}`);
|
||||
writeStdoutLine(standardIo, `[Command] setup -> ${settingsPath}`);
|
||||
await core.services.setting.applyExternalSettings(nextSettings, true);
|
||||
await core.services.control.applySettings();
|
||||
return true;
|
||||
@@ -348,8 +360,8 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
throw new Error("put requires one argument: <dst>");
|
||||
}
|
||||
const destinationDatabasePath = toDatabaseRelativePath(options.commandArgs[0], vaultPath);
|
||||
const content = await readStdinAsUtf8();
|
||||
console.log(`[Command] put stdin -> ${destinationDatabasePath}`);
|
||||
const content = await standardIo.readStdin();
|
||||
writeStdoutLine(standardIo, `[Command] put stdin -> ${destinationDatabasePath}`);
|
||||
return await core.serviceModules.databaseFileAccess.storeContent(
|
||||
destinationDatabasePath as FilePathWithPrefix,
|
||||
content
|
||||
@@ -361,7 +373,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
throw new Error("cat requires one argument: <src>");
|
||||
}
|
||||
const sourceDatabasePath = toDatabaseRelativePath(options.commandArgs[0], vaultPath);
|
||||
console.error(`[Command] cat ${sourceDatabasePath}`);
|
||||
writeStderrLine(standardIo, `[Command] cat ${sourceDatabasePath}`);
|
||||
const source = await core.serviceModules.databaseFileAccess.fetch(
|
||||
sourceDatabasePath as FilePathWithPrefix,
|
||||
undefined,
|
||||
@@ -372,10 +384,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}
|
||||
const body = source.body;
|
||||
if (body.type === "text/plain") {
|
||||
process.stdout.write(await body.text());
|
||||
standardIo.writeStdout(await body.text());
|
||||
} else {
|
||||
const buffer = Buffer.from(await body.arrayBuffer());
|
||||
process.stdout.write(new Uint8Array(buffer));
|
||||
standardIo.writeStdout(new Uint8Array(buffer));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -389,7 +401,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
if (!rev) {
|
||||
throw new Error("cat-rev requires a non-empty revision");
|
||||
}
|
||||
console.error(`[Command] cat-rev ${sourceDatabasePath} @ ${rev}`);
|
||||
writeStderrLine(standardIo, `[Command] cat-rev ${sourceDatabasePath} @ ${rev}`);
|
||||
const source = await core.serviceModules.databaseFileAccess.fetch(
|
||||
sourceDatabasePath as FilePathWithPrefix,
|
||||
rev,
|
||||
@@ -400,10 +412,10 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}
|
||||
const body = source.body;
|
||||
if (body.type === "text/plain") {
|
||||
process.stdout.write(await body.text());
|
||||
standardIo.writeStdout(await body.text());
|
||||
} else {
|
||||
const buffer = Buffer.from(await body.arrayBuffer());
|
||||
process.stdout.write(new Uint8Array(buffer));
|
||||
standardIo.writeStdout(new Uint8Array(buffer));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -432,9 +444,9 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
|
||||
rows.sort((a, b) => a.path.localeCompare(b.path));
|
||||
if (rows.length > 0) {
|
||||
process.stdout.write(rows.map((e) => e.line).join("\n") + "\n");
|
||||
standardIo.writeStdout(rows.map((e) => e.line).join("\n") + "\n");
|
||||
} else {
|
||||
process.stderr.write("[Info] No documents found in the local database.\n");
|
||||
standardIo.writeStderr("[Info] No documents found in the local database.\n");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -475,11 +487,11 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
chunks: children.length,
|
||||
children: children,
|
||||
};
|
||||
process.stdout.write(JSON.stringify(out, null, 2) + "\n");
|
||||
standardIo.writeStdout(JSON.stringify(out, null, 2) + "\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
process.stderr.write(`[Info] File not found: ${targetPath}\n`);
|
||||
standardIo.writeStderr(`[Info] File not found: ${targetPath}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -488,7 +500,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
throw new Error("rm requires one argument: <path>");
|
||||
}
|
||||
const targetPath = toDatabaseRelativePath(options.commandArgs[0], vaultPath);
|
||||
console.error(`[Command] rm ${targetPath}`);
|
||||
writeStderrLine(standardIo, `[Command] rm ${targetPath}`);
|
||||
return await core.serviceModules.databaseFileAccess.delete(targetPath as FilePathWithPrefix);
|
||||
}
|
||||
|
||||
@@ -504,30 +516,30 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
|
||||
const currentMeta = await core.serviceModules.databaseFileAccess.fetchEntryMeta(targetPath, undefined, true);
|
||||
if (currentMeta === false || currentMeta._deleted || currentMeta.deleted) {
|
||||
process.stderr.write(`[Info] File not found: ${targetPath}\n`);
|
||||
standardIo.writeStderr(`[Info] File not found: ${targetPath}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
const conflicts = await core.serviceModules.databaseFileAccess.getConflictedRevs(targetPath);
|
||||
const candidateRevisions = [currentMeta._rev, ...conflicts];
|
||||
if (!candidateRevisions.includes(revisionToKeep)) {
|
||||
process.stderr.write(`[Info] Revision not found for ${targetPath}: ${revisionToKeep}\n`);
|
||||
standardIo.writeStderr(`[Info] Revision not found for ${targetPath}: ${revisionToKeep}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (conflicts.length === 0 && currentMeta._rev === revisionToKeep) {
|
||||
console.error(`[Command] resolve ${targetPath} keep ${revisionToKeep} (already resolved)`);
|
||||
writeStderrLine(standardIo, `[Command] resolve ${targetPath} keep ${revisionToKeep} (already resolved)`);
|
||||
return true;
|
||||
}
|
||||
|
||||
console.error(`[Command] resolve ${targetPath} keep ${revisionToKeep}`);
|
||||
writeStderrLine(standardIo, `[Command] resolve ${targetPath} keep ${revisionToKeep}`);
|
||||
for (const revision of candidateRevisions) {
|
||||
if (revision === revisionToKeep) {
|
||||
continue;
|
||||
}
|
||||
const resolved = await core.services.conflict.resolveByDeletingRevision(targetPath, revision ?? "", "CLI");
|
||||
if (!resolved) {
|
||||
process.stderr.write(`[Info] Failed to delete revision ${revision} for ${targetPath}\n`);
|
||||
standardIo.writeStderr(`[Info] Failed to delete revision ${revision} for ${targetPath}\n`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -535,9 +547,9 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}
|
||||
|
||||
if (options.command === "mirror") {
|
||||
console.error("[Command] mirror");
|
||||
const log = (msg: unknown) => console.error(`[Mirror] ${msg}`);
|
||||
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle);
|
||||
writeStderrLine(standardIo, "[Command] mirror");
|
||||
const log = (msg: unknown) => writeStderrLine(standardIo, `[Mirror] ${String(msg)}`);
|
||||
const errorManager = new UnresolvedErrorManager(core.services.appLifecycle, core.services.context.events);
|
||||
return await performFullScan(core, log, errorManager, false, true);
|
||||
}
|
||||
|
||||
@@ -579,7 +591,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
await core.services.control.applySettings();
|
||||
}
|
||||
|
||||
process.stdout.write(`${id}\t${name}\t${redactConnectionString(canonicalUri)}\n`);
|
||||
standardIo.writeStdout(`${id}\t${name}\t${redactConnectionString(canonicalUri)}\n`);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -594,7 +606,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
|
||||
const current = core.services.setting.currentSettings();
|
||||
if (!current.remoteConfigurations?.[id]) {
|
||||
process.stderr.write(`[Info] Remote configuration not found: ${id}\n`);
|
||||
standardIo.writeStderr(`[Info] Remote configuration not found: ${id}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -624,7 +636,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
await core.services.control.applySettings();
|
||||
}
|
||||
|
||||
console.error(`[Command] remote-rm ${id}`);
|
||||
writeStderrLine(standardIo, `[Command] remote-rm ${id}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -634,7 +646,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
configs.sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
if (configs.length === 0) {
|
||||
process.stderr.write("[Info] No remote configurations found.\n");
|
||||
standardIo.writeStderr("[Info] No remote configurations found.\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -642,7 +654,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
const status = config.id === settings.activeConfigurationId ? "active" : "inactive";
|
||||
return `${config.id}\t${config.name}\t${status}\t${redactConnectionString(config.uri)}`;
|
||||
});
|
||||
process.stdout.write(lines.join("\n") + "\n");
|
||||
standardIo.writeStdout(lines.join("\n") + "\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -657,11 +669,11 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
|
||||
const config = core.services.setting.currentSettings().remoteConfigurations?.[id];
|
||||
if (!config) {
|
||||
process.stderr.write(`[Info] Remote configuration not found: ${id}\n`);
|
||||
standardIo.writeStderr(`[Info] Remote configuration not found: ${id}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
process.stdout.write(`${config.uri}\n`);
|
||||
standardIo.writeStdout(`${config.uri}\n`);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -701,7 +713,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
|
||||
const updated = core.services.setting.currentSettings().remoteConfigurations?.[id];
|
||||
if (!updated) {
|
||||
process.stderr.write(`[Info] Remote configuration not found: ${id}\n`);
|
||||
standardIo.writeStderr(`[Info] Remote configuration not found: ${id}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -709,7 +721,7 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
await core.services.control.applySettings();
|
||||
}
|
||||
|
||||
console.error(`[Command] remote-set ${id}`);
|
||||
writeStderrLine(standardIo, `[Command] remote-set ${id}`);
|
||||
return true;
|
||||
}
|
||||
if (options.command === "remote-activate") {
|
||||
@@ -732,12 +744,12 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}, true);
|
||||
|
||||
if (!switched) {
|
||||
process.stderr.write(`[Info] Failed to activate remote configuration: ${id}\n`);
|
||||
standardIo.writeStderr(`[Info] Failed to activate remote configuration: ${id}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
await core.services.control.applySettings();
|
||||
console.error(`[Command] remote-activate ${id}`);
|
||||
writeStderrLine(standardIo, `[Command] remote-activate ${id}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -755,14 +767,14 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}, false);
|
||||
|
||||
if (!switched) {
|
||||
process.stderr.write(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
|
||||
standardIo.writeStderr(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
await core.services.control.applySettings();
|
||||
}
|
||||
|
||||
console.error(`[Command] mark-resolved${id ? ` ${id}` : ""}`);
|
||||
writeStderrLine(standardIo, `[Command] mark-resolved${id ? ` ${id}` : ""}`);
|
||||
await core.services.replication.markResolved();
|
||||
const settings = core.services.setting.currentSettings();
|
||||
await verifyRemoteState(core, settings);
|
||||
@@ -783,14 +795,14 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}, false);
|
||||
|
||||
if (!switched) {
|
||||
process.stderr.write(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
|
||||
standardIo.writeStderr(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
await core.services.control.applySettings();
|
||||
}
|
||||
|
||||
console.error(`[Command] unlock-remote${id ? ` ${id}` : ""}`);
|
||||
writeStderrLine(standardIo, `[Command] unlock-remote${id ? ` ${id}` : ""}`);
|
||||
await core.services.replication.markUnlocked();
|
||||
const settings = core.services.setting.currentSettings();
|
||||
await verifyRemoteState(core, settings);
|
||||
@@ -811,14 +823,14 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}, false);
|
||||
|
||||
if (!switched) {
|
||||
process.stderr.write(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
|
||||
standardIo.writeStderr(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
await core.services.control.applySettings();
|
||||
}
|
||||
|
||||
console.error(`[Command] lock-remote${id ? ` ${id}` : ""}`);
|
||||
writeStderrLine(standardIo, `[Command] lock-remote${id ? ` ${id}` : ""}`);
|
||||
await core.services.replication.markLocked();
|
||||
const settings = core.services.setting.currentSettings();
|
||||
await verifyRemoteState(core, settings);
|
||||
@@ -839,26 +851,26 @@ export async function runCommand(options: CLIOptions, context: CLICommandContext
|
||||
}, false);
|
||||
|
||||
if (!switched) {
|
||||
process.stderr.write(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
|
||||
standardIo.writeStderr(`[Info] Failed to temporarily activate remote configuration: ${id}\n`);
|
||||
return false;
|
||||
}
|
||||
|
||||
await core.services.control.applySettings();
|
||||
}
|
||||
|
||||
console.error(`[Command] remote-status${id ? ` ${id}` : ""}`);
|
||||
writeStderrLine(standardIo, `[Command] remote-status${id ? ` ${id}` : ""}`);
|
||||
const replicator = core.services.replicator.getActiveReplicator();
|
||||
if (!replicator) {
|
||||
process.stderr.write("[Error] No active replicator found\n");
|
||||
standardIo.writeStderr("[Error] No active replicator found\n");
|
||||
return false;
|
||||
}
|
||||
const settings = core.services.setting.currentSettings();
|
||||
const status = await replicator.getRemoteStatus(settings);
|
||||
if (status === false) {
|
||||
process.stderr.write("[Error] Failed to fetch remote status\n");
|
||||
standardIo.writeStderr("[Error] Failed to fetch remote status\n");
|
||||
return false;
|
||||
}
|
||||
process.stdout.write(JSON.stringify(status, null, 2) + "\n");
|
||||
standardIo.writeStdout(JSON.stringify(status, null, 2) + "\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,20 @@
|
||||
import * as path from "path";
|
||||
import * as fs from "fs/promises";
|
||||
import * as os from "os";
|
||||
import * as processSetting from "@lib/API/processSetting";
|
||||
import { ConnectionStringParser } from "@lib/common/ConnectionString";
|
||||
import { configURIBase } from "@lib/common/models/shared.const";
|
||||
import { DEFAULT_SETTINGS, REMOTE_COUCHDB, REMOTE_MINIO, REMOTE_P2P } from "@lib/common/types";
|
||||
import { fsPromises as fs, os, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import * as processSetting from "@vrtmrz/livesync-commonlib/compat/API/processSetting";
|
||||
import { ConnectionStringParser } from "@vrtmrz/livesync-commonlib/compat/common/ConnectionString";
|
||||
import { configURIBase } from "@vrtmrz/livesync-commonlib/compat/common/models/shared.const";
|
||||
import { DEFAULT_SETTINGS, REMOTE_COUCHDB, REMOTE_MINIO, REMOTE_P2P } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
|
||||
import { runCommand } from "./runCommand";
|
||||
import type { CLIOptions } from "./types";
|
||||
import * as commandUtils from "./utils";
|
||||
|
||||
function createStandardIoMock() {
|
||||
return {
|
||||
readStdin: vi.fn(async () => ""),
|
||||
prompt: vi.fn(async () => ""),
|
||||
writeStdout: vi.fn((_chunk: string | Uint8Array) => undefined),
|
||||
writeStderr: vi.fn((_chunk: string | Uint8Array) => undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function createCoreMock() {
|
||||
const liveSettings = {
|
||||
@@ -19,6 +25,9 @@ function createCoreMock() {
|
||||
} as any;
|
||||
return {
|
||||
services: {
|
||||
context: {
|
||||
standardIo: createStandardIoMock(),
|
||||
},
|
||||
control: {
|
||||
activated: Promise.resolve(),
|
||||
applySettings: vi.fn(async () => {}),
|
||||
@@ -71,6 +80,7 @@ function createCoreMock() {
|
||||
},
|
||||
databaseFileAccess: {
|
||||
fetch: vi.fn(async () => undefined),
|
||||
storeContent: vi.fn(async () => true),
|
||||
},
|
||||
},
|
||||
} as any;
|
||||
@@ -98,20 +108,20 @@ async function createSetupURI(passphrase: string): Promise<string> {
|
||||
return await processSetting.encodeSettingsToSetupURI(settings, passphrase);
|
||||
}
|
||||
|
||||
function captureStdout() {
|
||||
const writes: string[] = [];
|
||||
const spy = vi.spyOn(process.stdout, "write").mockImplementation((chunk: any) => {
|
||||
writes.push(typeof chunk === "string" ? chunk : String(chunk));
|
||||
return true;
|
||||
});
|
||||
function captureStdout(core: ReturnType<typeof createCoreMock>) {
|
||||
const spy = core.services.context.standardIo.writeStdout;
|
||||
spy.mockClear();
|
||||
return {
|
||||
spy,
|
||||
lines: () =>
|
||||
writes
|
||||
spy.mock.calls
|
||||
.map(([chunk]: [string | Uint8Array]) =>
|
||||
typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk)
|
||||
)
|
||||
.join("")
|
||||
.split("\n")
|
||||
.map((e) => e.trim())
|
||||
.filter((e) => e.length > 0),
|
||||
.map((entry: string) => entry.trim())
|
||||
.filter((entry: string) => entry.length > 0),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -306,7 +316,7 @@ describe("runCommand abnormal cases", () => {
|
||||
|
||||
it("setup rejects empty passphrase", async () => {
|
||||
const core = createCoreMock();
|
||||
vi.spyOn(commandUtils, "promptForPassphrase").mockRejectedValue(new Error("Passphrase is required"));
|
||||
core.services.context.standardIo.prompt.mockResolvedValue("");
|
||||
|
||||
await expect(
|
||||
runCommand(makeOptions("setup", [`${configURIBase}dummy`]), {
|
||||
@@ -320,7 +330,7 @@ describe("runCommand abnormal cases", () => {
|
||||
const core = createCoreMock();
|
||||
const passphrase = "correct-passphrase";
|
||||
const setupURI = await createSetupURI(passphrase);
|
||||
vi.spyOn(commandUtils, "promptForPassphrase").mockResolvedValue(passphrase);
|
||||
core.services.context.standardIo.prompt.mockResolvedValue(passphrase);
|
||||
|
||||
const result = await runCommand(makeOptions("setup", [setupURI]), {
|
||||
...context,
|
||||
@@ -341,7 +351,7 @@ describe("runCommand abnormal cases", () => {
|
||||
it("setup rejects encoded URI when passphrase is wrong", async () => {
|
||||
const core = createCoreMock();
|
||||
const setupURI = await createSetupURI("correct-passphrase");
|
||||
vi.spyOn(commandUtils, "promptForPassphrase").mockResolvedValue("wrong-passphrase");
|
||||
core.services.context.standardIo.prompt.mockResolvedValue("wrong-passphrase");
|
||||
|
||||
await expect(
|
||||
runCommand(makeOptions("setup", [setupURI]), {
|
||||
@@ -354,9 +364,61 @@ describe("runCommand abnormal cases", () => {
|
||||
expect(core.services.control.applySettings).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("put reads content from the injected standard input", async () => {
|
||||
const core = createCoreMock();
|
||||
core.services.context.standardIo.readStdin.mockResolvedValue("content from stdin");
|
||||
|
||||
const result = await runCommand(makeOptions("put", ["notes/input.md"]), {
|
||||
...context,
|
||||
core,
|
||||
});
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(core.services.context.standardIo.readStdin).toHaveBeenCalledOnce();
|
||||
expect(core.serviceModules.databaseFileAccess.storeContent).toHaveBeenCalledWith(
|
||||
"notes/input.md",
|
||||
"content from stdin"
|
||||
);
|
||||
});
|
||||
|
||||
it("cat writes text to the injected standard output without adding a delimiter", async () => {
|
||||
const core = createCoreMock();
|
||||
core.serviceModules.databaseFileAccess.fetch.mockResolvedValue({
|
||||
deleted: false,
|
||||
body: new Blob(["exact text"], { type: "text/plain" }),
|
||||
});
|
||||
|
||||
const result = await runCommand(makeOptions("cat", ["notes/output.md"]), {
|
||||
...context,
|
||||
core,
|
||||
});
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(core.services.context.standardIo.writeStdout).toHaveBeenCalledWith("exact text");
|
||||
});
|
||||
|
||||
it("cat preserves binary bytes through the injected standard output", async () => {
|
||||
const core = createCoreMock();
|
||||
const expected = Uint8Array.from([0x00, 0x7f, 0x80, 0xff]);
|
||||
core.serviceModules.databaseFileAccess.fetch.mockResolvedValue({
|
||||
deleted: false,
|
||||
body: new Blob([expected], { type: "application/octet-stream" }),
|
||||
});
|
||||
|
||||
const result = await runCommand(makeOptions("cat", ["binary/output.bin"]), {
|
||||
...context,
|
||||
core,
|
||||
});
|
||||
|
||||
expect(result).toBe(true);
|
||||
const chunk = core.services.context.standardIo.writeStdout.mock.calls.at(-1)?.[0];
|
||||
expect(chunk).toBeInstanceOf(Uint8Array);
|
||||
expect([...(chunk as Uint8Array)]).toEqual([...expected]);
|
||||
});
|
||||
|
||||
it("remote-add stores canonical URI and prints the created id", async () => {
|
||||
const core = createCoreMock();
|
||||
const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
|
||||
const stdout = core.services.context.standardIo.writeStdout;
|
||||
|
||||
const result = await runCommand(makeOptions("remote-add", ["my-remote", "sls+https://example.com/db"]), {
|
||||
...context,
|
||||
@@ -437,7 +499,7 @@ describe("runCommand abnormal cases", () => {
|
||||
uri: "sls+https://example.com/db?db=vault",
|
||||
isEncrypted: false,
|
||||
};
|
||||
const stdout = captureStdout();
|
||||
const stdout = captureStdout(core);
|
||||
|
||||
const result = await runCommand(makeOptions("remote-export", ["r1"]), {
|
||||
...context,
|
||||
@@ -567,7 +629,7 @@ describe("runCommand abnormal cases", () => {
|
||||
])("remote command round-trip works for %s", async (_protocol, initialConnStr) => {
|
||||
const core = createCoreMock();
|
||||
|
||||
const addOut = captureStdout();
|
||||
const addOut = captureStdout(core);
|
||||
const addResult = await runCommand(makeOptions("remote-add", ["rt", initialConnStr]), {
|
||||
...context,
|
||||
core,
|
||||
@@ -576,7 +638,7 @@ describe("runCommand abnormal cases", () => {
|
||||
const remoteId = parseAddedRemoteIdFromLines(addOut.lines());
|
||||
expect(remoteId).not.toBe("");
|
||||
|
||||
const export1Out = captureStdout();
|
||||
const export1Out = captureStdout(core);
|
||||
const export1Result = await runCommand(makeOptions("remote-export", [remoteId]), {
|
||||
...context,
|
||||
core,
|
||||
@@ -593,7 +655,7 @@ describe("runCommand abnormal cases", () => {
|
||||
});
|
||||
expect(setResult).toBe(true);
|
||||
|
||||
const export2Out = captureStdout();
|
||||
const export2Out = captureStdout(core);
|
||||
const export2Result = await runCommand(makeOptions("remote-export", [remoteId]), {
|
||||
...context,
|
||||
core,
|
||||
@@ -742,13 +804,13 @@ describe("runCommand abnormal cases", () => {
|
||||
|
||||
it("remote-status without args outputs status of active remote configuration", async () => {
|
||||
const core = createCoreMock();
|
||||
const stdout = captureStdout();
|
||||
const stdout = captureStdout(core);
|
||||
const result = await runCommand(makeOptions("remote-status", []), {
|
||||
...context,
|
||||
core,
|
||||
});
|
||||
expect(result).toBe(true);
|
||||
const fullOutput = stdout.spy.mock.calls.map((c) => c[0]).join("");
|
||||
const fullOutput = stdout.spy.mock.calls.map((call: [string | Uint8Array]) => call[0]).join("");
|
||||
const parsedStatus = JSON.parse(fullOutput);
|
||||
expect(parsedStatus.db_name).toBe("test-db");
|
||||
expect(parsedStatus.doc_count).toBe(42);
|
||||
@@ -763,13 +825,13 @@ describe("runCommand abnormal cases", () => {
|
||||
uri: "sls+https://example.com/db1",
|
||||
isEncrypted: false,
|
||||
};
|
||||
const stdout = captureStdout();
|
||||
const stdout = captureStdout(core);
|
||||
const result = await runCommand(makeOptions("remote-status", ["r1"]), {
|
||||
...context,
|
||||
core,
|
||||
});
|
||||
expect(result).toBe(true);
|
||||
const fullOutput = stdout.spy.mock.calls.map((c) => c[0]).join("");
|
||||
const fullOutput = stdout.spy.mock.calls.map((call: [string | Uint8Array]) => call[0]).join("");
|
||||
const parsedStatus = JSON.parse(fullOutput);
|
||||
expect(parsedStatus.db_name).toBe("test-db");
|
||||
expect(parsedStatus.doc_count).toBe(42);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
|
||||
import { ServiceContext } from "@lib/services/base/ServiceBase";
|
||||
import type { ObsidianLiveSyncSettings } from "@lib/common/types";
|
||||
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { NodeServiceContext } from "@/apps/cli/services/NodeServiceContext";
|
||||
import type { UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/UseP2PReplicatorResult";
|
||||
|
||||
export type CLICommand =
|
||||
| "daemon"
|
||||
@@ -47,7 +48,9 @@ export interface CLIOptions {
|
||||
export interface CLICommandContext {
|
||||
databasePath: string;
|
||||
vaultPath: string;
|
||||
core: LiveSyncBaseCore<ServiceContext, never>;
|
||||
core: LiveSyncBaseCore<NodeServiceContext, never>;
|
||||
/** Current-result contract owned by the P2P service feature. */
|
||||
p2pReplicator?: UseP2PReplicatorResult;
|
||||
settingsPath: string;
|
||||
originalSyncSettings: Pick<
|
||||
ObsidianLiveSyncSettings,
|
||||
@@ -91,3 +94,7 @@ export const VALID_COMMANDS = new Set([
|
||||
"remote-status",
|
||||
"init-settings",
|
||||
] as const);
|
||||
|
||||
export function isCLICommand(value: string): value is CLICommand {
|
||||
return (VALID_COMMANDS as ReadonlySet<string>).has(value);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { path, readline } from "@/apps/cli/node-compat";
|
||||
import { path } from "@vrtmrz/livesync-commonlib/node";
|
||||
|
||||
export function toArrayBuffer(data: Buffer): ArrayBuffer {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion -- required in environments where Buffer.buffer is ArrayBufferLike
|
||||
return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer;
|
||||
}
|
||||
|
||||
@@ -23,28 +22,3 @@ export function toDatabaseRelativePath(inputPath: string, databasePath: string):
|
||||
}
|
||||
return rel.replace(/\\/g, "/");
|
||||
}
|
||||
|
||||
export async function readStdinAsUtf8(): Promise<string> {
|
||||
const chunks = [];
|
||||
for await (const chunk of process.stdin) {
|
||||
if (typeof chunk === "string") {
|
||||
chunks.push(Buffer.from(chunk, "utf-8"));
|
||||
} else {
|
||||
chunks.push(chunk as Buffer);
|
||||
}
|
||||
}
|
||||
return Buffer.concat(chunks as Uint8Array[]).toString("utf-8");
|
||||
}
|
||||
|
||||
export async function promptForPassphrase(prompt = "Enter setup URI passphrase: "): Promise<string> {
|
||||
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
||||
try {
|
||||
const passphrase = await rl.question(prompt);
|
||||
if (!passphrase) {
|
||||
throw new Error("Passphrase is required");
|
||||
}
|
||||
return passphrase;
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import * as path from "path";
|
||||
import { path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { toDatabaseRelativePath } from "./utils";
|
||||
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const dockerfile = readFileSync(new URL("./Dockerfile", import.meta.url), "utf8");
|
||||
|
||||
describe("CLI Docker image", () => {
|
||||
it("sets a deterministic readable and executable entrypoint mode", () => {
|
||||
expect(dockerfile).toContain("COPY --chmod=755 src/apps/cli/docker-entrypoint.sh /usr/local/bin/livesync-cli");
|
||||
expect(dockerfile).not.toContain("RUN chmod +x /usr/local/bin/livesync-cli");
|
||||
});
|
||||
});
|
||||
@@ -1,20 +1,21 @@
|
||||
#!/usr/bin/env node
|
||||
// eslint-disable -- This is the entry point for the CLI application.
|
||||
import * as polyfill from "werift";
|
||||
import { RTCPeerConnection } from "werift";
|
||||
import { main } from "./main";
|
||||
import { compatGlobal } from "@lib/common/coreEnvFunctions";
|
||||
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
import { createNodeStandardIo } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { writeStderrLine } from "./cliOutput";
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Polyfill
|
||||
const rtcPolyfillCtor = (polyfill as any).RTCPeerConnection;
|
||||
if (
|
||||
typeof (compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection === "undefined" &&
|
||||
typeof rtcPolyfillCtor === "function"
|
||||
typeof RTCPeerConnection === "function"
|
||||
) {
|
||||
// Fill only the standard WebRTC global in Node CLI runtime.
|
||||
(compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection = rtcPolyfillCtor;
|
||||
(compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection = RTCPeerConnection;
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`[Fatal Error]`, error);
|
||||
const standardIo = createNodeStandardIo();
|
||||
|
||||
main(standardIo).catch((error) => {
|
||||
writeStderrLine(standardIo, `[Fatal Error]`, error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
@@ -8,11 +8,8 @@ import LevelDBAdapter from "pouchdb-adapter-leveldb";
|
||||
|
||||
import find from "pouchdb-find";
|
||||
import transform from "transform-pouch";
|
||||
//@ts-ignore
|
||||
import { findPathToLeaf } from "pouchdb-merge";
|
||||
//@ts-ignore
|
||||
import { findPathToLeaf, type RevisionTreeNode } from "pouchdb-merge";
|
||||
import { adapterFun } from "pouchdb-utils";
|
||||
//@ts-ignore
|
||||
import { createError, MISSING_DOC, UNKNOWN_ERROR } from "pouchdb-errors";
|
||||
import { mapAllTasksWithConcurrencyLimit, unwrapTaskResult } from "octagonal-wheels/concurrency/task";
|
||||
|
||||
@@ -24,113 +21,145 @@ type PurgeMultiResult = {
|
||||
documentWasRemovedCompletely: boolean;
|
||||
};
|
||||
type PurgeMultiParam = [docId: string, rev$$1: string];
|
||||
function appendPurgeSeqs(db: PouchDB.Database, docs: PurgeMultiParam[]) {
|
||||
return (
|
||||
db
|
||||
.get("_local/purges")
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Internal method patching.
|
||||
.then(function (doc: any) {
|
||||
for (const [docId, rev$$1] of docs) {
|
||||
const purgeSeq = doc.purgeSeq + 1;
|
||||
doc.purges.push({
|
||||
docId,
|
||||
rev: rev$$1,
|
||||
purgeSeq,
|
||||
});
|
||||
//@ts-ignore : missing type def
|
||||
if (doc.purges.length > db.purged_infos_limit) {
|
||||
//@ts-ignore : missing type def
|
||||
doc.purges.splice(0, doc.purges.length - db.purged_infos_limit);
|
||||
}
|
||||
doc.purgeSeq = purgeSeq;
|
||||
type PurgeLogDocument = {
|
||||
purgeSeq: number;
|
||||
purges: Array<{ docId: string; rev: string; purgeSeq: number }>;
|
||||
};
|
||||
type PurgeMultiResultMap = Record<string, unknown>;
|
||||
|
||||
interface PouchDBPrivateDatabase extends PouchDB.Database {
|
||||
adapter: string;
|
||||
purged_infos_limit: number;
|
||||
_getRevisionTree(
|
||||
documentId: string,
|
||||
callback: (error: Error | undefined, revisions?: RevisionTreeNode[]) => void
|
||||
): void;
|
||||
_purge(
|
||||
documentId: string,
|
||||
revisionPath: string[],
|
||||
callback: (error: Error | undefined, result?: PurgeMultiResult) => void
|
||||
): void;
|
||||
purgeMulti(documents: PurgeMultiParam[]): Promise<PurgeMultiResultMap>;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
function isSuccessfulPurge(value: unknown): value is PurgeMultiResult {
|
||||
return isRecord(value) && value.ok === true;
|
||||
}
|
||||
|
||||
function appendPurgeSeqs(db: PouchDBPrivateDatabase, docs: PurgeMultiParam[]) {
|
||||
return db
|
||||
.get<PurgeLogDocument>("_local/purges")
|
||||
.then(function (doc) {
|
||||
for (const [docId, rev$$1] of docs) {
|
||||
const purgeSeq = doc.purgeSeq + 1;
|
||||
doc.purges.push({
|
||||
docId,
|
||||
rev: rev$$1,
|
||||
purgeSeq,
|
||||
});
|
||||
if (doc.purges.length > db.purged_infos_limit) {
|
||||
doc.purges.splice(0, doc.purges.length - db.purged_infos_limit);
|
||||
}
|
||||
return doc;
|
||||
})
|
||||
.catch(function (err) {
|
||||
if (err.status !== 404) {
|
||||
throw err;
|
||||
}
|
||||
return {
|
||||
_id: "_local/purges",
|
||||
purges: docs.map(([docId, rev$$1], idx) => ({
|
||||
docId,
|
||||
rev: rev$$1,
|
||||
purgeSeq: idx,
|
||||
})),
|
||||
purgeSeq: docs.length,
|
||||
};
|
||||
})
|
||||
.then(function (doc) {
|
||||
return db.put(doc);
|
||||
})
|
||||
);
|
||||
doc.purgeSeq = purgeSeq;
|
||||
}
|
||||
return doc;
|
||||
})
|
||||
.catch(function (error: unknown) {
|
||||
if (!isRecord(error) || error.status !== 404) {
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
_id: "_local/purges",
|
||||
purges: docs.map(([docId, rev$$1], idx) => ({
|
||||
docId,
|
||||
rev: rev$$1,
|
||||
purgeSeq: idx,
|
||||
})),
|
||||
purgeSeq: docs.length,
|
||||
};
|
||||
})
|
||||
.then(function (doc) {
|
||||
return db.put(doc);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* purge multiple documents at once.
|
||||
*/
|
||||
PouchDB.prototype.purgeMulti = adapterFun(
|
||||
const pouchDBPrototype = (PouchDB as typeof PouchDB & { prototype: PouchDBPrivateDatabase }).prototype;
|
||||
|
||||
pouchDBPrototype.purgeMulti = adapterFun<PouchDBPrivateDatabase, [documents: PurgeMultiParam[]], PurgeMultiResultMap>(
|
||||
"_purgeMulti",
|
||||
function (
|
||||
this: PouchDBPrivateDatabase,
|
||||
docs: PurgeMultiParam[],
|
||||
callback: (
|
||||
error: Error,
|
||||
result?: {
|
||||
[x: string]: PurgeMultiResult | Error;
|
||||
}
|
||||
) => void
|
||||
callback: (error?: Error, result?: PurgeMultiResultMap) => void
|
||||
) {
|
||||
//@ts-ignore
|
||||
if (typeof this._purge === "undefined") {
|
||||
return callback(
|
||||
//@ts-ignore: this ts-ignore might be hiding a `this` bug where we don't have "this" conext.
|
||||
createError(UNKNOWN_ERROR, "Purge is not implemented in the " + this.adapter + " adapter.")
|
||||
);
|
||||
}
|
||||
//@ts-ignore
|
||||
// eslint-disable-next-line @typescript-eslint/no-this-alias
|
||||
// eslint-disable-next-line @typescript-eslint/no-this-alias -- The adapter task callbacks must retain this PouchDB instance.
|
||||
const self = this;
|
||||
const tasks = docs.map(
|
||||
(param) => () =>
|
||||
new Promise<[PurgeMultiParam, PurgeMultiResult | Error]>((res, rej) => {
|
||||
new Promise<[PurgeMultiParam, unknown]>((res) => {
|
||||
const [docId, rev$$1] = param;
|
||||
self._getRevisionTree(docId, (error: Error, revs: string[]) => {
|
||||
self._getRevisionTree(docId, (error, revs) => {
|
||||
if (error) {
|
||||
return res([param, error]);
|
||||
}
|
||||
if (!revs) {
|
||||
return res([param, createError(MISSING_DOC)]);
|
||||
}
|
||||
let path;
|
||||
let path: string[];
|
||||
try {
|
||||
path = findPathToLeaf(revs, rev$$1);
|
||||
} catch (error) {
|
||||
//@ts-ignore
|
||||
return res([param, error.message || error]);
|
||||
} catch (caught: unknown) {
|
||||
const failure = caught instanceof Error && caught.message ? caught.message : caught;
|
||||
return res([param, failure]);
|
||||
}
|
||||
self._purge(docId, path, (error: Error, result: PurgeMultiResult) => {
|
||||
self._purge(docId, path, (error, result) => {
|
||||
if (error) {
|
||||
return res([param, error]);
|
||||
} else {
|
||||
return res([param, result]);
|
||||
}
|
||||
return res([param, result]);
|
||||
});
|
||||
});
|
||||
})
|
||||
);
|
||||
(async () => {
|
||||
const ret = await mapAllTasksWithConcurrencyLimit(1, tasks);
|
||||
const retAll = ret.map((e) => unwrapTaskResult(e)) as [PurgeMultiParam, PurgeMultiResult | Error][];
|
||||
await appendPurgeSeqs(
|
||||
self,
|
||||
retAll.filter((e) => "ok" in e[1]).map((e) => e[0])
|
||||
);
|
||||
const result = Object.fromEntries(retAll.map((e) => [e[0][0], e[1]]));
|
||||
const retAll: Array<[PurgeMultiParam, unknown]> = [];
|
||||
for (const entry of ret) {
|
||||
const outcome = unwrapTaskResult(entry);
|
||||
if (outcome instanceof Error) {
|
||||
throw outcome;
|
||||
}
|
||||
retAll.push(outcome);
|
||||
}
|
||||
const successfullyPurged: PurgeMultiParam[] = [];
|
||||
const resultEntries: Array<[string, unknown]> = [];
|
||||
for (const [document, outcome] of retAll) {
|
||||
if (isSuccessfulPurge(outcome)) {
|
||||
successfullyPurged.push(document);
|
||||
}
|
||||
resultEntries.push([document[0], outcome]);
|
||||
}
|
||||
await appendPurgeSeqs(self, successfullyPurged);
|
||||
const result: PurgeMultiResultMap = Object.fromEntries(resultEntries);
|
||||
return result;
|
||||
})()
|
||||
//@ts-ignore
|
||||
.then((result) => callback(undefined, result))
|
||||
.catch((error) => callback(error));
|
||||
.catch((caught: unknown) => {
|
||||
const error = caught instanceof Error ? caught : new Error(String(caught));
|
||||
callback(error);
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
+105
-79
@@ -2,9 +2,13 @@ import { NodeServiceContext, NodeServiceHub } from "./services/NodeServiceHub";
|
||||
import { configureNodeLocalStorage, ensureGlobalNodeLocalStorage } from "./services/NodeLocalStorage";
|
||||
import { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
|
||||
import { initialiseServiceModulesCLI } from "./serviceModules/CLIServiceModules";
|
||||
import { DEFAULT_SETTINGS, LOG_LEVEL_VERBOSE, type LOG_LEVEL, type ObsidianLiveSyncSettings } from "@lib/common/types";
|
||||
import type { InjectableServiceHub } from "@lib/services/implements/injectable/InjectableServiceHub";
|
||||
import type { InjectableSettingService } from "@lib/services/implements/injectable/InjectableSettingService";
|
||||
import {
|
||||
LOG_LEVEL_VERBOSE,
|
||||
type LOG_LEVEL,
|
||||
type ObsidianLiveSyncSettings,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub";
|
||||
import type { InjectableSettingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableSettingService";
|
||||
import {
|
||||
LOG_LEVEL_DEBUG,
|
||||
setGlobalLogFunction,
|
||||
@@ -14,20 +18,29 @@ import {
|
||||
LOG_LEVEL_NOTICE,
|
||||
} from "octagonal-wheels/common/logger";
|
||||
import { runCommand } from "./commands/runCommand";
|
||||
import { VALID_COMMANDS } from "./commands/types";
|
||||
import type { CLICommand, CLIOptions } from "./commands/types";
|
||||
import { getPathFromUXFileInfo } from "@lib/common/typeUtils";
|
||||
import { stripAllPrefixes } from "@lib/string_and_binary/path";
|
||||
import { isCLICommand } from "./commands/types";
|
||||
import type { CLICommand, CLICommandContext, CLIOptions } from "./commands/types";
|
||||
import { getPathFromUXFileInfo } from "@vrtmrz/livesync-commonlib/compat/common/typeUtils";
|
||||
import { stripAllPrefixes } from "@vrtmrz/livesync-commonlib/compat/string_and_binary/path";
|
||||
import { IgnoreRules } from "./serviceModules/IgnoreRules";
|
||||
import { useP2PReplicatorFeature } from "@lib/replication/trystero/useP2PReplicatorFeature";
|
||||
import { fsPromises as fs, path, fs as fsSync } from "./node-compat";
|
||||
import { useP2PReplicatorFeature } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/useP2PReplicatorFeature";
|
||||
import type { UseP2PReplicatorResult } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/UseP2PReplicatorResult";
|
||||
import { createNodeStandardIo, fsPromises as fs, path, fs as fsSync } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { StandardIo } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { writeStderrLine, writeStdoutLine } from "./cliOutput";
|
||||
import { createDefaultCliSettings } from "./cliSettingsDefaults";
|
||||
|
||||
const SETTINGS_FILE = ".livesync/settings.json";
|
||||
ensureGlobalNodeLocalStorage();
|
||||
defaultLoggerEnv.minLogLevel = LOG_LEVEL_DEBUG;
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`
|
||||
/** Injectable command boundary used by CLI integration probes. */
|
||||
export type CliCommandRunner = (options: CLIOptions, context: CLICommandContext) => Promise<boolean>;
|
||||
|
||||
function printHelp(standardIo: StandardIo): void {
|
||||
writeStdoutLine(
|
||||
standardIo,
|
||||
`
|
||||
Self-hosted LiveSync CLI
|
||||
|
||||
Usage:
|
||||
@@ -110,14 +123,15 @@ Examples:
|
||||
livesync-cli ./my-database remote-status remote-abc123
|
||||
livesync-cli init-settings ./data.json
|
||||
livesync-cli ./my-database --verbose
|
||||
`);
|
||||
`
|
||||
);
|
||||
}
|
||||
|
||||
export function parseArgs(): CLIOptions {
|
||||
export function parseArgs(standardIo: StandardIo = createNodeStandardIo()): CLIOptions {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length === 0 || args.includes("--help") || args.includes("-h")) {
|
||||
printHelp();
|
||||
printHelp(standardIo);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
@@ -138,7 +152,7 @@ export function parseArgs(): CLIOptions {
|
||||
case "-V": {
|
||||
i++;
|
||||
if (!args[i]) {
|
||||
console.error(`Error: Missing value for ${token}`);
|
||||
writeStderrLine(standardIo, `Error: Missing value for ${token}`);
|
||||
process.exit(1);
|
||||
}
|
||||
vaultPath = args[i];
|
||||
@@ -148,7 +162,7 @@ export function parseArgs(): CLIOptions {
|
||||
case "-s": {
|
||||
i++;
|
||||
if (!args[i]) {
|
||||
console.error(`Error: Missing value for ${token}`);
|
||||
writeStderrLine(standardIo, `Error: Missing value for ${token}`);
|
||||
process.exit(1);
|
||||
}
|
||||
settingsPath = args[i];
|
||||
@@ -158,12 +172,12 @@ export function parseArgs(): CLIOptions {
|
||||
case "-i": {
|
||||
i++;
|
||||
if (!args[i]) {
|
||||
console.error(`Error: Missing value for ${token}`);
|
||||
writeStderrLine(standardIo, `Error: Missing value for ${token}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const n = parseInt(args[i], 10);
|
||||
if (!Number.isInteger(n) || n <= 0) {
|
||||
console.error(`Error: --interval requires a positive integer, got '${args[i]}'`);
|
||||
writeStderrLine(standardIo, `Error: --interval requires a positive integer, got '${args[i]}'`);
|
||||
process.exit(1);
|
||||
}
|
||||
interval = n;
|
||||
@@ -173,7 +187,8 @@ export function parseArgs(): CLIOptions {
|
||||
case "-d":
|
||||
// debugging automatically enables verbose logging, as it is intended for debugging issues.
|
||||
debug = true;
|
||||
// falls through
|
||||
verbose = true;
|
||||
break;
|
||||
case "--verbose":
|
||||
case "-v":
|
||||
verbose = true;
|
||||
@@ -184,9 +199,8 @@ export function parseArgs(): CLIOptions {
|
||||
break;
|
||||
default: {
|
||||
if (!databasePath) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Set checking
|
||||
if (command === "daemon" && VALID_COMMANDS.has(token as any)) {
|
||||
command = token as CLICommand;
|
||||
if (command === "daemon" && isCLICommand(token)) {
|
||||
command = token;
|
||||
break;
|
||||
}
|
||||
if (command === "init-settings") {
|
||||
@@ -196,9 +210,8 @@ export function parseArgs(): CLIOptions {
|
||||
databasePath = token;
|
||||
break;
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Set checking
|
||||
if (command === "daemon" && VALID_COMMANDS.has(token as any)) {
|
||||
command = token as CLICommand;
|
||||
if (command === "daemon" && isCLICommand(token)) {
|
||||
command = token;
|
||||
break;
|
||||
}
|
||||
commandArgs.push(token);
|
||||
@@ -208,12 +221,12 @@ export function parseArgs(): CLIOptions {
|
||||
}
|
||||
|
||||
if (!databasePath && command !== "init-settings") {
|
||||
console.error("Error: database-path is required");
|
||||
writeStderrLine(standardIo, "Error: database-path is required");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (command === "daemon" && commandArgs.length > 0) {
|
||||
console.error(`Error: Unknown command '${commandArgs[0]}'`);
|
||||
writeStderrLine(standardIo, `Error: Unknown command '${commandArgs[0]}'`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -230,7 +243,7 @@ export function parseArgs(): CLIOptions {
|
||||
};
|
||||
}
|
||||
|
||||
async function createDefaultSettingsFile(options: CLIOptions) {
|
||||
async function createDefaultSettingsFile(options: CLIOptions, standardIo: StandardIo) {
|
||||
const targetPath = options.settingsPath
|
||||
? path.resolve(options.settingsPath)
|
||||
: options.commandArgs[0]
|
||||
@@ -248,20 +261,20 @@ async function createDefaultSettingsFile(options: CLIOptions) {
|
||||
}
|
||||
}
|
||||
|
||||
const settings = {
|
||||
...DEFAULT_SETTINGS,
|
||||
useIndexedDBAdapter: false,
|
||||
} as ObsidianLiveSyncSettings;
|
||||
const settings = createDefaultCliSettings();
|
||||
|
||||
await fs.mkdir(path.dirname(targetPath), { recursive: true });
|
||||
await fs.writeFile(targetPath, JSON.stringify(settings, null, 2), "utf-8");
|
||||
console.log(`[Done] Created settings file: ${targetPath}`);
|
||||
writeStdoutLine(standardIo, `[Done] Created settings file: ${targetPath}`);
|
||||
}
|
||||
|
||||
export async function main() {
|
||||
const options = parseArgs();
|
||||
export async function main(
|
||||
standardIo: StandardIo = createNodeStandardIo(),
|
||||
commandRunner: CliCommandRunner = runCommand
|
||||
) {
|
||||
const options = parseArgs(standardIo);
|
||||
if (options.interval && options.command !== "daemon") {
|
||||
console.error(`Warning: --interval is only used in daemon mode, ignored for '${options.command}'`);
|
||||
writeStderrLine(standardIo, `Warning: --interval is only used in daemon mode, ignored for '${options.command}'`);
|
||||
}
|
||||
const avoidStdoutNoise =
|
||||
options.command === "cat" ||
|
||||
@@ -278,13 +291,13 @@ export async function main() {
|
||||
options.command === "unlock-remote" ||
|
||||
options.command === "lock-remote" ||
|
||||
options.command === "remote-status";
|
||||
const infoLog = avoidStdoutNoise ? console.error : console.log;
|
||||
const infoLog = (...values: readonly unknown[]) => {
|
||||
const writeLine = avoidStdoutNoise ? writeStderrLine : writeStdoutLine;
|
||||
writeLine(standardIo, ...values);
|
||||
};
|
||||
if (options.debug) {
|
||||
setGlobalLogFunction((msg, level) => {
|
||||
console.error(`[${level}] ${typeof msg === "string" ? msg : JSON.stringify(msg)}`);
|
||||
if (msg instanceof Error) {
|
||||
console.error(msg);
|
||||
}
|
||||
writeStderrLine(standardIo, `[${level}]`, msg);
|
||||
});
|
||||
} else {
|
||||
setGlobalLogFunction((msg, level) => {
|
||||
@@ -292,7 +305,7 @@ export async function main() {
|
||||
});
|
||||
}
|
||||
if (options.command === "init-settings") {
|
||||
await createDefaultSettingsFile(options);
|
||||
await createDefaultSettingsFile(options, standardIo);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -302,11 +315,11 @@ export async function main() {
|
||||
try {
|
||||
const stat = await fs.stat(databasePath);
|
||||
if (!stat.isDirectory()) {
|
||||
console.error(`Error: ${databasePath} is not a directory`);
|
||||
writeStderrLine(standardIo, `Error: ${databasePath} is not a directory`);
|
||||
process.exit(1);
|
||||
}
|
||||
} catch {
|
||||
console.error(`Error: Database directory ${databasePath} does not exist`);
|
||||
writeStderrLine(standardIo, `Error: Database directory ${databasePath} does not exist`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -332,11 +345,11 @@ export async function main() {
|
||||
try {
|
||||
const stat = await fs.stat(vaultPath);
|
||||
if (!stat.isDirectory()) {
|
||||
console.error(`Error: Vault path ${vaultPath} is not a directory`);
|
||||
writeStderrLine(standardIo, `Error: Vault path ${vaultPath} is not a directory`);
|
||||
process.exit(1);
|
||||
}
|
||||
} catch {
|
||||
console.error(`Error: Vault directory ${vaultPath} does not exist`);
|
||||
writeStderrLine(standardIo, `Error: Vault directory ${vaultPath} does not exist`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -347,14 +360,20 @@ export async function main() {
|
||||
infoLog("");
|
||||
let ignoreRules: IgnoreRules | undefined;
|
||||
if (options.command === "daemon" || options.command === "mirror") {
|
||||
ignoreRules = new IgnoreRules(vaultPath);
|
||||
ignoreRules = new IgnoreRules(vaultPath, (message, detail) => {
|
||||
if (detail === undefined) {
|
||||
writeStderrLine(standardIo, message);
|
||||
} else {
|
||||
writeStderrLine(standardIo, message, detail);
|
||||
}
|
||||
});
|
||||
await ignoreRules.load();
|
||||
}
|
||||
|
||||
// Create service context and hub
|
||||
const context = new NodeServiceContext(databasePath);
|
||||
const context = new NodeServiceContext(databasePath, standardIo);
|
||||
const serviceHubInstance = new NodeServiceHub<NodeServiceContext>(databasePath, context);
|
||||
serviceHubInstance.API.addLog.setHandler((message: string, level: LOG_LEVEL) => {
|
||||
serviceHubInstance.API.addLog.setHandler((message: unknown, level: LOG_LEVEL) => {
|
||||
let levelStr = "";
|
||||
switch (level) {
|
||||
case LOG_LEVEL_DEBUG:
|
||||
@@ -373,19 +392,19 @@ export async function main() {
|
||||
levelStr = "Urgent";
|
||||
break;
|
||||
default:
|
||||
levelStr = `${level}`;
|
||||
levelStr = String(level);
|
||||
}
|
||||
const prefix = `(${levelStr})`;
|
||||
if (level <= LOG_LEVEL_INFO) {
|
||||
if (!options.verbose) return;
|
||||
}
|
||||
console.error(`${prefix} ${message}`);
|
||||
writeStderrLine(standardIo, prefix, message);
|
||||
});
|
||||
// Prevent replication result from being processed automatically in non-daemon commands.
|
||||
// In daemon mode the default handler must run so changes are applied to the filesystem.
|
||||
if (options.command !== "daemon") {
|
||||
serviceHubInstance.replication.processSynchroniseResult.addHandler(async () => {
|
||||
console.error(`[Info] Replication result received, but not processed automatically in CLI mode.`);
|
||||
writeStderrLine(standardIo, `[Info] Replication result received, but not processed automatically in CLI mode.`);
|
||||
return await Promise.resolve(true);
|
||||
}, -100);
|
||||
}
|
||||
@@ -398,10 +417,10 @@ export async function main() {
|
||||
try {
|
||||
await fs.writeFile(settingsPath, JSON.stringify(data, null, 2), "utf-8");
|
||||
if (options.verbose) {
|
||||
console.error(`[Settings] Saved to ${settingsPath}`);
|
||||
writeStderrLine(standardIo, `[Settings] Saved to ${settingsPath}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[Settings] Failed to save:`, error);
|
||||
writeStderrLine(standardIo, `[Settings] Failed to save:`, error);
|
||||
}
|
||||
}
|
||||
);
|
||||
@@ -410,16 +429,15 @@ export async function main() {
|
||||
async (): Promise<ObsidianLiveSyncSettings | undefined> => {
|
||||
try {
|
||||
const content = await fs.readFile(settingsPath, "utf-8");
|
||||
const data = JSON.parse(content);
|
||||
const data = JSON.parse(content) as ObsidianLiveSyncSettings;
|
||||
if (options.verbose) {
|
||||
console.error(`[Settings] Loaded from ${settingsPath}`);
|
||||
writeStderrLine(standardIo, `[Settings] Loaded from ${settingsPath}`);
|
||||
}
|
||||
// Force disable IndexedDB adapter in CLI environment
|
||||
data.useIndexedDBAdapter = false;
|
||||
return data;
|
||||
// Force disable IndexedDB adapter in CLI environment without mutating the loaded settings object.
|
||||
return { ...data, useIndexedDBAdapter: false };
|
||||
} catch {
|
||||
if (options.verbose) {
|
||||
console.error(`[Settings] File not found, using defaults`);
|
||||
writeStderrLine(standardIo, `[Settings] File not found, using defaults`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -427,6 +445,7 @@ export async function main() {
|
||||
);
|
||||
|
||||
// Create LiveSync core
|
||||
let p2pReplicator: UseP2PReplicatorResult | undefined;
|
||||
const core = new LiveSyncBaseCore(
|
||||
serviceHubInstance,
|
||||
(core: LiveSyncBaseCore<NodeServiceContext, never>, serviceHub: InjectableServiceHub<NodeServiceContext>) => {
|
||||
@@ -436,7 +455,7 @@ export async function main() {
|
||||
() => [], // No add-ons
|
||||
(core) => {
|
||||
// Register P2P replicator feature.
|
||||
useP2PReplicatorFeature(core);
|
||||
p2pReplicator = useP2PReplicatorFeature(core);
|
||||
// Add target filter to prevent internal files are handled
|
||||
core.services.vault.isTargetFile.addHandler(async (target) => {
|
||||
const targetPath = stripAllPrefixes(getPathFromUXFileInfo(target));
|
||||
@@ -469,14 +488,14 @@ export async function main() {
|
||||
|
||||
// Setup signal handlers for graceful shutdown
|
||||
const shutdown = async (signal: string) => {
|
||||
console.log();
|
||||
console.log(`[Shutdown] Received ${signal}, shutting down gracefully...`);
|
||||
writeStdoutLine(standardIo);
|
||||
writeStdoutLine(standardIo, `[Shutdown] Received ${signal}, shutting down gracefully...`);
|
||||
try {
|
||||
await core.services.control.onUnload();
|
||||
console.log(`[Shutdown] Complete`);
|
||||
writeStdoutLine(standardIo, `[Shutdown] Complete`);
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error(`[Shutdown] Error:`, error);
|
||||
writeStderrLine(standardIo, `[Shutdown] Error:`, error);
|
||||
process.exit(1);
|
||||
}
|
||||
};
|
||||
@@ -498,7 +517,7 @@ export async function main() {
|
||||
fsSync.writeFileSync(tmpPath, settingsBackup, "utf-8");
|
||||
fsSync.renameSync(tmpPath, settingsPath);
|
||||
} catch (err) {
|
||||
console.error("[Settings] Failed to restore settings on exit:", err);
|
||||
writeStderrLine(standardIo, "[Settings] Failed to restore settings on exit:", err);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -509,7 +528,7 @@ export async function main() {
|
||||
|
||||
const loadResult = await core.services.control.onLoad();
|
||||
if (!loadResult) {
|
||||
console.error(`[Error] Failed to initialize LiveSync`);
|
||||
writeStderrLine(standardIo, `[Error] Failed to initialize LiveSync`);
|
||||
process.exit(1);
|
||||
}
|
||||
// Capture sync settings before suspendAllSync() clobbers them.
|
||||
@@ -534,24 +553,31 @@ export async function main() {
|
||||
// Check if configured
|
||||
const settings = core.services.setting.currentSettings();
|
||||
if (!settings.isConfigured) {
|
||||
console.warn(`[Warning] LiveSync is not configured yet`);
|
||||
console.warn(`[Warning] Please edit ${settingsPath} to configure CouchDB connection`);
|
||||
console.warn();
|
||||
console.warn(`Required settings:`);
|
||||
console.warn(` - couchDB_URI: CouchDB server URL`);
|
||||
console.warn(` - couchDB_USER: CouchDB username`);
|
||||
console.warn(` - couchDB_PASSWORD: CouchDB password`);
|
||||
console.warn(` - couchDB_DBNAME: Database name`);
|
||||
console.warn();
|
||||
writeStderrLine(standardIo, `[Warning] LiveSync is not configured yet`);
|
||||
writeStderrLine(standardIo, `[Warning] Please edit ${settingsPath} to configure CouchDB connection`);
|
||||
writeStderrLine(standardIo);
|
||||
writeStderrLine(standardIo, `Required settings:`);
|
||||
writeStderrLine(standardIo, ` - couchDB_URI: CouchDB server URL`);
|
||||
writeStderrLine(standardIo, ` - couchDB_USER: CouchDB username`);
|
||||
writeStderrLine(standardIo, ` - couchDB_PASSWORD: CouchDB password`);
|
||||
writeStderrLine(standardIo, ` - couchDB_DBNAME: Database name`);
|
||||
writeStderrLine(standardIo);
|
||||
} else {
|
||||
infoLog(`[Info] LiveSync is configured and ready`);
|
||||
infoLog(`[Info] Database: ${settings.couchDB_URI}/${settings.couchDB_DBNAME}`);
|
||||
infoLog("");
|
||||
}
|
||||
|
||||
const result = await runCommand(options, { databasePath, vaultPath, core, settingsPath, originalSyncSettings });
|
||||
const result = await commandRunner(options, {
|
||||
databasePath,
|
||||
vaultPath,
|
||||
core,
|
||||
p2pReplicator,
|
||||
settingsPath,
|
||||
originalSyncSettings,
|
||||
});
|
||||
if (!result) {
|
||||
console.error(`[Error] Command '${options.command}' failed`);
|
||||
writeStderrLine(standardIo, `[Error] Command '${options.command}' failed`);
|
||||
process.exitCode = 1;
|
||||
} else if (options.command !== "daemon") {
|
||||
infoLog(`[Done] Command '${options.command}' completed`);
|
||||
@@ -564,7 +590,7 @@ export async function main() {
|
||||
await core.services.control.onUnload();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[Error] Failed to start:`, error);
|
||||
writeStderrLine(standardIo, `[Error] Failed to start:`, error);
|
||||
process.exit(1);
|
||||
}
|
||||
// To prevent unexpected hanging in webRTC connections.
|
||||
|
||||
@@ -1,5 +1,14 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { parseArgs } from "./main";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { parseArgs as parseCliArgs } from "./main";
|
||||
|
||||
function createStandardIoMock() {
|
||||
return {
|
||||
readStdin: vi.fn(async () => ""),
|
||||
prompt: vi.fn(async () => ""),
|
||||
writeStdout: vi.fn(),
|
||||
writeStderr: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function mockProcessExit() {
|
||||
const exitMock = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
|
||||
@@ -10,6 +19,13 @@ function mockProcessExit() {
|
||||
|
||||
describe("CLI parseArgs", () => {
|
||||
const originalArgv = process.argv.slice();
|
||||
let standardIo: ReturnType<typeof createStandardIoMock>;
|
||||
|
||||
beforeEach(() => {
|
||||
standardIo = createStandardIoMock();
|
||||
});
|
||||
|
||||
const parseArgs = () => parseCliArgs(standardIo);
|
||||
|
||||
afterEach(() => {
|
||||
process.argv = originalArgv.slice();
|
||||
@@ -19,42 +35,38 @@ describe("CLI parseArgs", () => {
|
||||
it("exits 1 when --settings has no value", () => {
|
||||
process.argv = ["node", "livesync-cli", "./databasePath", "--settings"];
|
||||
const exitMock = mockProcessExit();
|
||||
const stderr = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
expect(() => parseArgs()).toThrowError("__EXIT__:1");
|
||||
expect(exitMock).toHaveBeenCalledWith(1);
|
||||
expect(stderr).toHaveBeenCalledWith("Error: Missing value for --settings");
|
||||
expect(standardIo.writeStderr).toHaveBeenCalledWith("Error: Missing value for --settings\n");
|
||||
});
|
||||
|
||||
it("exits 1 when database-path is missing", () => {
|
||||
process.argv = ["node", "livesync-cli", "sync"];
|
||||
const exitMock = mockProcessExit();
|
||||
const stderr = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
expect(() => parseArgs()).toThrowError("__EXIT__:1");
|
||||
expect(exitMock).toHaveBeenCalledWith(1);
|
||||
expect(stderr).toHaveBeenCalledWith("Error: database-path is required");
|
||||
expect(standardIo.writeStderr).toHaveBeenCalledWith("Error: database-path is required\n");
|
||||
});
|
||||
|
||||
it("exits 1 for unknown command after database-path", () => {
|
||||
process.argv = ["node", "livesync-cli", "./databasePath", "unknown-cmd"];
|
||||
const exitMock = mockProcessExit();
|
||||
const stderr = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
expect(() => parseArgs()).toThrowError("__EXIT__:1");
|
||||
expect(exitMock).toHaveBeenCalledWith(1);
|
||||
expect(stderr).toHaveBeenCalledWith("Error: Unknown command 'unknown-cmd'");
|
||||
expect(standardIo.writeStderr).toHaveBeenCalledWith("Error: Unknown command 'unknown-cmd'\n");
|
||||
});
|
||||
|
||||
it("exits 0 and prints help for --help", () => {
|
||||
process.argv = ["node", "livesync-cli", "--help"];
|
||||
const exitMock = mockProcessExit();
|
||||
const stdout = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
|
||||
expect(() => parseArgs()).toThrowError("__EXIT__:0");
|
||||
expect(exitMock).toHaveBeenCalledWith(0);
|
||||
expect(stdout).toHaveBeenCalled();
|
||||
const combined = stdout.mock.calls.flat().join("\n");
|
||||
expect(standardIo.writeStdout).toHaveBeenCalled();
|
||||
const combined = standardIo.writeStdout.mock.calls.flat().join("");
|
||||
expect(combined).toContain("Usage:");
|
||||
expect(combined).toContain("livesync-cli <database-path> [options] <command> [command-args]");
|
||||
});
|
||||
@@ -152,7 +164,6 @@ describe("CLI parseArgs", () => {
|
||||
it("exits 1 when --interval has no value", () => {
|
||||
process.argv = ["node", "livesync-cli", "./vault", "--interval"];
|
||||
const exitMock = mockProcessExit();
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
expect(() => parseArgs()).toThrowError("__EXIT__:1");
|
||||
expect(exitMock).toHaveBeenCalledWith(1);
|
||||
});
|
||||
@@ -160,22 +171,19 @@ describe("CLI parseArgs", () => {
|
||||
it("exits 1 when --interval is not a positive integer", () => {
|
||||
process.argv = ["node", "livesync-cli", "./vault", "--interval", "0"];
|
||||
const exitMock = mockProcessExit();
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
expect(() => parseArgs()).toThrowError("__EXIT__:1");
|
||||
expect(exitMock).toHaveBeenCalledWith(1);
|
||||
});
|
||||
|
||||
it("exits 1 when --interval is negative", () => {
|
||||
process.argv = ["node", "livesync-cli", "./vault", "--interval", "-5"];
|
||||
const exitMock = mockProcessExit();
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
mockProcessExit();
|
||||
expect(() => parseArgs()).toThrowError("__EXIT__:1");
|
||||
});
|
||||
|
||||
it("exits 1 when --interval is not numeric", () => {
|
||||
process.argv = ["node", "livesync-cli", "./vault", "--interval", "abc"];
|
||||
const exitMock = mockProcessExit();
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
mockProcessExit();
|
||||
expect(() => parseArgs()).toThrowError("__EXIT__:1");
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { FilePath, UXFileInfoStub, UXInternalFileInfoStub } from "@lib/common/types";
|
||||
import type { FileEventItem } from "@lib/common/types";
|
||||
import type { IStorageEventManagerAdapter } from "@lib/managers/adapters";
|
||||
import type { FilePath, UXFileInfoStub, UXInternalFileInfoStub } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { FileEventItem } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { IStorageEventManagerAdapter } from "@vrtmrz/livesync-commonlib/compat/managers/adapters";
|
||||
import type {
|
||||
IStorageEventTypeGuardAdapter,
|
||||
IStorageEventPersistenceAdapter,
|
||||
@@ -8,12 +8,13 @@ import type {
|
||||
IStorageEventStatusAdapter,
|
||||
IStorageEventConverterAdapter,
|
||||
IStorageEventWatchHandlers,
|
||||
} from "@lib/managers/adapters";
|
||||
import type { FileEventItemSentinel } from "@lib/managers/StorageEventManager";
|
||||
} from "@vrtmrz/livesync-commonlib/compat/managers/adapters";
|
||||
import type { FileEventItemSentinel } from "@vrtmrz/livesync-commonlib/compat/managers/StorageEventManager";
|
||||
import type { NodeFile, NodeFolder } from "@/apps/cli/adapters/NodeTypes";
|
||||
import { watch as chokidarWatch, type FSWatcher } from "chokidar";
|
||||
import type { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules";
|
||||
import { fsPromises as fs, path, type Stats } from "@/apps/cli/node-compat";
|
||||
import { fsPromises as fs, path, type Stats } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput";
|
||||
|
||||
/**
|
||||
* CLI-specific type guard adapter
|
||||
@@ -45,7 +46,10 @@ class CLITypeGuardAdapter implements IStorageEventTypeGuardAdapter<NodeFile, Nod
|
||||
class CLIPersistenceAdapter implements IStorageEventPersistenceAdapter {
|
||||
private snapshotPath: string;
|
||||
|
||||
constructor(basePath: string) {
|
||||
constructor(
|
||||
basePath: string,
|
||||
private reportDiagnostic: CliDiagnosticReporter = () => undefined
|
||||
) {
|
||||
this.snapshotPath = path.join(basePath, ".livesync-snapshot.json");
|
||||
}
|
||||
|
||||
@@ -53,14 +57,14 @@ class CLIPersistenceAdapter implements IStorageEventPersistenceAdapter {
|
||||
try {
|
||||
await fs.writeFile(this.snapshotPath, JSON.stringify(snapshot, null, 2), "utf-8");
|
||||
} catch (error) {
|
||||
console.error("Failed to save snapshot:", error);
|
||||
this.reportDiagnostic("Failed to save snapshot:", error);
|
||||
}
|
||||
}
|
||||
|
||||
async loadSnapshot(): Promise<(FileEventItem | FileEventItemSentinel)[] | null> {
|
||||
try {
|
||||
const content = await fs.readFile(this.snapshotPath, "utf-8");
|
||||
return JSON.parse(content);
|
||||
return JSON.parse(content) as (FileEventItem | FileEventItemSentinel)[];
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
@@ -109,7 +113,8 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter {
|
||||
constructor(
|
||||
private basePath: string,
|
||||
private ignoreRules?: IgnoreRules,
|
||||
private watchEnabled: boolean = false
|
||||
private watchEnabled: boolean = false,
|
||||
private reportDiagnostic: CliDiagnosticReporter = () => undefined
|
||||
) {}
|
||||
|
||||
private _toNodeFile(filePath: string, stats: Stats | undefined): NodeFile {
|
||||
@@ -180,8 +185,8 @@ class CLIWatchAdapter implements IStorageEventWatchAdapter {
|
||||
});
|
||||
|
||||
watcher.on("error", (err) => {
|
||||
console.error("[CLIWatchAdapter] Fatal watcher error — file watching stopped:", err);
|
||||
console.error("[CLIWatchAdapter] Exiting for systemd restart.");
|
||||
this.reportDiagnostic("[CLIWatchAdapter] Fatal watcher error — file watching stopped:", err);
|
||||
this.reportDiagnostic("[CLIWatchAdapter] Exiting for systemd restart.");
|
||||
void watcher.close();
|
||||
this._watcher = undefined;
|
||||
// Use exit(1) rather than SIGTERM so systemd Restart=on-failure engages.
|
||||
@@ -210,10 +215,15 @@ export class CLIStorageEventManagerAdapter implements IStorageEventManagerAdapte
|
||||
readonly status: CLIStatusAdapter;
|
||||
readonly converter: CLIConverterAdapter;
|
||||
|
||||
constructor(basePath: string, ignoreRules?: IgnoreRules, watchEnabled: boolean = false) {
|
||||
constructor(
|
||||
basePath: string,
|
||||
ignoreRules?: IgnoreRules,
|
||||
watchEnabled: boolean = false,
|
||||
reportDiagnostic: CliDiagnosticReporter = () => undefined
|
||||
) {
|
||||
this.typeGuard = new CLITypeGuardAdapter();
|
||||
this.persistence = new CLIPersistenceAdapter(basePath);
|
||||
this.watch = new CLIWatchAdapter(basePath, ignoreRules, watchEnabled);
|
||||
this.persistence = new CLIPersistenceAdapter(basePath, reportDiagnostic);
|
||||
this.watch = new CLIWatchAdapter(basePath, ignoreRules, watchEnabled, reportDiagnostic);
|
||||
this.status = new CLIStatusAdapter();
|
||||
this.converter = new CLIConverterAdapter();
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
import type { IStorageEventWatchHandlers } from "@lib/managers/adapters";
|
||||
import type { IStorageEventWatchHandlers } from "@vrtmrz/livesync-commonlib/compat/managers/adapters";
|
||||
import type { NodeFile } from "@/apps/cli/adapters/NodeTypes";
|
||||
|
||||
// ── chokidar mock ──────────────────────────────────────────────────────────────
|
||||
@@ -103,7 +103,8 @@ describe("CLIStorageEventManagerAdapter", () => {
|
||||
});
|
||||
|
||||
it("error event triggers process.exit(1)", async () => {
|
||||
const adapter = new CLIStorageEventManagerAdapter("/base", undefined, true);
|
||||
const reportDiagnostic = vi.fn();
|
||||
const adapter = new CLIStorageEventManagerAdapter("/base", undefined, true, reportDiagnostic);
|
||||
const handlers = makeHandlers();
|
||||
|
||||
await adapter.watch.beginWatch(handlers);
|
||||
@@ -117,6 +118,11 @@ describe("CLIStorageEventManagerAdapter", () => {
|
||||
errorCallback(new Error("disk failure"));
|
||||
|
||||
expect(processExitSpy).toHaveBeenCalledWith(1);
|
||||
expect(reportDiagnostic).toHaveBeenCalledWith(
|
||||
"[CLIWatchAdapter] Fatal watcher error — file watching stopped:",
|
||||
expect.any(Error)
|
||||
);
|
||||
expect(reportDiagnostic).toHaveBeenCalledWith("[CLIWatchAdapter] Exiting for systemd restart.");
|
||||
|
||||
processExitSpy.mockRestore();
|
||||
});
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { StorageEventManagerBase, type StorageEventManagerBaseDependencies } from "@lib/managers/StorageEventManager";
|
||||
import { StorageEventManagerBase, type StorageEventManagerBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/managers/StorageEventManager";
|
||||
import { CLIStorageEventManagerAdapter } from "./CLIStorageEventManagerAdapter";
|
||||
import type { IMinimumLiveSyncCommands, LiveSyncBaseCore } from "@/LiveSyncBaseCore";
|
||||
import type { ServiceContext } from "@lib/services/base/ServiceBase";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import type { IgnoreRules } from "@/apps/cli/serviceModules/IgnoreRules";
|
||||
// import type { IMinimumLiveSyncCommands } from "@lib/services/base/IService";
|
||||
import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
|
||||
// import type { IMinimumLiveSyncCommands } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
|
||||
|
||||
export class StorageEventManagerCLI extends StorageEventManagerBase<CLIStorageEventManagerAdapter> {
|
||||
core: LiveSyncBaseCore<ServiceContext, IMinimumLiveSyncCommands>;
|
||||
@@ -15,7 +16,12 @@ export class StorageEventManagerCLI extends StorageEventManagerBase<CLIStorageEv
|
||||
ignoreRules?: IgnoreRules,
|
||||
watchEnabled?: boolean
|
||||
) {
|
||||
const adapter = new CLIStorageEventManagerAdapter(basePath, ignoreRules, watchEnabled);
|
||||
const adapter = new CLIStorageEventManagerAdapter(basePath, ignoreRules, watchEnabled, (message, detail) => {
|
||||
dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE);
|
||||
if (detail !== undefined) {
|
||||
dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE);
|
||||
}
|
||||
});
|
||||
super(adapter, dependencies);
|
||||
this.core = core;
|
||||
}
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
|
||||
import * as nodeFs from "node:fs";
|
||||
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
|
||||
import * as nodeFsPromises from "node:fs/promises";
|
||||
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
|
||||
import * as nodePath from "node:path";
|
||||
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
|
||||
import * as nodeReadlinePromises from "node:readline/promises";
|
||||
// eslint-disable-next-line obsidianmd/no-nodejs-builtins -- This file is used to provide Node.js built-in modules in the CLI environment, which is not running in a browser context.
|
||||
import type { Stats } from "node:fs";
|
||||
export { nodeFs as fs, nodeFsPromises as fsPromises, nodePath as path, nodeReadlinePromises as readline, type Stats };
|
||||
@@ -1,18 +1,17 @@
|
||||
{
|
||||
"name": "self-hosted-livesync-cli",
|
||||
"private": true,
|
||||
"version": "0.25.80-cli",
|
||||
"version": "1.0.0-cli",
|
||||
"main": "dist/index.cjs",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"prebuild": "node scripts/check-submodule.mjs",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"cli": "node dist/index.cjs",
|
||||
"buildRun": "npm run build && npm run cli --",
|
||||
"build:docker": "docker build -f Dockerfile -t livesync-cli ../../..",
|
||||
"check": "svelte-check --tsconfig ./tsconfig.app.json && tsc -p tsconfig.node.json",
|
||||
"check": "tsc -p tsconfig.json",
|
||||
"test:unit": "cd ../../.. && npx vitest run --config vitest.config.unit.ts src/apps/cli/main.unit.spec.ts src/apps/cli/commands/utils.unit.spec.ts src/apps/cli/commands/runCommand.unit.spec.ts src/apps/cli/commands/p2p.unit.spec.ts",
|
||||
"test:e2e:two-vaults": "bash test/test-e2e-two-vaults-with-docker-linux.sh",
|
||||
"test:e2e:two-vaults:common": "bash test/test-e2e-two-vaults-common.sh",
|
||||
@@ -20,28 +19,25 @@
|
||||
"test:e2e:push-pull": "bash test/test-push-pull-linux.sh",
|
||||
"test:e2e:setup-put-cat": "bash test/test-setup-put-cat-linux.sh",
|
||||
"test:e2e:sync-two-local": "bash test/test-sync-two-local-databases-linux.sh",
|
||||
"test:e2e:p2p": "bash test/test-p2p-three-nodes-conflict-linux.sh",
|
||||
"test:e2e:p2p-upload-download-repro": "bash test/test-p2p-upload-download-repro-linux.sh",
|
||||
"test:e2e:p2p-host": "bash test/test-p2p-host-linux.sh",
|
||||
"test:e2e:p2p-sync": "bash test/test-p2p-sync-linux.sh",
|
||||
"pretest:e2e:ci": "npm run build",
|
||||
"test:e2e:ci": "deno task --cwd testdeno test:ci",
|
||||
"test:e2e:p2p": "deno task --cwd testdeno test:p2p:compose",
|
||||
"test:e2e:mirror": "bash test/test-mirror-linux.sh",
|
||||
"test:e2e:remote-commands": "bash test/test-remote-commands-linux.sh",
|
||||
"pretest:e2e:all": "npm run build",
|
||||
"test:e2e:all": " export RUN_BUILD=0 && npm run test:e2e:setup-put-cat && npm run test:e2e:push-pull && npm run test:e2e:sync-two-local && npm run test:e2e:p2p && npm run test:e2e:mirror && npm run test:e2e:two-vaults && npm run test:e2e:remote-commands",
|
||||
"test:e2e:all": "deno task --cwd testdeno test:ci && deno task --cwd testdeno test:p2p:compose",
|
||||
"pretest:e2e:docker:all": "npm run build:docker",
|
||||
"test:e2e:docker:push-pull": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-push-pull-linux.sh",
|
||||
"test:e2e:docker:setup-put-cat": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-setup-put-cat-linux.sh",
|
||||
"test:e2e:docker:mirror": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-mirror-linux.sh",
|
||||
"test:e2e:docker:remote-commands": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-remote-commands-linux.sh",
|
||||
"test:e2e:docker:sync-two-local": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-sync-two-local-databases-linux.sh",
|
||||
"test:e2e:docker:p2p": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-p2p-three-nodes-conflict-linux.sh",
|
||||
"test:e2e:docker:p2p-sync": "RUN_BUILD=0 LIVESYNC_TEST_DOCKER=1 bash test/test-p2p-sync-linux.sh",
|
||||
"test:e2e:docker:all": "export RUN_BUILD=0 && npm run test:e2e:docker:setup-put-cat && npm run test:e2e:docker:push-pull && npm run test:e2e:docker:sync-two-local && npm run test:e2e:docker:mirror && npm run test:e2e:docker:remote-commands"
|
||||
},
|
||||
"dependencies": {
|
||||
"chokidar": "^4.0.0",
|
||||
"minimatch": "^10.2.5",
|
||||
"octagonal-wheels": "^0.1.47",
|
||||
"octagonal-wheels": "^0.1.51",
|
||||
"pouchdb-adapter-http": "^9.0.0",
|
||||
"pouchdb-adapter-leveldb": "^9.0.0",
|
||||
"pouchdb-core": "^9.0.0",
|
||||
@@ -55,7 +51,6 @@
|
||||
"werift": "^0.23.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@sveltejs/vite-plugin-svelte": "^7.1.2",
|
||||
"typescript": "5.9.3",
|
||||
"vite": "^8.0.16",
|
||||
"vitest": "^4.1.8"
|
||||
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
declare module "pouchdb-merge" {
|
||||
export interface RevisionTreeNode {
|
||||
pos: number;
|
||||
ids: [revision: string, metadata: Record<string, unknown>, branches: RevisionTreeNode["ids"][]];
|
||||
}
|
||||
|
||||
export function findPathToLeaf(revisions: RevisionTreeNode[], targetRevision: string): string[];
|
||||
}
|
||||
|
||||
declare module "pouchdb-utils" {
|
||||
export function adapterFun<TThis, TArguments extends unknown[], TResult>(
|
||||
name: string,
|
||||
callback: (this: TThis, ...args: [...TArguments, callback: (error?: Error, result?: TResult) => void]) => void
|
||||
): (this: TThis, ...args: TArguments) => Promise<TResult>;
|
||||
}
|
||||
|
||||
declare module "pouchdb-errors" {
|
||||
export const MISSING_DOC: unknown;
|
||||
export const UNKNOWN_ERROR: unknown;
|
||||
export function createError(error: unknown, reason?: string): Error;
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
const cliDir = process.cwd();
|
||||
const repoRoot = path.resolve(cliDir, "../../..");
|
||||
const requiredFiles = [
|
||||
path.join(repoRoot, "src/lib/src/common/types.ts"),
|
||||
];
|
||||
|
||||
const missingFiles = requiredFiles.filter((filePath) => !fs.existsSync(filePath));
|
||||
|
||||
if (missingFiles.length === 0) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.error("[CLI Build Error] Required shared sources were not found.");
|
||||
console.error("This repository uses Git submodules, and the CLI depends on src/lib.");
|
||||
console.error("");
|
||||
console.error("Missing file(s):");
|
||||
for (const filePath of missingFiles) {
|
||||
console.error(` - ${path.relative(repoRoot, filePath)}`);
|
||||
}
|
||||
console.error("");
|
||||
console.error("Initialize submodules, then retry the CLI build:");
|
||||
console.error(" git submodule update --init --recursive");
|
||||
console.error("");
|
||||
console.error("For a fresh clone, prefer:");
|
||||
console.error(" git clone --recurse-submodules <repository-url>");
|
||||
console.error("");
|
||||
console.error("Then run:");
|
||||
console.error(" npm install");
|
||||
console.error(" cd src/apps/cli");
|
||||
console.error(" npm run build");
|
||||
|
||||
process.exit(1);
|
||||
@@ -1,15 +1,16 @@
|
||||
import type { InjectableServiceHub } from "@lib/services/implements/injectable/InjectableServiceHub";
|
||||
import { ServiceRebuilder } from "@lib/serviceModules/Rebuilder";
|
||||
import type { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub";
|
||||
import { ServiceRebuilder } from "@vrtmrz/livesync-commonlib/compat/serviceModules/Rebuilder";
|
||||
import { ServiceFileHandler } from "@/serviceModules/FileHandler";
|
||||
import { StorageAccessManager } from "@lib/managers/StorageProcessingManager";
|
||||
import { StorageAccessManager } from "@vrtmrz/livesync-commonlib/compat/managers/StorageProcessingManager";
|
||||
import type { LiveSyncBaseCore } from "@/LiveSyncBaseCore";
|
||||
import type { ServiceContext } from "@lib/services/base/ServiceBase";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { FileAccessCLI } from "./FileAccessCLI";
|
||||
import { ServiceFileAccessCLI } from "./ServiceFileAccessImpl";
|
||||
import { ServiceDatabaseFileAccessCLI } from "./DatabaseFileAccess";
|
||||
import { StorageEventManagerCLI } from "@/apps/cli/managers/StorageEventManagerCLI";
|
||||
import type { ServiceModules } from "@lib/interfaces/ServiceModule";
|
||||
import type { ServiceModules } from "@vrtmrz/livesync-commonlib/compat/interfaces/ServiceModule";
|
||||
import type { IgnoreRules } from "./IgnoreRules";
|
||||
import { createFileReflectionProvenance } from "@/serviceModules/FileReflectionProvenance";
|
||||
|
||||
/**
|
||||
* Initialize service modules for CLI version
|
||||
@@ -73,6 +74,7 @@ export function initialiseServiceModulesCLI(
|
||||
|
||||
// Database file access (platform-independent)
|
||||
const databaseFileAccess = new ServiceDatabaseFileAccessCLI({
|
||||
events: services.context.events,
|
||||
API: services.API,
|
||||
database: services.database,
|
||||
path: services.path,
|
||||
@@ -82,6 +84,7 @@ export function initialiseServiceModulesCLI(
|
||||
|
||||
// File handler (platform-independent)
|
||||
const fileHandler = new ServiceFileHandler({
|
||||
events: services.context.events,
|
||||
API: services.API,
|
||||
databaseFileAccess: databaseFileAccess,
|
||||
conflict: services.conflict,
|
||||
@@ -91,10 +94,12 @@ export function initialiseServiceModulesCLI(
|
||||
path: services.path,
|
||||
replication: services.replication,
|
||||
storageAccess: storageAccess,
|
||||
fileReflectionProvenance: createFileReflectionProvenance(services.keyValueDB),
|
||||
});
|
||||
|
||||
// Rebuilder (platform-independent)
|
||||
const rebuilder = new ServiceRebuilder({
|
||||
events: services.context.events,
|
||||
API: services.API,
|
||||
database: services.database,
|
||||
appLifecycle: services.appLifecycle,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import {
|
||||
ServiceDatabaseFileAccessBase,
|
||||
type ServiceDatabaseFileAccessDependencies,
|
||||
} from "@lib/serviceModules/ServiceDatabaseFileAccessBase";
|
||||
import type { DatabaseFileAccess } from "@lib/interfaces/DatabaseFileAccess";
|
||||
} from "@vrtmrz/livesync-commonlib/compat/serviceModules/ServiceDatabaseFileAccessBase";
|
||||
import type { DatabaseFileAccess } from "@vrtmrz/livesync-commonlib/compat/interfaces/DatabaseFileAccess";
|
||||
|
||||
/**
|
||||
* CLI-specific implementation of ServiceDatabaseFileAccess
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { FileAccessBase, type FileAccessBaseDependencies } from "@lib/serviceModules/FileAccessBase";
|
||||
import { FileAccessBase, type FileAccessBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/serviceModules/FileAccessBase";
|
||||
import { NodeFileSystemAdapter } from "@/apps/cli/adapters/NodeFileSystemAdapter";
|
||||
import { LOG_LEVEL_NOTICE } from "octagonal-wheels/common/logger";
|
||||
|
||||
/**
|
||||
* CLI-specific implementation of FileAccessBase
|
||||
@@ -7,7 +8,12 @@ import { NodeFileSystemAdapter } from "@/apps/cli/adapters/NodeFileSystemAdapter
|
||||
*/
|
||||
export class FileAccessCLI extends FileAccessBase<NodeFileSystemAdapter> {
|
||||
constructor(basePath: string, dependencies: FileAccessBaseDependencies) {
|
||||
const adapter = new NodeFileSystemAdapter(basePath);
|
||||
const adapter = new NodeFileSystemAdapter(basePath, (message, detail) => {
|
||||
dependencies.APIService.addLog(message, LOG_LEVEL_NOTICE);
|
||||
if (detail !== undefined) {
|
||||
dependencies.APIService.addLog(detail, LOG_LEVEL_NOTICE);
|
||||
}
|
||||
});
|
||||
super(adapter, dependencies);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { minimatch } from "minimatch";
|
||||
import { fsPromises as fs, path } from "@/apps/cli/node-compat";
|
||||
import { Minimatch } from "minimatch";
|
||||
import { fsPromises as fs, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { CliDiagnosticReporter } from "@/apps/cli/cliOutput";
|
||||
|
||||
/**
|
||||
* Loads and evaluates ignore rules from `.livesync/ignore` inside the vault.
|
||||
@@ -17,9 +18,12 @@ import { fsPromises as fs, path } from "@/apps/cli/node-compat";
|
||||
* Missing files (`.livesync/ignore` or `.gitignore`) are silently skipped.
|
||||
*/
|
||||
export class IgnoreRules {
|
||||
private patterns: string[] = [];
|
||||
private patterns: Minimatch[] = [];
|
||||
|
||||
constructor(private vaultPath: string) {}
|
||||
constructor(
|
||||
private vaultPath: string,
|
||||
private reportDiagnostic: CliDiagnosticReporter = () => undefined
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Reads `.livesync/ignore` (and optionally `.gitignore`) and populates the
|
||||
@@ -53,7 +57,7 @@ export class IgnoreRules {
|
||||
continue;
|
||||
}
|
||||
if (trimmed.startsWith("import:")) {
|
||||
console.error(
|
||||
this.reportDiagnostic(
|
||||
`[IgnoreRules] Warning: unrecognised directive '${trimmed}' — only 'import: .gitignore' is supported`
|
||||
);
|
||||
continue;
|
||||
@@ -61,7 +65,7 @@ export class IgnoreRules {
|
||||
this._addPattern(trimmed);
|
||||
}
|
||||
if (this.patterns.length > 0) {
|
||||
console.error(`[IgnoreRules] Loaded ${this.patterns.length} ignore patterns`);
|
||||
this.reportDiagnostic(`[IgnoreRules] Loaded ${this.patterns.length} ignore patterns`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,7 +112,7 @@ export class IgnoreRules {
|
||||
`Remove it from .livesync/ignore or use a separate include/exclude file.`
|
||||
);
|
||||
}
|
||||
this.patterns.push(this._normalisePattern(raw));
|
||||
this.patterns.push(new Minimatch(this._normalisePattern(raw), { dot: true }));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -124,6 +128,6 @@ export class IgnoreRules {
|
||||
}
|
||||
// Normalise to forward slashes for minimatch.
|
||||
const normalised = relativePath.replace(/\\/g, "/");
|
||||
return this.patterns.some((p) => minimatch(normalised, p, { dot: true }));
|
||||
return this.patterns.some((pattern) => pattern.match(normalised));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,26 @@
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { fsPromises as fs, os, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const minimatchStats = vi.hoisted(() => ({ constructions: 0 }));
|
||||
|
||||
vi.mock("minimatch", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("minimatch")>();
|
||||
|
||||
class CountingMinimatch extends actual.Minimatch {
|
||||
constructor(pattern: string, options?: import("minimatch").MinimatchOptions) {
|
||||
super(pattern, options);
|
||||
minimatchStats.constructions++;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...actual,
|
||||
Minimatch: CountingMinimatch,
|
||||
minimatch: (path: string, pattern: string, options?: import("minimatch").MinimatchOptions) =>
|
||||
new CountingMinimatch(pattern, options).match(path),
|
||||
};
|
||||
});
|
||||
|
||||
import { IgnoreRules } from "./IgnoreRules";
|
||||
|
||||
describe("IgnoreRules", () => {
|
||||
@@ -19,6 +38,10 @@ describe("IgnoreRules", () => {
|
||||
await fs.writeFile(path.join(ignoreDir, "ignore"), content, "utf-8");
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
minimatchStats.constructions = 0;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
@@ -55,6 +78,20 @@ describe("IgnoreRules", () => {
|
||||
});
|
||||
|
||||
describe("shouldIgnore", () => {
|
||||
it("compiles loaded patterns once", async () => {
|
||||
const vaultPath = await createVault();
|
||||
await writeIgnoreFile(vaultPath, "*.tmp\nbuild/\n");
|
||||
const rules = new IgnoreRules(vaultPath);
|
||||
await rules.load();
|
||||
|
||||
expect(rules.shouldIgnore("notes/readme.md")).toBe(false);
|
||||
expect(rules.shouldIgnore("notes/scratch.tmp")).toBe(true);
|
||||
expect(rules.shouldIgnore("build/output.js")).toBe(true);
|
||||
expect(rules.shouldIgnore("other.md")).toBe(false);
|
||||
|
||||
expect(minimatchStats.constructions).toBe(2);
|
||||
});
|
||||
|
||||
it("matches **/*.tmp against notes/scratch.tmp", async () => {
|
||||
const vaultPath = await createVault();
|
||||
await writeIgnoreFile(vaultPath, "*.tmp\n");
|
||||
@@ -101,11 +138,13 @@ describe("IgnoreRules", () => {
|
||||
const vaultPath = await createVault();
|
||||
// Typo: "import:.gitignore" instead of "import: .gitignore"
|
||||
await writeIgnoreFile(vaultPath, "*.tmp\nimport:.gitignore\n");
|
||||
const rules = new IgnoreRules(vaultPath);
|
||||
const reportDiagnostic = vi.fn();
|
||||
const rules = new IgnoreRules(vaultPath, reportDiagnostic);
|
||||
await rules.load();
|
||||
// *.tmp still loaded; import:.gitignore is skipped (not treated as a literal pattern)
|
||||
expect(rules.shouldIgnore("scratch.tmp")).toBe(true);
|
||||
expect(rules.shouldIgnore("import:.gitignore")).toBe(false);
|
||||
expect(reportDiagnostic).toHaveBeenCalledWith(expect.stringContaining("unrecognised directive"));
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ServiceFileAccessBase, type StorageAccessBaseDependencies } from "@lib/serviceModules/ServiceFileAccessBase";
|
||||
import { ServiceFileAccessBase, type StorageAccessBaseDependencies } from "@vrtmrz/livesync-commonlib/compat/serviceModules/ServiceFileAccessBase";
|
||||
import { NodeFileSystemAdapter } from "@/apps/cli/adapters/NodeFileSystemAdapter";
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@lib/common/logger";
|
||||
import type { KeyValueDatabase } from "@lib/interfaces/KeyValueDatabase";
|
||||
import type { IKeyValueDBService } from "@lib/services/base/IService";
|
||||
import { ServiceBase, type ServiceContext } from "@lib/services/base/ServiceBase";
|
||||
import type { InjectableAppLifecycleService } from "@lib/services/implements/injectable/InjectableAppLifecycleService";
|
||||
import type { InjectableDatabaseEventService } from "@lib/services/implements/injectable/InjectableDatabaseEventService";
|
||||
import type { IVaultService } from "@lib/services/base/IService";
|
||||
import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "@vrtmrz/livesync-commonlib/compat/common/logger";
|
||||
import type { KeyValueDatabase } from "@vrtmrz/livesync-commonlib/compat/interfaces/KeyValueDatabase";
|
||||
import type { IKeyValueDBService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { ServiceBase } from "@vrtmrz/livesync-commonlib/compat/services/base/ServiceBase";
|
||||
import type { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAppLifecycleService";
|
||||
import type { InjectableDatabaseEventService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableDatabaseEventService";
|
||||
import type { IVaultService } from "@vrtmrz/livesync-commonlib/compat/services/base/IService";
|
||||
import type { SimpleStore } from "octagonal-wheels/databases/SimpleStoreBase";
|
||||
import { createInstanceLogFunction } from "@lib/services/lib/logUtils";
|
||||
import { fs as nodeFs, path as nodePath } from "@/apps/cli/node-compat";
|
||||
import { createInstanceLogFunction } from "@vrtmrz/livesync-commonlib/compat/services/lib/logUtils";
|
||||
import { fs as nodeFs, path as nodePath } from "@vrtmrz/livesync-commonlib/node";
|
||||
|
||||
const NODE_KV_TYPED_KEY = "__nodeKvType";
|
||||
const NODE_KV_VALUES_KEY = "values";
|
||||
@@ -81,6 +82,17 @@ function deserializeFromNodeKV(value: unknown): unknown {
|
||||
return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, deserializeFromNodeKV(v)]));
|
||||
}
|
||||
|
||||
function asKeyString(key: unknown): string {
|
||||
if (typeof key === "string") {
|
||||
return key;
|
||||
}
|
||||
const serialised = JSON.stringify(key);
|
||||
if (typeof serialised !== "string") {
|
||||
throw new TypeError("The IndexedDB key could not be serialised");
|
||||
}
|
||||
return serialised;
|
||||
}
|
||||
|
||||
class NodeFileKeyValueDatabase implements KeyValueDatabase {
|
||||
private filePath: string;
|
||||
private data = new Map<string, unknown>();
|
||||
@@ -90,13 +102,6 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase {
|
||||
this.load();
|
||||
}
|
||||
|
||||
private asKeyString(key: IDBValidKey): string {
|
||||
if (typeof key === "string") {
|
||||
return key;
|
||||
}
|
||||
return JSON.stringify(key);
|
||||
}
|
||||
|
||||
private load() {
|
||||
try {
|
||||
const loaded = JSON.parse(nodeFs.readFileSync(this.filePath, "utf-8")) as Record<string, unknown>;
|
||||
@@ -115,17 +120,17 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase {
|
||||
}
|
||||
|
||||
async get<T>(key: IDBValidKey): Promise<T> {
|
||||
return this.data.get(this.asKeyString(key)) as T;
|
||||
return this.data.get(asKeyString(key)) as T;
|
||||
}
|
||||
|
||||
async set<T>(key: IDBValidKey, value: T): Promise<IDBValidKey> {
|
||||
this.data.set(this.asKeyString(key), value);
|
||||
this.data.set(asKeyString(key), value);
|
||||
this.flush();
|
||||
return key;
|
||||
}
|
||||
|
||||
async del(key: IDBValidKey): Promise<void> {
|
||||
this.data.delete(this.asKeyString(key));
|
||||
this.data.delete(asKeyString(key));
|
||||
this.flush();
|
||||
}
|
||||
|
||||
@@ -143,11 +148,12 @@ class NodeFileKeyValueDatabase implements KeyValueDatabase {
|
||||
let filtered = allKeys;
|
||||
if (typeof query !== "undefined") {
|
||||
if (this.isIDBKeyRangeLike(query)) {
|
||||
const lower = query.lower?.toString() ?? "";
|
||||
const upper = query.upper?.toString() ?? "\uffff";
|
||||
const lower = query.lower === undefined ? "" : String(query.lower);
|
||||
const upper = query.upper === undefined ? "\uffff" : String(query.upper);
|
||||
filtered = filtered.filter((key) => key >= lower && key <= upper);
|
||||
} else {
|
||||
const exact = query.toString();
|
||||
const exactValue: unknown = query;
|
||||
const exact = String(exactValue);
|
||||
filtered = filtered.filter((key) => key === exact);
|
||||
}
|
||||
}
|
||||
@@ -253,6 +259,14 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
|
||||
}
|
||||
|
||||
openSimpleStore<T>(kind: string): SimpleStore<T> {
|
||||
// Service modules are composed before onSettingLoaded opens the file-
|
||||
// backed database, so handle creation must not touch it. Actual store
|
||||
// operations are deliberately fail-fast: the sequential lifecycle opens
|
||||
// the database before scans, watchers, or replication start. Waiting here
|
||||
// could hang forever after failed initialisation, or deadlock if a future
|
||||
// initialisation handler tried to use the store it was waiting to open.
|
||||
// Reset is likewise a transient unavailable boundary, not a wait state;
|
||||
// callers must avoid store work there because an operation may fail.
|
||||
const getDB = () => {
|
||||
if (!this._kvDB) {
|
||||
throw new Error("KeyValueDB is not initialized yet");
|
||||
@@ -271,7 +285,15 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
|
||||
await getDB().del(`${prefix}${key}`);
|
||||
},
|
||||
keys: async (from: string | undefined, to: string | undefined, count?: number): Promise<string[]> => {
|
||||
const allKeys = (await getDB().keys(undefined, count)).map((e) => e.toString());
|
||||
const rawKeys: unknown = await getDB().keys(undefined, count);
|
||||
if (!Array.isArray(rawKeys)) {
|
||||
throw new TypeError("The key-value database returned an invalid key list");
|
||||
}
|
||||
const keyList: unknown[] = rawKeys;
|
||||
const allKeys: string[] = [];
|
||||
for (const key of keyList) {
|
||||
allKeys.push(String(key));
|
||||
}
|
||||
const lower = `${prefix}${from ?? ""}`;
|
||||
const upper = `${prefix}${to ?? "\uffff"}`;
|
||||
return allKeys
|
||||
@@ -279,7 +301,9 @@ export class NodeKeyValueDBService<T extends ServiceContext = ServiceContext>
|
||||
.filter((key) => key >= lower && key <= upper)
|
||||
.map((key) => key.substring(prefix.length));
|
||||
},
|
||||
db: Promise.resolve(getDB()),
|
||||
get db() {
|
||||
return Promise.resolve(getDB());
|
||||
},
|
||||
} satisfies SimpleStore<T>;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createServiceContext } from "@vrtmrz/livesync-commonlib/compat/services/base/ServiceBase";
|
||||
import type { NodeKeyValueDBDependencies } from "./NodeKeyValueDBService";
|
||||
import { NodeKeyValueDBService } from "./NodeKeyValueDBService";
|
||||
|
||||
describe("NodeKeyValueDBService.openSimpleStore", () => {
|
||||
it("creates a namespaced store handle before the backing database is initialised", () => {
|
||||
const dependencies = {
|
||||
appLifecycle: { onSettingLoaded: { addHandler: vi.fn() } },
|
||||
databaseEvents: {
|
||||
onResetDatabase: { addHandler: vi.fn() },
|
||||
onDatabaseInitialisation: { addHandler: vi.fn() },
|
||||
onUnloadDatabase: { addHandler: vi.fn() },
|
||||
onCloseDatabase: { addHandler: vi.fn() },
|
||||
},
|
||||
vault: {},
|
||||
} as unknown as NodeKeyValueDBDependencies;
|
||||
const service = new NodeKeyValueDBService(
|
||||
createServiceContext(),
|
||||
dependencies,
|
||||
"/tmp/obsidian-livesync-node-kv-handle-test.json"
|
||||
);
|
||||
|
||||
expect(() => service.openSimpleStore("early-composition")).not.toThrow();
|
||||
});
|
||||
|
||||
it("fails store operations promptly instead of waiting for lifecycle initialisation", async () => {
|
||||
const dependencies = {
|
||||
appLifecycle: { onSettingLoaded: { addHandler: vi.fn() } },
|
||||
databaseEvents: {
|
||||
onResetDatabase: { addHandler: vi.fn() },
|
||||
onDatabaseInitialisation: { addHandler: vi.fn() },
|
||||
onUnloadDatabase: { addHandler: vi.fn() },
|
||||
onCloseDatabase: { addHandler: vi.fn() },
|
||||
},
|
||||
vault: {},
|
||||
} as unknown as NodeKeyValueDBDependencies;
|
||||
const service = new NodeKeyValueDBService(
|
||||
createServiceContext(),
|
||||
dependencies,
|
||||
"/tmp/obsidian-livesync-node-kv-uninitialised-test.json"
|
||||
);
|
||||
const store = service.openSimpleStore("early-composition");
|
||||
|
||||
await expect(store.get("key")).rejects.toThrow("KeyValueDB is not initialized yet");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { fs as nodeFs, path as nodePath } from "@/apps/cli/node-compat";
|
||||
import { compatGlobal } from "@lib/common/coreEnvFunctions";
|
||||
import { fs as nodeFs, path as nodePath } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
|
||||
type LocalStorageShape = {
|
||||
getItem(key: string): string | null;
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { fs, os, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
clearNodeLocalStorage,
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { eventHub } from "@/common/events";
|
||||
import { translateLiveSyncMessage } from "@/common/translation";
|
||||
import { ServiceContext, type StandardIo } from "@vrtmrz/livesync-commonlib/context";
|
||||
|
||||
/** Host capabilities owned by one Self-hosted LiveSync CLI composition. */
|
||||
export class NodeServiceContext extends ServiceContext {
|
||||
constructor(
|
||||
readonly databasePath: string,
|
||||
readonly standardIo: StandardIo
|
||||
) {
|
||||
super({ events: eventHub, translate: translateLiveSyncMessage });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { eventHub } from "@/common/events";
|
||||
import { translateLiveSyncMessage } from "@/common/translation";
|
||||
import {
|
||||
observeServiceComposition,
|
||||
observeServiceContext,
|
||||
SERVICE_CONTEXT_MEMBERS,
|
||||
} from "../../../../test/contracts/serviceContext";
|
||||
import { NodeServiceContext } from "./NodeServiceContext";
|
||||
import { NodeServiceHub } from "./NodeServiceHub";
|
||||
import type { StandardIo } from "@vrtmrz/livesync-commonlib/context";
|
||||
|
||||
const TRANSLATION_KEY = "Replicator.Message.InitialiseFatalError";
|
||||
|
||||
describe("NodeServiceContext contract", () => {
|
||||
it("preserves the CLI capabilities and host-neutral API results", () => {
|
||||
const standardIo: StandardIo = {
|
||||
readStdin: async () => "input",
|
||||
prompt: async () => "answer",
|
||||
writeStdout: () => undefined,
|
||||
writeStderr: () => undefined,
|
||||
};
|
||||
const context = new NodeServiceContext("/tmp/livesync-context-contract", standardIo);
|
||||
|
||||
expect(observeServiceContext(context, TRANSLATION_KEY)).toEqual({
|
||||
translation: translateLiveSyncMessage(TRANSLATION_KEY),
|
||||
receivedEvents: ["context-contract-event"],
|
||||
});
|
||||
expect(context.events).toBe(eventHub);
|
||||
expect(context.databasePath).toBe("/tmp/livesync-context-contract");
|
||||
expect(context.standardIo).toBe(standardIo);
|
||||
|
||||
const hub = new NodeServiceHub(context.databasePath, context);
|
||||
const composition = observeServiceComposition(hub, context);
|
||||
expect(composition.hubUsesExpectedContext).toBe(true);
|
||||
expect(SERVICE_CONTEXT_MEMBERS.filter((member) => !composition.servicesUsingExpectedContext[member])).toEqual(
|
||||
[]
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,39 +1,36 @@
|
||||
import type { AppLifecycleService, AppLifecycleServiceDependencies } from "@lib/services/base/AppLifecycleService";
|
||||
import { ServiceContext } from "@lib/services/base/ServiceBase";
|
||||
import { ConfigServiceBrowserCompat } from "@lib/services/implements/browser/ConfigServiceBrowserCompat";
|
||||
import { SvelteDialogManagerBase, type ComponentHasResult } from "@lib/services/implements/base/SvelteDialog";
|
||||
import { UIService } from "@lib/services/implements/base/UIService";
|
||||
import { InjectableServiceHub } from "@lib/services/implements/injectable/InjectableServiceHub";
|
||||
import { InjectableAppLifecycleService } from "@lib/services/implements/injectable/InjectableAppLifecycleService";
|
||||
import { InjectableConflictService } from "@lib/services/implements/injectable/InjectableConflictService";
|
||||
import { InjectableDatabaseEventService } from "@lib/services/implements/injectable/InjectableDatabaseEventService";
|
||||
import { InjectableFileProcessingService } from "@lib/services/implements/injectable/InjectableFileProcessingService";
|
||||
import { PathServiceCompat } from "@lib/services/implements/injectable/InjectablePathService";
|
||||
import { InjectableRemoteService } from "@lib/services/implements/injectable/InjectableRemoteService";
|
||||
import { InjectableReplicationService } from "@lib/services/implements/injectable/InjectableReplicationService";
|
||||
import { InjectableReplicatorService } from "@lib/services/implements/injectable/InjectableReplicatorService";
|
||||
import { InjectableTestService } from "@lib/services/implements/injectable/InjectableTestService";
|
||||
import { InjectableTweakValueService } from "@lib/services/implements/injectable/InjectableTweakValueService";
|
||||
import { InjectableVaultServiceCompat } from "@lib/services/implements/injectable/InjectableVaultService";
|
||||
import { ControlService } from "@lib/services/base/ControlService";
|
||||
import type { IControlService } from "@lib/services/base/IService";
|
||||
import { HeadlessAPIService } from "@lib/services/implements/headless/HeadlessAPIService";
|
||||
// import { HeadlessDatabaseService } from "@lib/services/implements/headless/HeadlessDatabaseService";
|
||||
import type { ServiceInstances } from "@lib/services/ServiceHub";
|
||||
import type { AppLifecycleServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/AppLifecycleService";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { ConfigServiceBrowserCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/browser/ConfigServiceBrowserCompat";
|
||||
import type {
|
||||
ComponentHasResult,
|
||||
SvelteDialogManager,
|
||||
} from "@vrtmrz/livesync-commonlib/compat/services/implements/base/SvelteDialog";
|
||||
import { UIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/base/UIService";
|
||||
import { InjectableServiceHub } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableServiceHub";
|
||||
import { InjectableAppLifecycleService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableAppLifecycleService";
|
||||
import { InjectableConflictService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableConflictService";
|
||||
import { InjectableDatabaseEventService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableDatabaseEventService";
|
||||
import { InjectableFileProcessingService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableFileProcessingService";
|
||||
import { PathServiceCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectablePathService";
|
||||
import { InjectableRemoteService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableRemoteService";
|
||||
import { InjectableReplicationService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableReplicationService";
|
||||
import { InjectableReplicatorService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableReplicatorService";
|
||||
import { InjectableTestService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableTestService";
|
||||
import { InjectableTweakValueService } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableTweakValueService";
|
||||
import { InjectableVaultServiceCompat } from "@vrtmrz/livesync-commonlib/compat/services/implements/injectable/InjectableVaultService";
|
||||
import { ControlService } from "@vrtmrz/livesync-commonlib/compat/services/base/ControlService";
|
||||
import { HeadlessAPIService } from "@vrtmrz/livesync-commonlib/compat/services/implements/headless/HeadlessAPIService";
|
||||
import { NodeKeyValueDBService } from "./NodeKeyValueDBService";
|
||||
import { NodeSettingService } from "./NodeSettingService";
|
||||
import { DatabaseService } from "@lib/services/base/DatabaseService";
|
||||
import type { ObsidianLiveSyncSettings } from "@lib/common/types";
|
||||
import { path as nodePath } from "@/apps/cli/node-compat";
|
||||
import type { KeyValueDBService } from "@lib/services/base/KeyValueDBService";
|
||||
import { DatabaseService } from "@vrtmrz/livesync-commonlib/compat/services/base/DatabaseService";
|
||||
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { path as nodePath } from "@vrtmrz/livesync-commonlib/node";
|
||||
import type { KeyValueDBService } from "@vrtmrz/livesync-commonlib/compat/services/base/KeyValueDBService";
|
||||
import { PouchDB } from "@/apps/cli/lib/pouchdb-node";
|
||||
import { NodeServiceContext } from "./NodeServiceContext";
|
||||
import { setLang } from "@/common/translation";
|
||||
|
||||
export class NodeServiceContext extends ServiceContext {
|
||||
databasePath: string;
|
||||
constructor(databasePath: string) {
|
||||
super();
|
||||
this.databasePath = databasePath;
|
||||
}
|
||||
}
|
||||
export { NodeServiceContext } from "./NodeServiceContext";
|
||||
|
||||
class NodeAppLifecycleService<T extends ServiceContext> extends InjectableAppLifecycleService<T> {
|
||||
constructor(context: T, dependencies: AppLifecycleServiceDependencies) {
|
||||
@@ -41,21 +38,24 @@ class NodeAppLifecycleService<T extends ServiceContext> extends InjectableAppLif
|
||||
}
|
||||
}
|
||||
|
||||
class NodeSvelteDialogManager<T extends ServiceContext> extends SvelteDialogManagerBase<T> {
|
||||
openSvelteDialog<TValue, UInitial>(
|
||||
component: ComponentHasResult<TValue, UInitial>,
|
||||
initialData?: UInitial
|
||||
class NodeDialogManager<T extends ServiceContext> implements SvelteDialogManager<T> {
|
||||
open<TValue, UInitial>(
|
||||
_component: ComponentHasResult<TValue, UInitial>,
|
||||
_initialData?: UInitial
|
||||
): Promise<TValue | undefined> {
|
||||
throw new Error("Method not implemented.");
|
||||
return Promise.reject(new Error("Interactive dialogues are not available in the CLI."));
|
||||
}
|
||||
|
||||
openWithExplicitCancel<TValue, UInitial>(
|
||||
_component: ComponentHasResult<TValue, UInitial>,
|
||||
_initialData?: UInitial
|
||||
): Promise<TValue> {
|
||||
return Promise.reject(new Error("Interactive dialogues are not available in the CLI."));
|
||||
}
|
||||
}
|
||||
|
||||
type NodeUIServiceDependencies<T extends ServiceContext = ServiceContext> = {
|
||||
appLifecycle: AppLifecycleService<T>;
|
||||
config: ConfigServiceBrowserCompat<T>;
|
||||
replicator: InjectableReplicatorService<T>;
|
||||
APIService: HeadlessAPIService<T>;
|
||||
control: IControlService;
|
||||
};
|
||||
class NodeDatabaseService<T extends NodeServiceContext> extends DatabaseService<T> {
|
||||
protected override modifyDatabaseOptions(
|
||||
@@ -77,17 +77,9 @@ class NodeUIService<T extends ServiceContext> extends UIService<T> {
|
||||
}
|
||||
|
||||
constructor(context: T, dependencies: NodeUIServiceDependencies<T>) {
|
||||
const headlessConfirm = dependencies.APIService.confirm;
|
||||
const dialogManager = new NodeSvelteDialogManager<T>(context, {
|
||||
confirm: headlessConfirm,
|
||||
appLifecycle: dependencies.appLifecycle,
|
||||
config: dependencies.config,
|
||||
replicator: dependencies.replicator,
|
||||
control: dependencies.control,
|
||||
});
|
||||
const dialogManager = new NodeDialogManager<T>();
|
||||
|
||||
super(context, {
|
||||
appLifecycle: dependencies.appLifecycle,
|
||||
dialogManager,
|
||||
APIService: dependencies.APIService,
|
||||
});
|
||||
@@ -95,7 +87,7 @@ class NodeUIService<T extends ServiceContext> extends UIService<T> {
|
||||
}
|
||||
|
||||
export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServiceHub<T> {
|
||||
constructor(basePath: string, context: T = new NodeServiceContext(basePath) as T) {
|
||||
constructor(basePath: string, context: T) {
|
||||
const runtimeDir = nodePath.join(basePath, ".livesync", "runtime");
|
||||
const localStoragePath = nodePath.join(runtimeDir, "local-storage.json");
|
||||
const keyValueDBPath = nodePath.join(runtimeDir, "keyvalue-db.json");
|
||||
@@ -104,13 +96,18 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
|
||||
const conflict = new InjectableConflictService(context);
|
||||
const fileProcessing = new InjectableFileProcessingService(context);
|
||||
|
||||
const setting = new NodeSettingService(context, { APIService: API }, localStoragePath);
|
||||
const setting = new NodeSettingService(
|
||||
context,
|
||||
{ APIService: API, onDisplayLanguageChanged: setLang },
|
||||
localStoragePath
|
||||
);
|
||||
|
||||
const appLifecycle = new NodeAppLifecycleService<T>(context, {
|
||||
settingService: setting,
|
||||
});
|
||||
|
||||
const remote = new InjectableRemoteService(context, {
|
||||
pouchDB: PouchDB,
|
||||
APIService: API,
|
||||
appLifecycle,
|
||||
setting,
|
||||
@@ -128,6 +125,7 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
|
||||
});
|
||||
|
||||
const database = new NodeDatabaseService<T>(context, {
|
||||
pouchDB: PouchDB,
|
||||
API: API,
|
||||
path,
|
||||
vault,
|
||||
@@ -174,14 +172,10 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
|
||||
});
|
||||
|
||||
const ui = new NodeUIService<T>(context, {
|
||||
appLifecycle,
|
||||
config,
|
||||
replicator,
|
||||
APIService: API,
|
||||
control,
|
||||
});
|
||||
|
||||
const serviceInstancesToInit: Required<ServiceInstances<T>> = {
|
||||
const serviceInstancesToInit = {
|
||||
appLifecycle,
|
||||
conflict,
|
||||
database,
|
||||
@@ -201,7 +195,6 @@ export class NodeServiceHub<T extends NodeServiceContext> extends InjectableServ
|
||||
keyValueDB: keyValueDB as unknown as KeyValueDBService<T>,
|
||||
control,
|
||||
};
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- (Forcibly )
|
||||
super(context, serviceInstancesToInit as any);
|
||||
super(context, serviceInstancesToInit);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import { EVENT_SETTING_SAVED } from "@lib/events/coreEvents";
|
||||
import { EVENT_SETTING_SAVED } from "@vrtmrz/livesync-commonlib/compat/events/coreEvents";
|
||||
import { EVENT_REQUEST_RELOAD_SETTING_TAB } from "@/common/events";
|
||||
import { eventHub } from "@lib/hub/hub";
|
||||
import { handlers } from "@lib/services/lib/HandlerUtils";
|
||||
import type { ObsidianLiveSyncSettings } from "@lib/common/types";
|
||||
import type { ServiceContext } from "@lib/services/base/ServiceBase";
|
||||
import { SettingService, type SettingServiceDependencies } from "@lib/services/base/SettingService";
|
||||
import { handlers } from "@vrtmrz/livesync-commonlib/compat/services/lib/HandlerUtils";
|
||||
import type { ObsidianLiveSyncSettings } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { ServiceContext } from "@vrtmrz/livesync-commonlib/context";
|
||||
import { SettingService, type SettingServiceDependencies } from "@vrtmrz/livesync-commonlib/compat/services/base/SettingService";
|
||||
import {
|
||||
configureNodeLocalStorage,
|
||||
deleteNodeLocalStorageItem,
|
||||
@@ -17,11 +16,11 @@ export class NodeSettingService<T extends ServiceContext> extends SettingService
|
||||
super(context, dependencies);
|
||||
configureNodeLocalStorage(storagePath);
|
||||
this.onSettingSaved.addHandler((settings) => {
|
||||
eventHub.emitEvent(EVENT_SETTING_SAVED, settings);
|
||||
this.context.events.emitEvent(EVENT_SETTING_SAVED, settings);
|
||||
return Promise.resolve(true);
|
||||
});
|
||||
this.onSettingLoaded.addHandler((settings) => {
|
||||
eventHub.emitEvent(EVENT_REQUEST_RELOAD_SETTING_TAB);
|
||||
this.context.events.emitEvent(EVENT_REQUEST_RELOAD_SETTING_TAB);
|
||||
return Promise.resolve(true);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const setupPutCatHelper = readFileSync(new URL("./test/test-setup-put-cat-linux.sh", import.meta.url), "utf8");
|
||||
|
||||
describe("CLI setup URI E2E helper", () => {
|
||||
it("evaluates Commonlib package imports as ESM", () => {
|
||||
expect(setupPutCatHelper).toContain("node --input-type=module -e");
|
||||
expect(setupPutCatHelper).not.toContain("npx tsx -e");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env node
|
||||
import { RTCPeerConnection } from "werift";
|
||||
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
import { createNodeStandardIo } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { writeStderrLine } from "@/apps/cli/cliOutput";
|
||||
import { main, type CliCommandRunner } from "@/apps/cli/main";
|
||||
import { parseTimeoutSeconds } from "@/apps/cli/commands/p2p";
|
||||
import { runP2PReplicatorReplacementProbe } from "./p2p-replicator-replacement";
|
||||
|
||||
if (
|
||||
typeof (compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection === "undefined" &&
|
||||
typeof RTCPeerConnection === "function"
|
||||
) {
|
||||
(compatGlobal as unknown as Record<string, unknown>).RTCPeerConnection = RTCPeerConnection;
|
||||
}
|
||||
|
||||
const standardIo = createNodeStandardIo();
|
||||
const runLifecycleProbe: CliCommandRunner = async (options, context) => {
|
||||
if (options.command !== "p2p-sync" || options.commandArgs.length < 2) {
|
||||
throw new Error("The P2P lifecycle test entry requires: p2p-sync <peer> <timeout> [note-path] [note-content]");
|
||||
}
|
||||
const peerToken = options.commandArgs[0].trim();
|
||||
if (!peerToken) {
|
||||
throw new Error("The P2P lifecycle test entry requires a non-empty peer");
|
||||
}
|
||||
const timeoutSec = parseTimeoutSeconds(options.commandArgs[1], "P2P lifecycle test entry");
|
||||
return await runP2PReplicatorReplacementProbe(
|
||||
context,
|
||||
peerToken,
|
||||
timeoutSec * 1000,
|
||||
options.commandArgs[2],
|
||||
options.commandArgs[3]
|
||||
);
|
||||
};
|
||||
|
||||
main(standardIo, runLifecycleProbe).catch((error) => {
|
||||
writeStderrLine(standardIo, "[Fatal Error]", error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,132 @@
|
||||
import type { FilePathWithPrefix } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import { LiveSyncTrysteroReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/LiveSyncTrysteroReplicator";
|
||||
import { compatGlobal } from "@vrtmrz/livesync-commonlib/compat/common/coreEnvFunctions";
|
||||
import type { CLICommandContext } from "@/apps/cli/commands/types";
|
||||
import { openP2PHost } from "@/apps/cli/commands/p2p";
|
||||
|
||||
const DEFAULT_NOTE_PATH = "p2p-replicator-replacement.md";
|
||||
const DEFAULT_NOTE_CONTENT = "Replicated after replacing the active P2P replicator.";
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => compatGlobal.setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function describeError(value: unknown): string {
|
||||
return value instanceof Error ? (value.stack ?? value.message) : String(value);
|
||||
}
|
||||
|
||||
async function waitForServing(replicator: LiveSyncTrysteroReplicator, timeoutMs: number): Promise<void> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started <= timeoutMs) {
|
||||
if (replicator.server?.isServing) return;
|
||||
await delay(200);
|
||||
}
|
||||
throw new Error("The replacement P2P replicator did not start serving within the timeout");
|
||||
}
|
||||
|
||||
async function waitForPeer(
|
||||
replicator: LiveSyncTrysteroReplicator,
|
||||
targetPeer: string,
|
||||
timeoutMs: number
|
||||
): Promise<{ peerId: string; name: string }> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started <= timeoutMs) {
|
||||
const peer = replicator.knownAdvertisements.find(
|
||||
(candidate) => candidate.name === targetPeer || candidate.peerId === targetPeer
|
||||
);
|
||||
if (peer) return peer;
|
||||
await delay(200);
|
||||
}
|
||||
const knownPeers = replicator.knownAdvertisements.map((peer) => `${peer.name} (${peer.peerId})`).join(", ");
|
||||
throw new Error(
|
||||
`Peer '${targetPeer}' was not discovered within the timeout. Known peers: ${knownPeers || "none"}`
|
||||
);
|
||||
}
|
||||
|
||||
function assertPullSucceeded(result: unknown): void {
|
||||
if (result && typeof result === "object" && "error" in result && result.error) {
|
||||
throw new Error(`P2P pull failed: ${describeError(result.error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function communicateWithPeer(
|
||||
replicator: LiveSyncTrysteroReplicator,
|
||||
targetPeer: string,
|
||||
timeoutMs: number
|
||||
): Promise<{ peerId: string; name: string }> {
|
||||
await replicator.open();
|
||||
await waitForServing(replicator, timeoutMs);
|
||||
const peer = await waitForPeer(replicator, targetPeer, timeoutMs);
|
||||
assertPullSucceeded(await replicator.replicateFrom(peer.peerId, false));
|
||||
const pushResult = await replicator.requestSynchroniseToPeer(peer.peerId);
|
||||
if (!pushResult || pushResult.ok !== true) {
|
||||
throw new Error(`P2P push failed: ${describeError(pushResult?.error)}`);
|
||||
}
|
||||
return peer;
|
||||
}
|
||||
|
||||
/** Runs the real-transport lifecycle probe used by the Deno and Compose P2P suites. */
|
||||
export async function runP2PReplicatorReplacementProbe(
|
||||
context: CLICommandContext,
|
||||
targetPeer: string,
|
||||
timeoutMs: number,
|
||||
notePath = DEFAULT_NOTE_PATH,
|
||||
noteContent = DEFAULT_NOTE_CONTENT
|
||||
): Promise<boolean> {
|
||||
const { core, p2pReplicator } = context;
|
||||
if (!p2pReplicator) {
|
||||
throw new Error("The CLI did not expose its P2P service-feature result to the integration probe");
|
||||
}
|
||||
|
||||
const firstReplicator = await openP2PHost(core);
|
||||
if (p2pReplicator.replicator !== firstReplicator) {
|
||||
throw new Error("The P2P service feature did not expose the newly created replicator");
|
||||
}
|
||||
|
||||
const firstPeer = await communicateWithPeer(firstReplicator, targetPeer, timeoutMs);
|
||||
const initialised = await core.services.databaseEvents.initialiseDatabase(false, true, false);
|
||||
if (!initialised) {
|
||||
throw new Error("Database reinitialisation failed during the P2P replacement probe");
|
||||
}
|
||||
|
||||
const replacementReplicator = p2pReplicator.replicator;
|
||||
if (core.services.replicator.getActiveReplicator() !== replacementReplicator) {
|
||||
throw new Error("ReplicatorService did not activate the P2P service feature's replacement replicator");
|
||||
}
|
||||
if (replacementReplicator === firstReplicator) {
|
||||
throw new Error("Database reinitialisation retained the previous P2P replicator instance");
|
||||
}
|
||||
if (firstReplicator.server !== undefined) {
|
||||
throw new Error("The previous P2P replicator remained open after replacement");
|
||||
}
|
||||
|
||||
const settings = core.services.setting.currentSettings();
|
||||
settings.P2P_AutoStart = true;
|
||||
await core.services.control.applySettings();
|
||||
const resumedReplicator = p2pReplicator.replicator;
|
||||
await waitForServing(resumedReplicator, timeoutMs);
|
||||
if (firstReplicator.server !== undefined) {
|
||||
throw new Error("A setting event reopened the previous P2P replicator");
|
||||
}
|
||||
|
||||
const encoded = new TextEncoder().encode(noteContent);
|
||||
const noteBody = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength);
|
||||
const timestamp = Date.now();
|
||||
await core.serviceModules.storageAccess.writeFileAuto(notePath, noteBody, {
|
||||
ctime: timestamp,
|
||||
mtime: timestamp,
|
||||
});
|
||||
await core.serviceModules.fileHandler.storeFileToDB(notePath as FilePathWithPrefix, true);
|
||||
|
||||
const replacementPeer = await communicateWithPeer(resumedReplicator, targetPeer, timeoutMs);
|
||||
if (replacementPeer.name !== firstPeer.name) {
|
||||
throw new Error(
|
||||
`The replacement replicator reached '${replacementPeer.name}' instead of the original peer '${firstPeer.name}'`
|
||||
);
|
||||
}
|
||||
|
||||
core.services.context.standardIo.writeStdout(
|
||||
`[Probe] P2P replicator replaced, old transport stayed closed, and ${notePath} was sent through the replacement.\n`
|
||||
);
|
||||
return true;
|
||||
}
|
||||
@@ -93,9 +93,8 @@ data.encrypt = true;
|
||||
data.passphrase = process.env.PASSPHRASE_VAL;
|
||||
data.usePathObfuscation = true;
|
||||
data.handleFilenameCaseSensitive = false;
|
||||
data.customChunkSize = 50;
|
||||
data.customChunkSize = 60;
|
||||
data.usePluginSyncV2 = true;
|
||||
data.doNotUseFixedRevisionForChunks = false;
|
||||
data.P2P_DevicePeerName = process.env.DEVICE_NAME;
|
||||
data.isConfigured = true;
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
CLI_DIR="$(cd -- "$SCRIPT_DIR/.." && pwd)"
|
||||
REPO_ROOT="$(cd -- "$CLI_DIR/../../.." && pwd)"
|
||||
cd "$CLI_DIR"
|
||||
source "$SCRIPT_DIR/test-helpers.sh"
|
||||
display_test_info
|
||||
@@ -28,10 +27,10 @@ cli_test_init_settings_file "$SETTINGS_FILE"
|
||||
|
||||
echo "[INFO] creating setup URI from settings"
|
||||
SETUP_URI="$(
|
||||
REPO_ROOT="$REPO_ROOT" SETTINGS_FILE="$SETTINGS_FILE" SETUP_PASSPHRASE="$SETUP_PASSPHRASE" npx tsx -e '
|
||||
import fs from "node:fs";
|
||||
SETTINGS_FILE="$SETTINGS_FILE" SETUP_PASSPHRASE="$SETUP_PASSPHRASE" node --input-type=module -e '
|
||||
import { fs } from "@vrtmrz/livesync-commonlib/node";
|
||||
import { encodeSettingsToSetupURI } from "@vrtmrz/livesync-commonlib/compat/API/processSetting";
|
||||
(async () => {
|
||||
const { encodeSettingsToSetupURI } = await import(process.env.REPO_ROOT + "/src/lib/src/API/processSetting.ts");
|
||||
const settingsPath = process.env.SETTINGS_FILE;
|
||||
const setupPassphrase = process.env.SETUP_PASSPHRASE;
|
||||
const settings = JSON.parse(fs.readFileSync(settingsPath, "utf-8"));
|
||||
|
||||
@@ -107,6 +107,7 @@ Deno.test("feature: behaviour", async () => {
|
||||
- Re-run sync operations where the protocol is eventually consistent.
|
||||
- For network-sensitive commands, use `LIVESYNC_CLI_RETRY` during debugging.
|
||||
- Keep Docker container reuse disabled by default unless debugging.
|
||||
- Use `npm run test:e2e:cli:p2p` for canonical P2P validation. It runs the Deno scenario in Compose because host networking and WebRTC candidate selection are not reproducible across environments. Individual `deno task test:p2p-*` tasks remain available when explicitly invoked for cross-platform diagnostics, but are not selected by the default suite or CI.
|
||||
|
||||
## Environment variables
|
||||
|
||||
|
||||
@@ -0,0 +1,603 @@
|
||||
import { join } from "@std/path";
|
||||
import { TempDir } from "./helpers/temp.ts";
|
||||
import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
|
||||
import { assertFilesEqual } from "./helpers/cli.ts";
|
||||
import { runMeasuredCliOrFail, type CliProcessMeasurement } from "./helpers/measuredCli.ts";
|
||||
import { createCouchdbDatabase, startCouchdb, stopCouchdb } from "./helpers/docker.ts";
|
||||
import {
|
||||
createCompressionBenchmarkDataset,
|
||||
type CompressionDataset,
|
||||
type CompressionDatasetEntry,
|
||||
} from "./helpers/compressionDataset.ts";
|
||||
import { computeDatasetDigestSha256 } from "./helpers/benchmarkVerification.ts";
|
||||
import { startCouchdbProxy, type CouchdbProxyCounters } from "./bench-couchdb.ts";
|
||||
import type { DatasetKind } from "./helpers/dataset.ts";
|
||||
|
||||
type CompressionCondition = {
|
||||
name: string;
|
||||
encrypt: boolean;
|
||||
enableCompression: boolean;
|
||||
};
|
||||
|
||||
type CouchDbSizes = {
|
||||
file: number;
|
||||
external: number;
|
||||
active: number;
|
||||
};
|
||||
|
||||
type PerKindRemoteMeasurement = {
|
||||
sourceFiles: number;
|
||||
sourceBytes: number;
|
||||
mappedFiles: number;
|
||||
uniqueReferencedChunks: number;
|
||||
storedChunkDataBytes: number;
|
||||
storedChunkJsonBytes: number;
|
||||
};
|
||||
|
||||
type RemoteMeasurement = {
|
||||
couchdbSizes: CouchDbSizes;
|
||||
documentCount: number;
|
||||
chunkDocumentCount: number;
|
||||
metadataDocumentCount: number;
|
||||
compressedMarkerCount: number;
|
||||
encryptedChunkCount: number;
|
||||
storedChunkDataBytes: number;
|
||||
storedChunkJsonBytes: number;
|
||||
perKind: Record<DatasetKind, PerKindRemoteMeasurement>;
|
||||
};
|
||||
|
||||
type RunResult = {
|
||||
condition: CompressionCondition;
|
||||
repeatIndex: number;
|
||||
executionOrder: number;
|
||||
databaseName: string;
|
||||
datasetDigestSha256: string;
|
||||
dataset: {
|
||||
totalFiles: number;
|
||||
totalBytes: number;
|
||||
filesByKind: CompressionDataset["filesByKind"];
|
||||
bytesByKind: CompressionDataset["bytesByKind"];
|
||||
jpegGenerator: string;
|
||||
};
|
||||
effectiveSettings: Record<string, unknown>;
|
||||
mirror: CliProcessMeasurement;
|
||||
upload: CliProcessMeasurement & { http: CouchdbProxyCounters };
|
||||
download: CliProcessMeasurement & { http: CouchdbProxyCounters };
|
||||
materialisation: CliProcessMeasurement & { http: CouchdbProxyCounters };
|
||||
verification: {
|
||||
verifiedFiles: number;
|
||||
complete: boolean;
|
||||
};
|
||||
remote: RemoteMeasurement;
|
||||
};
|
||||
|
||||
const CONDITIONS: CompressionCondition[] = [
|
||||
{ name: "plain", encrypt: false, enableCompression: false },
|
||||
{ name: "plain-compressed", encrypt: false, enableCompression: true },
|
||||
{ name: "e2ee", encrypt: true, enableCompression: false },
|
||||
{ name: "e2ee-compressed", encrypt: true, enableCompression: true },
|
||||
];
|
||||
|
||||
const DATASET_KINDS: DatasetKind[] = ["md", "jpg", "png", "json", "ts", "gz", "bin"];
|
||||
const COMPRESSED_MARKER = "\u000eLZ\u001d";
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
const value = Deno.env.get(name)?.trim();
|
||||
return value ? value : fallback;
|
||||
}
|
||||
|
||||
function readEnvPositiveInteger(name: string, fallback: number): number {
|
||||
const raw = Deno.env.get(name)?.trim();
|
||||
if (!raw) return fallback;
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value <= 0) {
|
||||
throw new Error(`${name} must be a positive integer, got '${raw}'`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function readEnvPositiveNumber(name: string, fallback: number): number {
|
||||
const raw = Deno.env.get(name)?.trim();
|
||||
if (!raw) return fallback;
|
||||
const value = Number(raw);
|
||||
if (!Number.isFinite(value) || value <= 0) {
|
||||
throw new Error(`${name} must be positive, got '${raw}'`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function readEnvBoolean(name: string, fallback: boolean): boolean {
|
||||
const raw = Deno.env.get(name)?.trim();
|
||||
if (!raw) return fallback;
|
||||
return /^(1|true|yes|on)$/i.test(raw);
|
||||
}
|
||||
|
||||
function byteLength(value: string): number {
|
||||
return new TextEncoder().encode(value).byteLength;
|
||||
}
|
||||
|
||||
function median(values: number[]): number {
|
||||
if (values.length === 0) return 0;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
if (sorted.length % 2 === 1) return sorted[middle];
|
||||
return (sorted[middle - 1] + sorted[middle]) / 2;
|
||||
}
|
||||
|
||||
function rounded(value: number): number {
|
||||
return Number(value.toFixed(4));
|
||||
}
|
||||
|
||||
function deltaPercent(enabled: number, disabled: number): number | null {
|
||||
if (disabled === 0) return null;
|
||||
return rounded(((enabled - disabled) / disabled) * 100);
|
||||
}
|
||||
|
||||
function reductionPercent(enabled: number, disabled: number): number | null {
|
||||
if (disabled === 0) return null;
|
||||
return rounded((1 - enabled / disabled) * 100);
|
||||
}
|
||||
|
||||
function basicAuth(user: string, password: string): string {
|
||||
return `Basic ${btoa(`${user}:${password}`)}`;
|
||||
}
|
||||
|
||||
async function couchRequest(
|
||||
baseUri: string,
|
||||
user: string,
|
||||
password: string,
|
||||
path: string,
|
||||
init: RequestInit = {},
|
||||
allowedStatuses: number[] = []
|
||||
): Promise<Response> {
|
||||
const response = await fetch(`${baseUri.replace(/\/$/, "")}${path}`, {
|
||||
...init,
|
||||
headers: {
|
||||
Authorization: basicAuth(user, password),
|
||||
...(init.method === "POST" || init.body ? { "Content-Type": "application/json" } : {}),
|
||||
...init.headers,
|
||||
},
|
||||
});
|
||||
if (!response.ok && !allowedStatuses.includes(response.status)) {
|
||||
throw new Error(`${init.method ?? "GET"} ${path}: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async function deleteDatabase(baseUri: string, user: string, password: string, databaseName: string): Promise<void> {
|
||||
const response = await couchRequest(
|
||||
baseUri,
|
||||
user,
|
||||
password,
|
||||
`/${encodeURIComponent(databaseName)}`,
|
||||
{ method: "DELETE" },
|
||||
[404]
|
||||
);
|
||||
await response.body?.cancel().catch(() => {});
|
||||
}
|
||||
|
||||
function blankPerKind(dataset: CompressionDataset): Record<DatasetKind, PerKindRemoteMeasurement> {
|
||||
return Object.fromEntries(
|
||||
DATASET_KINDS.map((kind) => [
|
||||
kind,
|
||||
{
|
||||
sourceFiles: dataset.filesByKind[kind],
|
||||
sourceBytes: dataset.bytesByKind[kind],
|
||||
mappedFiles: 0,
|
||||
uniqueReferencedChunks: 0,
|
||||
storedChunkDataBytes: 0,
|
||||
storedChunkJsonBytes: 0,
|
||||
},
|
||||
])
|
||||
) as Record<DatasetKind, PerKindRemoteMeasurement>;
|
||||
}
|
||||
|
||||
function findDatasetEntry(path: unknown, entries: CompressionDatasetEntry[]): CompressionDatasetEntry | undefined {
|
||||
if (typeof path !== "string") return undefined;
|
||||
return entries.find((entry) => path === entry.relativePath || path.endsWith(`/${entry.relativePath}`));
|
||||
}
|
||||
|
||||
async function inspectRemoteDatabase(options: {
|
||||
baseUri: string;
|
||||
user: string;
|
||||
password: string;
|
||||
databaseName: string;
|
||||
dataset: CompressionDataset;
|
||||
}): Promise<RemoteMeasurement> {
|
||||
const dbPath = `/${encodeURIComponent(options.databaseName)}`;
|
||||
await couchRequest(options.baseUri, options.user, options.password, `${dbPath}/_ensure_full_commit`, {
|
||||
method: "POST",
|
||||
body: "{}",
|
||||
});
|
||||
const [info, allDocs] = await Promise.all([
|
||||
couchRequest(options.baseUri, options.user, options.password, dbPath).then((response) => response.json()),
|
||||
couchRequest(options.baseUri, options.user, options.password, `${dbPath}/_all_docs?include_docs=true`).then(
|
||||
(response) => response.json()
|
||||
),
|
||||
]);
|
||||
const rows = (allDocs as { rows?: Array<{ doc?: Record<string, unknown> }> }).rows ?? [];
|
||||
const docs = rows.flatMap((row) => (row.doc ? [row.doc] : []));
|
||||
const docsById = new Map(docs.flatMap((doc) => (typeof doc._id === "string" ? [[doc._id, doc] as const] : [])));
|
||||
const metadataDocs = docs.filter((doc) => Array.isArray(doc.children) && typeof doc.path === "string");
|
||||
const chunkDocs = docs.filter(
|
||||
(doc) =>
|
||||
(doc.type === "leaf" || doc.type === "chunkpack") &&
|
||||
typeof doc.data === "string" &&
|
||||
typeof doc._id === "string"
|
||||
);
|
||||
const perKind = blankPerKind(options.dataset);
|
||||
const chunkIdsByKind = new Map(DATASET_KINDS.map((kind) => [kind, new Set<string>()] as const));
|
||||
|
||||
for (const doc of metadataDocs) {
|
||||
const entry = findDatasetEntry(doc.path, options.dataset.entries);
|
||||
if (!entry) continue;
|
||||
perKind[entry.kind].mappedFiles += 1;
|
||||
for (const child of doc.children as unknown[]) {
|
||||
if (typeof child === "string") chunkIdsByKind.get(entry.kind)!.add(child);
|
||||
}
|
||||
}
|
||||
for (const kind of DATASET_KINDS) {
|
||||
const chunkIds = chunkIdsByKind.get(kind)!;
|
||||
perKind[kind].uniqueReferencedChunks = chunkIds.size;
|
||||
for (const chunkId of chunkIds) {
|
||||
const chunk = docsById.get(chunkId);
|
||||
if (!chunk || typeof chunk.data !== "string") continue;
|
||||
perKind[kind].storedChunkDataBytes += byteLength(chunk.data);
|
||||
perKind[kind].storedChunkJsonBytes += byteLength(JSON.stringify(chunk));
|
||||
}
|
||||
}
|
||||
|
||||
const sizeInfo = (info as { sizes?: Partial<CouchDbSizes>; doc_count?: number }).sizes ?? {};
|
||||
return {
|
||||
couchdbSizes: {
|
||||
file: sizeInfo.file ?? 0,
|
||||
external: sizeInfo.external ?? 0,
|
||||
active: sizeInfo.active ?? 0,
|
||||
},
|
||||
documentCount: (info as { doc_count?: number }).doc_count ?? docs.length,
|
||||
chunkDocumentCount: chunkDocs.length,
|
||||
metadataDocumentCount: metadataDocs.length,
|
||||
compressedMarkerCount: chunkDocs.filter(
|
||||
(doc) => typeof doc.data === "string" && doc.data.startsWith(COMPRESSED_MARKER)
|
||||
).length,
|
||||
encryptedChunkCount: chunkDocs.filter((doc) => doc.e_ === true).length,
|
||||
storedChunkDataBytes: chunkDocs.reduce(
|
||||
(sum, doc) => sum + (typeof doc.data === "string" ? byteLength(doc.data) : 0),
|
||||
0
|
||||
),
|
||||
storedChunkJsonBytes: chunkDocs.reduce((sum, doc) => sum + byteLength(JSON.stringify(doc)), 0),
|
||||
perKind,
|
||||
};
|
||||
}
|
||||
|
||||
async function verifyDataset(
|
||||
workDir: TempDir,
|
||||
vaultB: string,
|
||||
settingsB: string,
|
||||
entries: CompressionDatasetEntry[]
|
||||
): Promise<CliProcessMeasurement> {
|
||||
const started = performance.now();
|
||||
const measurements: CliProcessMeasurement[] = [];
|
||||
for (const entry of entries) {
|
||||
const pulledPath = workDir.join(`verify-${entry.kind}-${entry.relativePath.split("/").at(-1)}`);
|
||||
measurements.push(
|
||||
await runMeasuredCliOrFail(vaultB, "--settings", settingsB, "pull", entry.relativePath, pulledPath)
|
||||
);
|
||||
await assertFilesEqual(entry.absolutePath, pulledPath, `compression benchmark mismatch: ${entry.relativePath}`);
|
||||
}
|
||||
const elapsedMs = performance.now() - started;
|
||||
const userCpuMs = measurements.reduce((sum, measurement) => sum + measurement.userCpuMs, 0);
|
||||
const systemCpuMs = measurements.reduce((sum, measurement) => sum + measurement.systemCpuMs, 0);
|
||||
const totalCpuMs = userCpuMs + systemCpuMs;
|
||||
return {
|
||||
elapsedMs: rounded(elapsedMs),
|
||||
userCpuMs: rounded(userCpuMs),
|
||||
systemCpuMs: rounded(systemCpuMs),
|
||||
totalCpuMs: rounded(totalCpuMs),
|
||||
cpuToWallRatio: rounded(totalCpuMs / elapsedMs),
|
||||
maxResidentSetKiB: Math.max(...measurements.map((measurement) => measurement.maxResidentSetKiB)),
|
||||
};
|
||||
}
|
||||
|
||||
function summariseResults(results: RunResult[]) {
|
||||
const byCondition = Object.fromEntries(
|
||||
CONDITIONS.map((condition) => {
|
||||
const runs = results.filter((result) => result.condition.name === condition.name);
|
||||
return [
|
||||
condition.name,
|
||||
{
|
||||
repeats: runs.length,
|
||||
remoteStoredChunkDataBytesMedian: median(runs.map((run) => run.remote.storedChunkDataBytes)),
|
||||
couchdbExternalBytesMedian: median(runs.map((run) => run.remote.couchdbSizes.external)),
|
||||
couchdbFileBytesMedian: median(runs.map((run) => run.remote.couchdbSizes.file)),
|
||||
uploadRequestBodyBytesMedian: median(runs.map((run) => run.upload.http.requestBodyBytes)),
|
||||
uploadResponseBodyBytesMedian: median(runs.map((run) => run.upload.http.responseBodyBytes)),
|
||||
downloadRequestBodyBytesMedian: median(runs.map((run) => run.download.http.requestBodyBytes)),
|
||||
downloadResponseBodyBytesMedian: median(runs.map((run) => run.download.http.responseBodyBytes)),
|
||||
uploadElapsedMsMedian: median(runs.map((run) => run.upload.elapsedMs)),
|
||||
uploadCpuMsMedian: median(runs.map((run) => run.upload.totalCpuMs)),
|
||||
downloadElapsedMsMedian: median(runs.map((run) => run.download.elapsedMs)),
|
||||
downloadCpuMsMedian: median(runs.map((run) => run.download.totalCpuMs)),
|
||||
materialisationElapsedMsMedian: median(runs.map((run) => run.materialisation.elapsedMs)),
|
||||
materialisationCpuMsMedian: median(runs.map((run) => run.materialisation.totalCpuMs)),
|
||||
materialisationResponseBodyBytesMedian: median(
|
||||
runs.map((run) => run.materialisation.http.responseBodyBytes)
|
||||
),
|
||||
completeDownloadResponseBodyBytesMedian: median(
|
||||
runs.map(
|
||||
(run) => run.download.http.responseBodyBytes + run.materialisation.http.responseBodyBytes
|
||||
)
|
||||
),
|
||||
completeDownloadElapsedMsMedian: median(
|
||||
runs.map((run) => run.download.elapsedMs + run.materialisation.elapsedMs)
|
||||
),
|
||||
completeDownloadCpuMsMedian: median(
|
||||
runs.map((run) => run.download.totalCpuMs + run.materialisation.totalCpuMs)
|
||||
),
|
||||
maxResidentSetKiBMedian: median(
|
||||
runs.map((run) =>
|
||||
Math.max(
|
||||
run.upload.maxResidentSetKiB,
|
||||
run.download.maxResidentSetKiB,
|
||||
run.materialisation.maxResidentSetKiB
|
||||
)
|
||||
)
|
||||
),
|
||||
perKindStoredChunkDataBytesMedian: Object.fromEntries(
|
||||
DATASET_KINDS.map((kind) => [
|
||||
kind,
|
||||
median(runs.map((run) => run.remote.perKind[kind].storedChunkDataBytes)),
|
||||
])
|
||||
),
|
||||
},
|
||||
];
|
||||
})
|
||||
);
|
||||
|
||||
const comparisons = [false, true].map((encrypt) => {
|
||||
const disabledName = encrypt ? "e2ee" : "plain";
|
||||
const enabledName = encrypt ? "e2ee-compressed" : "plain-compressed";
|
||||
const disabled = byCondition[disabledName] as Record<string, unknown>;
|
||||
const enabled = byCondition[enabledName] as Record<string, unknown>;
|
||||
const disabledPerKind = disabled.perKindStoredChunkDataBytesMedian as Record<DatasetKind, number>;
|
||||
const enabledPerKind = enabled.perKindStoredChunkDataBytesMedian as Record<DatasetKind, number>;
|
||||
return {
|
||||
encrypt,
|
||||
disabledCondition: disabledName,
|
||||
enabledCondition: enabledName,
|
||||
storedChunkDataReductionPercent: reductionPercent(
|
||||
enabled.remoteStoredChunkDataBytesMedian as number,
|
||||
disabled.remoteStoredChunkDataBytesMedian as number
|
||||
),
|
||||
couchdbExternalReductionPercent: reductionPercent(
|
||||
enabled.couchdbExternalBytesMedian as number,
|
||||
disabled.couchdbExternalBytesMedian as number
|
||||
),
|
||||
couchdbFileReductionPercent: reductionPercent(
|
||||
enabled.couchdbFileBytesMedian as number,
|
||||
disabled.couchdbFileBytesMedian as number
|
||||
),
|
||||
uploadRequestBodyReductionPercent: reductionPercent(
|
||||
enabled.uploadRequestBodyBytesMedian as number,
|
||||
disabled.uploadRequestBodyBytesMedian as number
|
||||
),
|
||||
completeDownloadResponseBodyReductionPercent: reductionPercent(
|
||||
enabled.completeDownloadResponseBodyBytesMedian as number,
|
||||
disabled.completeDownloadResponseBodyBytesMedian as number
|
||||
),
|
||||
uploadElapsedDeltaPercent: deltaPercent(
|
||||
enabled.uploadElapsedMsMedian as number,
|
||||
disabled.uploadElapsedMsMedian as number
|
||||
),
|
||||
uploadCpuDeltaPercent: deltaPercent(
|
||||
enabled.uploadCpuMsMedian as number,
|
||||
disabled.uploadCpuMsMedian as number
|
||||
),
|
||||
downloadElapsedDeltaPercent: deltaPercent(
|
||||
enabled.downloadElapsedMsMedian as number,
|
||||
disabled.downloadElapsedMsMedian as number
|
||||
),
|
||||
downloadCpuDeltaPercent: deltaPercent(
|
||||
enabled.downloadCpuMsMedian as number,
|
||||
disabled.downloadCpuMsMedian as number
|
||||
),
|
||||
completeDownloadElapsedDeltaPercent: deltaPercent(
|
||||
enabled.completeDownloadElapsedMsMedian as number,
|
||||
disabled.completeDownloadElapsedMsMedian as number
|
||||
),
|
||||
completeDownloadCpuDeltaPercent: deltaPercent(
|
||||
enabled.completeDownloadCpuMsMedian as number,
|
||||
disabled.completeDownloadCpuMsMedian as number
|
||||
),
|
||||
perKindStoredChunkDataReductionPercent: Object.fromEntries(
|
||||
DATASET_KINDS.map((kind) => [kind, reductionPercent(enabledPerKind[kind], disabledPerKind[kind])])
|
||||
),
|
||||
};
|
||||
});
|
||||
return { byCondition, comparisons };
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const backendUri = readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989");
|
||||
const proxyUri = readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989");
|
||||
const user = readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin"));
|
||||
const password = readEnvString("BENCH_COUCHDB_PASSWORD", readEnvString("password", "password"));
|
||||
const databasePrefix = readEnvString("BENCH_COUCHDB_DBNAME", `compression-bench-${Date.now()}`);
|
||||
const repeatCount = readEnvPositiveInteger("BENCH_COMPRESSION_REPEAT_COUNT", 1);
|
||||
const requestedRttMs = readEnvPositiveNumber("BENCH_COUCHDB_RTT_MS", 1);
|
||||
const managedCouchdb = readEnvBoolean("BENCH_COUCHDB_MANAGED", true);
|
||||
const passphrase = readEnvString("BENCH_PASSPHRASE", "compression-benchmark-passphrase");
|
||||
const resultRoot = readEnvString("BENCH_COMPRESSION_RESULT_ROOT", "bench-results");
|
||||
const resultPath =
|
||||
Deno.env.get("BENCH_COMPRESSION_RESULT_JSON")?.trim() ||
|
||||
join(resultRoot, `compression-${new Date().toISOString().replaceAll(":", "-")}.json`);
|
||||
const createdDatabases = new Set<string>();
|
||||
const results: RunResult[] = [];
|
||||
let managedStarted = false;
|
||||
|
||||
await Deno.mkdir(resultRoot, { recursive: true });
|
||||
const proxy = startCouchdbProxy({ backendUri, proxyUri, requestedRttMs });
|
||||
|
||||
try {
|
||||
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
|
||||
const rotation = (repeatIndex - 1) % CONDITIONS.length;
|
||||
const orderedConditions = [...CONDITIONS.slice(rotation), ...CONDITIONS.slice(0, rotation)];
|
||||
for (const [executionOffset, condition] of orderedConditions.entries()) {
|
||||
const databaseName = `${databasePrefix}-${repeatIndex}-${condition.name}`.toLowerCase();
|
||||
if (managedCouchdb && !managedStarted) {
|
||||
await startCouchdb(backendUri, user, password, databaseName);
|
||||
managedStarted = true;
|
||||
} else {
|
||||
await createCouchdbDatabase(backendUri, user, password, databaseName);
|
||||
}
|
||||
createdDatabases.add(databaseName);
|
||||
|
||||
await using workDir = await TempDir.create(`livesync-compression-${condition.name}`);
|
||||
const vaultA = workDir.join("vault-a");
|
||||
const vaultB = workDir.join("vault-b");
|
||||
const settingsA = workDir.join("settings-a.json");
|
||||
const settingsB = workDir.join("settings-b.json");
|
||||
await Deno.mkdir(vaultA, { recursive: true });
|
||||
await Deno.mkdir(vaultB, { recursive: true });
|
||||
await initSettingsFile(settingsA);
|
||||
await initSettingsFile(settingsB);
|
||||
await Promise.all(
|
||||
[settingsA, settingsB].map((settingsFile) =>
|
||||
applyRemoteSyncSettings(settingsFile, {
|
||||
remoteType: "COUCHDB",
|
||||
couchdbUri: proxyUri,
|
||||
couchdbUser: user,
|
||||
couchdbPassword: password,
|
||||
couchdbDbname: databaseName,
|
||||
encrypt: condition.encrypt,
|
||||
passphrase,
|
||||
enableCompression: condition.enableCompression,
|
||||
usePathObfuscation: false,
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
const dataset = await createCompressionBenchmarkDataset({ rootDir: vaultA });
|
||||
const datasetDigestSha256 = await computeDatasetDigestSha256(dataset.entries);
|
||||
const mirror = await runMeasuredCliOrFail(vaultA, "--settings", settingsA, "mirror");
|
||||
|
||||
proxy.resetCounters();
|
||||
const uploadMeasurement = await runMeasuredCliOrFail(vaultA, "--settings", settingsA, "sync");
|
||||
const upload = { ...uploadMeasurement, http: proxy.snapshotCounters() };
|
||||
|
||||
proxy.resetCounters();
|
||||
const downloadMeasurement = await runMeasuredCliOrFail(vaultB, "--settings", settingsB, "sync");
|
||||
const download = { ...downloadMeasurement, http: proxy.snapshotCounters() };
|
||||
|
||||
proxy.resetCounters();
|
||||
const materialisationMeasurement = await verifyDataset(workDir, vaultB, settingsB, dataset.entries);
|
||||
const materialisation = {
|
||||
...materialisationMeasurement,
|
||||
http: proxy.snapshotCounters(),
|
||||
};
|
||||
const remote = await inspectRemoteDatabase({
|
||||
baseUri: backendUri,
|
||||
user,
|
||||
password,
|
||||
databaseName,
|
||||
dataset,
|
||||
});
|
||||
const settings = JSON.parse(await Deno.readTextFile(settingsA)) as Record<string, unknown>;
|
||||
const effectiveSettings = Object.fromEntries(
|
||||
[
|
||||
"encrypt",
|
||||
"enableCompression",
|
||||
"E2EEAlgorithm",
|
||||
"usePathObfuscation",
|
||||
"chunkSplitterVersion",
|
||||
"customChunkSize",
|
||||
"minimumChunkSize",
|
||||
"hashAlg",
|
||||
].map((key) => [key, settings[key]])
|
||||
);
|
||||
|
||||
results.push({
|
||||
condition,
|
||||
repeatIndex,
|
||||
executionOrder: executionOffset + 1,
|
||||
databaseName,
|
||||
datasetDigestSha256,
|
||||
dataset: {
|
||||
totalFiles: dataset.totalFiles,
|
||||
totalBytes: dataset.totalBytes,
|
||||
filesByKind: dataset.filesByKind,
|
||||
bytesByKind: dataset.bytesByKind,
|
||||
jpegGenerator: dataset.jpegGenerator,
|
||||
},
|
||||
effectiveSettings,
|
||||
mirror,
|
||||
upload,
|
||||
download,
|
||||
materialisation,
|
||||
verification: { verifiedFiles: dataset.entries.length, complete: true },
|
||||
remote,
|
||||
});
|
||||
await deleteDatabase(backendUri, user, password, databaseName);
|
||||
createdDatabases.delete(databaseName);
|
||||
console.error(
|
||||
`[Compression benchmark] repeat ${repeatIndex}/${repeatCount} ${condition.name}: ` +
|
||||
`${dataset.totalFiles} files, ${remote.chunkDocumentCount} chunks, ` +
|
||||
`${remote.storedChunkDataBytes} stored chunk-data bytes`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const output = {
|
||||
schemaVersion: 1,
|
||||
mode: "couchdb-cli-compression-benchmark",
|
||||
generatedAt: new Date().toISOString(),
|
||||
commonlibVersion: (
|
||||
JSON.parse(
|
||||
await Deno.readTextFile(
|
||||
join(import.meta.dirname!, "../../../../node_modules/@vrtmrz/livesync-commonlib/package.json")
|
||||
)
|
||||
) as { version: string }
|
||||
).version,
|
||||
couchdbVersion: (
|
||||
(await couchRequest(backendUri, user, password, "/").then((response) => response.json())) as {
|
||||
version?: string;
|
||||
}
|
||||
).version,
|
||||
compressionImplementation: "Commonlib replicationFilter using fflate level 8 before E2EE V2",
|
||||
chunkingImplementation: "LiveSync CLI mirror using the effective settings recorded for each run",
|
||||
requestedRttMs,
|
||||
httpByteScope:
|
||||
"Decoded HTTP request and response body bytes observed by the local proxy; headers are excluded.",
|
||||
limitations: [
|
||||
"Synthetic JPEGs exercise a deterministic image-like fixture but are not a photographic corpus.",
|
||||
"PNG, Markdown, JSON, and TypeScript inputs are current repository files and therefore change with the source tree.",
|
||||
"Wall and CPU times include CLI process start-up; compare repeated medians rather than treating one run as a universal result.",
|
||||
"Full materialisation starts one CLI process per file and can repeat lazy chunk fetches; treat it as a CLI workflow measurement rather than a raw download lower bound.",
|
||||
"The benchmark uses a local CouchDB and a fixed latency proxy, not a contended production server or a real WAN.",
|
||||
"Path obfuscation is explicitly disabled so raw metadata can be mapped back to file kinds.",
|
||||
],
|
||||
repeatCount,
|
||||
conditions: CONDITIONS,
|
||||
summary: summariseResults(results),
|
||||
runs: results,
|
||||
};
|
||||
await Deno.writeTextFile(resultPath, JSON.stringify(output, null, 2));
|
||||
console.log(JSON.stringify(output, null, 2));
|
||||
console.error(`[Compression benchmark] wrote ${resultPath}`);
|
||||
} finally {
|
||||
await proxy.stop();
|
||||
for (const databaseName of createdDatabases) {
|
||||
await deleteDatabase(backendUri, user, password, databaseName).catch((error) => console.error(error));
|
||||
}
|
||||
if (managedCouchdb && managedStarted) {
|
||||
await stopCouchdb().catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error("[Compression benchmark fatal]", error);
|
||||
Deno.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,10 +1,25 @@
|
||||
import { TempDir } from "./helpers/temp.ts";
|
||||
import { applyRemoteSyncSettings, initSettingsFile } from "./helpers/settings.ts";
|
||||
import {
|
||||
applyRemoteSyncSettings,
|
||||
initSettingsFile,
|
||||
} from "./helpers/settings.ts";
|
||||
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
|
||||
import { startCouchdb, stopCouchdb } from "./helpers/docker.ts";
|
||||
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
|
||||
import {
|
||||
createCouchdbDatabase,
|
||||
startCouchdb,
|
||||
stopCouchdb,
|
||||
} from "./helpers/docker.ts";
|
||||
import {
|
||||
createDeterministicDataset,
|
||||
} from "./helpers/dataset.ts";
|
||||
import {
|
||||
type BenchmarkVerificationMode,
|
||||
parseBenchmarkVerificationMode,
|
||||
verifyBenchmarkDataset,
|
||||
} from "./helpers/benchmarkVerification.ts";
|
||||
|
||||
type BenchmarkConfig = {
|
||||
caseName: string;
|
||||
couchdbBackendUri: string;
|
||||
couchdbProxyUri: string;
|
||||
couchdbUser: string;
|
||||
@@ -21,6 +36,15 @@ type BenchmarkConfig = {
|
||||
requestedRttMs: number;
|
||||
passphrase: string;
|
||||
encrypt: boolean;
|
||||
managedCouchdb: boolean;
|
||||
simulationTier: string;
|
||||
networkProfile: string;
|
||||
networkModel: string;
|
||||
measurementScope: string;
|
||||
limitations: string[];
|
||||
verificationMode: BenchmarkVerificationMode;
|
||||
repeatIndex: number;
|
||||
repeatCount: number;
|
||||
};
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
@@ -49,6 +73,30 @@ function readEnvBool(name: string, fallback: boolean): boolean {
|
||||
return /^(1|true|yes|on)$/i.test(raw.trim());
|
||||
}
|
||||
|
||||
function readEnvStringArray(name: string, fallback: string[]): string[] {
|
||||
const raw = Deno.env.get(name)?.trim();
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (
|
||||
Array.isArray(parsed) &&
|
||||
parsed.every((item) => typeof item === "string")
|
||||
) {
|
||||
return parsed;
|
||||
}
|
||||
} catch {
|
||||
// Fall through to pipe-separated parsing for hand-written invocations.
|
||||
}
|
||||
|
||||
return raw
|
||||
.split("|")
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function nowMs(): number {
|
||||
return performance.now();
|
||||
}
|
||||
@@ -70,22 +118,63 @@ function formatBytes(value: number): string {
|
||||
|
||||
function buildConfig(): BenchmarkConfig {
|
||||
return {
|
||||
couchdbBackendUri: readEnvString("BENCH_COUCHDB_BACKEND_URI", "http://127.0.0.1:5989"),
|
||||
couchdbProxyUri: readEnvString("BENCH_COUCHDB_URI", "http://127.0.0.1:15989"),
|
||||
couchdbUser: readEnvString("BENCH_COUCHDB_USER", readEnvString("username", "admin")),
|
||||
couchdbPassword: readEnvString("BENCH_COUCHDB_PASSWORD", readEnvString("password", "password")),
|
||||
couchdbDbname: readEnvString("BENCH_COUCHDB_DBNAME", `bench-couchdb-${Date.now()}`),
|
||||
caseName: readEnvString("BENCH_CASE", "couchdb-baseline"),
|
||||
couchdbBackendUri: readEnvString(
|
||||
"BENCH_COUCHDB_BACKEND_URI",
|
||||
"http://127.0.0.1:5989",
|
||||
),
|
||||
couchdbProxyUri: readEnvString(
|
||||
"BENCH_COUCHDB_URI",
|
||||
"http://127.0.0.1:15989",
|
||||
),
|
||||
couchdbUser: readEnvString(
|
||||
"BENCH_COUCHDB_USER",
|
||||
readEnvString("username", "admin"),
|
||||
),
|
||||
couchdbPassword: readEnvString(
|
||||
"BENCH_COUCHDB_PASSWORD",
|
||||
readEnvString("password", "password"),
|
||||
),
|
||||
couchdbDbname: readEnvString(
|
||||
"BENCH_COUCHDB_DBNAME",
|
||||
`bench-couchdb-${Date.now()}`,
|
||||
),
|
||||
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
|
||||
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
|
||||
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
|
||||
mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)),
|
||||
mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)),
|
||||
mdMinSizeBytes: Math.floor(
|
||||
readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024),
|
||||
),
|
||||
mdMaxSizeBytes: Math.floor(
|
||||
readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024),
|
||||
),
|
||||
binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)),
|
||||
binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)),
|
||||
binSizeBytes: Math.floor(
|
||||
readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024),
|
||||
),
|
||||
syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240),
|
||||
requestedRttMs: Math.floor(readEnvNumber("BENCH_COUCHDB_RTT_MS", 50)),
|
||||
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
|
||||
encrypt: readEnvBool("BENCH_ENCRYPT", true),
|
||||
managedCouchdb: readEnvBool("BENCH_COUCHDB_MANAGED", true),
|
||||
simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"),
|
||||
networkProfile: readEnvString(
|
||||
"BENCH_NETWORK_PROFILE",
|
||||
"http-latency-proxy",
|
||||
),
|
||||
networkModel: readEnvString("BENCH_NETWORK_MODEL", "local-http-proxy"),
|
||||
measurementScope: readEnvString(
|
||||
"BENCH_MEASUREMENT_SCOPE",
|
||||
"Two one-shot synchronisation phases through a CouchDB-compatible remote-store path.",
|
||||
),
|
||||
limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
|
||||
"This benchmark result is scoped to the configured dataset, remote store, and network model.",
|
||||
]),
|
||||
verificationMode: parseBenchmarkVerificationMode(
|
||||
Deno.env.get("BENCH_VERIFY_MODE"),
|
||||
),
|
||||
repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)),
|
||||
repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -97,34 +186,41 @@ function readOptionalResultPath(): string | undefined {
|
||||
return raw;
|
||||
}
|
||||
|
||||
function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
|
||||
if (entries.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const md = entries.find((e) => e.kind === "md");
|
||||
const bin = entries.find((e) => e.kind === "bin");
|
||||
const middle = entries[Math.floor(entries.length / 2)];
|
||||
const last = entries[entries.length - 1];
|
||||
const unique = new Map<string, DatasetEntry>();
|
||||
for (const entry of [md, bin, middle, last]) {
|
||||
if (entry) {
|
||||
unique.set(entry.relativePath, entry);
|
||||
}
|
||||
}
|
||||
return [...unique.values()];
|
||||
}
|
||||
|
||||
type ProxyHandle = {
|
||||
export type CouchdbProxyHandle = {
|
||||
stop: () => Promise<void>;
|
||||
resetCounters: () => void;
|
||||
snapshotCounters: () => CouchdbProxyCounters;
|
||||
applied: boolean;
|
||||
note: string;
|
||||
directionalDelayMs: number;
|
||||
};
|
||||
|
||||
function startCouchdbProxy(options: { backendUri: string; proxyUri: string; requestedRttMs: number }): ProxyHandle {
|
||||
export type CouchdbProxyCounters = {
|
||||
requestCount: number;
|
||||
requestBodyBytes: number;
|
||||
responseBodyBytes: number;
|
||||
};
|
||||
|
||||
export function startCouchdbProxy(
|
||||
options: {
|
||||
backendUri: string;
|
||||
proxyUri: string;
|
||||
requestedRttMs: number;
|
||||
delay?: (milliseconds: number) => Promise<void>;
|
||||
},
|
||||
): CouchdbProxyHandle {
|
||||
const backend = new URL(options.backendUri);
|
||||
const proxy = new URL(options.proxyUri);
|
||||
const halfDelayMs = Math.max(1, Math.floor(options.requestedRttMs / 2));
|
||||
const halfDelayMs = options.requestedRttMs / 2;
|
||||
const delay = options.delay ??
|
||||
((milliseconds: number) =>
|
||||
new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
|
||||
const controller = new AbortController();
|
||||
const counters: CouchdbProxyCounters = {
|
||||
requestCount: 0,
|
||||
requestBodyBytes: 0,
|
||||
responseBodyBytes: 0,
|
||||
};
|
||||
|
||||
const listener = Deno.serve(
|
||||
{
|
||||
@@ -137,7 +233,7 @@ function startCouchdbProxy(options: { backendUri: string; proxyUri: string; requ
|
||||
},
|
||||
},
|
||||
async (request) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, halfDelayMs));
|
||||
await delay(halfDelayMs);
|
||||
|
||||
const targetUrl = new URL(request.url);
|
||||
targetUrl.protocol = backend.protocol;
|
||||
@@ -155,6 +251,8 @@ function startCouchdbProxy(options: { backendUri: string; proxyUri: string; requ
|
||||
requestBody = undefined;
|
||||
}
|
||||
}
|
||||
counters.requestCount += 1;
|
||||
counters.requestBodyBytes += requestBody?.byteLength ?? 0;
|
||||
|
||||
const upstream = await fetch(targetUrl, {
|
||||
method: request.method,
|
||||
@@ -166,18 +264,28 @@ function startCouchdbProxy(options: { backendUri: string; proxyUri: string; requ
|
||||
const responseHeaders = new Headers(upstream.headers);
|
||||
responseHeaders.delete("content-length");
|
||||
const responseBody = await upstream.arrayBuffer();
|
||||
counters.responseBodyBytes += responseBody.byteLength;
|
||||
await delay(halfDelayMs);
|
||||
|
||||
return new Response(responseBody, {
|
||||
status: upstream.status,
|
||||
statusText: upstream.statusText,
|
||||
headers: responseHeaders,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
applied: true,
|
||||
note: `local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms pre-forward delay`,
|
||||
directionalDelayMs: halfDelayMs,
|
||||
note:
|
||||
`local reverse proxy on ${proxy.origin} with ${halfDelayMs}ms request-path and ${halfDelayMs}ms response-path delay`,
|
||||
resetCounters: () => {
|
||||
counters.requestCount = 0;
|
||||
counters.requestBodyBytes = 0;
|
||||
counters.responseBodyBytes = 0;
|
||||
},
|
||||
snapshotCounters: () => ({ ...counters }),
|
||||
stop: async () => {
|
||||
controller.abort();
|
||||
await listener.finished.catch(() => {});
|
||||
@@ -200,7 +308,24 @@ async function main(): Promise<void> {
|
||||
await initSettingsFile(settingsA);
|
||||
await initSettingsFile(settingsB);
|
||||
|
||||
await startCouchdb(config.couchdbBackendUri, config.couchdbUser, config.couchdbPassword, config.couchdbDbname);
|
||||
if (config.managedCouchdb) {
|
||||
await startCouchdb(
|
||||
config.couchdbBackendUri,
|
||||
config.couchdbUser,
|
||||
config.couchdbPassword,
|
||||
config.couchdbDbname,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`[INFO] using externally managed CouchDB: ${config.couchdbBackendUri}`,
|
||||
);
|
||||
await createCouchdbDatabase(
|
||||
config.couchdbBackendUri,
|
||||
config.couchdbUser,
|
||||
config.couchdbPassword,
|
||||
config.couchdbDbname,
|
||||
);
|
||||
}
|
||||
|
||||
const proxy = startCouchdbProxy({
|
||||
backendUri: config.couchdbBackendUri,
|
||||
@@ -253,54 +378,99 @@ async function main(): Promise<void> {
|
||||
await runCliOrFail(vaultB, "--settings", settingsB, "sync");
|
||||
const syncBElapsed = nowMs() - syncBStart;
|
||||
|
||||
const sampleFiles = pickSampleFiles(seedFiles.entries);
|
||||
for (const sample of sampleFiles) {
|
||||
const pulledPath = workDir.join(`pulled-${sample.relativePath.split("/").join("_")}`);
|
||||
await runCliOrFail(vaultB, "--settings", settingsB, "pull", sample.relativePath, pulledPath);
|
||||
await assertFilesEqual(
|
||||
sample.absolutePath,
|
||||
pulledPath,
|
||||
`sample file mismatch after CouchDB sync: ${sample.relativePath}`
|
||||
);
|
||||
}
|
||||
const verification = await verifyBenchmarkDataset(
|
||||
seedFiles.entries,
|
||||
config.verificationMode,
|
||||
async (entry) => {
|
||||
const pulledPath = workDir.join(
|
||||
`pulled-${entry.relativePath.split("/").join("_")}`,
|
||||
);
|
||||
await runCliOrFail(
|
||||
vaultB,
|
||||
"--settings",
|
||||
settingsB,
|
||||
"pull",
|
||||
entry.relativePath,
|
||||
pulledPath,
|
||||
);
|
||||
await assertFilesEqual(
|
||||
entry.absolutePath,
|
||||
pulledPath,
|
||||
`file mismatch after CouchDB sync: ${entry.relativePath}`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
const result = {
|
||||
caseName: config.caseName,
|
||||
mode: "couchdb-cli-benchmark",
|
||||
couchdbBackendUri: config.couchdbBackendUri,
|
||||
couchdbProxyUri: config.couchdbProxyUri,
|
||||
couchdbDbname: config.couchdbDbname,
|
||||
managedCouchdb: config.managedCouchdb,
|
||||
simulationTier: config.simulationTier,
|
||||
networkProfile: config.networkProfile,
|
||||
networkModel: config.networkModel,
|
||||
measurementScope: config.measurementScope,
|
||||
limitations: config.limitations,
|
||||
repeatIndex: config.repeatIndex,
|
||||
repeatCount: config.repeatCount,
|
||||
rttRequestedMs: config.requestedRttMs,
|
||||
proxyApplied: proxy.applied,
|
||||
proxyNote: proxy.note,
|
||||
proxyDirectionalDelayMs: proxy.directionalDelayMs,
|
||||
proxyConfiguredRttMs: proxy.directionalDelayMs * 2,
|
||||
proxyDelayApplication: "request-and-response",
|
||||
datasetSeed: config.datasetSeed,
|
||||
datasetDirName: config.datasetDirName,
|
||||
totalFiles: seedFiles.totalFiles,
|
||||
totalBytes: seedFiles.totalBytes,
|
||||
mdFileCount: seedFiles.mdCount,
|
||||
binFileCount: seedFiles.binCount,
|
||||
...verification,
|
||||
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
|
||||
syncAElapsedMs: Number(syncAElapsed.toFixed(1)),
|
||||
syncBElapsedMs: Number(syncBElapsed.toFixed(1)),
|
||||
totalSyncElapsedMs: Number((syncAElapsed + syncBElapsed).toFixed(1)),
|
||||
throughputBytesPerSec: Number((seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000)).toFixed(2)),
|
||||
totalSyncElapsedMs: Number(
|
||||
(syncAElapsed + syncBElapsed).toFixed(1),
|
||||
),
|
||||
throughputBytesPerSec: Number(
|
||||
(seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000))
|
||||
.toFixed(
|
||||
2,
|
||||
),
|
||||
),
|
||||
throughputMiBPerSec: Number(
|
||||
(seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) / 1024 / 1024).toFixed(4)
|
||||
(seedFiles.totalBytes / ((syncAElapsed + syncBElapsed) / 1000) /
|
||||
1024 /
|
||||
1024).toFixed(4),
|
||||
),
|
||||
};
|
||||
|
||||
if (resultPath) {
|
||||
await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2));
|
||||
await Deno.writeTextFile(
|
||||
resultPath,
|
||||
JSON.stringify(result, null, 2),
|
||||
);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
console.error(
|
||||
`[Benchmark] couchdb mirrored ${seedFiles.totalFiles} files (${formatBytes(seedFiles.totalBytes)}) in ${formatMs(
|
||||
mirrorElapsed
|
||||
)}, synced in ${formatMs(syncAElapsed + syncBElapsed)} (${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`
|
||||
`[Benchmark] couchdb mirrored ${seedFiles.totalFiles} files (${
|
||||
formatBytes(seedFiles.totalBytes)
|
||||
}) in ${
|
||||
formatMs(
|
||||
mirrorElapsed,
|
||||
)
|
||||
}, synced in ${
|
||||
formatMs(syncAElapsed + syncBElapsed)
|
||||
} (${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`,
|
||||
);
|
||||
} finally {
|
||||
await proxy.stop();
|
||||
await stopCouchdb().catch(() => {});
|
||||
if (config.managedCouchdb) {
|
||||
await stopCouchdb().catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
type SweepResult = {
|
||||
name: string;
|
||||
runner: "p2p" | "couchdb";
|
||||
rttMs?: number;
|
||||
repeatIndex: number;
|
||||
repeatCount: number;
|
||||
result: Record<string, unknown>;
|
||||
};
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
const value = Deno.env.get(name)?.trim();
|
||||
return value && value.length > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function timestamp(): string {
|
||||
const d = new Date();
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
return (
|
||||
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}-` +
|
||||
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}`
|
||||
);
|
||||
}
|
||||
|
||||
function readEnvInteger(name: string, fallback: number): number {
|
||||
const raw = readEnvString(name, String(fallback));
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isInteger(parsed) || parsed < 1) {
|
||||
throw new Error(`${name} must be a positive integer, got '${raw}'`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseRttList(raw: string): number[] {
|
||||
const values = raw
|
||||
.split(",")
|
||||
.map((value) => Number(value.trim()))
|
||||
.filter((value) => Number.isFinite(value) && value > 0)
|
||||
.map((value) => Math.floor(value));
|
||||
if (values.length === 0) {
|
||||
throw new Error(`BENCH_SWEEP_RTT_MS must contain at least one positive number, got '${raw}'`);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
function buildBaseEnv(): Record<string, string> {
|
||||
return {
|
||||
BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
|
||||
BENCH_MD_MIN_SIZE_BYTES: readEnvString("BENCH_MD_MIN_SIZE_BYTES", "512"),
|
||||
BENCH_MD_MAX_SIZE_BYTES: readEnvString("BENCH_MD_MAX_SIZE_BYTES", "2048"),
|
||||
BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
|
||||
BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
|
||||
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
|
||||
BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
|
||||
BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
|
||||
BENCH_VERIFY_MODE: readEnvString("BENCH_VERIFY_MODE", "all"),
|
||||
LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
|
||||
};
|
||||
}
|
||||
|
||||
async function runBenchmark(options: {
|
||||
taskName: "bench:p2p" | "bench:couchdb";
|
||||
name: string;
|
||||
outputDir: string;
|
||||
env: Record<string, string>;
|
||||
repeatIndex: number;
|
||||
repeatCount: number;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
const suffix = options.repeatCount > 1
|
||||
? `-r${String(options.repeatIndex).padStart(2, "0")}`
|
||||
: "";
|
||||
const resultPath = `${options.outputDir}/${options.name}${suffix}.json`;
|
||||
const env = {
|
||||
...Deno.env.toObject(),
|
||||
...options.env,
|
||||
BENCH_RESULT_JSON: resultPath,
|
||||
BENCH_REPEAT_INDEX: String(options.repeatIndex),
|
||||
BENCH_REPEAT_COUNT: String(options.repeatCount),
|
||||
};
|
||||
|
||||
console.log(`[latency-sweep] running ${options.name}`);
|
||||
const child = new Deno.Command("deno", {
|
||||
args: ["task", options.taskName],
|
||||
cwd: import.meta.dirname,
|
||||
env,
|
||||
stdin: "null",
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
}).spawn();
|
||||
const status = await child.status;
|
||||
if (status.code !== 0) {
|
||||
throw new Error(`benchmark failed: ${options.name} (exit ${status.code})`);
|
||||
}
|
||||
return JSON.parse(await Deno.readTextFile(resultPath)) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const outRoot = readEnvString("BENCH_SWEEP_ROOT", `${import.meta.dirname}/bench-results`);
|
||||
const outputDir = `${outRoot}/latency-sweep-${timestamp()}`;
|
||||
const rtts = parseRttList(readEnvString("BENCH_SWEEP_RTT_MS", "20,50,100,150,300"));
|
||||
const repeatCount = readEnvInteger("BENCH_REPEAT_COUNT", 1);
|
||||
const base = buildBaseEnv();
|
||||
|
||||
await Deno.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const results: SweepResult[] = [];
|
||||
if (readEnvString("BENCH_SWEEP_INCLUDE_P2P", "true") !== "false") {
|
||||
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
|
||||
const p2pResult = await runBenchmark({
|
||||
taskName: "bench:p2p",
|
||||
name: "p2p-direct-local",
|
||||
outputDir,
|
||||
repeatIndex,
|
||||
repeatCount,
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-direct-local",
|
||||
BENCH_TURN_SERVERS: "",
|
||||
},
|
||||
});
|
||||
results.push({
|
||||
name: "p2p-direct-local",
|
||||
runner: "p2p",
|
||||
repeatIndex,
|
||||
repeatCount,
|
||||
result: p2pResult,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const rtt of rtts) {
|
||||
const name = `couchdb-rtt-${rtt}ms`;
|
||||
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
|
||||
const couchdbResult = await runBenchmark({
|
||||
taskName: "bench:couchdb",
|
||||
name,
|
||||
outputDir,
|
||||
repeatIndex,
|
||||
repeatCount,
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: name,
|
||||
BENCH_COUCHDB_RTT_MS: String(rtt),
|
||||
},
|
||||
});
|
||||
results.push({
|
||||
name,
|
||||
runner: "couchdb",
|
||||
rttMs: rtt,
|
||||
repeatIndex,
|
||||
repeatCount,
|
||||
result: couchdbResult,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const summary = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
outputDir,
|
||||
note:
|
||||
"This sweep applies half of each requested CouchDB RTT before forwarding requests and half before returning responses. It is not a full netem model of jitter, loss, MTU, bandwidth, or VPN encapsulation.",
|
||||
rtts,
|
||||
repeatCount,
|
||||
results,
|
||||
};
|
||||
await Deno.writeTextFile(`${outputDir}/summary.json`, JSON.stringify(summary, null, 2));
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
console.log(`[latency-sweep] result directory: ${outputDir}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error("[Fatal Error]", error);
|
||||
Deno.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
export type BenchmarkCase = {
|
||||
name: string;
|
||||
runner: "p2p" | "couchdb";
|
||||
description: string;
|
||||
dataPath: string;
|
||||
trustBoundary: string;
|
||||
measurementScope: string;
|
||||
limitations: string[];
|
||||
env: Record<string, string>;
|
||||
};
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
const value = Deno.env.get(name)?.trim();
|
||||
return value && value.length > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function readEnvInteger(name: string, fallback: number): number {
|
||||
const value = readEnvString(name, String(fallback));
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < 1) {
|
||||
throw new Error(`${name} must be a positive integer, got '${value}'`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function timestamp(): string {
|
||||
const d = new Date();
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
return (
|
||||
`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${
|
||||
pad(d.getUTCDate())
|
||||
}-` +
|
||||
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${
|
||||
pad(d.getUTCSeconds())
|
||||
}`
|
||||
);
|
||||
}
|
||||
|
||||
function buildBaseEnv(): Record<string, string> {
|
||||
return {
|
||||
BENCH_MD_FILE_COUNT: readEnvString("BENCH_MD_FILE_COUNT", "20"),
|
||||
BENCH_MD_MIN_SIZE_BYTES: readEnvString(
|
||||
"BENCH_MD_MIN_SIZE_BYTES",
|
||||
"512",
|
||||
),
|
||||
BENCH_MD_MAX_SIZE_BYTES: readEnvString(
|
||||
"BENCH_MD_MAX_SIZE_BYTES",
|
||||
"2048",
|
||||
),
|
||||
BENCH_BIN_FILE_COUNT: readEnvString("BENCH_BIN_FILE_COUNT", "5"),
|
||||
BENCH_BIN_SIZE_BYTES: readEnvString("BENCH_BIN_SIZE_BYTES", "8192"),
|
||||
BENCH_SYNC_TIMEOUT: readEnvString("BENCH_SYNC_TIMEOUT", "300"),
|
||||
BENCH_PEERS_TIMEOUT: readEnvString("BENCH_PEERS_TIMEOUT", "60"),
|
||||
BENCH_SEED: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
|
||||
BENCH_VERIFY_MODE: readEnvString("BENCH_VERIFY_MODE", "all"),
|
||||
LIVESYNC_TEST_TEE: readEnvString("BENCH_LIVESYNC_TEST_TEE", "0"),
|
||||
};
|
||||
}
|
||||
|
||||
function withScopeEnv(
|
||||
env: Record<string, string>,
|
||||
options: Pick<BenchmarkCase, "measurementScope" | "limitations">,
|
||||
): Record<string, string> {
|
||||
return {
|
||||
...env,
|
||||
BENCH_MEASUREMENT_SCOPE: options.measurementScope,
|
||||
BENCH_LIMITATIONS_JSON: JSON.stringify(options.limitations),
|
||||
};
|
||||
}
|
||||
|
||||
function defineCase(testCase: BenchmarkCase): BenchmarkCase {
|
||||
return {
|
||||
...testCase,
|
||||
env: withScopeEnv(testCase.env, testCase),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildCases(): BenchmarkCase[] {
|
||||
const base = buildBaseEnv();
|
||||
const couchdbRtt = readEnvString("BENCH_COUCHDB_RTT_MS", "20");
|
||||
const tetheringVpnRtt = readEnvString("BENCH_TETHERING_VPN_RTT_MS", "120");
|
||||
const localTurnServers = readEnvString(
|
||||
"BENCH_LOCAL_TURN_SERVERS",
|
||||
"turn:127.0.0.1:3478",
|
||||
);
|
||||
const shimCouchdbUri = readEnvString(
|
||||
"BENCH_SHIM_COUCHDB_URI",
|
||||
"http://couchdb-shim:5984",
|
||||
);
|
||||
const signallingShimRelay = readEnvString(
|
||||
"BENCH_SIGNAL_SHIM_RELAY",
|
||||
"ws://p2p-signalling-shim:7777/",
|
||||
);
|
||||
|
||||
return [
|
||||
defineCase({
|
||||
name: "couchdb-baseline",
|
||||
runner: "couchdb",
|
||||
description:
|
||||
"Standard self-hosted CouchDB path through a local latency proxy.",
|
||||
dataPath: "Device A -> CouchDB -> Device B",
|
||||
trustBoundary: "CouchDB operator and network path",
|
||||
measurementScope:
|
||||
"Two one-shot synchronisation phases through a CouchDB-compatible remote-store path with a local HTTP latency proxy.",
|
||||
limitations: [
|
||||
"This is not a full netem model of packet loss, jitter, MTU, bandwidth limits, or VPN encapsulation.",
|
||||
"This result should be compared with P2P only as a remote-store baseline under the same deterministic dataset.",
|
||||
],
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "couchdb-baseline",
|
||||
BENCH_COUCHDB_RTT_MS: couchdbRtt,
|
||||
},
|
||||
}),
|
||||
defineCase({
|
||||
name: "p2p-direct-local",
|
||||
runner: "p2p",
|
||||
description:
|
||||
"Preferred direct WebRTC P2P path with Nostr signalling and TURN disabled.",
|
||||
dataPath: "Device A -> Device B",
|
||||
trustBoundary: "Nostr relay for signalling metadata; no TURN relay",
|
||||
measurementScope:
|
||||
"One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment, with TURN disabled; the earlier peer-list observation command is excluded.",
|
||||
limitations: [
|
||||
"The timed command includes its own signalling and connection establishment, but not the earlier peer-list observation window.",
|
||||
"This does not measure public relay operation, mobile carrier behaviour, or TURN-relayed throughput.",
|
||||
"This small-dataset run should not be treated as a WAN, VPN, or large binary initial synchronisation measurement.",
|
||||
],
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-direct-local",
|
||||
BENCH_TURN_SERVERS: "",
|
||||
BENCH_SIMULATION_TIER: "1",
|
||||
BENCH_NETWORK_PROFILE: "local-direct",
|
||||
BENCH_NETWORK_MODEL: "local-runner-webrtc",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
|
||||
"turn-disabled-but-selected-ice-pair-not-collected",
|
||||
},
|
||||
}),
|
||||
defineCase({
|
||||
name: "couchdb-tethering-vpn-proxy",
|
||||
runner: "couchdb",
|
||||
description:
|
||||
"Approximate smartphone tethering/VPN remote-database path using an HTTP latency proxy. This does not model loss, jitter, MTU, or VPN encapsulation.",
|
||||
dataPath:
|
||||
"Device A -> VPN/network path -> CouchDB -> VPN/network path -> Device B",
|
||||
trustBoundary: "VPN/network path and CouchDB operator",
|
||||
measurementScope:
|
||||
"Two one-shot CouchDB synchronisation phases with additional requested RTT through the local HTTP proxy.",
|
||||
limitations: [
|
||||
"This approximates request latency only and does not model loss, jitter, MTU, bandwidth limits, carrier NAT, or VPN encapsulation.",
|
||||
"Use the Tier 2 netem shim cases for a stronger constrained-network fixture.",
|
||||
],
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "couchdb-tethering-vpn-proxy",
|
||||
BENCH_COUCHDB_RTT_MS: tetheringVpnRtt,
|
||||
},
|
||||
}),
|
||||
defineCase({
|
||||
name: "couchdb-netem-home-wifi",
|
||||
runner: "couchdb",
|
||||
description:
|
||||
"Tier 2 CouchDB path through the Compose netem TCP shim using the home-wifi profile.",
|
||||
dataPath:
|
||||
"Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
|
||||
trustBoundary: "CouchDB operator and constrained network shim",
|
||||
measurementScope:
|
||||
"Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the home-wifi netem profile.",
|
||||
limitations: [
|
||||
"This shapes the CouchDB TCP path, not the WebRTC P2P data path.",
|
||||
"The fixture remains a reproducible network emulation, not a field measurement on a real home network.",
|
||||
],
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "couchdb-netem-home-wifi",
|
||||
BENCH_COUCHDB_BACKEND_URI: shimCouchdbUri,
|
||||
BENCH_COUCHDB_RTT_MS: "1",
|
||||
BENCH_SIMULATION_TIER: "2",
|
||||
BENCH_NETWORK_PROFILE: "home-wifi",
|
||||
BENCH_NETWORK_MODEL: "compose-netem-tcp-shim",
|
||||
},
|
||||
}),
|
||||
defineCase({
|
||||
name: "couchdb-netem-tethering-vpn",
|
||||
runner: "couchdb",
|
||||
description:
|
||||
"Tier 2 CouchDB path through the Compose netem TCP shim using a tethering-vpn profile.",
|
||||
dataPath:
|
||||
"Device A -> netem TCP shim -> CouchDB -> netem TCP shim -> Device B",
|
||||
trustBoundary:
|
||||
"CouchDB operator and constrained smartphone/VPN-like network shim",
|
||||
measurementScope:
|
||||
"Tier 2 CouchDB synchronisation through a Compose TCP shim that applies the tethering-vpn netem profile.",
|
||||
limitations: [
|
||||
"This shapes the CouchDB TCP path, not the WebRTC P2P data path.",
|
||||
"The profile approximates smartphone/VPN constraints but is not a field measurement on a real tethered VPN connection.",
|
||||
],
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "couchdb-netem-tethering-vpn",
|
||||
BENCH_COUCHDB_BACKEND_URI: shimCouchdbUri,
|
||||
BENCH_COUCHDB_RTT_MS: "1",
|
||||
BENCH_SIMULATION_TIER: "2",
|
||||
BENCH_NETWORK_PROFILE: "tethering-vpn",
|
||||
BENCH_NETWORK_MODEL: "compose-netem-tcp-shim",
|
||||
},
|
||||
}),
|
||||
defineCase({
|
||||
name: "p2p-smartphone-vpn-direct",
|
||||
runner: "p2p",
|
||||
description:
|
||||
"Direct P2P case name for smartphone tethering/VPN measurements. In this local runner it is unshaped and should be treated as a wiring check unless executed on that network.",
|
||||
dataPath:
|
||||
"Device A -> Device B when WebRTC direct connectivity succeeds",
|
||||
trustBoundary:
|
||||
"Smartphone/VPN routing policy plus Nostr signalling metadata",
|
||||
measurementScope:
|
||||
"Structural placeholder for direct P2P measurements on a real smartphone tethering/VPN path.",
|
||||
limitations: [
|
||||
"In the local runner this is unshaped and must not be reported as smartphone, VPN, WAN, or Tier 2 evidence.",
|
||||
"Use only when the command is executed on the intended real network path and the selected ICE candidate pair is recorded.",
|
||||
],
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-smartphone-vpn-direct",
|
||||
BENCH_TURN_SERVERS: "",
|
||||
BENCH_SIMULATION_TIER: "unmeasured",
|
||||
BENCH_NETWORK_PROFILE: "smartphone-vpn-direct-placeholder",
|
||||
BENCH_NETWORK_MODEL: "local-runner-no-netem",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
|
||||
"structural-placeholder-only; selected ICE pair may be collected, but the path is not shaped",
|
||||
},
|
||||
}),
|
||||
defineCase({
|
||||
name: "p2p-signalling-netem-home-wifi",
|
||||
runner: "p2p",
|
||||
description:
|
||||
"Tier 2 P2P path with only the Nostr signalling relay accessed through the home-wifi netem shim.",
|
||||
dataPath:
|
||||
"Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
|
||||
trustBoundary:
|
||||
"Nostr signalling metadata through constrained network shim; no TURN relay",
|
||||
measurementScope:
|
||||
"One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the home-wifi netem profile; the selected WebRTC note-data path is unshaped.",
|
||||
limitations: [
|
||||
"The timed p2p-sync command includes signalling and WebRTC connection establishment.",
|
||||
"This does not shape the selected WebRTC DataChannel note-data path.",
|
||||
"This supports only the claim that constrained signalling access does not place note data on the relay path when a non-relayed ICE path is selected.",
|
||||
],
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-signalling-netem-home-wifi",
|
||||
BENCH_RELAY: signallingShimRelay,
|
||||
BENCH_TURN_SERVERS: "",
|
||||
BENCH_SIMULATION_TIER: "2",
|
||||
BENCH_NETWORK_PROFILE: "home-wifi",
|
||||
BENCH_NETWORK_MODEL: "compose-netem-signalling-shim",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
|
||||
"selected ICE pair collected; only Nostr signalling path is shaped",
|
||||
},
|
||||
}),
|
||||
defineCase({
|
||||
name: "p2p-signalling-netem-tethering-vpn",
|
||||
runner: "p2p",
|
||||
description:
|
||||
"Tier 2 P2P path with only the Nostr signalling relay accessed through the tethering-vpn netem shim.",
|
||||
dataPath:
|
||||
"Device A -> Device B over WebRTC DataChannel; Nostr signalling through netem shim",
|
||||
trustBoundary:
|
||||
"Nostr signalling metadata through constrained smartphone/VPN-like network shim; no TURN relay",
|
||||
measurementScope:
|
||||
"One fresh CLI p2p-sync command where only Nostr signalling access is shaped by the tethering-vpn netem profile; the selected WebRTC note-data path is unshaped.",
|
||||
limitations: [
|
||||
"The timed p2p-sync command includes signalling and WebRTC connection establishment.",
|
||||
"This does not shape the selected WebRTC DataChannel note-data path.",
|
||||
"The profile approximates constrained relay access and is not a field measurement on a real tethered VPN connection.",
|
||||
],
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-signalling-netem-tethering-vpn",
|
||||
BENCH_RELAY: signallingShimRelay,
|
||||
BENCH_TURN_SERVERS: "",
|
||||
BENCH_SIMULATION_TIER: "2",
|
||||
BENCH_NETWORK_PROFILE: "tethering-vpn",
|
||||
BENCH_NETWORK_MODEL: "compose-netem-signalling-shim",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
|
||||
"selected ICE pair collected; only Nostr signalling path is shaped",
|
||||
},
|
||||
}),
|
||||
defineCase({
|
||||
name: "p2p-user-turn",
|
||||
runner: "p2p",
|
||||
description:
|
||||
"Optional fallback path through a local user-controlled TURN server.",
|
||||
dataPath: "Device A -> user-controlled TURN -> Device B",
|
||||
trustBoundary: "User-controlled TURN server",
|
||||
measurementScope:
|
||||
"Optional local TURN fallback wiring check with a user-controlled TURN server configured.",
|
||||
limitations: [
|
||||
"TURN configuration does not prove that the selected ICE path was relayed; interpret the recorded candidate pair.",
|
||||
"This is not evidence for public TURN relay privacy, throughput, or availability.",
|
||||
],
|
||||
env: {
|
||||
...base,
|
||||
BENCH_CASE: "p2p-user-turn",
|
||||
BENCH_TURN_SERVERS: localTurnServers,
|
||||
BENCH_SIMULATION_TIER: "1",
|
||||
BENCH_NETWORK_PROFILE: "local-turn-fallback",
|
||||
BENCH_NETWORK_MODEL: "local-runner-webrtc-turn-configured",
|
||||
BENCH_P2P_CANDIDATE_PATH_VERIFICATION:
|
||||
"turn-configured; selected ICE pair may still be direct or relayed, so interpret the recorded candidate types",
|
||||
},
|
||||
}),
|
||||
];
|
||||
}
|
||||
|
||||
async function runCase(
|
||||
testCase: BenchmarkCase,
|
||||
outputDir: string,
|
||||
repeatIndex: number,
|
||||
repeatCount: number,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const suffix = repeatCount > 1
|
||||
? `-r${String(repeatIndex).padStart(2, "0")}`
|
||||
: "";
|
||||
const resultPath = `${outputDir}/${testCase.name}${suffix}.json`;
|
||||
const taskName = testCase.runner === "p2p" ? "bench:p2p" : "bench:couchdb";
|
||||
const env = {
|
||||
...Deno.env.toObject(),
|
||||
...testCase.env,
|
||||
BENCH_RESULT_JSON: resultPath,
|
||||
BENCH_REPEAT_INDEX: String(repeatIndex),
|
||||
BENCH_REPEAT_COUNT: String(repeatCount),
|
||||
};
|
||||
|
||||
const repeatLabel = repeatCount > 1
|
||||
? ` (${repeatIndex}/${repeatCount})`
|
||||
: "";
|
||||
console.log(
|
||||
`[bench-cases] running ${testCase.name}${repeatLabel}: ${testCase.description}`,
|
||||
);
|
||||
const command = new Deno.Command("deno", {
|
||||
args: ["task", taskName],
|
||||
cwd: import.meta.dirname,
|
||||
env,
|
||||
stdin: "null",
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
});
|
||||
|
||||
const child = command.spawn();
|
||||
const status = await child.status;
|
||||
if (status.code !== 0) {
|
||||
throw new Error(`case failed: ${testCase.name} (exit ${status.code})`);
|
||||
}
|
||||
|
||||
const result = JSON.parse(await Deno.readTextFile(resultPath)) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
return {
|
||||
...testCase,
|
||||
repeatIndex,
|
||||
repeatCount,
|
||||
resultPath,
|
||||
result,
|
||||
};
|
||||
}
|
||||
|
||||
function selectCases(allCases: BenchmarkCase[]): BenchmarkCase[] {
|
||||
const requested = readEnvString(
|
||||
"BENCH_CASES",
|
||||
"couchdb-baseline,p2p-direct-local",
|
||||
);
|
||||
const names = requested
|
||||
.split(",")
|
||||
.map((v) => v.trim())
|
||||
.filter((v) => v.length > 0);
|
||||
const byName = new Map(allCases.map((c) => [c.name, c]));
|
||||
return names.map((name) => {
|
||||
const found = byName.get(name);
|
||||
if (!found) {
|
||||
throw new Error(
|
||||
`Unknown BENCH_CASES entry '${name}'. Available: ${
|
||||
allCases.map((c) => c.name).join(", ")
|
||||
}`,
|
||||
);
|
||||
}
|
||||
return found;
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const outRoot = readEnvString(
|
||||
"BENCH_CASES_ROOT",
|
||||
`${import.meta.dirname}/bench-results`,
|
||||
);
|
||||
const outputDir = `${outRoot}/cases-${timestamp()}`;
|
||||
await Deno.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const allCases = buildCases();
|
||||
const cases = selectCases(allCases);
|
||||
const repeatCount = readEnvInteger("BENCH_REPEAT_COUNT", 1);
|
||||
await Deno.writeTextFile(
|
||||
`${outputDir}/case-manifest.json`,
|
||||
JSON.stringify(
|
||||
{
|
||||
generatedAt: new Date().toISOString(),
|
||||
repeatCount,
|
||||
selectedCases: cases,
|
||||
availableCases: allCases,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
const results: Record<string, unknown>[] = [];
|
||||
for (const testCase of cases) {
|
||||
for (let repeatIndex = 1; repeatIndex <= repeatCount; repeatIndex++) {
|
||||
results.push(
|
||||
await runCase(testCase, outputDir, repeatIndex, repeatCount),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const summary = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
outputDir,
|
||||
repeatCount,
|
||||
results,
|
||||
};
|
||||
await Deno.writeTextFile(
|
||||
`${outputDir}/summary.json`,
|
||||
JSON.stringify(summary, null, 2),
|
||||
);
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
console.log(`[bench-cases] result directory: ${outputDir}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error("[Fatal Error]", error);
|
||||
Deno.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,435 @@
|
||||
import { join } from "@std/path";
|
||||
import { startCliInBackground } from "./helpers/backgroundCli.ts";
|
||||
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
|
||||
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
|
||||
import { discoverPeer } from "./helpers/p2p.ts";
|
||||
import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts";
|
||||
|
||||
type Role = "host" | "client";
|
||||
|
||||
type NetemSummary = {
|
||||
enabled: boolean;
|
||||
profile: string;
|
||||
interface: string;
|
||||
delayMs: number;
|
||||
jitterMs: number;
|
||||
lossPercent: number;
|
||||
bandwidthMbit: number;
|
||||
mtu: number;
|
||||
tcQdisc?: string;
|
||||
ipAddr?: string;
|
||||
ipRoute?: string;
|
||||
};
|
||||
|
||||
type HostReady = {
|
||||
generatedAt: string;
|
||||
totalFiles: number;
|
||||
totalBytes: number;
|
||||
mdFileCount: number;
|
||||
binFileCount: number;
|
||||
mirrorElapsedMs: number;
|
||||
netem: NetemSummary;
|
||||
};
|
||||
|
||||
type P2PConnectionStats = {
|
||||
candidatePathCollected: boolean;
|
||||
selectedPath: string;
|
||||
localCandidate?: { candidateType: string; protocol: string; relayProtocol: string };
|
||||
remoteCandidate?: { candidateType: string; protocol: string; relayProtocol: string };
|
||||
};
|
||||
|
||||
function errorToRecord(error: unknown): Record<string, unknown> {
|
||||
if (error instanceof Error) {
|
||||
return {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
};
|
||||
}
|
||||
return {
|
||||
name: "UnknownError",
|
||||
message: String(error),
|
||||
};
|
||||
}
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
const value = Deno.env.get(name)?.trim();
|
||||
return value && value.length > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function readEnvNumber(name: string, fallback: number): number {
|
||||
const raw = Deno.env.get(name);
|
||||
if (raw === undefined || raw.trim() === "") {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed < 0) {
|
||||
throw new Error(`${name} must be a non-negative number, got '${raw}'`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function nowMs(): number {
|
||||
return performance.now();
|
||||
}
|
||||
|
||||
async function commandOutput(command: string, args: string[]): Promise<string> {
|
||||
const output = await new Deno.Command(command, {
|
||||
args,
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
}).output();
|
||||
const stdout = new TextDecoder().decode(output.stdout);
|
||||
const stderr = new TextDecoder().decode(output.stderr);
|
||||
if (!output.success) {
|
||||
throw new Error(`${command} ${args.join(" ")} failed\nstdout: ${stdout}\nstderr: ${stderr}`);
|
||||
}
|
||||
return stdout.trim();
|
||||
}
|
||||
|
||||
async function commandOk(command: string, args: string[]): Promise<void> {
|
||||
await commandOutput(command, args);
|
||||
}
|
||||
|
||||
async function applyNetemIfRequested(): Promise<NetemSummary> {
|
||||
const enabled = readEnvString("BENCH_NETEM_ENABLED", "0") === "1";
|
||||
const profile = readEnvString("NETEM_PROFILE", "home-wifi");
|
||||
const iface = readEnvString("NETEM_INTERFACE", "eth0");
|
||||
const delayMs = readEnvNumber("NETEM_DELAY_MS", 20);
|
||||
const jitterMs = readEnvNumber("NETEM_JITTER_MS", 5);
|
||||
const lossPercent = readEnvNumber("NETEM_LOSS_PERCENT", 0.1);
|
||||
const bandwidthMbit = readEnvNumber("NETEM_BANDWIDTH_MBIT", 100);
|
||||
const mtu = readEnvNumber("NETEM_MTU", 1500);
|
||||
|
||||
const summary: NetemSummary = {
|
||||
enabled,
|
||||
profile,
|
||||
interface: iface,
|
||||
delayMs,
|
||||
jitterMs,
|
||||
lossPercent,
|
||||
bandwidthMbit,
|
||||
mtu,
|
||||
};
|
||||
|
||||
if (!enabled) {
|
||||
return summary;
|
||||
}
|
||||
|
||||
await commandOk("ip", ["link", "set", "dev", iface, "mtu", String(mtu)]);
|
||||
await new Deno.Command("tc", { args: ["qdisc", "del", "dev", iface, "root"] }).output();
|
||||
await commandOk("tc", [
|
||||
"qdisc",
|
||||
"add",
|
||||
"dev",
|
||||
iface,
|
||||
"root",
|
||||
"netem",
|
||||
"delay",
|
||||
`${delayMs}ms`,
|
||||
`${jitterMs}ms`,
|
||||
"loss",
|
||||
`${lossPercent}%`,
|
||||
"rate",
|
||||
`${bandwidthMbit}mbit`,
|
||||
]);
|
||||
summary.tcQdisc = await commandOutput("tc", ["qdisc", "show", "dev", iface]);
|
||||
summary.ipAddr = await commandOutput("ip", ["addr", "show", iface]);
|
||||
summary.ipRoute = await commandOutput("ip", ["route"]);
|
||||
return summary;
|
||||
}
|
||||
|
||||
async function waitForFile(path: string, timeoutMs: number): Promise<void> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
try {
|
||||
const stat = await Deno.stat(path);
|
||||
if (stat.isFile) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// wait
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${path}`);
|
||||
}
|
||||
|
||||
async function readJsonFile<T>(path: string): Promise<T> {
|
||||
return JSON.parse(await Deno.readTextFile(path)) as T;
|
||||
}
|
||||
|
||||
function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
|
||||
const unique = new Map<string, DatasetEntry>();
|
||||
for (const entry of [entries.find((e) => e.kind === "md"), entries.find((e) => e.kind === "bin"), entries.at(-1)]) {
|
||||
if (entry) {
|
||||
unique.set(entry.relativePath, entry);
|
||||
}
|
||||
}
|
||||
return [...unique.values()];
|
||||
}
|
||||
|
||||
async function readLatestP2PConnectionStats(path: string): Promise<P2PConnectionStats | undefined> {
|
||||
try {
|
||||
const lines = (await Deno.readTextFile(path))
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0);
|
||||
return lines.length === 0 ? undefined : (JSON.parse(lines.at(-1)!) as P2PConnectionStats);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function buildCommonConfig() {
|
||||
const runId = readEnvString("BENCH_SPLIT_RUN_ID", readEnvString("BENCH_ROOM_ID", "bench-split-run"));
|
||||
const baseWorkRoot = readEnvString("BENCH_SPLIT_WORK_ROOT", "/p2p-work");
|
||||
return {
|
||||
runId,
|
||||
workRoot: join(baseWorkRoot, runId),
|
||||
resultRoot: readEnvString("BENCH_SPLIT_RESULT_ROOT", "/workspace/src/apps/cli/testdeno/bench-results"),
|
||||
relay: readEnvString("BENCH_RELAY", "ws://nostr-relay:7777/"),
|
||||
appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"),
|
||||
roomId: readEnvString("BENCH_ROOM_ID", "bench-split-room"),
|
||||
passphrase: readEnvString("BENCH_PASSPHRASE", "bench-split-passphrase"),
|
||||
turnServers: readEnvString("BENCH_TURN_SERVERS", ""),
|
||||
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
|
||||
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
|
||||
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 20)),
|
||||
mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 512)),
|
||||
mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 2048)),
|
||||
binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 5)),
|
||||
binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 8192)),
|
||||
peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 60),
|
||||
syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 300),
|
||||
nodeTimeoutMs: readEnvNumber("BENCH_SPLIT_NODE_TIMEOUT_MS", 360_000),
|
||||
profile: readEnvString("BENCH_NETWORK_PROFILE", readEnvString("NETEM_PROFILE", "split-compose")),
|
||||
};
|
||||
}
|
||||
|
||||
async function prepareP2PSettings(
|
||||
settingsPath: string,
|
||||
peerName: string,
|
||||
config: ReturnType<typeof buildCommonConfig>
|
||||
) {
|
||||
await initSettingsFile(settingsPath);
|
||||
await applyP2pSettings(
|
||||
settingsPath,
|
||||
config.roomId,
|
||||
config.passphrase,
|
||||
config.appId,
|
||||
config.relay,
|
||||
"~.*",
|
||||
config.turnServers
|
||||
);
|
||||
await applyP2pTestTweaks(settingsPath, peerName, config.passphrase);
|
||||
}
|
||||
|
||||
async function runHost(): Promise<void> {
|
||||
const config = buildCommonConfig();
|
||||
const netem = await applyNetemIfRequested();
|
||||
await Deno.mkdir(config.workRoot, { recursive: true });
|
||||
await Deno.mkdir(config.resultRoot, { recursive: true });
|
||||
|
||||
const hostVault = join(config.workRoot, "vault-host");
|
||||
const hostSettings = join(config.workRoot, "settings-host.json");
|
||||
await Deno.mkdir(hostVault, { recursive: true });
|
||||
await prepareP2PSettings(hostSettings, "p2p-split-host", config);
|
||||
|
||||
const seedFiles = await createDeterministicDataset({
|
||||
rootDir: hostVault,
|
||||
datasetDirName: config.datasetDirName,
|
||||
seed: config.datasetSeed,
|
||||
mdCount: config.mdFileCount,
|
||||
mdMinSizeBytes: config.mdMinSizeBytes,
|
||||
mdMaxSizeBytes: config.mdMaxSizeBytes,
|
||||
binCount: config.binFileCount,
|
||||
binSizeBytes: config.binSizeBytes,
|
||||
});
|
||||
await Deno.writeTextFile(
|
||||
join(config.workRoot, "sample-files.json"),
|
||||
JSON.stringify(pickSampleFiles(seedFiles.entries), null, 2)
|
||||
);
|
||||
|
||||
const mirrorStart = nowMs();
|
||||
await runCliOrFail(hostVault, "--settings", hostSettings, "mirror");
|
||||
const mirrorElapsedMs = Number((nowMs() - mirrorStart).toFixed(1));
|
||||
const hostReady: HostReady = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
totalFiles: seedFiles.totalFiles,
|
||||
totalBytes: seedFiles.totalBytes,
|
||||
mdFileCount: seedFiles.mdCount,
|
||||
binFileCount: seedFiles.binCount,
|
||||
mirrorElapsedMs,
|
||||
netem,
|
||||
};
|
||||
await Deno.writeTextFile(join(config.workRoot, "host-ready.json"), JSON.stringify(hostReady, null, 2));
|
||||
|
||||
const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
|
||||
try {
|
||||
await host.waitUntilContains("P2P host is running", 20_000);
|
||||
await Deno.writeTextFile(
|
||||
join(config.workRoot, "p2p-host-ready.json"),
|
||||
JSON.stringify({ generatedAt: new Date().toISOString() })
|
||||
);
|
||||
await waitForFile(join(config.workRoot, "client-done.json"), config.nodeTimeoutMs);
|
||||
} finally {
|
||||
await host.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function runClient(): Promise<void> {
|
||||
const config = buildCommonConfig();
|
||||
const netem = await applyNetemIfRequested();
|
||||
await Deno.mkdir(config.resultRoot, { recursive: true });
|
||||
await waitForFile(join(config.workRoot, "host-ready.json"), config.nodeTimeoutMs);
|
||||
await waitForFile(join(config.workRoot, "p2p-host-ready.json"), config.nodeTimeoutMs);
|
||||
|
||||
const clientVault = join(config.workRoot, "vault-client");
|
||||
const clientSettings = join(config.workRoot, "settings-client.json");
|
||||
const statsPath = join(config.workRoot, "p2p-connection-stats.jsonl");
|
||||
await Deno.mkdir(clientVault, { recursive: true });
|
||||
await prepareP2PSettings(clientSettings, "p2p-split-client", config);
|
||||
|
||||
const hostReady = await readJsonFile<HostReady>(join(config.workRoot, "host-ready.json"));
|
||||
const timestamp = new Date().toISOString().replace(/[-:]/g, "").slice(0, 15);
|
||||
const outputDir = join(config.resultRoot, `p2p-split-${config.profile}-${timestamp}`);
|
||||
await Deno.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const previousStatsPath = Deno.env.get("LIVESYNC_P2P_STATS_JSONL");
|
||||
Deno.env.set("LIVESYNC_P2P_STATS_JSONL", statsPath);
|
||||
let stage = "peer-discovery";
|
||||
let peerDiscoveryCommandElapsedMs: number | undefined;
|
||||
let syncElapsedMs: number | undefined;
|
||||
try {
|
||||
const peerDiscoveryCommandStart = nowMs();
|
||||
const peer = await discoverPeer(clientVault, clientSettings, config.peersTimeoutSeconds);
|
||||
peerDiscoveryCommandElapsedMs = Number((nowMs() - peerDiscoveryCommandStart).toFixed(1));
|
||||
|
||||
stage = "p2p-sync";
|
||||
const syncStart = nowMs();
|
||||
await runCliOrFail(
|
||||
clientVault,
|
||||
"--settings",
|
||||
clientSettings,
|
||||
"p2p-sync",
|
||||
peer.id,
|
||||
String(config.syncTimeoutSeconds)
|
||||
);
|
||||
syncElapsedMs = Number((nowMs() - syncStart).toFixed(1));
|
||||
|
||||
stage = "sample-verification";
|
||||
const samples = await readJsonFile<DatasetEntry[]>(join(config.workRoot, "sample-files.json"));
|
||||
for (const sample of samples) {
|
||||
const pulledPath = join(config.workRoot, `pulled-${sample.relativePath.replaceAll("/", "_")}`);
|
||||
await runCliOrFail(clientVault, "--settings", clientSettings, "pull", sample.relativePath, pulledPath);
|
||||
await assertFilesEqual(
|
||||
sample.absolutePath,
|
||||
pulledPath,
|
||||
`sample file mismatch after split sync: ${sample.relativePath}`
|
||||
);
|
||||
}
|
||||
|
||||
const p2pConnectionStats = await readLatestP2PConnectionStats(statsPath);
|
||||
const result = {
|
||||
ok: true,
|
||||
generatedAt: new Date().toISOString(),
|
||||
caseName: "p2p-split-compose",
|
||||
mode: "p2p-split-compose-benchmark",
|
||||
runId: config.runId,
|
||||
simulationTier: Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "2" : "1",
|
||||
networkProfile: config.profile,
|
||||
networkModel:
|
||||
Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "split-compose-egress-netem" : "split-compose-no-netem",
|
||||
relay: config.relay,
|
||||
turnServers: config.turnServers,
|
||||
turnEnabled: config.turnServers.trim().length > 0,
|
||||
p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true,
|
||||
p2pConnectionStats,
|
||||
hostNetem: hostReady.netem,
|
||||
clientNetem: netem,
|
||||
totalFiles: hostReady.totalFiles,
|
||||
totalBytes: hostReady.totalBytes,
|
||||
mdFileCount: hostReady.mdFileCount,
|
||||
binFileCount: hostReady.binFileCount,
|
||||
mirrorElapsedMs: hostReady.mirrorElapsedMs,
|
||||
peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
|
||||
peerDiscoveryCommandElapsedMs,
|
||||
syncElapsedMs,
|
||||
throughputBytesPerSec: Number((hostReady.totalBytes / (syncElapsedMs / 1000)).toFixed(2)),
|
||||
throughputMiBPerSec: Number((hostReady.totalBytes / (syncElapsedMs / 1000) / 1024 / 1024).toFixed(4)),
|
||||
};
|
||||
await Deno.writeTextFile(join(outputDir, "summary.json"), JSON.stringify(result, null, 2));
|
||||
await Deno.writeTextFile(
|
||||
join(config.workRoot, "client-done.json"),
|
||||
JSON.stringify({ generatedAt: new Date().toISOString(), outputDir, ok: true })
|
||||
);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} catch (error) {
|
||||
const p2pConnectionStats = await readLatestP2PConnectionStats(statsPath);
|
||||
const result = {
|
||||
ok: false,
|
||||
generatedAt: new Date().toISOString(),
|
||||
caseName: "p2p-split-compose",
|
||||
mode: "p2p-split-compose-benchmark",
|
||||
runId: config.runId,
|
||||
simulationTier: Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "2" : "1",
|
||||
networkProfile: config.profile,
|
||||
networkModel:
|
||||
Deno.env.get("BENCH_NETEM_ENABLED") === "1" ? "split-compose-egress-netem" : "split-compose-no-netem",
|
||||
relay: config.relay,
|
||||
turnServers: config.turnServers,
|
||||
turnEnabled: config.turnServers.trim().length > 0,
|
||||
p2pCandidatePathVerified: p2pConnectionStats?.candidatePathCollected === true,
|
||||
p2pConnectionStats,
|
||||
hostNetem: hostReady.netem,
|
||||
clientNetem: netem,
|
||||
totalFiles: hostReady.totalFiles,
|
||||
totalBytes: hostReady.totalBytes,
|
||||
mdFileCount: hostReady.mdFileCount,
|
||||
binFileCount: hostReady.binFileCount,
|
||||
mirrorElapsedMs: hostReady.mirrorElapsedMs,
|
||||
peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
|
||||
peerDiscoveryCommandElapsedMs,
|
||||
syncElapsedMs,
|
||||
failure: {
|
||||
stage,
|
||||
...errorToRecord(error),
|
||||
},
|
||||
};
|
||||
await Deno.writeTextFile(join(outputDir, "summary.json"), JSON.stringify(result, null, 2));
|
||||
await Deno.writeTextFile(
|
||||
join(config.workRoot, "client-done.json"),
|
||||
JSON.stringify({ generatedAt: new Date().toISOString(), outputDir, ok: false })
|
||||
);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
throw error;
|
||||
} finally {
|
||||
if (previousStatsPath === undefined) {
|
||||
Deno.env.delete("LIVESYNC_P2P_STATS_JSONL");
|
||||
} else {
|
||||
Deno.env.set("LIVESYNC_P2P_STATS_JSONL", previousStatsPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const role = readEnvString("BENCH_P2P_SPLIT_ROLE", "") as Role;
|
||||
if (role === "host") {
|
||||
await runHost();
|
||||
return;
|
||||
}
|
||||
if (role === "client") {
|
||||
await runClient();
|
||||
return;
|
||||
}
|
||||
throw new Error("BENCH_P2P_SPLIT_ROLE must be 'host' or 'client'");
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error("[Fatal Error]", error);
|
||||
Deno.exit(1);
|
||||
});
|
||||
}
|
||||
+330
-108
@@ -1,15 +1,34 @@
|
||||
import { TempDir } from "./helpers/temp.ts";
|
||||
import { applyP2pSettings, applyP2pTestTweaks, initSettingsFile } from "./helpers/settings.ts";
|
||||
import {
|
||||
applyP2pSettings,
|
||||
applyP2pTestTweaks,
|
||||
initSettingsFile,
|
||||
} from "./helpers/settings.ts";
|
||||
import { startCliInBackground } from "./helpers/backgroundCli.ts";
|
||||
import { discoverPeer, maybeStartLocalRelay, stopLocalRelayIfStarted } from "./helpers/p2p.ts";
|
||||
import {
|
||||
discoverPeer,
|
||||
maybeStartCoturn,
|
||||
maybeStartLocalRelay,
|
||||
stopCoturnIfStarted,
|
||||
stopLocalRelayIfStarted,
|
||||
} from "./helpers/p2p.ts";
|
||||
import { assertFilesEqual, runCliOrFail } from "./helpers/cli.ts";
|
||||
import { createDeterministicDataset, type DatasetEntry } from "./helpers/dataset.ts";
|
||||
import {
|
||||
createDeterministicDataset,
|
||||
} from "./helpers/dataset.ts";
|
||||
import {
|
||||
type BenchmarkVerificationMode,
|
||||
parseBenchmarkVerificationMode,
|
||||
verifyBenchmarkDataset,
|
||||
} from "./helpers/benchmarkVerification.ts";
|
||||
|
||||
type BenchmarkConfig = {
|
||||
caseName: string;
|
||||
relay: string;
|
||||
appId: string;
|
||||
roomId: string;
|
||||
passphrase: string;
|
||||
turnServers: string;
|
||||
datasetDirName: string;
|
||||
datasetSeed: string;
|
||||
mdFileCount: number;
|
||||
@@ -19,6 +38,47 @@ type BenchmarkConfig = {
|
||||
binSizeBytes: number;
|
||||
peersTimeoutSeconds: number;
|
||||
syncTimeoutSeconds: number;
|
||||
simulationTier: string;
|
||||
networkProfile: string;
|
||||
networkModel: string;
|
||||
candidatePathVerification: string;
|
||||
measurementScope: string;
|
||||
limitations: string[];
|
||||
verificationMode: BenchmarkVerificationMode;
|
||||
repeatIndex: number;
|
||||
repeatCount: number;
|
||||
};
|
||||
|
||||
type P2PConnectionStats = {
|
||||
generatedAt: string;
|
||||
command: string;
|
||||
peerId: string;
|
||||
peerName: string;
|
||||
candidatePathCollected: boolean;
|
||||
selectedPath: string;
|
||||
selectedPair?: {
|
||||
id: string;
|
||||
state: string;
|
||||
currentRoundTripTime: number | "unknown";
|
||||
totalRoundTripTime: number | "unknown";
|
||||
requestsSent: number | "unknown";
|
||||
responsesReceived: number | "unknown";
|
||||
packetsDiscardedOnSend: number | "unknown";
|
||||
bytesSent: number | "unknown";
|
||||
bytesReceived: number | "unknown";
|
||||
};
|
||||
localCandidate?: {
|
||||
id: string;
|
||||
candidateType: string;
|
||||
protocol: string;
|
||||
relayProtocol: string;
|
||||
};
|
||||
remoteCandidate?: {
|
||||
id: string;
|
||||
candidateType: string;
|
||||
protocol: string;
|
||||
relayProtocol: string;
|
||||
};
|
||||
};
|
||||
|
||||
function readEnvString(name: string, fallback: string): string {
|
||||
@@ -39,6 +99,30 @@ function readEnvNumber(name: string, fallback: number): number {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readEnvStringArray(name: string, fallback: string[]): string[] {
|
||||
const raw = Deno.env.get(name)?.trim();
|
||||
if (!raw) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (
|
||||
Array.isArray(parsed) &&
|
||||
parsed.every((item) => typeof item === "string")
|
||||
) {
|
||||
return parsed;
|
||||
}
|
||||
} catch {
|
||||
// Fall through to comma-separated parsing for hand-written invocations.
|
||||
}
|
||||
|
||||
return raw
|
||||
.split("|")
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function nowMs(): number {
|
||||
return performance.now();
|
||||
}
|
||||
@@ -61,19 +145,52 @@ function formatBytes(value: number): string {
|
||||
|
||||
function buildConfig(): BenchmarkConfig {
|
||||
return {
|
||||
caseName: readEnvString("BENCH_CASE", "p2p-direct-local"),
|
||||
relay: readEnvString("BENCH_RELAY", "ws://localhost:4000/"),
|
||||
appId: readEnvString("BENCH_APP_ID", "self-hosted-livesync-cli-benchmark"),
|
||||
appId: readEnvString(
|
||||
"BENCH_APP_ID",
|
||||
"self-hosted-livesync-cli-benchmark",
|
||||
),
|
||||
roomId: readEnvString("BENCH_ROOM_ID", `bench-room-${Date.now()}`),
|
||||
passphrase: readEnvString("BENCH_PASSPHRASE", `bench-${Date.now()}`),
|
||||
turnServers: readEnvString("BENCH_TURN_SERVERS", ""),
|
||||
datasetDirName: readEnvString("BENCH_DATASET_DIR", "bench-dataset"),
|
||||
datasetSeed: readEnvString("BENCH_SEED", "livesync-benchmark-seed"),
|
||||
mdFileCount: Math.floor(readEnvNumber("BENCH_MD_FILE_COUNT", 1500)),
|
||||
mdMinSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024)),
|
||||
mdMaxSizeBytes: Math.floor(readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024)),
|
||||
mdMinSizeBytes: Math.floor(
|
||||
readEnvNumber("BENCH_MD_MIN_SIZE_BYTES", 1024),
|
||||
),
|
||||
mdMaxSizeBytes: Math.floor(
|
||||
readEnvNumber("BENCH_MD_MAX_SIZE_BYTES", 20 * 1024),
|
||||
),
|
||||
binFileCount: Math.floor(readEnvNumber("BENCH_BIN_FILE_COUNT", 500)),
|
||||
binSizeBytes: Math.floor(readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024)),
|
||||
binSizeBytes: Math.floor(
|
||||
readEnvNumber("BENCH_BIN_SIZE_BYTES", 100 * 1024),
|
||||
),
|
||||
peersTimeoutSeconds: readEnvNumber("BENCH_PEERS_TIMEOUT", 20),
|
||||
syncTimeoutSeconds: readEnvNumber("BENCH_SYNC_TIMEOUT", 240),
|
||||
simulationTier: readEnvString("BENCH_SIMULATION_TIER", "1"),
|
||||
networkProfile: readEnvString("BENCH_NETWORK_PROFILE", "local-direct"),
|
||||
networkModel: readEnvString(
|
||||
"BENCH_NETWORK_MODEL",
|
||||
"local-runner-webrtc",
|
||||
),
|
||||
candidatePathVerification: readEnvString(
|
||||
"BENCH_P2P_CANDIDATE_PATH_VERIFICATION",
|
||||
"not-collected",
|
||||
),
|
||||
measurementScope: readEnvString(
|
||||
"BENCH_MEASUREMENT_SCOPE",
|
||||
"One fresh CLI p2p-sync command, including process start-up and WebRTC connection establishment; the earlier peer-list observation command is excluded.",
|
||||
),
|
||||
limitations: readEnvStringArray("BENCH_LIMITATIONS_JSON", [
|
||||
"This benchmark result is scoped to the configured dataset, network model, and selected ICE path.",
|
||||
]),
|
||||
verificationMode: parseBenchmarkVerificationMode(
|
||||
Deno.env.get("BENCH_VERIFY_MODE"),
|
||||
),
|
||||
repeatIndex: Math.floor(readEnvNumber("BENCH_REPEAT_INDEX", 1)),
|
||||
repeatCount: Math.floor(readEnvNumber("BENCH_REPEAT_COUNT", 1)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -85,21 +202,22 @@ function readOptionalResultPath(): string | undefined {
|
||||
return raw;
|
||||
}
|
||||
|
||||
function pickSampleFiles(entries: DatasetEntry[]): DatasetEntry[] {
|
||||
if (entries.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const md = entries.find((e) => e.kind === "md");
|
||||
const bin = entries.find((e) => e.kind === "bin");
|
||||
const middle = entries[Math.floor(entries.length / 2)];
|
||||
const last = entries[entries.length - 1];
|
||||
const unique = new Map<string, DatasetEntry>();
|
||||
for (const entry of [md, bin, middle, last]) {
|
||||
if (entry) {
|
||||
unique.set(entry.relativePath, entry);
|
||||
async function readLatestP2PConnectionStats(
|
||||
statsPath: string,
|
||||
): Promise<P2PConnectionStats | undefined> {
|
||||
try {
|
||||
const text = await Deno.readTextFile(statsPath);
|
||||
const lines = text
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0);
|
||||
if (lines.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return JSON.parse(lines[lines.length - 1]) as P2PConnectionStats;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
return [...unique.values()];
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
@@ -107,110 +225,214 @@ async function main(): Promise<void> {
|
||||
const resultPath = readOptionalResultPath();
|
||||
|
||||
const relayStarted = await maybeStartLocalRelay(config.relay);
|
||||
const coturnStarted = await maybeStartCoturn(config.turnServers);
|
||||
await using workDir = await TempDir.create("livesync-cli-p2p-bench");
|
||||
|
||||
const hostVault = workDir.join("vault-host");
|
||||
const clientVault = workDir.join("vault-client");
|
||||
const hostSettings = workDir.join("settings-host.json");
|
||||
const clientSettings = workDir.join("settings-client.json");
|
||||
const p2pStatsPath = workDir.join("p2p-connection-stats.jsonl");
|
||||
const previousStatsPath = Deno.env.get("LIVESYNC_P2P_STATS_JSONL");
|
||||
Deno.env.set("LIVESYNC_P2P_STATS_JSONL", p2pStatsPath);
|
||||
|
||||
await Promise.all([
|
||||
Deno.mkdir(hostVault, { recursive: true }),
|
||||
Deno.mkdir(clientVault, { recursive: true }),
|
||||
initSettingsFile(hostSettings),
|
||||
initSettingsFile(clientSettings),
|
||||
]);
|
||||
|
||||
await Promise.all([
|
||||
applyP2pSettings(hostSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"),
|
||||
applyP2pSettings(clientSettings, config.roomId, config.passphrase, config.appId, config.relay, "~.*"),
|
||||
]);
|
||||
|
||||
await Promise.all([
|
||||
applyP2pTestTweaks(hostSettings, "p2p-bench-host", config.passphrase),
|
||||
applyP2pTestTweaks(clientSettings, "p2p-bench-client", config.passphrase),
|
||||
]);
|
||||
|
||||
const seedFiles = await createDeterministicDataset({
|
||||
rootDir: hostVault,
|
||||
datasetDirName: config.datasetDirName,
|
||||
seed: config.datasetSeed,
|
||||
mdCount: config.mdFileCount,
|
||||
mdMinSizeBytes: config.mdMinSizeBytes,
|
||||
mdMaxSizeBytes: config.mdMaxSizeBytes,
|
||||
binCount: config.binFileCount,
|
||||
binSizeBytes: config.binSizeBytes,
|
||||
});
|
||||
|
||||
const mirrorStart = nowMs();
|
||||
await runCliOrFail(hostVault, "--settings", hostSettings, "mirror");
|
||||
const mirrorElapsed = nowMs() - mirrorStart;
|
||||
|
||||
const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
|
||||
try {
|
||||
const hostReadyStart = nowMs();
|
||||
await host.waitUntilContains("P2P host is running", 20000);
|
||||
const hostReadyElapsed = nowMs() - hostReadyStart;
|
||||
await Promise.all([
|
||||
Deno.mkdir(hostVault, { recursive: true }),
|
||||
Deno.mkdir(clientVault, { recursive: true }),
|
||||
initSettingsFile(hostSettings),
|
||||
initSettingsFile(clientSettings),
|
||||
]);
|
||||
|
||||
const peerDiscoveryStart = nowMs();
|
||||
const peer = await discoverPeer(clientVault, clientSettings, config.peersTimeoutSeconds);
|
||||
const peerDiscoveryElapsed = nowMs() - peerDiscoveryStart;
|
||||
await Promise.all([
|
||||
applyP2pSettings(
|
||||
hostSettings,
|
||||
config.roomId,
|
||||
config.passphrase,
|
||||
config.appId,
|
||||
config.relay,
|
||||
"~.*",
|
||||
config.turnServers,
|
||||
),
|
||||
applyP2pSettings(
|
||||
clientSettings,
|
||||
config.roomId,
|
||||
config.passphrase,
|
||||
config.appId,
|
||||
config.relay,
|
||||
"~.*",
|
||||
config.turnServers,
|
||||
),
|
||||
]);
|
||||
|
||||
const syncStart = nowMs();
|
||||
await runCliOrFail(
|
||||
clientVault,
|
||||
"--settings",
|
||||
clientSettings,
|
||||
"p2p-sync",
|
||||
peer.id,
|
||||
String(config.syncTimeoutSeconds)
|
||||
);
|
||||
const syncElapsed = nowMs() - syncStart;
|
||||
await Promise.all([
|
||||
applyP2pTestTweaks(
|
||||
hostSettings,
|
||||
"p2p-bench-host",
|
||||
config.passphrase,
|
||||
),
|
||||
applyP2pTestTweaks(
|
||||
clientSettings,
|
||||
"p2p-bench-client",
|
||||
config.passphrase,
|
||||
),
|
||||
]);
|
||||
|
||||
const sampleFiles = pickSampleFiles(seedFiles.entries);
|
||||
for (const sample of sampleFiles) {
|
||||
const pulledPath = workDir.join(`pulled-${sample.relativePath.replaceAll("/", "_")}`);
|
||||
await runCliOrFail(clientVault, "--settings", clientSettings, "pull", sample.relativePath, pulledPath);
|
||||
await assertFilesEqual(
|
||||
sample.absolutePath,
|
||||
pulledPath,
|
||||
`sample file mismatch after sync: ${sample.relativePath}`
|
||||
);
|
||||
}
|
||||
|
||||
const result = {
|
||||
mode: "p2p-cli-benchmark",
|
||||
relay: config.relay,
|
||||
appId: config.appId,
|
||||
roomId: config.roomId,
|
||||
datasetSeed: config.datasetSeed,
|
||||
const seedFiles = await createDeterministicDataset({
|
||||
rootDir: hostVault,
|
||||
datasetDirName: config.datasetDirName,
|
||||
peerId: peer.id,
|
||||
peerName: peer.name,
|
||||
totalFiles: seedFiles.totalFiles,
|
||||
totalBytes: seedFiles.totalBytes,
|
||||
mdFileCount: seedFiles.mdCount,
|
||||
binFileCount: seedFiles.binCount,
|
||||
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
|
||||
hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)),
|
||||
peerDiscoveryElapsedMs: Number(peerDiscoveryElapsed.toFixed(1)),
|
||||
syncElapsedMs: Number(syncElapsed.toFixed(1)),
|
||||
throughputBytesPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2)),
|
||||
throughputMiBPerSec: Number((seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024).toFixed(4)),
|
||||
};
|
||||
seed: config.datasetSeed,
|
||||
mdCount: config.mdFileCount,
|
||||
mdMinSizeBytes: config.mdMinSizeBytes,
|
||||
mdMaxSizeBytes: config.mdMaxSizeBytes,
|
||||
binCount: config.binFileCount,
|
||||
binSizeBytes: config.binSizeBytes,
|
||||
});
|
||||
|
||||
if (resultPath) {
|
||||
await Deno.writeTextFile(resultPath, JSON.stringify(result, null, 2));
|
||||
}
|
||||
const mirrorStart = nowMs();
|
||||
await runCliOrFail(hostVault, "--settings", hostSettings, "mirror");
|
||||
const mirrorElapsed = nowMs() - mirrorStart;
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
console.error(
|
||||
`[Benchmark] mirrored ${seedFiles.totalFiles} files (${formatBytes(seedFiles.totalBytes)}) in ${formatMs(mirrorElapsed)}, ` +
|
||||
`synced in ${formatMs(syncElapsed)} ` +
|
||||
`(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`
|
||||
const host = startCliInBackground(
|
||||
hostVault,
|
||||
"--settings",
|
||||
hostSettings,
|
||||
"p2p-host",
|
||||
);
|
||||
try {
|
||||
const hostReadyStart = nowMs();
|
||||
await host.waitUntilContains("P2P host is running", 20000);
|
||||
const hostReadyElapsed = nowMs() - hostReadyStart;
|
||||
|
||||
const peerDiscoveryCommandStart = nowMs();
|
||||
const peer = await discoverPeer(
|
||||
clientVault,
|
||||
clientSettings,
|
||||
config.peersTimeoutSeconds,
|
||||
);
|
||||
const peerDiscoveryCommandElapsed = nowMs() -
|
||||
peerDiscoveryCommandStart;
|
||||
|
||||
const syncStart = nowMs();
|
||||
await runCliOrFail(
|
||||
clientVault,
|
||||
"--settings",
|
||||
clientSettings,
|
||||
"p2p-sync",
|
||||
peer.id,
|
||||
String(config.syncTimeoutSeconds),
|
||||
);
|
||||
const syncElapsed = nowMs() - syncStart;
|
||||
|
||||
const verification = await verifyBenchmarkDataset(
|
||||
seedFiles.entries,
|
||||
config.verificationMode,
|
||||
async (entry) => {
|
||||
const pulledPath = workDir.join(
|
||||
`pulled-${entry.relativePath.replaceAll("/", "_")}`,
|
||||
);
|
||||
await runCliOrFail(
|
||||
clientVault,
|
||||
"--settings",
|
||||
clientSettings,
|
||||
"pull",
|
||||
entry.relativePath,
|
||||
pulledPath,
|
||||
);
|
||||
await assertFilesEqual(
|
||||
entry.absolutePath,
|
||||
pulledPath,
|
||||
`file mismatch after P2P sync: ${entry.relativePath}`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
const p2pConnectionStats = await readLatestP2PConnectionStats(
|
||||
p2pStatsPath,
|
||||
);
|
||||
const result = {
|
||||
caseName: config.caseName,
|
||||
mode: "p2p-cli-benchmark",
|
||||
relay: config.relay,
|
||||
turnServers: config.turnServers,
|
||||
turnEnabled: config.turnServers.trim().length > 0,
|
||||
simulationTier: config.simulationTier,
|
||||
networkProfile: config.networkProfile,
|
||||
networkModel: config.networkModel,
|
||||
measurementScope: config.measurementScope,
|
||||
limitations: config.limitations,
|
||||
repeatIndex: config.repeatIndex,
|
||||
repeatCount: config.repeatCount,
|
||||
p2pCandidatePathVerified:
|
||||
p2pConnectionStats?.candidatePathCollected === true,
|
||||
p2pCandidatePathVerification:
|
||||
p2pConnectionStats?.candidatePathCollected
|
||||
? "selected ICE candidate pair collected from RTCPeerConnection.getStats"
|
||||
: config.candidatePathVerification,
|
||||
p2pCandidatePathNote: p2pConnectionStats?.candidatePathCollected
|
||||
? "The selected ICE candidate pair was collected by the CLI benchmark. Interpret the path from the candidate types; do not infer TURN use from configuration alone."
|
||||
: config.turnServers.trim().length > 0
|
||||
? "TURN is configured, so the selected WebRTC path may be direct, server-reflexive, or relayed. The selected ICE candidate pair was not exported by this run."
|
||||
: "TURN is disabled, so a TURN-relayed path is not expected. The selected ICE candidate pair was not exported by this run.",
|
||||
p2pConnectionStats,
|
||||
appId: config.appId,
|
||||
roomId: config.roomId,
|
||||
datasetSeed: config.datasetSeed,
|
||||
datasetDirName: config.datasetDirName,
|
||||
peerId: peer.id,
|
||||
peerName: peer.name,
|
||||
totalFiles: seedFiles.totalFiles,
|
||||
totalBytes: seedFiles.totalBytes,
|
||||
mdFileCount: seedFiles.mdCount,
|
||||
binFileCount: seedFiles.binCount,
|
||||
...verification,
|
||||
mirrorElapsedMs: Number(mirrorElapsed.toFixed(1)),
|
||||
hostReadyElapsedMs: Number(hostReadyElapsed.toFixed(1)),
|
||||
peerDiscoveryTimeoutSeconds: config.peersTimeoutSeconds,
|
||||
peerDiscoveryCommandElapsedMs: Number(
|
||||
peerDiscoveryCommandElapsed.toFixed(1),
|
||||
),
|
||||
peerDiscoveryNote:
|
||||
"p2p-peers waits for the requested timeout before printing discovered peers, so this is command duration, not first-peer latency.",
|
||||
syncElapsedMs: Number(syncElapsed.toFixed(1)),
|
||||
throughputBytesPerSec: Number(
|
||||
(seedFiles.totalBytes / (syncElapsed / 1000)).toFixed(2),
|
||||
),
|
||||
throughputMiBPerSec: Number(
|
||||
(seedFiles.totalBytes / (syncElapsed / 1000) / 1024 / 1024)
|
||||
.toFixed(
|
||||
4,
|
||||
),
|
||||
),
|
||||
};
|
||||
|
||||
if (resultPath) {
|
||||
await Deno.writeTextFile(
|
||||
resultPath,
|
||||
JSON.stringify(result, null, 2),
|
||||
);
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
console.error(
|
||||
`[Benchmark] mirrored ${seedFiles.totalFiles} files (${
|
||||
formatBytes(
|
||||
seedFiles.totalBytes,
|
||||
)
|
||||
}) in ${formatMs(mirrorElapsed)}, ` +
|
||||
`synced in ${formatMs(syncElapsed)} ` +
|
||||
`(${result.throughputBytesPerSec} B/s, ${result.throughputMiBPerSec} MiB/s)`,
|
||||
);
|
||||
} finally {
|
||||
await host.stop();
|
||||
}
|
||||
} finally {
|
||||
await host.stop();
|
||||
if (previousStatsPath === undefined) {
|
||||
Deno.env.delete("LIVESYNC_P2P_STATS_JSONL");
|
||||
} else {
|
||||
Deno.env.set("LIVESYNC_P2P_STATS_JSONL", previousStatsPath);
|
||||
}
|
||||
await stopCoturnIfStarted(coturnStarted);
|
||||
await stopLocalRelayIfStarted(relayStarted);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
{
|
||||
"tasks": {
|
||||
"test": "deno test --env-file=.test.env -A --no-check test-*.ts",
|
||||
"test": "deno task test:ci",
|
||||
"test:ci": "deno run -A --no-check run-ci-suite.ts",
|
||||
"test:p2p:compose": "deno run -A --no-check run-compose-p2p.ts",
|
||||
"test:local": "deno test --env-file=.test.env -A --no-check test-setup-put-cat.ts test-mirror.ts test-daemon.ts",
|
||||
"test:daemon": "deno test --env-file=.test.env -A --no-check test-daemon.ts",
|
||||
"test:decoupled-vault": "deno test --env-file=.test.env -A --no-check test-decoupled-vault.ts",
|
||||
@@ -13,10 +15,18 @@
|
||||
"test:p2p-host": "deno test --env-file=.test.env -A --no-check test-p2p-host.ts",
|
||||
"test:p2p-peers": "deno test --env-file=.test.env -A --no-check test-p2p-peers-local-relay.ts",
|
||||
"test:p2p-sync": "deno test --env-file=.test.env -A --no-check test-p2p-sync.ts",
|
||||
"test:p2p-replacement": "deno test --env-file=.test.env -A --no-check test-p2p-replicator-replacement.ts",
|
||||
"test:p2p-relay-disconnect": "deno test --env-file=.test.env -A --no-check test-p2p-relay-disconnect.ts",
|
||||
"test:p2p:ci": "deno test --env-file=.test.env -A --no-check test-p2p-sync.ts test-p2p-replicator-replacement.ts test-p2p-relay-disconnect.ts",
|
||||
"test:p2p-three-nodes": "deno test --env-file=.test.env -A --no-check test-p2p-three-nodes-conflict.ts",
|
||||
"test:p2p-upload-download": "deno test --env-file=.test.env -A --no-check test-p2p-upload-download-repro.ts",
|
||||
"test:benchmark-contract": "deno test --env-file=.test.env -A --no-check test-benchmark-contract.ts",
|
||||
"bench:p2p": "deno run --env-file=.test.env -A --no-check bench-p2p.ts",
|
||||
"bench:couchdb": "deno run --env-file=.test.env -A --no-check bench-couchdb.ts",
|
||||
"bench:compression": "deno run --env-file=.test.env -A --no-check bench-compression.ts",
|
||||
"bench:cases": "deno run --env-file=.test.env -A --no-check bench-network-cases.ts",
|
||||
"bench:latency-sweep": "deno run --env-file=.test.env -A --no-check bench-latency-sweep.ts",
|
||||
"bench:p2p-split-node": "deno run --env-file=.test.env -A --no-check bench-p2p-split-node.ts",
|
||||
"bench:item1": "bash ./bench-run-item1.sh",
|
||||
"bench:item1:full": "BENCH_MD_FILE_COUNT=1500 BENCH_MD_MIN_SIZE_BYTES=1024 BENCH_MD_MAX_SIZE_BYTES=20480 BENCH_BIN_FILE_COUNT=500 BENCH_BIN_SIZE_BYTES=102400 BENCH_COUCHDB_RTT_MS=50 bash ./bench-run-item1.sh",
|
||||
"test:e2e-couchdb": "deno test --env-file=.test.env -A --no-check test-e2e-two-vaults-couchdb.ts",
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import type { DatasetEntry } from "./dataset.ts";
|
||||
|
||||
export type BenchmarkVerificationMode = "all" | "sample";
|
||||
|
||||
export type BenchmarkVerificationResult = {
|
||||
verificationMode: BenchmarkVerificationMode;
|
||||
verifiedFiles: number;
|
||||
verificationComplete: boolean;
|
||||
datasetDigestSha256: string;
|
||||
};
|
||||
|
||||
function toHex(bytes: ArrayBuffer): string {
|
||||
return [...new Uint8Array(bytes)]
|
||||
.map((value) => value.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
}
|
||||
|
||||
async function sha256(bytes: Uint8Array): Promise<string> {
|
||||
const input = new ArrayBuffer(bytes.byteLength);
|
||||
new Uint8Array(input).set(bytes);
|
||||
return toHex(await crypto.subtle.digest("SHA-256", input));
|
||||
}
|
||||
|
||||
export function parseBenchmarkVerificationMode(
|
||||
raw: string | undefined,
|
||||
fallback: BenchmarkVerificationMode = "sample",
|
||||
): BenchmarkVerificationMode {
|
||||
const value = raw?.trim().toLowerCase();
|
||||
if (!value) return fallback;
|
||||
if (value === "all" || value === "sample") return value;
|
||||
throw new Error(`BENCH_VERIFY_MODE must be 'all' or 'sample', got '${raw}'`);
|
||||
}
|
||||
|
||||
export function selectVerificationEntries(
|
||||
entries: DatasetEntry[],
|
||||
mode: BenchmarkVerificationMode,
|
||||
): DatasetEntry[] {
|
||||
if (mode === "all" || entries.length === 0) return [...entries];
|
||||
|
||||
const md = entries.find((entry) => entry.kind === "md");
|
||||
const bin = entries.find((entry) => entry.kind === "bin");
|
||||
const middle = entries[Math.floor(entries.length / 2)];
|
||||
const last = entries[entries.length - 1];
|
||||
const selected = new Map<string, DatasetEntry>();
|
||||
for (const entry of [md, bin, middle, last]) {
|
||||
if (entry) selected.set(entry.relativePath, entry);
|
||||
}
|
||||
return [...selected.values()];
|
||||
}
|
||||
|
||||
export async function computeDatasetDigestSha256(
|
||||
entries: DatasetEntry[],
|
||||
): Promise<string> {
|
||||
const manifest: string[] = [];
|
||||
for (const entry of entries) {
|
||||
const contentDigest = await sha256(await Deno.readFile(entry.absolutePath));
|
||||
manifest.push(
|
||||
`${entry.kind}\t${entry.relativePath}\t${entry.size}\t${contentDigest}`,
|
||||
);
|
||||
}
|
||||
return await sha256(new TextEncoder().encode(manifest.join("\n")));
|
||||
}
|
||||
|
||||
export async function verifyBenchmarkDataset(
|
||||
entries: DatasetEntry[],
|
||||
mode: BenchmarkVerificationMode,
|
||||
verifyEntry: (entry: DatasetEntry) => Promise<void>,
|
||||
): Promise<BenchmarkVerificationResult> {
|
||||
const selected = selectVerificationEntries(entries, mode);
|
||||
for (const entry of selected) {
|
||||
await verifyEntry(entry);
|
||||
}
|
||||
|
||||
return {
|
||||
verificationMode: mode,
|
||||
verifiedFiles: selected.length,
|
||||
verificationComplete: selected.length === entries.length,
|
||||
datasetDigestSha256: await computeDatasetDigestSha256(entries),
|
||||
};
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import { join } from "@std/path";
|
||||
// CLI root (src/apps/cli/) is two levels up.
|
||||
// import.meta.dirname is available in Deno 1.40+ as an OS-native path string.
|
||||
export const CLI_DIR: string = join(import.meta.dirname!, "..", "..");
|
||||
const CLI_DIST = join(CLI_DIR, "dist", "index.cjs");
|
||||
export const CLI_DIST = join(CLI_DIR, "dist", "index.cjs");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result type
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import { join } from "@std/path";
|
||||
import type { DatasetEntry, DatasetKind } from "./dataset.ts";
|
||||
|
||||
export type CompressionDatasetEntry = DatasetEntry & {
|
||||
source: string;
|
||||
};
|
||||
|
||||
export type CompressionDataset = {
|
||||
entries: CompressionDatasetEntry[];
|
||||
totalFiles: number;
|
||||
totalBytes: number;
|
||||
bytesByKind: Record<DatasetKind, number>;
|
||||
filesByKind: Record<DatasetKind, number>;
|
||||
jpegGenerator: string;
|
||||
};
|
||||
|
||||
export type JpegEncoder = (inputPpm: string, outputJpeg: string) => Promise<string>;
|
||||
|
||||
const ALL_KINDS: DatasetKind[] = ["md", "jpg", "png", "json", "ts", "gz", "bin"];
|
||||
|
||||
const REPOSITORY_ROOT = join(import.meta.dirname!, "..", "..", "..", "..", "..");
|
||||
|
||||
function fnv1a32(input: string): number {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
hash ^= input.charCodeAt(i) & 0xff;
|
||||
hash = Math.imul(hash, 0x01000193);
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
function createXorshift32(seed: number): () => number {
|
||||
let state = seed || 0x9e3779b9;
|
||||
return () => {
|
||||
state ^= state << 13;
|
||||
state ^= state >>> 17;
|
||||
state ^= state << 5;
|
||||
return state >>> 0;
|
||||
};
|
||||
}
|
||||
|
||||
function createSyntheticPpm(width: number, height: number, seed: string, textured: boolean): Uint8Array {
|
||||
const header = new TextEncoder().encode(`P6\n${width} ${height}\n255\n`);
|
||||
const pixels = new Uint8Array(width * height * 3);
|
||||
const nextRandom = createXorshift32(fnv1a32(seed));
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const offset = (y * width + x) * 3;
|
||||
const noise = textured ? (nextRandom() & 0x3f) - 32 : 0;
|
||||
pixels[offset] = Math.max(0, Math.min(255, Math.floor((x * 255) / (width - 1)) + noise));
|
||||
pixels[offset + 1] = Math.max(0, Math.min(255, Math.floor((y * 255) / (height - 1)) + noise));
|
||||
pixels[offset + 2] = Math.max(0, Math.min(255, Math.floor(((x + y) * 255) / (width + height - 2)) - noise));
|
||||
}
|
||||
}
|
||||
const result = new Uint8Array(header.length + pixels.length);
|
||||
result.set(header);
|
||||
result.set(pixels, header.length);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function gzip(input: Uint8Array): Promise<Uint8Array> {
|
||||
const copied = new Uint8Array(input.byteLength);
|
||||
copied.set(input);
|
||||
const stream = new Blob([copied.buffer]).stream().pipeThrough(new CompressionStream("gzip"));
|
||||
return new Uint8Array(await new Response(stream).arrayBuffer());
|
||||
}
|
||||
|
||||
export async function encodeJpegWithCjpeg(inputPpm: string, outputJpeg: string): Promise<string> {
|
||||
const command = new Deno.Command("cjpeg", {
|
||||
args: ["-quality", "85", "-optimize", "-outfile", outputJpeg, inputPpm],
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
});
|
||||
let result: Deno.CommandOutput;
|
||||
try {
|
||||
result = await command.output();
|
||||
} catch (error) {
|
||||
if (error instanceof Deno.errors.NotFound) {
|
||||
throw new Error(
|
||||
"cjpeg is required for the compression benchmark. Use the Compose runner or install libjpeg tools."
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!result.success) {
|
||||
throw new Error(`cjpeg failed: ${new TextDecoder().decode(result.stderr)}`);
|
||||
}
|
||||
return "cjpeg quality=85, optimise=true, synthetic PPM 640x480";
|
||||
}
|
||||
|
||||
async function writeRandomBinary(path: string, size: number, seed: string): Promise<void> {
|
||||
const bytes = new Uint8Array(size);
|
||||
const nextRandom = createXorshift32(fnv1a32(seed));
|
||||
for (let index = 0; index < bytes.length; index++) {
|
||||
bytes[index] = nextRandom() & 0xff;
|
||||
}
|
||||
await Deno.writeFile(path, bytes);
|
||||
}
|
||||
|
||||
export async function createCompressionBenchmarkDataset(options: {
|
||||
rootDir: string;
|
||||
datasetDirName?: string;
|
||||
repositoryRoot?: string;
|
||||
seed?: string;
|
||||
jpegEncoder?: JpegEncoder;
|
||||
}): Promise<CompressionDataset> {
|
||||
const datasetDirName = options.datasetDirName ?? "compression-benchmark";
|
||||
const repositoryRoot = options.repositoryRoot ?? REPOSITORY_ROOT;
|
||||
const seed = options.seed ?? "livesync-compression-benchmark";
|
||||
const jpegEncoder = options.jpegEncoder ?? encodeJpegWithCjpeg;
|
||||
const datasetRoot = join(options.rootDir, datasetDirName);
|
||||
const entries: CompressionDatasetEntry[] = [];
|
||||
let jpegGenerator = "";
|
||||
|
||||
for (const kind of ALL_KINDS) {
|
||||
await Deno.mkdir(join(datasetRoot, kind), { recursive: true });
|
||||
}
|
||||
|
||||
const addFile = async (kind: DatasetKind, absolutePath: string, source: string) => {
|
||||
const relativePath = absolutePath
|
||||
.slice(options.rootDir.length + 1)
|
||||
.split("\\")
|
||||
.join("/");
|
||||
const size = (await Deno.stat(absolutePath)).size;
|
||||
entries.push({ kind, relativePath, absolutePath, size, source });
|
||||
};
|
||||
|
||||
const copyRepositoryFile = async (kind: DatasetKind, sourcePath: string, targetName: string) => {
|
||||
const destination = join(datasetRoot, kind, targetName);
|
||||
await Deno.copyFile(join(repositoryRoot, sourcePath), destination);
|
||||
await addFile(kind, destination, sourcePath);
|
||||
};
|
||||
|
||||
await copyRepositoryFile("md", "docs/settings.md", "settings.md");
|
||||
await copyRepositoryFile("md", "docs/quick_setup.md", "quick-setup.md");
|
||||
await copyRepositoryFile("md", "updates.md", "updates.md");
|
||||
await copyRepositoryFile("png", "instruction_images/cloudant_1.png", "cloudant-1.png");
|
||||
await copyRepositoryFile(
|
||||
"png",
|
||||
"images/quick-setup/guide-quick-setup-first-setup-uri.png",
|
||||
"quick-setup-first-setup-uri.png"
|
||||
);
|
||||
await copyRepositoryFile("json", "package.json", "package.json");
|
||||
await copyRepositoryFile("json", "manifest.json", "manifest.json");
|
||||
await copyRepositoryFile("ts", "src/modules/core/ModuleReplicator.ts", "ModuleReplicator.ts");
|
||||
await copyRepositoryFile("ts", "src/modules/core/ReplicateResultProcessor.ts", "ReplicateResultProcessor.ts");
|
||||
|
||||
const markdownBytes = await Deno.readFile(join(repositoryRoot, "docs/settings.md"));
|
||||
const gzipPath = join(datasetRoot, "gz", "settings.md.gz");
|
||||
await Deno.writeFile(gzipPath, await gzip(markdownBytes));
|
||||
await addFile("gz", gzipPath, "generated gzip of docs/settings.md");
|
||||
|
||||
const randomPath = join(datasetRoot, "bin", "deterministic-random.bin");
|
||||
await writeRandomBinary(randomPath, 256 * 1024, seed);
|
||||
await addFile("bin", randomPath, `deterministic xorshift32 seed=${seed}`);
|
||||
|
||||
for (const [name, textured] of [
|
||||
["smooth-gradient.jpg", false],
|
||||
["textured-gradient.jpg", true],
|
||||
] as const) {
|
||||
const ppmPath = await Deno.makeTempFile({ dir: options.rootDir, prefix: "compression-jpeg-", suffix: ".ppm" });
|
||||
const jpegPath = join(datasetRoot, "jpg", name);
|
||||
try {
|
||||
await Deno.writeFile(ppmPath, createSyntheticPpm(640, 480, `${seed}-${name}`, textured));
|
||||
jpegGenerator = await jpegEncoder(ppmPath, jpegPath);
|
||||
} finally {
|
||||
await Deno.remove(ppmPath).catch(() => {});
|
||||
}
|
||||
await addFile("jpg", jpegPath, `${jpegGenerator}; ${textured ? "textured" : "smooth"}`);
|
||||
}
|
||||
|
||||
const bytesByKind = Object.fromEntries(ALL_KINDS.map((kind) => [kind, 0])) as Record<DatasetKind, number>;
|
||||
const filesByKind = Object.fromEntries(ALL_KINDS.map((kind) => [kind, 0])) as Record<DatasetKind, number>;
|
||||
for (const entry of entries) {
|
||||
bytesByKind[entry.kind] += entry.size;
|
||||
filesByKind[entry.kind] += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
entries,
|
||||
totalFiles: entries.length,
|
||||
totalBytes: entries.reduce((sum, entry) => sum + entry.size, 0),
|
||||
bytesByKind,
|
||||
filesByKind,
|
||||
jpegGenerator,
|
||||
};
|
||||
}
|
||||
@@ -9,8 +9,10 @@ export type DeterministicDatasetConfig = {
|
||||
binSizeBytes: number;
|
||||
};
|
||||
|
||||
export type DatasetKind = "md" | "jpg" | "png" | "json" | "ts" | "gz" | "bin";
|
||||
|
||||
export type DatasetEntry = {
|
||||
kind: "md" | "bin";
|
||||
kind: DatasetKind;
|
||||
relativePath: string;
|
||||
absolutePath: string;
|
||||
size: number;
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { CLI_DIR, CLI_DIST } from "./cli.ts";
|
||||
|
||||
export type CliProcessMeasurement = {
|
||||
elapsedMs: number;
|
||||
userCpuMs: number;
|
||||
systemCpuMs: number;
|
||||
totalCpuMs: number;
|
||||
cpuToWallRatio: number;
|
||||
maxResidentSetKiB: number;
|
||||
};
|
||||
|
||||
const MARKER = "__LIVESYNC_GNU_TIME__";
|
||||
|
||||
export async function runMeasuredCliOrFail(...args: string[]): Promise<CliProcessMeasurement> {
|
||||
const started = performance.now();
|
||||
let output: Deno.CommandOutput;
|
||||
try {
|
||||
output = await new Deno.Command("/usr/bin/time", {
|
||||
args: ["-f", `${MARKER}%U\t%S\t%M`, "node", CLI_DIST, ...args],
|
||||
cwd: CLI_DIR,
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
}).output();
|
||||
} catch (error) {
|
||||
if (error instanceof Deno.errors.NotFound) {
|
||||
throw new Error(
|
||||
"GNU /usr/bin/time is required for the compression benchmark. Use the Compose runner or install GNU time."
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const elapsedMs = performance.now() - started;
|
||||
const stderr = new TextDecoder().decode(output.stderr);
|
||||
const stdout = new TextDecoder().decode(output.stdout);
|
||||
const measurementLine = stderr.split(/\r?\n/).find((line) => line.startsWith(MARKER));
|
||||
if (!output.success) {
|
||||
throw new Error(`CLI exited with code ${output.code}\nstdout: ${stdout}\nstderr: ${stderr}`);
|
||||
}
|
||||
if (!measurementLine) {
|
||||
throw new Error(`GNU time did not emit the expected measurement marker\nstderr: ${stderr}`);
|
||||
}
|
||||
const [userSeconds, systemSeconds, maxResidentSetKiB] = measurementLine
|
||||
.slice(MARKER.length)
|
||||
.split("\t")
|
||||
.map(Number);
|
||||
if (![userSeconds, systemSeconds, maxResidentSetKiB].every(Number.isFinite)) {
|
||||
throw new Error(`Could not parse GNU time measurement: ${measurementLine}`);
|
||||
}
|
||||
const userCpuMs = userSeconds * 1000;
|
||||
const systemCpuMs = systemSeconds * 1000;
|
||||
const totalCpuMs = userCpuMs + systemCpuMs;
|
||||
return {
|
||||
elapsedMs: Number(elapsedMs.toFixed(1)),
|
||||
userCpuMs: Number(userCpuMs.toFixed(1)),
|
||||
systemCpuMs: Number(systemCpuMs.toFixed(1)),
|
||||
totalCpuMs: Number(totalCpuMs.toFixed(1)),
|
||||
cpuToWallRatio: Number((totalCpuMs / elapsedMs).toFixed(4)),
|
||||
maxResidentSetKiB,
|
||||
};
|
||||
}
|
||||
@@ -9,7 +9,7 @@ function sleep(ms: number): Promise<void> {
|
||||
}
|
||||
|
||||
async function connectWithTimeout(hostname: string, port: number, timeoutMs: number): Promise<void> {
|
||||
let timer: number | undefined;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const connPromise = Deno.connect({ hostname, port });
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
|
||||
@@ -76,8 +76,10 @@ export async function discoverPeer(
|
||||
}
|
||||
|
||||
export async function maybeStartLocalRelay(relay: string): Promise<boolean> {
|
||||
if (!isLocalP2pRelay(relay)) return false;
|
||||
await startP2pRelay();
|
||||
const shouldStart = isLocalP2pRelay(relay);
|
||||
if (shouldStart) {
|
||||
await startP2pRelay();
|
||||
}
|
||||
const endpoint = parseRelayEndpoint(relay);
|
||||
await waitForPort(endpoint.hostname, endpoint.port, {
|
||||
timeoutMs: Number(Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000"),
|
||||
@@ -86,8 +88,10 @@ export async function maybeStartLocalRelay(relay: string): Promise<boolean> {
|
||||
});
|
||||
// Docker proxy accepts TCP connections instantly before the container's internal process is fully ready.
|
||||
// Wait an additional few seconds to ensure strfry is actually accepting WebSockets.
|
||||
await sleep(3000);
|
||||
return true;
|
||||
if (shouldStart) {
|
||||
await sleep(3000);
|
||||
}
|
||||
return shouldStart;
|
||||
}
|
||||
|
||||
export async function stopLocalRelayIfStarted(started: boolean): Promise<void> {
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { join } from "@std/path";
|
||||
import { CLI_DIR, runCliOrFail } from "./cli.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -11,18 +10,14 @@ export async function initSettingsFile(settingsFile: string): Promise<void> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a full setup URI from a settings file via src/lib API.
|
||||
* Generate a full setup URI from a settings file via the Commonlib package API.
|
||||
* Mirrors the bash flow in test-setup-put-cat-linux.sh.
|
||||
*/
|
||||
export async function generateSetupUriFromSettings(settingsFile: string, setupPassphrase: string): Promise<string> {
|
||||
const repoRoot = join(CLI_DIR, "..", "..", "..");
|
||||
const script = [
|
||||
"import fs from 'node:fs';",
|
||||
"import { pathToFileURL } from 'node:url';",
|
||||
"import { fs } from '@vrtmrz/livesync-commonlib/node';",
|
||||
"import { encodeSettingsToSetupURI } from '@vrtmrz/livesync-commonlib/compat/API/processSetting';",
|
||||
"(async () => {",
|
||||
" const modulePath = process.env.REPO_ROOT + '/src/lib/src/API/processSetting.ts';",
|
||||
" const moduleUrl = pathToFileURL(modulePath).href;",
|
||||
" const { encodeSettingsToSetupURI } = await import(moduleUrl);",
|
||||
" const settingsPath = process.env.SETTINGS_FILE;",
|
||||
" const passphrase = process.env.SETUP_PASSPHRASE;",
|
||||
" const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf-8'));",
|
||||
@@ -39,6 +34,7 @@ export async function generateSetupUriFromSettings(settingsFile: string, setupPa
|
||||
].join("\n");
|
||||
|
||||
const scriptPath = await Deno.makeTempFile({
|
||||
dir: CLI_DIR,
|
||||
prefix: "livesync-setup-uri-",
|
||||
suffix: ".mts",
|
||||
});
|
||||
@@ -49,7 +45,6 @@ export async function generateSetupUriFromSettings(settingsFile: string, setupPa
|
||||
args: ["tsx", scriptPath],
|
||||
cwd: CLI_DIR,
|
||||
env: {
|
||||
REPO_ROOT: repoRoot,
|
||||
SETTINGS_FILE: settingsFile,
|
||||
SETUP_PASSPHRASE: setupPassphrase,
|
||||
},
|
||||
@@ -128,6 +123,8 @@ export async function applyRemoteSyncSettings(
|
||||
minioSecretKey?: string;
|
||||
encrypt?: boolean;
|
||||
passphrase?: string;
|
||||
enableCompression?: boolean;
|
||||
usePathObfuscation?: boolean;
|
||||
}
|
||||
): Promise<void> {
|
||||
const data = JSON.parse(await Deno.readTextFile(settingsFile));
|
||||
@@ -154,6 +151,12 @@ export async function applyRemoteSyncSettings(
|
||||
data.usePluginSync = false;
|
||||
data.encrypt = options.encrypt === true;
|
||||
data.passphrase = options.encrypt ? (options.passphrase ?? "") : "";
|
||||
if (options.enableCompression !== undefined) {
|
||||
data.enableCompression = options.enableCompression;
|
||||
}
|
||||
if (options.usePathObfuscation !== undefined) {
|
||||
data.usePathObfuscation = options.usePathObfuscation;
|
||||
}
|
||||
data.isConfigured = true;
|
||||
await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2));
|
||||
}
|
||||
@@ -200,9 +203,8 @@ export async function applyP2pTestTweaks(settingsFile: string, deviceName: strin
|
||||
data.passphrase = passphrase;
|
||||
data.usePathObfuscation = true;
|
||||
data.handleFilenameCaseSensitive = false;
|
||||
data.customChunkSize = 50;
|
||||
data.customChunkSize = 60;
|
||||
data.usePluginSyncV2 = true;
|
||||
data.doNotUseFixedRevisionForChunks = false;
|
||||
data.P2P_DevicePeerName = deviceName;
|
||||
data.isConfigured = true;
|
||||
await Deno.writeTextFile(settingsFile, JSON.stringify(data, null, 2));
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import { createRequire } from "node:module";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { RTCPeerConnection } from "werift";
|
||||
import { TrysteroReplicator } from "@vrtmrz/livesync-commonlib/compat/replication/trystero/TrysteroReplicator";
|
||||
|
||||
const requireFromProbe = createRequire(import.meta.url);
|
||||
const commonlibEntry = requireFromProbe.resolve("@vrtmrz/livesync-commonlib/context");
|
||||
const requireFromCommonlib = createRequire(commonlibEntry);
|
||||
const nostrEntry = requireFromCommonlib.resolve("@trystero-p2p/nostr");
|
||||
const { getRelaySockets, joinRoom, pauseRelayReconnection } = await import(pathToFileURL(nostrEntry).href);
|
||||
|
||||
const relayUrl = process.env.RELAY ?? "ws://nostr-relay:7777/";
|
||||
const timeoutMs = Number(process.env.RELAY_TIMEOUT_MS ?? 15_000);
|
||||
|
||||
function delay(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function waitFor(description, predicate) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() <= deadline) {
|
||||
if (predicate()) return;
|
||||
await delay(50);
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${description}`);
|
||||
}
|
||||
|
||||
async function waitForSocketOpen(socket, description) {
|
||||
if (socket.readyState === WebSocket.OPEN) {
|
||||
// Node can expose OPEN before it dispatches the event. Yield once so
|
||||
// Trystero's previously registered onopen handler has completed before
|
||||
// this probe starts the close handshake.
|
||||
await delay(0);
|
||||
if (socket.readyState === WebSocket.OPEN) return;
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error(`Timed out waiting for ${description}; readyState=${socket.readyState}`));
|
||||
}, timeoutMs);
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
socket.removeEventListener("open", onOpen);
|
||||
socket.removeEventListener("close", onClose);
|
||||
socket.removeEventListener("error", onError);
|
||||
};
|
||||
const onOpen = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onClose = () => {
|
||||
cleanup();
|
||||
reject(new Error(`${description} closed before opening`));
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
reject(new Error(`${description} failed before opening`));
|
||||
};
|
||||
|
||||
// Trystero installs its onopen handler while constructing the socket,
|
||||
// before this observer is registered. Waiting for the actual event
|
||||
// therefore establishes transport readiness without a fixed delay.
|
||||
socket.addEventListener("open", onOpen, { once: true });
|
||||
socket.addEventListener("close", onClose, { once: true });
|
||||
socket.addEventListener("error", onError, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
const room = joinRoom(
|
||||
{
|
||||
appId: `livesync-relay-disconnect-probe-${Date.now()}`,
|
||||
password: "local-test-only",
|
||||
relayConfig: {
|
||||
urls: [relayUrl],
|
||||
manualReconnection: true,
|
||||
},
|
||||
rtcPolyfill: RTCPeerConnection,
|
||||
},
|
||||
"disconnect-probe"
|
||||
);
|
||||
|
||||
try {
|
||||
await waitFor("the relay WebSocket to be registered", () => Object.values(getRelaySockets()).length > 0);
|
||||
const originalSockets = Object.values(getRelaySockets());
|
||||
await Promise.all(
|
||||
originalSockets.map((socket, index) => waitForSocketOpen(socket, `relay WebSocket ${index + 1} to open`))
|
||||
);
|
||||
|
||||
const replicator = new TrysteroReplicator(
|
||||
{},
|
||||
{
|
||||
close: async () => undefined,
|
||||
dispatchConnectionStatus: async () => undefined,
|
||||
}
|
||||
);
|
||||
replicator.disconnectFromServer();
|
||||
|
||||
await waitFor("all relay WebSockets to close", () =>
|
||||
originalSockets.every((socket) => socket.readyState === WebSocket.CLOSED)
|
||||
);
|
||||
await delay(4_000);
|
||||
if (!originalSockets.every((socket) => socket.readyState === WebSocket.CLOSED)) {
|
||||
throw new Error("A relay WebSocket reconnected while reconnection was paused");
|
||||
}
|
||||
|
||||
replicator.allowReconnection();
|
||||
await waitFor("a replacement relay WebSocket to be registered", () =>
|
||||
Object.values(getRelaySockets()).some((socket) => !originalSockets.includes(socket))
|
||||
);
|
||||
const replacementSockets = Object.values(getRelaySockets()).filter((socket) => !originalSockets.includes(socket));
|
||||
await Promise.all(
|
||||
replacementSockets.map((socket, index) =>
|
||||
waitForSocketOpen(socket, `replacement relay WebSocket ${index + 1} to open`)
|
||||
)
|
||||
);
|
||||
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
socketsClosed: originalSockets.length,
|
||||
stayedDisconnectedWhilePaused: true,
|
||||
reconnectedAfterResume: true,
|
||||
})
|
||||
);
|
||||
} finally {
|
||||
await room.leave();
|
||||
pauseRelayReconnection();
|
||||
for (const socket of Object.values(getRelaySockets())) socket.close();
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
const TASKS = [
|
||||
"test:setup-put-cat",
|
||||
"test:mirror",
|
||||
"test:daemon",
|
||||
"test:push-pull",
|
||||
"test:decoupled-vault",
|
||||
"test:sync-two-local",
|
||||
"test:sync-locked-remote",
|
||||
"test:remote-commands",
|
||||
"test:e2e-matrix:couchdb-enc0",
|
||||
"test:e2e-matrix:couchdb-enc1",
|
||||
"test:e2e-matrix:minio-enc0",
|
||||
"test:e2e-matrix:minio-enc1",
|
||||
] as const;
|
||||
|
||||
for (const [index, task] of TASKS.entries()) {
|
||||
console.log(`\n[CLI E2E ${index + 1}/${TASKS.length}] ${task}`);
|
||||
const child = new Deno.Command(Deno.execPath(), {
|
||||
args: ["task", task],
|
||||
cwd: import.meta.dirname,
|
||||
stdin: "inherit",
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
}).spawn();
|
||||
const status = await child.status;
|
||||
if (!status.success) {
|
||||
console.error(`[CLI E2E] ${task} failed with exit code ${status.code}.`);
|
||||
Deno.exit(status.code);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n[CLI E2E] CI suite passed (${TASKS.length} tasks).`);
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
TASK="${CLI_E2E_TASK:-test:p2p:ci}"
|
||||
|
||||
case "$TASK" in
|
||||
test:p2p-host|test:p2p-peers|test:p2p-sync|test:p2p-replacement|test:p2p-relay-disconnect|test:p2p:ci|test:p2p-three-nodes|test:p2p-upload-download)
|
||||
exec deno task "$TASK"
|
||||
;;
|
||||
*)
|
||||
echo "Unknown CLI_E2E_TASK: $TASK" >&2
|
||||
echo "Expected one of: test:p2p-host, test:p2p-peers, test:p2p-sync, test:p2p-replacement, test:p2p-relay-disconnect, test:p2p:ci, test:p2p-three-nodes, test:p2p-upload-download" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,50 @@
|
||||
const repositoryRoot = await Deno.realPath(new URL("../../../../", import.meta.url));
|
||||
const composeArgs = ["compose", "-f", "test/bench-network/compose.yml"];
|
||||
const p2pEnvironment = {
|
||||
CLI_E2E_TASK: Deno.env.get("CLI_E2E_TASK") ?? "test:p2p:ci",
|
||||
RELAY: Deno.env.get("RELAY") ?? "ws://nostr-relay:7777/",
|
||||
PEERS_TIMEOUT: Deno.env.get("PEERS_TIMEOUT") ?? "20",
|
||||
SYNC_TIMEOUT: Deno.env.get("SYNC_TIMEOUT") ?? "60",
|
||||
LIVESYNC_USE_COTURN: Deno.env.get("LIVESYNC_USE_COTURN") ?? "0",
|
||||
TURN_SERVERS: Deno.env.get("TURN_SERVERS") ?? "none",
|
||||
LIVESYNC_P2P_PEERS_RETRY: Deno.env.get("LIVESYNC_P2P_PEERS_RETRY") ?? "1",
|
||||
LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS: Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "60000",
|
||||
BENCH_LIVESYNC_TEST_TEE: Deno.env.get("BENCH_LIVESYNC_TEST_TEE") ?? "0",
|
||||
LIVESYNC_CLI_DEBUG: Deno.env.get("LIVESYNC_CLI_DEBUG") ?? "0",
|
||||
LIVESYNC_CLI_VERBOSE: Deno.env.get("LIVESYNC_CLI_VERBOSE") ?? "0",
|
||||
};
|
||||
|
||||
async function runDocker(args: string[], env?: Record<string, string>): Promise<Deno.CommandStatus> {
|
||||
return await new Deno.Command("docker", {
|
||||
args,
|
||||
cwd: repositoryRoot,
|
||||
env,
|
||||
stdin: "inherit",
|
||||
stdout: "inherit",
|
||||
stderr: "inherit",
|
||||
}).spawn().status;
|
||||
}
|
||||
|
||||
let testStatus: Deno.CommandStatus | undefined;
|
||||
try {
|
||||
testStatus = await runDocker(
|
||||
[...composeArgs, "run", "--build", "--rm", "bench-runner", "run-livesync-cli-e2e"],
|
||||
p2pEnvironment
|
||||
);
|
||||
} finally {
|
||||
const cleanupStatus = await runDocker([...composeArgs, "down", "-v", "--remove-orphans"]);
|
||||
if (!cleanupStatus.success) {
|
||||
console.error(`[CLI E2E] Compose cleanup failed with exit code ${cleanupStatus.code}.`);
|
||||
if (testStatus?.success) {
|
||||
Deno.exit(cleanupStatus.code);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!testStatus?.success) {
|
||||
const code = testStatus?.code ?? 1;
|
||||
console.error(`[CLI E2E] Compose P2P suite failed with exit code ${code}.`);
|
||||
Deno.exit(code);
|
||||
}
|
||||
|
||||
console.log("\n[CLI E2E] Compose P2P suite passed.");
|
||||
@@ -0,0 +1,334 @@
|
||||
import { assert, assertEquals, assertStringIncludes } from "@std/assert";
|
||||
import { type BenchmarkCase, buildCases } from "./bench-network-cases.ts";
|
||||
import { startCouchdbProxy } from "./bench-couchdb.ts";
|
||||
import {
|
||||
parseBenchmarkVerificationMode,
|
||||
selectVerificationEntries,
|
||||
} from "./helpers/benchmarkVerification.ts";
|
||||
import type { DatasetEntry } from "./helpers/dataset.ts";
|
||||
import { createCompressionBenchmarkDataset } from "./helpers/compressionDataset.ts";
|
||||
|
||||
function getFreePort(): number {
|
||||
const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 });
|
||||
try {
|
||||
return (listener.addr as Deno.NetAddr).port;
|
||||
} finally {
|
||||
listener.close();
|
||||
}
|
||||
}
|
||||
|
||||
function getCase(cases: BenchmarkCase[], name: string): BenchmarkCase {
|
||||
const found = cases.find((testCase) => testCase.name === name);
|
||||
assert(found, `missing benchmark case: ${name}`);
|
||||
return found;
|
||||
}
|
||||
|
||||
function parsedLimitations(testCase: BenchmarkCase): string[] {
|
||||
const raw = testCase.env.BENCH_LIMITATIONS_JSON;
|
||||
assert(
|
||||
raw,
|
||||
`${testCase.name} must pass BENCH_LIMITATIONS_JSON to benchmark result output`,
|
||||
);
|
||||
const parsed = JSON.parse(raw);
|
||||
assert(
|
||||
Array.isArray(parsed),
|
||||
`${testCase.name} limitations must be an array`,
|
||||
);
|
||||
assert(
|
||||
parsed.every((item) =>
|
||||
typeof item === "string" && item.trim().length > 0
|
||||
),
|
||||
);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
Deno.test("benchmark cases record scope and limitations for paper use", () => {
|
||||
const cases = buildCases();
|
||||
assert(cases.length > 0);
|
||||
|
||||
for (const testCase of cases) {
|
||||
assert(
|
||||
testCase.description.trim().length > 0,
|
||||
`${testCase.name} must describe the case`,
|
||||
);
|
||||
assert(
|
||||
testCase.dataPath.trim().length > 0,
|
||||
`${testCase.name} must describe the data path`,
|
||||
);
|
||||
assert(
|
||||
testCase.trustBoundary.trim().length > 0,
|
||||
`${testCase.name} must describe the trust boundary`,
|
||||
);
|
||||
assert(
|
||||
testCase.measurementScope.trim().length > 0,
|
||||
`${testCase.name} must describe the measurement scope`,
|
||||
);
|
||||
assert(
|
||||
testCase.limitations.length > 0,
|
||||
`${testCase.name} must list limitations`,
|
||||
);
|
||||
assertEquals(
|
||||
testCase.env.BENCH_MEASUREMENT_SCOPE,
|
||||
testCase.measurementScope,
|
||||
);
|
||||
assertEquals(parsedLimitations(testCase), testCase.limitations);
|
||||
assertEquals(
|
||||
testCase.env.BENCH_VERIFY_MODE,
|
||||
"all",
|
||||
`${testCase.name} must verify the complete dataset`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("CouchDB latency proxy applies half the requested RTT in each direction", async () => {
|
||||
const backendPort = getFreePort();
|
||||
const proxyPort = getFreePort();
|
||||
const delays: number[] = [];
|
||||
const backend = Deno.serve(
|
||||
{
|
||||
hostname: "127.0.0.1",
|
||||
port: backendPort,
|
||||
onListen() {},
|
||||
},
|
||||
() => new Response("ok"),
|
||||
);
|
||||
const proxy = startCouchdbProxy({
|
||||
backendUri: `http://127.0.0.1:${backendPort}`,
|
||||
proxyUri: `http://127.0.0.1:${proxyPort}`,
|
||||
requestedRttMs: 20,
|
||||
delay: (milliseconds) => {
|
||||
delays.push(milliseconds);
|
||||
return Promise.resolve();
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${proxyPort}/probe`);
|
||||
assertEquals(await response.text(), "ok");
|
||||
assertEquals(proxy.directionalDelayMs, 10);
|
||||
assertEquals(delays, [10, 10]);
|
||||
assertEquals(proxy.snapshotCounters(), {
|
||||
requestCount: 1,
|
||||
requestBodyBytes: 0,
|
||||
responseBodyBytes: 2,
|
||||
});
|
||||
proxy.resetCounters();
|
||||
const posted = await fetch(`http://127.0.0.1:${proxyPort}/probe`, {
|
||||
method: "POST",
|
||||
body: "abc",
|
||||
});
|
||||
assertEquals(await posted.text(), "ok");
|
||||
assertEquals(proxy.snapshotCounters(), {
|
||||
requestCount: 1,
|
||||
requestBodyBytes: 3,
|
||||
responseBodyBytes: 2,
|
||||
});
|
||||
} finally {
|
||||
await proxy.stop();
|
||||
await backend.shutdown();
|
||||
}
|
||||
|
||||
const halfMillisecondProxy = startCouchdbProxy({
|
||||
backendUri: "http://127.0.0.1:1",
|
||||
proxyUri: `http://127.0.0.1:${getFreePort()}`,
|
||||
requestedRttMs: 1,
|
||||
delay: () => Promise.resolve(),
|
||||
});
|
||||
try {
|
||||
assertEquals(halfMillisecondProxy.directionalDelayMs, 0.5);
|
||||
} finally {
|
||||
await halfMillisecondProxy.stop();
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("compression benchmark dataset covers representative file kinds deterministically", async () => {
|
||||
const fixtureRoot = await Deno.makeTempDir({
|
||||
prefix: "livesync-compression-contract-",
|
||||
});
|
||||
const repositoryRoot = `${fixtureRoot}/repository`;
|
||||
const vaultA = `${fixtureRoot}/vault-a`;
|
||||
const vaultB = `${fixtureRoot}/vault-b`;
|
||||
const repositoryFiles = [
|
||||
"docs/settings.md",
|
||||
"docs/quick_setup.md",
|
||||
"updates.md",
|
||||
"instruction_images/cloudant_1.png",
|
||||
"images/quick-setup/guide-quick-setup-first-setup-uri.png",
|
||||
"package.json",
|
||||
"manifest.json",
|
||||
"src/modules/core/ModuleReplicator.ts",
|
||||
"src/modules/core/ReplicateResultProcessor.ts",
|
||||
];
|
||||
try {
|
||||
for (const [index, relativePath] of repositoryFiles.entries()) {
|
||||
const absolutePath = `${repositoryRoot}/${relativePath}`;
|
||||
await Deno.mkdir(
|
||||
absolutePath.slice(0, absolutePath.lastIndexOf("/")),
|
||||
{ recursive: true },
|
||||
);
|
||||
await Deno.writeFile(
|
||||
absolutePath,
|
||||
new TextEncoder().encode(
|
||||
`fixture-${index}-${relativePath}\n`.repeat(20),
|
||||
),
|
||||
);
|
||||
}
|
||||
await Deno.mkdir(vaultA, { recursive: true });
|
||||
await Deno.mkdir(vaultB, { recursive: true });
|
||||
const jpegEncoder = async (_input: string, output: string) => {
|
||||
await Deno.writeFile(
|
||||
output,
|
||||
new Uint8Array([
|
||||
0xff,
|
||||
0xd8,
|
||||
0xff,
|
||||
0xdb,
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
0xff,
|
||||
0xd9,
|
||||
]),
|
||||
);
|
||||
return "contract JPEG encoder";
|
||||
};
|
||||
const first = await createCompressionBenchmarkDataset({
|
||||
rootDir: vaultA,
|
||||
repositoryRoot,
|
||||
seed: "contract-seed",
|
||||
jpegEncoder,
|
||||
});
|
||||
const second = await createCompressionBenchmarkDataset({
|
||||
rootDir: vaultB,
|
||||
repositoryRoot,
|
||||
seed: "contract-seed",
|
||||
jpegEncoder,
|
||||
});
|
||||
|
||||
assertEquals(first.filesByKind, {
|
||||
md: 3,
|
||||
jpg: 2,
|
||||
png: 2,
|
||||
json: 2,
|
||||
ts: 2,
|
||||
gz: 1,
|
||||
bin: 1,
|
||||
});
|
||||
assertEquals(first.totalFiles, 13);
|
||||
assertEquals(first.jpegGenerator, "contract JPEG encoder");
|
||||
assertEquals(
|
||||
first.entries.map((entry) => [
|
||||
entry.kind,
|
||||
entry.relativePath,
|
||||
entry.size,
|
||||
]),
|
||||
second.entries.map((entry) => [
|
||||
entry.kind,
|
||||
entry.relativePath,
|
||||
entry.size,
|
||||
]),
|
||||
);
|
||||
assert(first.entries.every((entry) => entry.size > 0));
|
||||
} finally {
|
||||
await Deno.remove(fixtureRoot, { recursive: true }).catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("benchmark verification mode selects either all files or a labelled sample", () => {
|
||||
const entries: DatasetEntry[] = [
|
||||
{ kind: "md", relativePath: "a.md", absolutePath: "/a", size: 1 },
|
||||
{ kind: "md", relativePath: "b.md", absolutePath: "/b", size: 1 },
|
||||
{ kind: "bin", relativePath: "c.bin", absolutePath: "/c", size: 1 },
|
||||
{ kind: "md", relativePath: "d.md", absolutePath: "/d", size: 1 },
|
||||
{ kind: "bin", relativePath: "e.bin", absolutePath: "/e", size: 1 },
|
||||
];
|
||||
|
||||
assertEquals(parseBenchmarkVerificationMode("ALL"), "all");
|
||||
assertEquals(selectVerificationEntries(entries, "all").length, entries.length);
|
||||
const sample = selectVerificationEntries(entries, "sample");
|
||||
assert(sample.length > 0 && sample.length < entries.length);
|
||||
assert(sample.some((entry) => entry.kind === "md"));
|
||||
assert(sample.some((entry) => entry.kind === "bin"));
|
||||
});
|
||||
|
||||
Deno.test("P2P signalling-shim cases do not claim to shape the note-data path", () => {
|
||||
const cases = buildCases();
|
||||
for (
|
||||
const name of [
|
||||
"p2p-signalling-netem-home-wifi",
|
||||
"p2p-signalling-netem-tethering-vpn",
|
||||
]
|
||||
) {
|
||||
const testCase = getCase(cases, name);
|
||||
assertEquals(testCase.runner, "p2p");
|
||||
assertEquals(testCase.env.BENCH_TURN_SERVERS, "");
|
||||
assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2");
|
||||
assertEquals(
|
||||
testCase.env.BENCH_NETWORK_MODEL,
|
||||
"compose-netem-signalling-shim",
|
||||
);
|
||||
assertStringIncludes(testCase.dataPath, "WebRTC DataChannel");
|
||||
assertStringIncludes(testCase.dataPath, "Nostr signalling");
|
||||
assertStringIncludes(testCase.measurementScope, "fresh CLI p2p-sync");
|
||||
assert(
|
||||
testCase.limitations.some((limitation) =>
|
||||
limitation.includes("connection establishment")
|
||||
),
|
||||
`${name} must state that connection establishment is timed`,
|
||||
);
|
||||
assert(
|
||||
testCase.limitations.some((limitation) =>
|
||||
limitation.includes("does not shape the selected WebRTC")
|
||||
),
|
||||
`${name} must avoid claiming that the P2P note-data path was shaped`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("placeholder and TURN cases are clearly non-evidence for broad P2P performance", () => {
|
||||
const cases = buildCases();
|
||||
|
||||
const smartphone = getCase(cases, "p2p-smartphone-vpn-direct");
|
||||
assertEquals(smartphone.env.BENCH_SIMULATION_TIER, "unmeasured");
|
||||
assertEquals(smartphone.env.BENCH_NETWORK_MODEL, "local-runner-no-netem");
|
||||
assert(
|
||||
smartphone.limitations.some((limitation) =>
|
||||
limitation.includes("must not be reported as smartphone")
|
||||
),
|
||||
"smartphone/VPN placeholder must not be usable as field evidence by accident",
|
||||
);
|
||||
|
||||
const turn = getCase(cases, "p2p-user-turn");
|
||||
assertStringIncludes(turn.env.BENCH_TURN_SERVERS, "turn:");
|
||||
assert(
|
||||
turn.limitations.some((limitation) =>
|
||||
limitation.includes(
|
||||
"does not prove that the selected ICE path was relayed",
|
||||
)
|
||||
),
|
||||
"TURN case must require selected ICE candidate interpretation",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("CouchDB netem cases are marked as remote-store baselines", () => {
|
||||
const cases = buildCases();
|
||||
for (
|
||||
const name of ["couchdb-netem-home-wifi", "couchdb-netem-tethering-vpn"]
|
||||
) {
|
||||
const testCase = getCase(cases, name);
|
||||
assertEquals(testCase.runner, "couchdb");
|
||||
assertEquals(testCase.env.BENCH_SIMULATION_TIER, "2");
|
||||
assertEquals(
|
||||
testCase.env.BENCH_NETWORK_MODEL,
|
||||
"compose-netem-tcp-shim",
|
||||
);
|
||||
assertStringIncludes(testCase.measurementScope, "CouchDB");
|
||||
assert(
|
||||
testCase.limitations.some((limitation) =>
|
||||
limitation.includes("not the WebRTC P2P data path")
|
||||
),
|
||||
`${name} must remain scoped to the CouchDB remote-store path`,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
import { assert, assertEquals } from "@std/assert";
|
||||
|
||||
Deno.test("p2p lifecycle: explicit disconnect closes and pauses relay WebSockets", async () => {
|
||||
const command = new Deno.Command("node", {
|
||||
args: [new URL("./relay-disconnect-probe.mjs", import.meta.url).pathname],
|
||||
env: {
|
||||
RELAY: Deno.env.get("RELAY") ?? "ws://nostr-relay:7777/",
|
||||
RELAY_TIMEOUT_MS: Deno.env.get("LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS") ?? "15000",
|
||||
},
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
});
|
||||
const result = await command.output();
|
||||
const stdout = new TextDecoder().decode(result.stdout).trim();
|
||||
const stderr = new TextDecoder().decode(result.stderr).trim();
|
||||
|
||||
assert(result.success, `Relay disconnect probe failed\nstdout: ${stdout}\nstderr: ${stderr}`);
|
||||
const report = JSON.parse(stdout.split("\n").at(-1) ?? "{}") as {
|
||||
socketsClosed?: number;
|
||||
stayedDisconnectedWhilePaused?: boolean;
|
||||
reconnectedAfterResume?: boolean;
|
||||
};
|
||||
assert((report.socketsClosed ?? 0) > 0, "The probe did not observe an open relay WebSocket");
|
||||
assertEquals(report.stayedDisconnectedWhilePaused, true);
|
||||
assertEquals(report.reconnectedAfterResume, true);
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
import { assert, assertEquals, assertStringIncludes } from "@std/assert";
|
||||
import { join } from "@std/path";
|
||||
import { TempDir } from "./helpers/temp.ts";
|
||||
import { initSettingsFile, applyP2pSettings, applyP2pTestTweaks } from "./helpers/settings.ts";
|
||||
import { startCliInBackground } from "./helpers/backgroundCli.ts";
|
||||
import { maybeStartLocalRelay, stopLocalRelayIfStarted, maybeStartCoturn, stopCoturnIfStarted } from "./helpers/p2p.ts";
|
||||
import { CLI_DIR, runCli, sanitiseCatStdout } from "./helpers/cli.ts";
|
||||
|
||||
const NOTE_PATH = "p2p-replicator-replacement.md";
|
||||
const NOTE_CONTENT = "Replicated after replacing the active P2P replicator.";
|
||||
|
||||
async function runReplacementProbe(
|
||||
vaultPath: string,
|
||||
settingsPath: string,
|
||||
targetPeer: string,
|
||||
timeoutMs: number
|
||||
): Promise<{ code: number; stdout: string; stderr: string }> {
|
||||
const command = new Deno.Command("node", {
|
||||
args: [
|
||||
join(CLI_DIR, "dist", "p2p-lifecycle-test.cjs"),
|
||||
vaultPath,
|
||||
"--settings",
|
||||
settingsPath,
|
||||
"p2p-sync",
|
||||
targetPeer,
|
||||
String(timeoutMs / 1000),
|
||||
NOTE_PATH,
|
||||
NOTE_CONTENT,
|
||||
],
|
||||
cwd: CLI_DIR,
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
});
|
||||
const result = await command.output();
|
||||
return {
|
||||
code: result.code,
|
||||
stdout: new TextDecoder().decode(result.stdout),
|
||||
stderr: new TextDecoder().decode(result.stderr),
|
||||
};
|
||||
}
|
||||
|
||||
Deno.test("p2p lifecycle: replacement keeps real CLI communication on the current replicator", async () => {
|
||||
const relay = Deno.env.get("RELAY") ?? "ws://localhost:4000/";
|
||||
const peersTimeout = Number(Deno.env.get("PEERS_TIMEOUT") ?? "20");
|
||||
const syncTimeout = Number(Deno.env.get("SYNC_TIMEOUT") ?? "60");
|
||||
const probeTimeoutMs = Math.max(peersTimeout, syncTimeout) * 1000;
|
||||
const nonce = `${Date.now()}-${crypto.randomUUID().slice(0, 8)}`;
|
||||
const roomId = Deno.env.get("ROOM_ID") ?? `replacement-room-${nonce}`;
|
||||
const passphrase = Deno.env.get("PASSPHRASE") ?? `replacement-pass-${nonce}`;
|
||||
const appId = "self-hosted-livesync-cli-replacement-test";
|
||||
const hostPeerName = `p2p-replacement-host-${nonce}`;
|
||||
const probePeerName = `p2p-replacement-probe-${nonce}`;
|
||||
const verifierPeerName = `p2p-replacement-verifier-${nonce}`;
|
||||
const useCoturn = Deno.env.get("LIVESYNC_USE_COTURN") !== "0";
|
||||
const turnServers = Deno.env.get("TURN_SERVERS") ?? (useCoturn ? "turn:127.0.0.1:3478" : "none");
|
||||
|
||||
await using workDir = await TempDir.create("livesync-cli-p2p-replacement");
|
||||
const hostVault = workDir.join("vault-host");
|
||||
const probeVault = workDir.join("vault-probe");
|
||||
const verifierVault = workDir.join("vault-verifier");
|
||||
const hostSettings = workDir.join("settings-host.json");
|
||||
const probeSettings = workDir.join("settings-probe.json");
|
||||
const verifierSettings = workDir.join("settings-verifier.json");
|
||||
await Promise.all([
|
||||
Deno.mkdir(hostVault, { recursive: true }),
|
||||
Deno.mkdir(probeVault, { recursive: true }),
|
||||
Deno.mkdir(verifierVault, { recursive: true }),
|
||||
]);
|
||||
|
||||
const relayStarted = await maybeStartLocalRelay(relay);
|
||||
const coturnStarted = await maybeStartCoturn(turnServers);
|
||||
try {
|
||||
for (const settingsPath of [hostSettings, probeSettings, verifierSettings]) {
|
||||
await initSettingsFile(settingsPath);
|
||||
await applyP2pSettings(settingsPath, roomId, passphrase, appId, relay, "~.*", turnServers);
|
||||
}
|
||||
await applyP2pTestTweaks(hostSettings, hostPeerName, passphrase);
|
||||
await applyP2pTestTweaks(probeSettings, probePeerName, passphrase);
|
||||
await applyP2pTestTweaks(verifierSettings, verifierPeerName, passphrase);
|
||||
|
||||
const host = startCliInBackground(hostVault, "--settings", hostSettings, "p2p-host");
|
||||
try {
|
||||
await host.waitUntilContains("P2P host is running", 20000);
|
||||
const probe = await runReplacementProbe(probeVault, probeSettings, hostPeerName, probeTimeoutMs);
|
||||
assert(
|
||||
probe.code === 0,
|
||||
`P2P replacement probe failed\nstdout: ${probe.stdout}\nstderr: ${probe.stderr}`
|
||||
);
|
||||
assertStringIncludes(probe.stdout, "[Probe] P2P replicator replaced");
|
||||
|
||||
const syncResult = await runCli(
|
||||
verifierVault,
|
||||
"--settings",
|
||||
verifierSettings,
|
||||
"p2p-sync",
|
||||
hostPeerName,
|
||||
String(syncTimeout)
|
||||
);
|
||||
assert(
|
||||
syncResult.code === 0,
|
||||
`Verifier P2P sync failed\nstdout: ${syncResult.stdout}\nstderr: ${syncResult.stderr}`
|
||||
);
|
||||
|
||||
const catResult = await runCli(verifierVault, "--settings", verifierSettings, "cat", NOTE_PATH);
|
||||
assert(
|
||||
catResult.code === 0,
|
||||
`Verifier could not read ${NOTE_PATH}\nstdout: ${catResult.stdout}\nstderr: ${catResult.stderr}`
|
||||
);
|
||||
assertEquals(sanitiseCatStdout(catResult.stdout).trim(), NOTE_CONTENT);
|
||||
} finally {
|
||||
await host.stop();
|
||||
}
|
||||
} finally {
|
||||
await stopLocalRelayIfStarted(relayStarted);
|
||||
await stopCoturnIfStarted(coturnStarted);
|
||||
}
|
||||
});
|
||||
@@ -1,13 +1,13 @@
|
||||
# CLI Deno Test Development Notes
|
||||
|
||||
This document provides an overview of the Deno-based compatibility tests under `src/apps/cli/testdeno/`.
|
||||
The existing bash tests under `src/apps/cli/test/` are preserved, while a Windows-friendly suite is maintained in parallel.
|
||||
The Deno suite is the canonical CLI E2E entry point. P2P scenarios run through the repository Compose entry point so that networking, signalling, and the runner environment are reproducible. Existing Bash tests under `src/apps/cli/test/` remain as legacy implementation references, but are not exposed as supported P2P entry points.
|
||||
|
||||
---
|
||||
|
||||
## Goals
|
||||
|
||||
- Keep existing bash tests intact.
|
||||
- Keep the existing Bash tests as migration references while using Deno and Compose for supported execution.
|
||||
- Provide direct execution from Windows PowerShell.
|
||||
- Establish a TypeScript (Deno) foundation for core end-to-end and integration scenarios.
|
||||
|
||||
@@ -18,6 +18,8 @@ The existing bash tests under `src/apps/cli/test/` are preserved, while a Window
|
||||
```
|
||||
src/apps/cli/testdeno/
|
||||
deno.json
|
||||
run-ci-suite.ts
|
||||
run-compose-p2p.ts
|
||||
CONTRIBUTING_TESTS.md
|
||||
helpers/
|
||||
backgroundCli.ts
|
||||
@@ -56,6 +58,8 @@ src/apps/cli/testdeno/
|
||||
Main tasks:
|
||||
|
||||
- `deno task test`
|
||||
- `deno task test:ci`
|
||||
- `deno task test:p2p:compose`
|
||||
- `deno task test:local`
|
||||
- `deno task test:daemon`
|
||||
- `deno task test:decoupled-vault`
|
||||
@@ -73,6 +77,8 @@ Main tasks:
|
||||
- `deno task test:e2e-couchdb`
|
||||
- `deno task test:e2e-matrix`
|
||||
|
||||
`deno task test` is an alias for the non-P2P `test:ci` suite. The individual P2P tasks are explicit host-direct entry points for cross-platform diagnostics; they are never selected by the default suite or CI. Use `test:p2p:compose` for canonical P2P verification.
|
||||
|
||||
### `helpers/cli.ts`
|
||||
|
||||
- CLI execution wrappers.
|
||||
@@ -206,11 +212,23 @@ Both CouchDB and P2P relay flows are bash-independent.
|
||||
|
||||
## Running tests (PowerShell)
|
||||
|
||||
From the repository root, use the canonical package scripts. `test:e2e:cli` runs the same non-P2P task set selected by the default CLI CI workflow. P2P validation runs in Compose so peer discovery does not depend on host loopback, firewall, or WebRTC candidate behaviour.
|
||||
|
||||
```powershell
|
||||
npm run test:e2e:cli
|
||||
npm run test:e2e:cli:p2p
|
||||
npm run test:e2e:cli:all
|
||||
```
|
||||
|
||||
From `src/apps/cli/testdeno`:
|
||||
|
||||
```powershell
|
||||
cd src/apps/cli/testdeno
|
||||
|
||||
# Canonical suites
|
||||
deno task test:ci
|
||||
deno task test:p2p:compose
|
||||
|
||||
# Local-only set
|
||||
deno task test:local
|
||||
|
||||
@@ -227,7 +245,8 @@ deno task test:decoupled-vault
|
||||
deno task test:remote-commands
|
||||
deno task test:e2e-couchdb
|
||||
|
||||
# P2P-based tests
|
||||
# Explicit host-direct P2P diagnostics for cross-platform investigations.
|
||||
# These are not part of the default suite or release evidence.
|
||||
deno task test:p2p-host
|
||||
deno task test:p2p-peers
|
||||
deno task test:p2p-sync
|
||||
|
||||
@@ -23,8 +23,7 @@
|
||||
// "rootDir": "../../../",
|
||||
/* Path mapping */
|
||||
"paths": {
|
||||
"@/*": ["../../*"],
|
||||
"@lib/*": ["../../lib/src/*", "../../../_types/src/lib/src/*"]
|
||||
"@/*": ["../../*"]
|
||||
}
|
||||
},
|
||||
"include": ["*.ts", "**/*.ts", "**/*.tsx"],
|
||||
|
||||
+31
-21
@@ -1,13 +1,20 @@
|
||||
import { defineConfig } from "vite";
|
||||
import { defaultServerConditions, defaultServerMainFields, defineConfig } from "vite";
|
||||
import { svelte } from "@sveltejs/vite-plugin-svelte";
|
||||
import path from "node:path";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { fileURLToPath, fs, isBuiltin, path } from "@vrtmrz/livesync-commonlib/node";
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const resolve = (...args: string[]) => path.resolve(...args).replace(/\\/g, "/");
|
||||
const repoRoot = path.resolve(__dirname, "../../..");
|
||||
const packageJson = JSON.parse(readFileSync(path.resolve(repoRoot, "package.json"), "utf-8"));
|
||||
const manifestJson = JSON.parse(readFileSync(path.resolve(repoRoot, "manifest.json"), "utf-8"));
|
||||
|
||||
function readVersion(filePath: string): string | undefined {
|
||||
const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
if (typeof parsed !== "object" || parsed === null || !("version" in parsed)) {
|
||||
return undefined;
|
||||
}
|
||||
return typeof parsed.version === "string" ? parsed.version : undefined;
|
||||
}
|
||||
|
||||
const packageVersion = readVersion(path.resolve(repoRoot, "package.json"));
|
||||
const manifestVersion = readVersion(path.resolve(repoRoot, "manifest.json"));
|
||||
// https://vite.dev/config/
|
||||
const defaultExternal = [
|
||||
"obsidian",
|
||||
@@ -47,7 +54,7 @@ function injectBanner(): import("vite").Plugin {
|
||||
name: "inject-banner",
|
||||
generateBundle(_options, bundle) {
|
||||
for (const chunk of Object.values(bundle)) {
|
||||
if (chunk.type === "chunk" && chunk.fileName.startsWith("entrypoint")) {
|
||||
if (chunk.type === "chunk" && chunk.isEntry) {
|
||||
// Insert after the shebang line if present, otherwise at the top.
|
||||
if (chunk.code.startsWith("#!")) {
|
||||
const newline = chunk.code.indexOf("\n");
|
||||
@@ -62,20 +69,27 @@ function injectBanner(): import("vite").Plugin {
|
||||
};
|
||||
}
|
||||
|
||||
const buildInputs: Record<string, string> = {
|
||||
index: resolve(__dirname, "entrypoint.ts"),
|
||||
};
|
||||
if (process.env.LIVESYNC_CLI_TEST_SUPPORT === "1") {
|
||||
buildInputs["p2p-lifecycle-test"] = resolve(__dirname, "test-support/p2p-lifecycle-entrypoint.ts");
|
||||
}
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [svelte(), injectBanner()],
|
||||
resolve: {
|
||||
// This bundle runs in Node. Vite's client defaults include the `browser`
|
||||
// export condition, which would select Commonlib's inline Web Worker.
|
||||
conditions: [...defaultServerConditions],
|
||||
mainFields: [...defaultServerMainFields],
|
||||
alias: {
|
||||
"@lib/worker/bgWorker.ts": "../../lib/src/worker/bgWorker.mock.ts",
|
||||
"@lib/pouchdb/pouchdb-browser.ts": resolve(__dirname, "lib/pouchdb-node.ts"),
|
||||
// The CLI runs on Node.js; force AWS XML builder to its CJS Node entry
|
||||
// so Vite does not resolve the browser DOMParser-based XML parser.
|
||||
"@aws-sdk/xml-builder": resolve(__dirname, "../../../node_modules/@aws-sdk/xml-builder/dist-cjs/index.js"),
|
||||
// Force fflate to the Node CJS entry; browser entry expects Web Worker globals.
|
||||
fflate: resolve(__dirname, "../../../node_modules/fflate/lib/node.cjs"),
|
||||
"@": resolve(__dirname, "../../"),
|
||||
"@lib": resolve(__dirname, "../../lib/src"),
|
||||
"../../src/worker/bgWorker.ts": "../../src/worker/bgWorker.mock.ts",
|
||||
},
|
||||
},
|
||||
|
||||
@@ -85,26 +99,22 @@ export default defineConfig({
|
||||
emptyOutDir: true,
|
||||
minify: false,
|
||||
rollupOptions: {
|
||||
input: {
|
||||
index: resolve(__dirname, "entrypoint.ts"),
|
||||
},
|
||||
input: buildInputs,
|
||||
external: (id) => {
|
||||
if (isBuiltin(id)) return true;
|
||||
if (defaultExternal.includes(id)) return true;
|
||||
if (id.startsWith(".") || id.startsWith("/")) return false;
|
||||
if (id.startsWith("@/") || id.startsWith("@lib/")) return false;
|
||||
if (id.startsWith("@/")) return false;
|
||||
if (id.endsWith(".ts") || id.endsWith(".js")) return false;
|
||||
if (id === "fs" || id === "fs/promises" || id === "path" || id === "crypto" || id === "worker_threads")
|
||||
return true;
|
||||
if (id.startsWith("pouchdb-")) return true;
|
||||
if (id.startsWith("werift")) return true;
|
||||
if (id.startsWith("node:")) return true;
|
||||
return false;
|
||||
},
|
||||
},
|
||||
lib: {
|
||||
entry: resolve(__dirname, "entrypoint.ts"),
|
||||
formats: ["cjs"],
|
||||
fileName: "index",
|
||||
fileName: (_format, entryName) => `${entryName}.cjs`,
|
||||
},
|
||||
},
|
||||
define: {
|
||||
@@ -112,7 +122,7 @@ export default defineConfig({
|
||||
global: "globalThis",
|
||||
nonInteractive: "true",
|
||||
// localStorage: "undefined", // Prevent usage of localStorage in the CLI environment
|
||||
MANIFEST_VERSION: JSON.stringify(process.env.MANIFEST_VERSION || manifestJson.version || "0.0.0"),
|
||||
PACKAGE_VERSION: JSON.stringify(process.env.PACKAGE_VERSION || packageJson.version || "0.0.0"),
|
||||
MANIFEST_VERSION: JSON.stringify(process.env.MANIFEST_VERSION || manifestVersion || "0.0.0"),
|
||||
PACKAGE_VERSION: JSON.stringify(process.env.PACKAGE_VERSION || packageVersion || "0.0.0"),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
import type { LOG_LEVEL } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
|
||||
/** Diagnostic output supplied by the Webapp host. */
|
||||
export type WebAppLog = (message: unknown, level: LOG_LEVEL, key?: string) => void;
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { FilePath, UXFileInfoStub, UXFolderInfo } from "@lib/common/types";
|
||||
import type { IConversionAdapter } from "@lib/serviceModules/adapters";
|
||||
import type { FilePath, UXFileInfoStub, UXFolderInfo } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { IConversionAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
|
||||
import type { FSAPIFile, FSAPIFolder } from "./FSAPITypes";
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import type { FilePath, UXStat } from "@lib/common/types";
|
||||
import type { IFileSystemAdapter } from "@lib/serviceModules/adapters";
|
||||
import { LOG_LEVEL_NOTICE, type FilePath, type UXStat } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { IFileSystemAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
|
||||
import { FSAPIPathAdapter } from "./FSAPIPathAdapter";
|
||||
import { FSAPITypeGuardAdapter } from "./FSAPITypeGuardAdapter";
|
||||
import { FSAPIConversionAdapter } from "./FSAPIConversionAdapter";
|
||||
import { FSAPIStorageAdapter } from "./FSAPIStorageAdapter";
|
||||
import { FileSystemAccessStorageAdapter } from "@vrtmrz/livesync-commonlib/browser";
|
||||
import { FSAPIVaultAdapter } from "./FSAPIVaultAdapter";
|
||||
import type { FSAPIFile, FSAPIFolder, FSAPIStat } from "./FSAPITypes";
|
||||
import { shareRunningResult } from "octagonal-wheels/concurrency/lock_v2";
|
||||
import type { WebAppLog } from "@/apps/webapp/WebAppLog";
|
||||
|
||||
/**
|
||||
* Complete file system adapter implementation for FileSystem API
|
||||
@@ -15,29 +16,32 @@ export class FSAPIFileSystemAdapter implements IFileSystemAdapter<FSAPIFile, FSA
|
||||
readonly path: FSAPIPathAdapter;
|
||||
readonly typeGuard: FSAPITypeGuardAdapter;
|
||||
readonly conversion: FSAPIConversionAdapter;
|
||||
readonly storage: FSAPIStorageAdapter;
|
||||
readonly storage: FileSystemAccessStorageAdapter;
|
||||
readonly vault: FSAPIVaultAdapter;
|
||||
|
||||
private fileCache = new Map<string, FSAPIFile>();
|
||||
private handleCache = new Map<string, FileSystemFileHandle>();
|
||||
|
||||
constructor(private rootHandle: FileSystemDirectoryHandle) {
|
||||
constructor(
|
||||
private rootHandle: FileSystemDirectoryHandle,
|
||||
private readonly addLog: WebAppLog
|
||||
) {
|
||||
this.path = new FSAPIPathAdapter();
|
||||
this.typeGuard = new FSAPITypeGuardAdapter();
|
||||
this.conversion = new FSAPIConversionAdapter();
|
||||
this.storage = new FSAPIStorageAdapter(rootHandle);
|
||||
this.storage = new FileSystemAccessStorageAdapter(rootHandle);
|
||||
this.vault = new FSAPIVaultAdapter(rootHandle);
|
||||
}
|
||||
|
||||
private normalisePath(path: FilePath | string): string {
|
||||
return this.path.normalisePath(path as string);
|
||||
return this.path.normalisePath(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file handle for a given path
|
||||
*/
|
||||
private async getFileHandleByPath(p: FilePath | string): Promise<FileSystemFileHandle | null> {
|
||||
const pathStr = p as string;
|
||||
const pathStr = p;
|
||||
|
||||
// Check cache first
|
||||
const cached = this.handleCache.get(pathStr);
|
||||
@@ -53,9 +57,11 @@ export class FSAPIFileSystemAdapter implements IFileSystemAdapter<FSAPIFile, FSA
|
||||
// Navigate to the parent directory
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
currentHandle = await currentHandle.getDirectoryHandle(parts[i]);
|
||||
if (currentHandle.name !== parts[i]) return null;
|
||||
}
|
||||
|
||||
const fileHandle = await currentHandle.getFileHandle(fileName);
|
||||
if (fileHandle.name !== fileName) return null;
|
||||
this.handleCache.set(pathStr, fileHandle);
|
||||
return fileHandle;
|
||||
} catch {
|
||||
@@ -110,6 +116,14 @@ export class FSAPIFileSystemAdapter implements IFileSystemAdapter<FSAPIFile, FSA
|
||||
return Array.from(this.fileCache.values());
|
||||
}
|
||||
|
||||
async renameFile(file: FSAPIFile, newPath: string): Promise<FSAPIFile> {
|
||||
await this.vault.rename(file, newPath);
|
||||
this.clearCache();
|
||||
const renamedFile = await this.refreshFile(newPath);
|
||||
if (!renamedFile) throw new Error(`Could not find renamed file: ${newPath}`);
|
||||
return renamedFile;
|
||||
}
|
||||
|
||||
async statFromNative(file: FSAPIFile): Promise<UXStat> {
|
||||
// Refresh stat from the file handle
|
||||
try {
|
||||
@@ -203,7 +217,11 @@ export class FSAPIFileSystemAdapter implements IFileSystemAdapter<FSAPIFile, FSA
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error scanning directory ${relativePath}:`, error);
|
||||
this.addLog(
|
||||
`Error scanning directory '${relativePath || "."}': ${String(error)}`,
|
||||
LOG_LEVEL_NOTICE,
|
||||
"fsapi-scan"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { FilePath } from "@lib/common/types";
|
||||
import type { IPathAdapter } from "@lib/serviceModules/adapters";
|
||||
import type { FilePath } from "@vrtmrz/livesync-commonlib/compat/common/types";
|
||||
import type { IPathAdapter } from "@vrtmrz/livesync-commonlib/compat/serviceModules/adapters";
|
||||
import type { FSAPIFile } from "./FSAPITypes";
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
import type { UXDataWriteOptions } from "@lib/common/types";
|
||||
import type { IStorageAdapter } from "@lib/serviceModules/adapters";
|
||||
import type { FSAPIStat } from "./FSAPITypes";
|
||||
|
||||
/**
|
||||
* Storage adapter implementation for FileSystem API
|
||||
*/
|
||||
export class FSAPIStorageAdapter implements IStorageAdapter<FSAPIStat> {
|
||||
constructor(private rootHandle: FileSystemDirectoryHandle) {}
|
||||
|
||||
/**
|
||||
* Resolve a path to directory and file handles
|
||||
*/
|
||||
private async resolvePath(p: string): Promise<{
|
||||
dirHandle: FileSystemDirectoryHandle;
|
||||
fileName: string;
|
||||
} | null> {
|
||||
try {
|
||||
const parts = p.split("/").filter((part) => part !== "");
|
||||
if (parts.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let currentHandle = this.rootHandle;
|
||||
const fileName = parts[parts.length - 1];
|
||||
|
||||
// Navigate to the parent directory
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
currentHandle = await currentHandle.getDirectoryHandle(parts[i]);
|
||||
}
|
||||
|
||||
return { dirHandle: currentHandle, fileName };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file handle for a given path
|
||||
*/
|
||||
private async getFileHandle(p: string): Promise<FileSystemFileHandle | null> {
|
||||
const resolved = await this.resolvePath(p);
|
||||
if (!resolved) return null;
|
||||
|
||||
try {
|
||||
return await resolved.dirHandle.getFileHandle(resolved.fileName);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get directory handle for a given path
|
||||
*/
|
||||
private async getDirectoryHandle(p: string): Promise<FileSystemDirectoryHandle | null> {
|
||||
try {
|
||||
const parts = p.split("/").filter((part) => part !== "");
|
||||
if (parts.length === 0) {
|
||||
return this.rootHandle;
|
||||
}
|
||||
|
||||
let currentHandle = this.rootHandle;
|
||||
for (const part of parts) {
|
||||
currentHandle = await currentHandle.getDirectoryHandle(part);
|
||||
}
|
||||
|
||||
return currentHandle;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async exists(p: string): Promise<boolean> {
|
||||
const fileHandle = await this.getFileHandle(p);
|
||||
if (fileHandle) return true;
|
||||
|
||||
const dirHandle = await this.getDirectoryHandle(p);
|
||||
return dirHandle !== null;
|
||||
}
|
||||
|
||||
async trystat(p: string): Promise<FSAPIStat | null> {
|
||||
// Try as file first
|
||||
const fileHandle = await this.getFileHandle(p);
|
||||
if (fileHandle) {
|
||||
const file = await fileHandle.getFile();
|
||||
return {
|
||||
size: file.size,
|
||||
mtime: file.lastModified,
|
||||
ctime: file.lastModified,
|
||||
type: "file",
|
||||
};
|
||||
}
|
||||
|
||||
// Try as directory
|
||||
const dirHandle = await this.getDirectoryHandle(p);
|
||||
if (dirHandle) {
|
||||
return {
|
||||
size: 0,
|
||||
mtime: Date.now(),
|
||||
ctime: Date.now(),
|
||||
type: "folder",
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async stat(p: string): Promise<FSAPIStat | null> {
|
||||
return await this.trystat(p);
|
||||
}
|
||||
|
||||
async mkdir(p: string): Promise<void> {
|
||||
const parts = p.split("/").filter((part) => part !== "");
|
||||
let currentHandle = this.rootHandle;
|
||||
|
||||
for (const part of parts) {
|
||||
currentHandle = await currentHandle.getDirectoryHandle(part, { create: true });
|
||||
}
|
||||
}
|
||||
|
||||
async remove(p: string): Promise<void> {
|
||||
const resolved = await this.resolvePath(p);
|
||||
if (!resolved) return;
|
||||
|
||||
await resolved.dirHandle.removeEntry(resolved.fileName, { recursive: true });
|
||||
}
|
||||
|
||||
async read(p: string): Promise<string> {
|
||||
const fileHandle = await this.getFileHandle(p);
|
||||
if (!fileHandle) {
|
||||
throw new Error(`File not found: ${p}`);
|
||||
}
|
||||
|
||||
const file = await fileHandle.getFile();
|
||||
return await file.text();
|
||||
}
|
||||
|
||||
async readBinary(p: string): Promise<ArrayBuffer> {
|
||||
const fileHandle = await this.getFileHandle(p);
|
||||
if (!fileHandle) {
|
||||
throw new Error(`File not found: ${p}`);
|
||||
}
|
||||
|
||||
const file = await fileHandle.getFile();
|
||||
return await file.arrayBuffer();
|
||||
}
|
||||
|
||||
async write(p: string, data: string, options?: UXDataWriteOptions): Promise<void> {
|
||||
const resolved = await this.resolvePath(p);
|
||||
if (!resolved) {
|
||||
throw new Error(`Invalid path: ${p}`);
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
await this.mkdir(p.split("/").slice(0, -1).join("/"));
|
||||
|
||||
const fileHandle = await resolved.dirHandle.getFileHandle(resolved.fileName, { create: true });
|
||||
const writable = await fileHandle.createWritable();
|
||||
await writable.write(data);
|
||||
await writable.close();
|
||||
}
|
||||
|
||||
async writeBinary(p: string, data: ArrayBuffer, options?: UXDataWriteOptions): Promise<void> {
|
||||
const resolved = await this.resolvePath(p);
|
||||
if (!resolved) {
|
||||
throw new Error(`Invalid path: ${p}`);
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
await this.mkdir(p.split("/").slice(0, -1).join("/"));
|
||||
|
||||
const fileHandle = await resolved.dirHandle.getFileHandle(resolved.fileName, { create: true });
|
||||
const writable = await fileHandle.createWritable();
|
||||
await writable.write(data);
|
||||
await writable.close();
|
||||
}
|
||||
|
||||
async append(p: string, data: string, options?: UXDataWriteOptions): Promise<void> {
|
||||
const existing = await this.exists(p);
|
||||
if (existing) {
|
||||
const currentContent = await this.read(p);
|
||||
await this.write(p, currentContent + data, options);
|
||||
} else {
|
||||
await this.write(p, data, options);
|
||||
}
|
||||
}
|
||||
|
||||
async list(basePath: string): Promise<{ files: string[]; folders: string[] }> {
|
||||
const dirHandle = await this.getDirectoryHandle(basePath);
|
||||
if (!dirHandle) {
|
||||
return { files: [], folders: [] };
|
||||
}
|
||||
|
||||
const files: string[] = [];
|
||||
const folders: string[] = [];
|
||||
|
||||
// Use AsyncIterator instead of .values() for better compatibility
|
||||
for await (const [name, entry] of (
|
||||
dirHandle as unknown as {
|
||||
entries(): AsyncIterable<[string, FileSystemHandle]>;
|
||||
}
|
||||
).entries()) {
|
||||
const entryPath = basePath ? `${basePath}/${name}` : name;
|
||||
|
||||
if (entry.kind === "directory") {
|
||||
folders.push(entryPath);
|
||||
} else if (entry.kind === "file") {
|
||||
files.push(entryPath);
|
||||
}
|
||||
}
|
||||
|
||||
return { files, folders };
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user