mirror of
https://github.com/vrtmrz/obsidian-livesync.git
synced 2026-11-01 22:08:18 +00:00
Migrate CLI E2E tests to Node.js 26
Run the CLI subprocess suites and benchmark tools as native TypeScript with node:test, and move them under test/e2e-cli. Update CI and Compose runners, run local smoke tests on Linux, macOS, and Windows, and collect source-mapped CLI coverage with c8.
This commit is contained in:
vorotamoroz committed
1 parent
a0987016bb
commit
2433d117c2
91 files changed
+2219
-1615
No files matched your search
@@ -0,0 +1,125 @@
|
||||
import { writeSync } from "node:fs";
|
||||
import { spawnProcess, type RunningProcess } from "./process.ts";
|
||||
import { CLI_DIR, TEE_ENABLED, formatTeeCommand, createLineTeeWriter } from "./cli.ts";
|
||||
import { join } from "node:path";
|
||||
|
||||
const CLI_DIST = join(CLI_DIR, "dist", "index.cjs");
|
||||
const VERBOSE_ENABLED = process.env.LIVESYNC_CLI_VERBOSE === "1";
|
||||
const DEBUG_ENABLED = process.env.LIVESYNC_CLI_DEBUG === "1";
|
||||
|
||||
function decorateArgs(args: string[]): string[] {
|
||||
return DEBUG_ENABLED ? ["-d", ...args] : VERBOSE_ENABLED ? ["-v", ...args] : args;
|
||||
}
|
||||
|
||||
async function pump(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
sink: (text: string) => void,
|
||||
teeTarget: { write: (chunk: Uint8Array) => void; close: () => void } | null
|
||||
): Promise<void> {
|
||||
const reader = stream.getReader();
|
||||
const dec = new TextDecoder();
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value) continue;
|
||||
sink(dec.decode(value, { stream: true }));
|
||||
if (teeTarget) {
|
||||
teeTarget.write(value);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (teeTarget) teeTarget.close();
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
export class BackgroundCliProcess {
|
||||
#stdout = "";
|
||||
#stderr = "";
|
||||
#stdoutDone: Promise<void>;
|
||||
#stderrDone: Promise<void>;
|
||||
|
||||
readonly child: RunningProcess;
|
||||
readonly args: string[];
|
||||
|
||||
constructor(child: RunningProcess, args: string[]) {
|
||||
this.child = child;
|
||||
this.args = args;
|
||||
const cliArgs = decorateArgs(args);
|
||||
this.#stdoutDone = pump(
|
||||
child.stdout,
|
||||
(text) => {
|
||||
this.#stdout += text;
|
||||
},
|
||||
TEE_ENABLED ? createLineTeeWriter(child.pid, "stdout", (chunk) => writeSync(1, chunk)) : null
|
||||
);
|
||||
this.#stderrDone = pump(
|
||||
child.stderr,
|
||||
(text) => {
|
||||
this.#stderr += text;
|
||||
},
|
||||
TEE_ENABLED ? createLineTeeWriter(child.pid, "stderr", (chunk) => writeSync(2, chunk)) : null
|
||||
);
|
||||
}
|
||||
|
||||
get stdout(): string {
|
||||
return this.#stdout;
|
||||
}
|
||||
|
||||
get stderr(): string {
|
||||
return this.#stderr;
|
||||
}
|
||||
|
||||
get combined(): string {
|
||||
return this.#stdout + this.#stderr;
|
||||
}
|
||||
|
||||
async waitUntilContains(needle: string, timeoutMs = 15000): Promise<void> {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
if (this.combined.includes(needle)) return;
|
||||
const status = await Promise.race([
|
||||
this.child.status.then((s) => ({ type: "status" as const, status: s })),
|
||||
new Promise<{ type: "tick" }>((resolve) => setTimeout(() => resolve({ type: "tick" }), 100)),
|
||||
]);
|
||||
if (status.type === "status") {
|
||||
throw new Error(
|
||||
`Background CLI exited before '${needle}' appeared (code ${status.status.code})\n${this.combined}`
|
||||
);
|
||||
}
|
||||
}
|
||||
throw new Error(`Timed out waiting for '${needle}'\n${this.combined}`);
|
||||
}
|
||||
|
||||
async stop(): Promise<number> {
|
||||
try {
|
||||
this.child.kill("SIGTERM");
|
||||
} catch {
|
||||
// ignore already-exited processes
|
||||
}
|
||||
const status = await this.child.status;
|
||||
await Promise.all([this.#stdoutDone, this.#stderrDone]);
|
||||
return status.code;
|
||||
}
|
||||
}
|
||||
|
||||
export function startCliInBackground(...args: string[]): BackgroundCliProcess {
|
||||
const cliArgs = decorateArgs(args);
|
||||
const child = spawnProcess(process.execPath, {
|
||||
args: [CLI_DIST, ...cliArgs],
|
||||
cwd: CLI_DIR,
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
});
|
||||
|
||||
if (TEE_ENABLED) {
|
||||
writeSync(
|
||||
1,
|
||||
new TextEncoder().encode(`[CLI tee pid=${child.pid}] process(bg): ${formatTeeCommand(cliArgs)}\n`)
|
||||
);
|
||||
}
|
||||
|
||||
return new BackgroundCliProcess(child, args);
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { DatasetEntry } from "./dataset.ts";
|
||||
|
||||
export type BenchmarkVerificationMode = "all" | "sample";
|
||||
|
||||
export type BenchmarkVerificationResult = {
|
||||
verificationMode: BenchmarkVerificationMode;
|
||||
verifiedFiles: number;
|
||||
verificationComplete: boolean;
|
||||
datasetDigestSha256: string;
|
||||
};
|
||||
|
||||
function sha256(bytes: Uint8Array): string {
|
||||
return createHash("sha256").update(bytes).digest("hex");
|
||||
}
|
||||
|
||||
export function parseBenchmarkVerificationMode(
|
||||
raw: string | undefined,
|
||||
fallback: BenchmarkVerificationMode = "sample",
|
||||
): BenchmarkVerificationMode {
|
||||
const value = raw?.trim().toLowerCase();
|
||||
if (!value) return fallback;
|
||||
if (value === "all" || value === "sample") return value;
|
||||
throw new Error(`BENCH_VERIFY_MODE must be 'all' or 'sample', got '${raw}'`);
|
||||
}
|
||||
|
||||
export function selectVerificationEntries(
|
||||
entries: DatasetEntry[],
|
||||
mode: BenchmarkVerificationMode,
|
||||
): DatasetEntry[] {
|
||||
if (mode === "all" || entries.length === 0) return [...entries];
|
||||
|
||||
const md = entries.find((entry) => entry.kind === "md");
|
||||
const bin = entries.find((entry) => entry.kind === "bin");
|
||||
const middle = entries[Math.floor(entries.length / 2)];
|
||||
const last = entries[entries.length - 1];
|
||||
const selected = new Map<string, DatasetEntry>();
|
||||
for (const entry of [md, bin, middle, last]) {
|
||||
if (entry) selected.set(entry.relativePath, entry);
|
||||
}
|
||||
return [...selected.values()];
|
||||
}
|
||||
|
||||
export async function computeDatasetDigestSha256(
|
||||
entries: DatasetEntry[],
|
||||
): Promise<string> {
|
||||
const manifest: string[] = [];
|
||||
for (const entry of entries) {
|
||||
const contentDigest = sha256(await readFile(entry.absolutePath));
|
||||
manifest.push(
|
||||
`${entry.kind}\t${entry.relativePath}\t${entry.size}\t${contentDigest}`,
|
||||
);
|
||||
}
|
||||
return sha256(new TextEncoder().encode(manifest.join("\n")));
|
||||
}
|
||||
|
||||
export async function verifyBenchmarkDataset(
|
||||
entries: DatasetEntry[],
|
||||
mode: BenchmarkVerificationMode,
|
||||
verifyEntry: (entry: DatasetEntry) => Promise<void>,
|
||||
): Promise<BenchmarkVerificationResult> {
|
||||
const selected = selectVerificationEntries(entries, mode);
|
||||
for (const entry of selected) {
|
||||
await verifyEntry(entry);
|
||||
}
|
||||
|
||||
return {
|
||||
verificationMode: mode,
|
||||
verifiedFiles: selected.length,
|
||||
verificationComplete: selected.length === entries.length,
|
||||
datasetDigestSha256: await computeDatasetDigestSha256(entries),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
import { writeSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { spawnProcess } from "./process.ts";
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { CLI_DIR, CLI_DIST } from "./paths.ts";
|
||||
export { CLI_DIR, CLI_DIST } from "./paths.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result type
|
||||
// ---------------------------------------------------------------------------
|
||||
export interface CliResult {
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
/** stdout + stderr concatenated — useful for assertion messages. */
|
||||
combined: string;
|
||||
code: number;
|
||||
}
|
||||
|
||||
export const TEE_ENABLED = process.env.LIVESYNC_TEST_TEE === "1";
|
||||
const VERBOSE_ENABLED = process.env.LIVESYNC_CLI_VERBOSE === "1";
|
||||
const DEBUG_ENABLED = process.env.LIVESYNC_CLI_DEBUG === "1";
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function concatChunks(chunks: Uint8Array[]): Uint8Array {
|
||||
const total = chunks.reduce((n, c) => n + c.length, 0);
|
||||
const out = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, offset);
|
||||
offset += c.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function formatTeeCommand(args: string[]): string {
|
||||
return [process.execPath, CLI_DIST, ...args].map((part) => JSON.stringify(part)).join(" ");
|
||||
}
|
||||
|
||||
export function createLineTeeWriter(
|
||||
pid: number,
|
||||
streamName: "stdout" | "stderr",
|
||||
writer: (chunk: Uint8Array) => void
|
||||
): { write: (chunk: Uint8Array) => void; close: () => void } {
|
||||
const enc = new TextEncoder();
|
||||
const dec = new TextDecoder();
|
||||
let pending = "";
|
||||
let headerWritten = false;
|
||||
const emitLine = (line: string) => {
|
||||
if (!headerWritten) {
|
||||
writer(enc.encode(`[CLI tee pid=${pid}:${streamName}]\n`));
|
||||
headerWritten = true;
|
||||
}
|
||||
writer(enc.encode(`[CLI tee pid=${pid}:${streamName}] ${line}\n`));
|
||||
};
|
||||
|
||||
const flush = (final = false) => {
|
||||
let index = pending.indexOf("\n");
|
||||
while (index >= 0) {
|
||||
const line = pending.slice(0, index).replace(/\r$/, "");
|
||||
pending = pending.slice(index + 1);
|
||||
emitLine(line);
|
||||
index = pending.indexOf("\n");
|
||||
}
|
||||
if (final && pending.length > 0) {
|
||||
emitLine(pending.replace(/\r$/, ""));
|
||||
pending = "";
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
write(chunk: Uint8Array) {
|
||||
pending += dec.decode(chunk, { stream: true });
|
||||
flush(false);
|
||||
},
|
||||
close() {
|
||||
pending += dec.decode();
|
||||
flush(true);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function collectStream(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
teeTarget: { write: (chunk: Uint8Array) => void; close: () => void } | null
|
||||
): Promise<Uint8Array> {
|
||||
const reader = stream.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (value) {
|
||||
chunks.push(value);
|
||||
if (teeTarget) {
|
||||
teeTarget.write(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (teeTarget) {
|
||||
teeTarget.close();
|
||||
}
|
||||
reader.releaseLock();
|
||||
}
|
||||
return concatChunks(chunks);
|
||||
}
|
||||
|
||||
async function runNodeCommand(args: string[], stdinData?: Uint8Array): Promise<CliResult> {
|
||||
const cliArgs = DEBUG_ENABLED ? ["-d", ...args] : VERBOSE_ENABLED ? ["-v", ...args] : args;
|
||||
const child = spawnProcess(process.execPath, {
|
||||
args: [CLI_DIST, ...cliArgs],
|
||||
cwd: CLI_DIR,
|
||||
stdin: stdinData ? "piped" : "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
});
|
||||
|
||||
if (TEE_ENABLED) {
|
||||
writeSync(1, new TextEncoder().encode(`[CLI tee pid=${child.pid}] process: ${formatTeeCommand(cliArgs)}\n`));
|
||||
}
|
||||
|
||||
const stdoutPromise = collectStream(
|
||||
child.stdout,
|
||||
TEE_ENABLED ? createLineTeeWriter(child.pid, "stdout", (chunk) => writeSync(1, chunk)) : null
|
||||
);
|
||||
const stderrPromise = collectStream(
|
||||
child.stderr,
|
||||
TEE_ENABLED ? createLineTeeWriter(child.pid, "stderr", (chunk) => writeSync(2, chunk)) : null
|
||||
);
|
||||
|
||||
if (stdinData) {
|
||||
const w = child.stdin.getWriter();
|
||||
await w.write(stdinData);
|
||||
await w.close();
|
||||
}
|
||||
|
||||
const [status, stdout, stderr] = await Promise.all([child.status, stdoutPromise, stderrPromise]);
|
||||
|
||||
const dec = new TextDecoder();
|
||||
const out = dec.decode(stdout);
|
||||
const err = dec.decode(stderr);
|
||||
return { stdout: out, stderr: err, combined: out + err, code: status.code };
|
||||
}
|
||||
|
||||
function isTransientNetworkError(message: string): boolean {
|
||||
const m = message.toLowerCase();
|
||||
return (
|
||||
m.includes("fetch failed") ||
|
||||
m.includes("econnreset") ||
|
||||
m.includes("econnrefused") ||
|
||||
m.includes("und_err_socket") ||
|
||||
m.includes("other side closed")
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Core runners
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Run the CLI (node dist/index.cjs) with the supplied arguments.
|
||||
* Pass the vault / DB path as the first argument, exactly as the bash helpers
|
||||
* do. Does NOT throw on non-zero exit — check `.code` yourself.
|
||||
*/
|
||||
export async function runCli(...args: string[]): Promise<CliResult> {
|
||||
const retries = Number(process.env.LIVESYNC_CLI_RETRY ?? "0");
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const result = await runNodeCommand(args);
|
||||
if (result.code === 0) return result;
|
||||
|
||||
if (attempt >= retries || !isTransientNetworkError(result.combined)) {
|
||||
return result;
|
||||
}
|
||||
const waitMs = 400 * (attempt + 1);
|
||||
console.warn(`[WARN] transient CLI failure, retrying (${attempt + 1}/${retries}) in ${waitMs}ms`);
|
||||
await sleep(waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the CLI and throw if it exits non-zero. Returns stdout.
|
||||
*/
|
||||
export async function runCliOrFail(...args: string[]): Promise<string> {
|
||||
const r = await runCli(...args);
|
||||
if (r.code !== 0) {
|
||||
throw new Error(`CLI exited with code ${r.code}\nstdout: ${r.stdout}\nstderr: ${r.stderr}`);
|
||||
}
|
||||
return r.stdout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the CLI with data piped to stdin (equivalent to `echo … | run_cli …`
|
||||
* or `cat file | run_cli …`).
|
||||
*/
|
||||
export async function runCliWithInput(input: string | Uint8Array, ...args: string[]): Promise<CliResult> {
|
||||
const data = typeof input === "string" ? new TextEncoder().encode(input) : input;
|
||||
|
||||
const retries = Number(process.env.LIVESYNC_CLI_RETRY ?? "0");
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const result = await runNodeCommand(args, data);
|
||||
if (result.code === 0) return result;
|
||||
|
||||
if (attempt >= retries || !isTransientNetworkError(result.combined)) {
|
||||
return result;
|
||||
}
|
||||
const waitMs = 400 * (attempt + 1);
|
||||
console.warn(`[WARN] transient CLI(stdin) failure, retrying (${attempt + 1}/${retries}) in ${waitMs}ms`);
|
||||
await sleep(waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* runCliWithInput — throws on non-zero exit, returns stdout.
|
||||
*/
|
||||
export async function runCliWithInputOrFail(input: string | Uint8Array, ...args: string[]): Promise<string> {
|
||||
const r = await runCliWithInput(input, ...args);
|
||||
if (r.code !== 0) {
|
||||
throw new Error(`CLI (with stdin) exited with code ${r.code}\nstdout: ${r.stdout}\nstderr: ${r.stderr}`);
|
||||
}
|
||||
return r.stdout;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Output helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Strip the CLIWatchAdapter banner line that `cat` emits. */
|
||||
export function sanitiseCatStdout(raw: string): string {
|
||||
return raw
|
||||
.split("\n")
|
||||
.filter((l) => l !== "[CLIWatchAdapter] File watching is not enabled in CLI version")
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Assertions (parity with test-helpers.sh)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function assertContains(haystack: string, needle: string, message: string): void {
|
||||
if (!haystack.includes(needle)) {
|
||||
throw new Error(`[FAIL] ${message}\nExpected to find: ${JSON.stringify(needle)}\nActual output:\n${haystack}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function assertNotContains(haystack: string, needle: string, message: string): void {
|
||||
if (haystack.includes(needle)) {
|
||||
throw new Error(`[FAIL] ${message}\nDid NOT expect: ${JSON.stringify(needle)}\nActual output:\n${haystack}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertFilesEqual(expectedPath: string, actualPath: string, message: string): Promise<void> {
|
||||
const [expected, actual] = await Promise.all([readFile(expectedPath), readFile(actualPath)]);
|
||||
if (expected.length !== actual.length || expected.some((b, i) => b !== actual[i])) {
|
||||
const hex = (data: Uint8Array) => createHash("sha256").update(data).digest("hex");
|
||||
throw new Error(
|
||||
`[FAIL] ${message}\nexpected SHA-256: ${await hex(expected)}\nactual SHA-256: ${await hex(actual)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// JSON helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function readJsonFile<T = Record<string, unknown>>(filePath: string): Promise<T> {
|
||||
return JSON.parse(await readFile(filePath, "utf8")) as T;
|
||||
}
|
||||
|
||||
export function jsonStringField(jsonText: string, field: string): string {
|
||||
const data = JSON.parse(jsonText) as Record<string, unknown>;
|
||||
const value = data[field];
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
export function jsonFieldIsNa(data: Record<string, unknown>, field: string): boolean {
|
||||
return data[field] === "N/A";
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
import { copyFile, mkdtemp, mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { runProcess } from "./process.ts";
|
||||
import type { DatasetEntry, DatasetKind } from "./dataset.ts";
|
||||
import { REPOSITORY_ROOT } from "./paths.ts";
|
||||
|
||||
export type CompressionDatasetEntry = DatasetEntry & {
|
||||
source: string;
|
||||
};
|
||||
|
||||
export type CompressionDataset = {
|
||||
entries: CompressionDatasetEntry[];
|
||||
totalFiles: number;
|
||||
totalBytes: number;
|
||||
bytesByKind: Record<DatasetKind, number>;
|
||||
filesByKind: Record<DatasetKind, number>;
|
||||
jpegGenerator: string;
|
||||
};
|
||||
|
||||
export type JpegEncoder = (inputPpm: string, outputJpeg: string) => Promise<string>;
|
||||
|
||||
const ALL_KINDS: DatasetKind[] = ["md", "jpg", "png", "json", "ts", "gz", "bin"];
|
||||
|
||||
function fnv1a32(input: string): number {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
hash ^= input.charCodeAt(i) & 0xff;
|
||||
hash = Math.imul(hash, 0x01000193);
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
function createXorshift32(seed: number): () => number {
|
||||
let state = seed || 0x9e3779b9;
|
||||
return () => {
|
||||
state ^= state << 13;
|
||||
state ^= state >>> 17;
|
||||
state ^= state << 5;
|
||||
return state >>> 0;
|
||||
};
|
||||
}
|
||||
|
||||
function createSyntheticPpm(width: number, height: number, seed: string, textured: boolean): Uint8Array {
|
||||
const header = new TextEncoder().encode(`P6\n${width} ${height}\n255\n`);
|
||||
const pixels = new Uint8Array(width * height * 3);
|
||||
const nextRandom = createXorshift32(fnv1a32(seed));
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const offset = (y * width + x) * 3;
|
||||
const noise = textured ? (nextRandom() & 0x3f) - 32 : 0;
|
||||
pixels[offset] = Math.max(0, Math.min(255, Math.floor((x * 255) / (width - 1)) + noise));
|
||||
pixels[offset + 1] = Math.max(0, Math.min(255, Math.floor((y * 255) / (height - 1)) + noise));
|
||||
pixels[offset + 2] = Math.max(0, Math.min(255, Math.floor(((x + y) * 255) / (width + height - 2)) - noise));
|
||||
}
|
||||
}
|
||||
const result = new Uint8Array(header.length + pixels.length);
|
||||
result.set(header);
|
||||
result.set(pixels, header.length);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function gzip(input: Uint8Array): Promise<Uint8Array> {
|
||||
const copied = new Uint8Array(input.byteLength);
|
||||
copied.set(input);
|
||||
const stream = new Blob([copied.buffer]).stream().pipeThrough(new CompressionStream("gzip"));
|
||||
return new Uint8Array(await new Response(stream).arrayBuffer());
|
||||
}
|
||||
|
||||
export async function encodeJpegWithCjpeg(inputPpm: string, outputJpeg: string): Promise<string> {
|
||||
let result: Awaited<ReturnType<typeof runProcess>>;
|
||||
try {
|
||||
result = await runProcess("cjpeg", {
|
||||
args: ["-quality", "85", "-optimize", "-outfile", outputJpeg, inputPpm],
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
});
|
||||
} catch (error) {
|
||||
if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
|
||||
throw new Error(
|
||||
"cjpeg is required for the compression benchmark. Use the Compose runner or install libjpeg tools."
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!result.success) {
|
||||
throw new Error(`cjpeg failed: ${new TextDecoder().decode(result.stderr)}`);
|
||||
}
|
||||
return "cjpeg quality=85, optimise=true, synthetic PPM 640x480";
|
||||
}
|
||||
|
||||
async function writeRandomBinary(path: string, size: number, seed: string): Promise<void> {
|
||||
const bytes = new Uint8Array(size);
|
||||
const nextRandom = createXorshift32(fnv1a32(seed));
|
||||
for (let index = 0; index < bytes.length; index++) {
|
||||
bytes[index] = nextRandom() & 0xff;
|
||||
}
|
||||
await writeFile(path, bytes);
|
||||
}
|
||||
|
||||
export async function createCompressionBenchmarkDataset(options: {
|
||||
rootDir: string;
|
||||
datasetDirName?: string;
|
||||
repositoryRoot?: string;
|
||||
seed?: string;
|
||||
jpegEncoder?: JpegEncoder;
|
||||
}): Promise<CompressionDataset> {
|
||||
const datasetDirName = options.datasetDirName ?? "compression-benchmark";
|
||||
const repositoryRoot = options.repositoryRoot ?? REPOSITORY_ROOT;
|
||||
const seed = options.seed ?? "livesync-compression-benchmark";
|
||||
const jpegEncoder = options.jpegEncoder ?? encodeJpegWithCjpeg;
|
||||
const datasetRoot = join(options.rootDir, datasetDirName);
|
||||
const entries: CompressionDatasetEntry[] = [];
|
||||
let jpegGenerator = "";
|
||||
|
||||
for (const kind of ALL_KINDS) {
|
||||
await mkdir(join(datasetRoot, kind), { recursive: true });
|
||||
}
|
||||
|
||||
const addFile = async (kind: DatasetKind, absolutePath: string, source: string) => {
|
||||
const relativePath = absolutePath
|
||||
.slice(options.rootDir.length + 1)
|
||||
.split("\\")
|
||||
.join("/");
|
||||
const size = (await stat(absolutePath)).size;
|
||||
entries.push({ kind, relativePath, absolutePath, size, source });
|
||||
};
|
||||
|
||||
const copyRepositoryFile = async (kind: DatasetKind, sourcePath: string, targetName: string) => {
|
||||
const destination = join(datasetRoot, kind, targetName);
|
||||
await copyFile(join(repositoryRoot, sourcePath), destination);
|
||||
await addFile(kind, destination, sourcePath);
|
||||
};
|
||||
|
||||
await copyRepositoryFile("md", "docs/settings.md", "settings.md");
|
||||
await copyRepositoryFile("md", "docs/quick_setup.md", "quick-setup.md");
|
||||
await copyRepositoryFile("md", "updates.md", "updates.md");
|
||||
await copyRepositoryFile("png", "instruction_images/cloudant_1.png", "cloudant-1.png");
|
||||
await copyRepositoryFile(
|
||||
"png",
|
||||
"images/quick-setup/guide-quick-setup-first-setup-uri.png",
|
||||
"quick-setup-first-setup-uri.png"
|
||||
);
|
||||
await copyRepositoryFile("json", "package.json", "package.json");
|
||||
await copyRepositoryFile("json", "manifest.json", "manifest.json");
|
||||
await copyRepositoryFile("ts", "src/serviceFeatures/replication/index.ts", "replicationFeature.ts");
|
||||
await copyRepositoryFile(
|
||||
"ts",
|
||||
"src/serviceFeatures/replication/ReplicateResultProcessor.ts",
|
||||
"ReplicateResultProcessor.ts"
|
||||
);
|
||||
|
||||
const markdownBytes = await readFile(join(repositoryRoot, "docs/settings.md"));
|
||||
const gzipPath = join(datasetRoot, "gz", "settings.md.gz");
|
||||
await writeFile(gzipPath, await gzip(markdownBytes));
|
||||
await addFile("gz", gzipPath, "generated gzip of docs/settings.md");
|
||||
|
||||
const randomPath = join(datasetRoot, "bin", "deterministic-random.bin");
|
||||
await writeRandomBinary(randomPath, 256 * 1024, seed);
|
||||
await addFile("bin", randomPath, `deterministic xorshift32 seed=${seed}`);
|
||||
|
||||
for (const [name, textured] of [
|
||||
["smooth-gradient.jpg", false],
|
||||
["textured-gradient.jpg", true],
|
||||
] as const) {
|
||||
const temporaryDirectory = await mkdtemp(join(options.rootDir, "compression-jpeg-"));
|
||||
const ppmPath = join(temporaryDirectory, "input.ppm");
|
||||
const jpegPath = join(datasetRoot, "jpg", name);
|
||||
try {
|
||||
await writeFile(ppmPath, createSyntheticPpm(640, 480, `${seed}-${name}`, textured));
|
||||
jpegGenerator = await jpegEncoder(ppmPath, jpegPath);
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
await addFile("jpg", jpegPath, `${jpegGenerator}; ${textured ? "textured" : "smooth"}`);
|
||||
}
|
||||
|
||||
const bytesByKind = Object.fromEntries(ALL_KINDS.map((kind) => [kind, 0])) as Record<DatasetKind, number>;
|
||||
const filesByKind = Object.fromEntries(ALL_KINDS.map((kind) => [kind, 0])) as Record<DatasetKind, number>;
|
||||
for (const entry of entries) {
|
||||
bytesByKind[entry.kind] += entry.size;
|
||||
filesByKind[entry.kind] += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
entries,
|
||||
totalFiles: entries.length,
|
||||
totalBytes: entries.reduce((sum, entry) => sum + entry.size, 0),
|
||||
bytesByKind,
|
||||
filesByKind,
|
||||
jpegGenerator,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
export type DeterministicDatasetConfig = {
|
||||
rootDir: string;
|
||||
datasetDirName: string;
|
||||
seed: string;
|
||||
mdCount: number;
|
||||
mdMinSizeBytes: number;
|
||||
mdMaxSizeBytes: number;
|
||||
binCount: number;
|
||||
binSizeBytes: number;
|
||||
};
|
||||
|
||||
export type DatasetKind = "md" | "jpg" | "png" | "json" | "ts" | "gz" | "bin";
|
||||
|
||||
export type DatasetEntry = {
|
||||
kind: DatasetKind;
|
||||
relativePath: string;
|
||||
absolutePath: string;
|
||||
size: number;
|
||||
};
|
||||
|
||||
export type DeterministicDataset = {
|
||||
rootDir: string;
|
||||
datasetDirName: string;
|
||||
seed: string;
|
||||
entries: DatasetEntry[];
|
||||
totalFiles: number;
|
||||
totalBytes: number;
|
||||
mdCount: number;
|
||||
binCount: number;
|
||||
};
|
||||
|
||||
function fnv1a32(input: string): number {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
hash ^= input.charCodeAt(i) & 0xff;
|
||||
hash = Math.imul(hash, 0x01000193);
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
function createXorshift32(seed: number): () => number {
|
||||
let state = seed >>> 0;
|
||||
if (state === 0) {
|
||||
state = 0x9e3779b9;
|
||||
}
|
||||
return () => {
|
||||
state ^= state << 13;
|
||||
state ^= state >>> 17;
|
||||
state ^= state << 5;
|
||||
return state >>> 0;
|
||||
};
|
||||
}
|
||||
|
||||
function createTextBytes(size: number, fileIndex: number, seed: string): Uint8Array {
|
||||
const template =
|
||||
`# Bench file ${fileIndex}\n` +
|
||||
`seed: ${seed}\n` +
|
||||
"lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n";
|
||||
|
||||
const templateBytes = new TextEncoder().encode(template);
|
||||
const out = new Uint8Array(size);
|
||||
for (let i = 0; i < size; i++) {
|
||||
out[i] = templateBytes[i % templateBytes.length];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function createDeterministicDataset(config: DeterministicDatasetConfig): Promise<DeterministicDataset> {
|
||||
if (config.mdCount < 0 || config.binCount < 0) {
|
||||
throw new Error("mdCount and binCount must be non-negative");
|
||||
}
|
||||
if (config.mdMinSizeBytes <= 0 || config.mdMaxSizeBytes <= 0 || config.binSizeBytes <= 0) {
|
||||
throw new Error("all size values must be positive");
|
||||
}
|
||||
if (config.mdMinSizeBytes > config.mdMaxSizeBytes) {
|
||||
throw new Error("mdMinSizeBytes must be <= mdMaxSizeBytes");
|
||||
}
|
||||
|
||||
const datasetRoot = join(config.rootDir, config.datasetDirName);
|
||||
const mdDir = join(datasetRoot, "md");
|
||||
const binDir = join(datasetRoot, "bin");
|
||||
await mkdir(mdDir, { recursive: true });
|
||||
await mkdir(binDir, { recursive: true });
|
||||
|
||||
const nextRandom = createXorshift32(fnv1a32(config.seed));
|
||||
const mdRange = config.mdMaxSizeBytes - config.mdMinSizeBytes + 1;
|
||||
const entries: DatasetEntry[] = [];
|
||||
|
||||
for (let index = 0; index < config.mdCount; index++) {
|
||||
const size = config.mdMinSizeBytes + (nextRandom() % mdRange);
|
||||
const relativePath = `${config.datasetDirName}/md/file-${String(index).padStart(4, "0")}.md`;
|
||||
const absolutePath = join(config.rootDir, ...relativePath.split("/"));
|
||||
const body = createTextBytes(size, index, config.seed);
|
||||
await writeFile(absolutePath, body);
|
||||
entries.push({ kind: "md", relativePath, absolutePath, size });
|
||||
}
|
||||
|
||||
for (let index = 0; index < config.binCount; index++) {
|
||||
const size = config.binSizeBytes;
|
||||
const relativePath = `${config.datasetDirName}/bin/file-${String(index).padStart(4, "0")}.bin`;
|
||||
const absolutePath = join(config.rootDir, ...relativePath.split("/"));
|
||||
const body = new Uint8Array(size);
|
||||
for (let i = 0; i < size; i++) {
|
||||
body[i] = nextRandom() & 0xff;
|
||||
}
|
||||
await writeFile(absolutePath, body);
|
||||
entries.push({ kind: "bin", relativePath, absolutePath, size });
|
||||
}
|
||||
|
||||
const totalBytes = entries.reduce((sum, e) => sum + e.size, 0);
|
||||
return {
|
||||
rootDir: config.rootDir,
|
||||
datasetDirName: config.datasetDirName,
|
||||
seed: config.seed,
|
||||
entries,
|
||||
totalFiles: entries.length,
|
||||
totalBytes,
|
||||
mdCount: config.mdCount,
|
||||
binCount: config.binCount,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,687 @@
|
||||
import { writeSync } from "node:fs";
|
||||
import { spawnProcess } from "./process.ts";
|
||||
/**
|
||||
* Docker service management for tests.
|
||||
*
|
||||
* CouchDB start/stop/init is implemented directly using `docker` CLI commands
|
||||
* and the Fetch API, so it works on any platform where Docker (Desktop) is
|
||||
* available — including Windows — without needing bash.
|
||||
*/
|
||||
|
||||
type DockerInvoker = {
|
||||
bin: string;
|
||||
prefix: string[];
|
||||
label: string;
|
||||
};
|
||||
|
||||
let dockerInvokerPromise: Promise<DockerInvoker> | null = null;
|
||||
const DOCKER_TEE = process.env.LIVESYNC_DOCKER_TEE === "1" || process.env.LIVESYNC_TEST_TEE === "1";
|
||||
const trackedContainers = new Set<string>();
|
||||
const CLEANUP_SIGNALS: NodeJS.Signals[] = ["SIGINT", "SIGTERM"];
|
||||
let signalCleanupHandlersInstalled = false;
|
||||
let signalCleanupInProgress = false;
|
||||
const signalCleanupHandlers = new Map<NodeJS.Signals, () => void>();
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Low-level docker wrapper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseCommand(command: string): { bin: string; prefix: string[] } {
|
||||
const parts = command.trim().split(/\s+/).filter(Boolean);
|
||||
if (parts.length === 0) {
|
||||
throw new Error("LIVESYNC_DOCKER_COMMAND is empty");
|
||||
}
|
||||
return { bin: parts[0], prefix: parts.slice(1) };
|
||||
}
|
||||
|
||||
async function collectStream(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
teeTarget: ((chunk: Uint8Array) => void) | null
|
||||
): Promise<Uint8Array> {
|
||||
const reader = stream.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (!value) continue;
|
||||
chunks.push(value);
|
||||
if (teeTarget) {
|
||||
teeTarget(value);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const out = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
out.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function runCommand(bin: string, args: string[]): Promise<{ code: number; stdout: string; stderr: string }> {
|
||||
try {
|
||||
const child = spawnProcess(bin, {
|
||||
args,
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
});
|
||||
const stdoutPromise = collectStream(child.stdout, DOCKER_TEE ? (chunk) => writeSync(1, chunk) : null);
|
||||
const stderrPromise = collectStream(child.stderr, DOCKER_TEE ? (chunk) => writeSync(2, chunk) : null);
|
||||
const [status, stdout, stderr] = await Promise.all([child.status, stdoutPromise, stderrPromise]);
|
||||
const dec = new TextDecoder();
|
||||
const result = {
|
||||
code: status.code,
|
||||
stdout: dec.decode(stdout),
|
||||
stderr: dec.decode(stderr),
|
||||
};
|
||||
return result;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return {
|
||||
code: 127,
|
||||
stdout: "",
|
||||
stderr: `Command not found: ${bin}`,
|
||||
};
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveDockerInvoker(): Promise<DockerInvoker> {
|
||||
const custom = process.env.LIVESYNC_DOCKER_COMMAND?.trim();
|
||||
if (custom) {
|
||||
const parsed = parseCommand(custom);
|
||||
const runner: DockerInvoker = {
|
||||
...parsed,
|
||||
label: `custom(${custom})`,
|
||||
};
|
||||
|
||||
// Validate custom command eagerly so misconfiguration fails fast.
|
||||
const checkArgs = runner.prefix.length === 0 ? ["--version"] : [...runner.prefix, "docker", "--version"];
|
||||
const check = await runCommand(runner.bin, checkArgs);
|
||||
if (check.code !== 0) {
|
||||
throw new Error(`LIVESYNC_DOCKER_COMMAND is not usable: ${custom}\n${check.stderr || check.stdout}`);
|
||||
}
|
||||
return runner;
|
||||
}
|
||||
|
||||
const mode = (process.env.LIVESYNC_DOCKER_MODE ?? "auto").toLowerCase();
|
||||
const onWindows = process.platform === "win32";
|
||||
|
||||
const native: DockerInvoker = { bin: "docker", prefix: [], label: "docker" };
|
||||
const wsl: DockerInvoker = { bin: "wsl", prefix: [], label: "wsl docker" };
|
||||
|
||||
if (mode === "native") {
|
||||
return native;
|
||||
}
|
||||
if (mode === "wsl") {
|
||||
return wsl;
|
||||
}
|
||||
if (mode !== "auto") {
|
||||
throw new Error(`Unsupported LIVESYNC_DOCKER_MODE='${mode}'. Use auto, native, or wsl.`);
|
||||
}
|
||||
|
||||
// On Windows we prefer `wsl docker` first, then native docker.
|
||||
// This typically works better in setups where Docker is installed only in
|
||||
// WSL and not exposed as docker.exe on PATH.
|
||||
const candidates = onWindows ? [wsl, native] : [native, wsl];
|
||||
for (const c of candidates) {
|
||||
if (c.bin === "docker") {
|
||||
const r = await runCommand("docker", ["--version"]);
|
||||
if (r.code === 0) return c;
|
||||
continue;
|
||||
}
|
||||
const r = await runCommand("wsl", ["docker", "--version"]);
|
||||
if (r.code === 0) return c;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
[
|
||||
"Docker command is not available.",
|
||||
"Set one of:",
|
||||
"- LIVESYNC_DOCKER_MODE=native",
|
||||
"- LIVESYNC_DOCKER_MODE=wsl",
|
||||
"- LIVESYNC_DOCKER_COMMAND='docker'",
|
||||
"- LIVESYNC_DOCKER_COMMAND='wsl docker'",
|
||||
].join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
async function getDockerInvoker(): Promise<DockerInvoker> {
|
||||
if (!dockerInvokerPromise) {
|
||||
dockerInvokerPromise = resolveDockerInvoker().then((r) => {
|
||||
console.log(`[INFO] docker runner: ${r.label}`);
|
||||
return r;
|
||||
});
|
||||
}
|
||||
return await dockerInvokerPromise;
|
||||
}
|
||||
|
||||
async function docker(...args: string[]): Promise<{ code: number; stdout: string; stderr: string }> {
|
||||
const invoker = await getDockerInvoker();
|
||||
|
||||
// Either:
|
||||
// docker <args>
|
||||
// Or:
|
||||
// wsl docker <args>
|
||||
const finalArgs =
|
||||
invoker.prefix.length === 0
|
||||
? invoker.bin === "wsl"
|
||||
? ["docker", ...args]
|
||||
: args
|
||||
: [...invoker.prefix, ...args];
|
||||
|
||||
const r = await runCommand(invoker.bin, finalArgs);
|
||||
return { code: r.code, stdout: r.stdout, stderr: r.stderr };
|
||||
}
|
||||
|
||||
async function dockerOrFail(...args: string[]): Promise<string> {
|
||||
const r = await docker(...args);
|
||||
if (r.code !== 0) {
|
||||
throw new Error(`docker ${args[0]} failed (code ${r.code}): ${r.stderr.trim()}`);
|
||||
}
|
||||
return r.stdout;
|
||||
}
|
||||
|
||||
async function stopAndRemoveContainer(container: string): Promise<void> {
|
||||
await docker("stop", container).catch(() => {});
|
||||
await docker("rm", "-v", container).catch(() => {});
|
||||
}
|
||||
|
||||
async function cleanupTrackedContainers(reason: string): Promise<void> {
|
||||
const names = [...trackedContainers];
|
||||
if (names.length === 0) return;
|
||||
|
||||
console.warn(`[WARN] cleaning up tracked containers on ${reason}: ${names.join(", ")}`);
|
||||
for (const container of names.reverse()) {
|
||||
await stopAndRemoveContainer(container);
|
||||
trackedContainers.delete(container);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSignalCleanup(signal: NodeJS.Signals): Promise<void> {
|
||||
if (signalCleanupInProgress) return;
|
||||
signalCleanupInProgress = true;
|
||||
try {
|
||||
await cleanupTrackedContainers(`signal ${signal}`);
|
||||
} finally {
|
||||
process.exit(signal === "SIGINT" ? 130 : 143);
|
||||
}
|
||||
}
|
||||
|
||||
function ensureSignalCleanupHandlers(): void {
|
||||
if (signalCleanupHandlersInstalled) return;
|
||||
signalCleanupHandlersInstalled = true;
|
||||
for (const signal of CLEANUP_SIGNALS) {
|
||||
const listener = () => {
|
||||
void handleSignalCleanup(signal);
|
||||
};
|
||||
try {
|
||||
process.on(signal, listener);
|
||||
signalCleanupHandlers.set(signal, listener);
|
||||
} catch {
|
||||
// Unsupported signal on this platform.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function removeSignalCleanupHandlers(): void {
|
||||
if (!signalCleanupHandlersInstalled) return;
|
||||
for (const [signal, listener] of signalCleanupHandlers) {
|
||||
try {
|
||||
process.off(signal, listener);
|
||||
} catch {
|
||||
// Ignore if already removed or unsupported.
|
||||
}
|
||||
}
|
||||
signalCleanupHandlers.clear();
|
||||
signalCleanupHandlersInstalled = false;
|
||||
}
|
||||
|
||||
function trackContainer(container: string): void {
|
||||
ensureSignalCleanupHandlers();
|
||||
trackedContainers.add(container);
|
||||
}
|
||||
|
||||
function untrackContainer(container: string): void {
|
||||
trackedContainers.delete(container);
|
||||
if (trackedContainers.size === 0) {
|
||||
removeSignalCleanupHandlers();
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function waitForCouchdbStable(hostname: string, user: string, password: string): Promise<void> {
|
||||
const h = hostname.replace(/\/$/, "").replace("localhost", "127.0.0.1");
|
||||
const auth = btoa(`${user}:${password}`);
|
||||
const headers = { Authorization: `Basic ${auth}` };
|
||||
let consecutive = 0;
|
||||
for (let i = 0; i < 30; i++) {
|
||||
try {
|
||||
const r = await fetch(`${h}/_up`, {
|
||||
headers,
|
||||
signal: AbortSignal.timeout(3000),
|
||||
});
|
||||
if (r.ok) {
|
||||
consecutive++;
|
||||
if (consecutive >= 3) return;
|
||||
} else {
|
||||
consecutive = 0;
|
||||
}
|
||||
} catch {
|
||||
consecutive = 0;
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
throw new Error("CouchDB did not become stable in time");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fetch with retry (mirrors cli_test_curl_json() retry loop)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function fetchRetry(
|
||||
url: string,
|
||||
init: RequestInit,
|
||||
retries = 30,
|
||||
delayMs = 2000,
|
||||
allowStatus: number[] = []
|
||||
): Promise<void> {
|
||||
let lastError: unknown;
|
||||
let lastStatus: number | undefined;
|
||||
for (let i = 0; i < retries; i++) {
|
||||
try {
|
||||
const r = await fetch(url, {
|
||||
signal: AbortSignal.timeout(5000),
|
||||
...init,
|
||||
});
|
||||
lastStatus = r.status;
|
||||
await r.body?.cancel().catch(() => {});
|
||||
if (r.ok || allowStatus.includes(r.status)) return;
|
||||
lastError = `HTTP ${r.status}`;
|
||||
} catch (e) {
|
||||
lastError = e;
|
||||
}
|
||||
await sleep(delayMs);
|
||||
}
|
||||
throw new Error(
|
||||
`Could not reach ${url} after ${retries} retries: ${lastError} (last status: ${lastStatus ?? "N/A"})`
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CouchDB
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// TODO: these values could be configurable via environment variables.
|
||||
//
|
||||
const COUCHDB_CONTAINER = "couchdb-test";
|
||||
const COUCHDB_IMAGE = "couchdb:3.5.0";
|
||||
|
||||
const MINIO_CONTAINER = "minio-test";
|
||||
// RustFS provides the S3 backend for the existing MINIO test mode.
|
||||
const S3_IMAGE = "rustfs/rustfs:1.0.0-rc.6@sha256:97171b3d72cd47dc81000f92ea84de25608bfc35a94c965501afaeb5d99f6035";
|
||||
const S3_CLIENT_IMAGE = "rustfs/rc:v0.1.35@sha256:adb45b56539006120f1d790bcc17ee5f9b4d93c1d7e71ed0a24f10267f9d6914";
|
||||
const S3_BUCKET_CORS = `<CORSConfiguration>
|
||||
<CORSRule>
|
||||
<AllowedOrigin>*</AllowedOrigin>
|
||||
<AllowedMethod>GET</AllowedMethod>
|
||||
<AllowedMethod>PUT</AllowedMethod>
|
||||
<AllowedMethod>POST</AllowedMethod>
|
||||
<AllowedMethod>DELETE</AllowedMethod>
|
||||
<AllowedMethod>HEAD</AllowedMethod>
|
||||
<AllowedHeader>*</AllowedHeader>
|
||||
<AllowedHeader>authorization</AllowedHeader>
|
||||
<ExposeHeader>ETag</ExposeHeader>
|
||||
</CORSRule>
|
||||
</CORSConfiguration>`;
|
||||
|
||||
export async function stopCouchdb(): Promise<void> {
|
||||
await stopAndRemoveContainer(COUCHDB_CONTAINER);
|
||||
untrackContainer(COUCHDB_CONTAINER);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a CouchDB test container, initialise it, and create the test DB.
|
||||
* Mirrors cli_test_start_couchdb() from test-helpers.sh, using direct
|
||||
* docker / fetch calls instead of the bash util scripts.
|
||||
*/
|
||||
export async function startCouchdb(couchdbUri: string, user: string, password: string, dbname: string): Promise<void> {
|
||||
console.log("[INFO] stopping leftover CouchDB container if present");
|
||||
await stopCouchdb().catch(() => {});
|
||||
|
||||
console.log("[INFO] starting CouchDB test container");
|
||||
await dockerOrFail(
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
COUCHDB_CONTAINER,
|
||||
"-p",
|
||||
// TODO: port mapping should be configurable.
|
||||
"5989:5984",
|
||||
"-e",
|
||||
`COUCHDB_USER=${user}`,
|
||||
"-e",
|
||||
`COUCHDB_PASSWORD=${password}`,
|
||||
"-e",
|
||||
"COUCHDB_SINGLE_NODE=y",
|
||||
COUCHDB_IMAGE
|
||||
);
|
||||
trackContainer(COUCHDB_CONTAINER);
|
||||
|
||||
console.log("[INFO] initialising CouchDB");
|
||||
await initCouchdb(couchdbUri, user, password);
|
||||
|
||||
console.log("[INFO] waiting for CouchDB to become stable");
|
||||
await waitForCouchdbStable(couchdbUri, user, password);
|
||||
|
||||
console.log(`[INFO] creating test database: ${dbname}`);
|
||||
await createCouchdbDatabase(couchdbUri, user, password, dbname);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror couchdb-init.sh: configure single-node CouchDB via its REST API.
|
||||
*/
|
||||
async function initCouchdb(hostname: string, user: string, password: string, node = "_local"): Promise<void> {
|
||||
// Podman environments often resolve localhost to ::1; use 127.0.0.1 like
|
||||
// the bash script does.
|
||||
const h = hostname.replace(/\/$/, "").replace("localhost", "127.0.0.1");
|
||||
const auth = btoa(`${user}:${password}`);
|
||||
const headers = {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Basic ${auth}`,
|
||||
};
|
||||
|
||||
const calls: Array<[string, string, string]> = [
|
||||
[
|
||||
"POST",
|
||||
`${h}/_cluster_setup`,
|
||||
JSON.stringify({
|
||||
action: "enable_single_node",
|
||||
username: user,
|
||||
password,
|
||||
bind_address: "0.0.0.0",
|
||||
port: 5984,
|
||||
singlenode: true,
|
||||
}),
|
||||
],
|
||||
["PUT", `${h}/_node/${node}/_config/chttpd/require_valid_user`, '"true"'],
|
||||
["PUT", `${h}/_node/${node}/_config/chttpd_auth/require_valid_user`, '"true"'],
|
||||
["PUT", `${h}/_node/${node}/_config/httpd/WWW-Authenticate`, '"Basic realm=\\"couchdb\\""'],
|
||||
["PUT", `${h}/_node/${node}/_config/httpd/enable_cors`, '"true"'],
|
||||
["PUT", `${h}/_node/${node}/_config/chttpd/enable_cors`, '"true"'],
|
||||
["PUT", `${h}/_node/${node}/_config/chttpd/max_http_request_size`, '"4294967296"'],
|
||||
["PUT", `${h}/_node/${node}/_config/couchdb/max_document_size`, '"50000000"'],
|
||||
["PUT", `${h}/_node/${node}/_config/cors/credentials`, '"true"'],
|
||||
["PUT", `${h}/_node/${node}/_config/cors/origins`, '"*"'],
|
||||
];
|
||||
|
||||
for (const [method, url, body] of calls) {
|
||||
await fetchRetry(url, { method, headers, body });
|
||||
}
|
||||
}
|
||||
|
||||
export async function createCouchdbDatabase(
|
||||
hostname: string,
|
||||
user: string,
|
||||
password: string,
|
||||
dbname: string
|
||||
): Promise<void> {
|
||||
const h = hostname.replace(/\/$/, "").replace("localhost", "127.0.0.1");
|
||||
const auth = btoa(`${user}:${password}`);
|
||||
await fetchRetry(`${h}/${dbname}`, {
|
||||
method: "PUT",
|
||||
headers: { Authorization: `Basic ${auth}` },
|
||||
});
|
||||
}
|
||||
|
||||
/** Update a CouchDB document via PUT. Returns the updated document. */
|
||||
export async function updateCouchdbDoc(
|
||||
hostname: string,
|
||||
user: string,
|
||||
password: string,
|
||||
docUrl: string,
|
||||
updater: (doc: Record<string, unknown>) => Record<string, unknown>
|
||||
): Promise<void> {
|
||||
const h = hostname.replace(/\/$/, "").replace("localhost", "127.0.0.1");
|
||||
const auth = btoa(`${user}:${password}`);
|
||||
const headers = {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Basic ${auth}`,
|
||||
};
|
||||
const getRes = await fetch(`${h}/${docUrl}`, { headers });
|
||||
const current = (await getRes.json()) as Record<string, unknown>;
|
||||
const updated = updater(current);
|
||||
await fetchRetry(`${h}/${docUrl}`, {
|
||||
method: "PUT",
|
||||
headers,
|
||||
body: JSON.stringify(updated),
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// S3 (RustFS)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function shQuote(value: string): string {
|
||||
return `'${value.split("'").join(`'"'"'`)}'`;
|
||||
}
|
||||
|
||||
export async function stopMinio(): Promise<void> {
|
||||
await stopAndRemoveContainer(MINIO_CONTAINER);
|
||||
untrackContainer(MINIO_CONTAINER);
|
||||
}
|
||||
|
||||
async function initMinioBucket(
|
||||
minioEndpoint: string,
|
||||
accessKey: string,
|
||||
secretKey: string,
|
||||
bucket: string
|
||||
): Promise<boolean> {
|
||||
const cmd =
|
||||
`rc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` +
|
||||
`rc mb --ignore-existing myminio/${shQuote(bucket)} >/dev/null 2>&1 && ` +
|
||||
`printf %s ${shQuote(S3_BUCKET_CORS)} | rc cors set myminio/${shQuote(bucket)} - >/dev/null 2>&1`;
|
||||
const r = await docker("run", "--rm", "--network", "host", "--entrypoint", "/bin/sh", S3_CLIENT_IMAGE, "-c", cmd);
|
||||
return r.code === 0;
|
||||
}
|
||||
|
||||
async function waitForMinioBucket(
|
||||
minioEndpoint: string,
|
||||
accessKey: string,
|
||||
secretKey: string,
|
||||
bucket: string
|
||||
): Promise<void> {
|
||||
for (let i = 0; i < 30; i++) {
|
||||
const checkCmd =
|
||||
`rc alias set myminio ${shQuote(minioEndpoint)} ${shQuote(accessKey)} ${shQuote(secretKey)} >/dev/null 2>&1 && ` +
|
||||
`rc ls myminio/${shQuote(bucket)} >/dev/null 2>&1`;
|
||||
const check = await docker(
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
// Now I used host networking to access the container via localhost for some environments (Docker Desktop on Windows).
|
||||
// We need something good idea to work across all environments.
|
||||
"host",
|
||||
"--entrypoint",
|
||||
"/bin/sh",
|
||||
S3_CLIENT_IMAGE,
|
||||
"-c",
|
||||
checkCmd
|
||||
);
|
||||
if (check.code === 0) {
|
||||
return;
|
||||
}
|
||||
await initMinioBucket(minioEndpoint, accessKey, secretKey, bucket);
|
||||
await sleep(2000);
|
||||
}
|
||||
throw new Error(`S3 bucket not ready: ${bucket}`);
|
||||
}
|
||||
|
||||
export async function startMinio(
|
||||
minioEndpoint: string,
|
||||
accessKey: string,
|
||||
secretKey: string,
|
||||
bucket: string
|
||||
): Promise<void> {
|
||||
console.log("[INFO] stopping leftover S3 test container if present");
|
||||
await stopMinio().catch(() => {});
|
||||
|
||||
console.log("[INFO] starting RustFS test container");
|
||||
await dockerOrFail(
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
MINIO_CONTAINER,
|
||||
// TODO: Ports should be configurable.
|
||||
"-p",
|
||||
"9000:9000",
|
||||
"-p",
|
||||
"9001:9001",
|
||||
"-e",
|
||||
`RUSTFS_ACCESS_KEY=${accessKey}`,
|
||||
"-e",
|
||||
`RUSTFS_SECRET_KEY=${secretKey}`,
|
||||
"-e",
|
||||
"RUSTFS_CONSOLE_ENABLE=true",
|
||||
"-e",
|
||||
"RUSTFS_CORS_ALLOWED_ORIGINS=*",
|
||||
S3_IMAGE,
|
||||
"/data"
|
||||
);
|
||||
trackContainer(MINIO_CONTAINER);
|
||||
|
||||
console.log(`[INFO] initialising S3 test bucket: ${bucket}`);
|
||||
let initialised = false;
|
||||
for (let i = 0; i < 5; i++) {
|
||||
if (await initMinioBucket(minioEndpoint, accessKey, secretKey, bucket)) {
|
||||
initialised = true;
|
||||
break;
|
||||
}
|
||||
await sleep(2000);
|
||||
}
|
||||
if (!initialised) {
|
||||
throw new Error(`Could not initialise S3 bucket after retries: ${bucket}`);
|
||||
}
|
||||
|
||||
await waitForMinioBucket(minioEndpoint, accessKey, secretKey, bucket);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// P2P relay (strfry)
|
||||
// ---------------------------------------------------------------------------
|
||||
// TODO: these values could be configurable via environment variables.
|
||||
const P2P_RELAY_CONTAINER = "relay-test";
|
||||
const P2P_RELAY_IMAGE = "ghcr.io/hoytech/strfry:latest";
|
||||
const STRFRY_BOOTSTRAP_SH = String.raw`cat > /tmp/strfry.conf <<"EOF"
|
||||
db = "./strfry-db/"
|
||||
|
||||
relay {
|
||||
bind = "0.0.0.0"
|
||||
port = 7777
|
||||
nofiles = 100000
|
||||
|
||||
info {
|
||||
name = "livesync test relay"
|
||||
description = "local relay for livesync p2p tests"
|
||||
}
|
||||
|
||||
maxWebsocketPayloadSize = 131072
|
||||
autoPingSeconds = 55
|
||||
|
||||
writePolicy {
|
||||
plugin = ""
|
||||
}
|
||||
}
|
||||
EOF
|
||||
exec /app/strfry --config /tmp/strfry.conf relay`;
|
||||
|
||||
export async function stopP2pRelay(): Promise<void> {
|
||||
await stopAndRemoveContainer(P2P_RELAY_CONTAINER);
|
||||
untrackContainer(P2P_RELAY_CONTAINER);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the local P2P relay container through the same docker runner used
|
||||
* by CouchDB helpers. This keeps process ownership consistent across
|
||||
* start/stop on Windows, WSL, and native Linux/macOS.
|
||||
*/
|
||||
export async function startP2pRelay(): Promise<void> {
|
||||
console.log("[INFO] stopping leftover P2P relay container if present");
|
||||
await stopP2pRelay().catch(() => {});
|
||||
|
||||
console.log("[INFO] starting local P2P relay container");
|
||||
await dockerOrFail(
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
P2P_RELAY_CONTAINER,
|
||||
"-p",
|
||||
//TODO: port mapping should be configurable.
|
||||
"4000:7777",
|
||||
"--tmpfs",
|
||||
"/app/strfry-db:rw,size=256m,mode=1777",
|
||||
"--entrypoint",
|
||||
"sh",
|
||||
P2P_RELAY_IMAGE,
|
||||
"-lc",
|
||||
STRFRY_BOOTSTRAP_SH
|
||||
);
|
||||
trackContainer(P2P_RELAY_CONTAINER);
|
||||
}
|
||||
|
||||
export function isLocalP2pRelay(relayUrl: string): boolean {
|
||||
return relayUrl.includes("localhost") || relayUrl.includes("127.0.0.1") || relayUrl.includes("[::1]");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Coturn (STUN/TURN)
|
||||
// ---------------------------------------------------------------------------
|
||||
const COTURN_CONTAINER = "coturn-test";
|
||||
const COTURN_IMAGE = "coturn/coturn:latest";
|
||||
|
||||
export async function stopCoturn(): Promise<void> {
|
||||
await stopAndRemoveContainer(COTURN_CONTAINER);
|
||||
untrackContainer(COTURN_CONTAINER);
|
||||
}
|
||||
|
||||
export async function startCoturn(
|
||||
port = 3478,
|
||||
user = "testuser",
|
||||
pass = "testpass",
|
||||
realm = "livesync.test"
|
||||
): Promise<void> {
|
||||
console.log("[INFO] stopping leftover Coturn container if present");
|
||||
await stopCoturn().catch(() => {});
|
||||
|
||||
const { getOptimalLoopbackIp } = await import("./net.ts");
|
||||
const externalIp = await getOptimalLoopbackIp();
|
||||
|
||||
console.log(`[INFO] starting local Coturn container with external-ip ${externalIp}`);
|
||||
await dockerOrFail(
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
COTURN_CONTAINER,
|
||||
"-p",
|
||||
`${port}:${port}`,
|
||||
"-p",
|
||||
`${port}:${port}/udp`,
|
||||
COTURN_IMAGE,
|
||||
"--log-file=stdout",
|
||||
`--external-ip=${externalIp}`,
|
||||
`--user=${user}:${pass}`,
|
||||
`--realm=${realm}`
|
||||
);
|
||||
trackContainer(COTURN_CONTAINER);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
/**
|
||||
* Load a .env-style file (KEY=value per line) into a plain object.
|
||||
* Equivalent to `source $TEST_ENV_FILE; set -a` in bash.
|
||||
* Maybe we should use some library... now it is just the minimal implementation that covers our use cases.
|
||||
*
|
||||
* Supported value formats:
|
||||
* KEY=value
|
||||
* KEY='single quoted'
|
||||
* KEY="double quoted"
|
||||
* # comment lines are ignored
|
||||
*/
|
||||
export async function loadEnvFile(filePath: string): Promise<Record<string, string>> {
|
||||
const text = await readFile(filePath, "utf8");
|
||||
const result: Record<string, string> = {};
|
||||
for (const line of text.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith("#")) continue;
|
||||
const idx = trimmed.indexOf("=");
|
||||
if (idx < 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
const raw = trimmed.slice(idx + 1).trim();
|
||||
// Strip surrounding single or double quotes
|
||||
result[key] = raw.replace(/^(['"])(.*)\1$/, "$2");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { runProcess } from "./process.ts";
|
||||
import { CLI_DIR, CLI_DIST } from "./cli.ts";
|
||||
|
||||
export type CliProcessMeasurement = {
|
||||
elapsedMs: number;
|
||||
userCpuMs: number;
|
||||
systemCpuMs: number;
|
||||
totalCpuMs: number;
|
||||
cpuToWallRatio: number;
|
||||
maxResidentSetKiB: number;
|
||||
};
|
||||
|
||||
const MARKER = "__LIVESYNC_GNU_TIME__";
|
||||
|
||||
export async function runMeasuredCliOrFail(...args: string[]): Promise<CliProcessMeasurement> {
|
||||
const started = performance.now();
|
||||
let output: Awaited<ReturnType<typeof runProcess>>;
|
||||
try {
|
||||
output = await runProcess("/usr/bin/time", {
|
||||
args: ["-f", `${MARKER}%U\t%S\t%M`, process.execPath, CLI_DIST, ...args],
|
||||
cwd: CLI_DIR,
|
||||
stdin: "null",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
});
|
||||
} catch (error) {
|
||||
if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
|
||||
throw new Error(
|
||||
"GNU /usr/bin/time is required for the compression benchmark. Use the Compose runner or install GNU time."
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const elapsedMs = performance.now() - started;
|
||||
const stderr = new TextDecoder().decode(output.stderr);
|
||||
const stdout = new TextDecoder().decode(output.stdout);
|
||||
const measurementLine = stderr.split(/\r?\n/).find((line) => line.startsWith(MARKER));
|
||||
if (!output.success) {
|
||||
throw new Error(`CLI exited with code ${output.code}\nstdout: ${stdout}\nstderr: ${stderr}`);
|
||||
}
|
||||
if (!measurementLine) {
|
||||
throw new Error(`GNU time did not emit the expected measurement marker\nstderr: ${stderr}`);
|
||||
}
|
||||
const [userSeconds, systemSeconds, maxResidentSetKiB] = measurementLine
|
||||
.slice(MARKER.length)
|
||||
.split("\t")
|
||||
.map(Number);
|
||||
if (![userSeconds, systemSeconds, maxResidentSetKiB].every(Number.isFinite)) {
|
||||
throw new Error(`Could not parse GNU time measurement: ${measurementLine}`);
|
||||
}
|
||||
const userCpuMs = userSeconds * 1000;
|
||||
const systemCpuMs = systemSeconds * 1000;
|
||||
const totalCpuMs = userCpuMs + systemCpuMs;
|
||||
return {
|
||||
elapsedMs: Number(elapsedMs.toFixed(1)),
|
||||
userCpuMs: Number(userCpuMs.toFixed(1)),
|
||||
systemCpuMs: Number(systemCpuMs.toFixed(1)),
|
||||
totalCpuMs: Number(totalCpuMs.toFixed(1)),
|
||||
cpuToWallRatio: Number((totalCpuMs / elapsedMs).toFixed(4)),
|
||||
maxResidentSetKiB,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { createConnection, createServer } from "node:net";
|
||||
|
||||
type WaitForPortOptions = { timeoutMs?: number; intervalMs?: number; connectTimeoutMs?: number };
|
||||
|
||||
async function connectWithTimeout(hostname: string, port: number, timeoutMs: number): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const socket = createConnection({ host: hostname, port });
|
||||
socket.setTimeout(timeoutMs, () => socket.destroy(new Error(`connect timeout after ${timeoutMs}ms`)));
|
||||
socket.once("error", reject);
|
||||
socket.once("connect", () => {
|
||||
socket.destroy();
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function waitForPort(hostname: string, port: number, options: WaitForPortOptions = {}): Promise<void> {
|
||||
const timeoutMs = options.timeoutMs ?? 15000;
|
||||
const intervalMs = options.intervalMs ?? 250;
|
||||
const started = Date.now();
|
||||
let lastError: unknown;
|
||||
while (Date.now() - started < timeoutMs) {
|
||||
try {
|
||||
await connectWithTimeout(hostname, port, options.connectTimeoutMs ?? 1000);
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`Port ${hostname}:${port} did not become ready within ${timeoutMs}ms` +
|
||||
(lastError ? ` (last error: ${String(lastError)})` : "")
|
||||
);
|
||||
}
|
||||
|
||||
export async function getFreePort(hostname = "127.0.0.1"): Promise<number> {
|
||||
const server = createServer();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, hostname, resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
await new Promise<void>((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
|
||||
if (!address || typeof address === "string") throw new Error("The listener did not bind a TCP port");
|
||||
return address.port;
|
||||
}
|
||||
|
||||
export async function getOptimalLoopbackIp(): Promise<string> {
|
||||
for (const hostname of ["127.0.0.1", "::1"]) {
|
||||
try {
|
||||
await getFreePort(hostname);
|
||||
return hostname;
|
||||
} catch {
|
||||
/* Try the other loopback family. */
|
||||
}
|
||||
}
|
||||
return "127.0.0.1";
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
|
||||
import { runCli } from "./cli.ts";
|
||||
import { isLocalP2pRelay, startP2pRelay, stopP2pRelay, startCoturn, stopCoturn } from "./docker.ts";
|
||||
import { waitForPort } from "./net.ts";
|
||||
|
||||
export type PeerEntry = {
|
||||
id: string;
|
||||
name: string;
|
||||
};
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function parseRelayEndpoint(relay: string): { hostname: string; port: number } {
|
||||
const url = new URL(relay);
|
||||
const port = url.port ? Number(url.port) : url.protocol === "ws:" ? 80 : url.protocol === "wss:" ? 443 : NaN;
|
||||
if (!Number.isFinite(port)) {
|
||||
throw new Error(`Unsupported relay URL: ${relay}`);
|
||||
}
|
||||
const hostname = url.hostname === "localhost" ? "127.0.0.1" : url.hostname;
|
||||
return { hostname, port };
|
||||
}
|
||||
|
||||
export function parsePeerLines(output: string): PeerEntry[] {
|
||||
return output
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.split("\t"))
|
||||
.filter((parts) => parts.length >= 3 && parts[0] === "[peer]")
|
||||
.map((parts) => ({ id: parts[1], name: parts[2] }));
|
||||
}
|
||||
|
||||
export async function discoverPeer(
|
||||
vaultDir: string,
|
||||
settingsFile: string,
|
||||
timeoutSeconds: number,
|
||||
targetPeer?: string
|
||||
): Promise<PeerEntry> {
|
||||
const retries = Math.max(0, Number(process.env.LIVESYNC_P2P_PEERS_RETRY ?? "3"));
|
||||
let lastCombined = "";
|
||||
|
||||
for (let attempt = 0; attempt <= retries; attempt++) {
|
||||
const result = await runCli(vaultDir, "--settings", settingsFile, "p2p-peers", String(timeoutSeconds));
|
||||
lastCombined = result.combined;
|
||||
|
||||
if (result.code === 0) {
|
||||
const peers = parsePeerLines(result.stdout);
|
||||
if (targetPeer) {
|
||||
const matched = peers.find((peer) => peer.id === targetPeer || peer.name === targetPeer);
|
||||
if (matched) return matched;
|
||||
}
|
||||
if (peers.length > 0) {
|
||||
return peers[0];
|
||||
}
|
||||
|
||||
const fallback = result.combined.match(/Advertisement from\s+([^\s]+)/);
|
||||
if (fallback?.[1]) {
|
||||
return { id: fallback[1], name: fallback[1] };
|
||||
}
|
||||
}
|
||||
|
||||
if (attempt < retries) {
|
||||
const waitMs = 400 * (attempt + 1);
|
||||
console.warn(
|
||||
`[WARN] p2p-peers returned no usable peers, retrying (${attempt + 1}/${retries}) in ${waitMs}ms`
|
||||
);
|
||||
await sleep(waitMs);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
result.code !== 0 ? `p2p-peers failed\n${result.combined}` : `No peers discovered\n${result.combined}`
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(`No peers discovered\n${lastCombined}`);
|
||||
}
|
||||
|
||||
export async function maybeStartLocalRelay(relay: string): Promise<boolean> {
|
||||
const shouldStart = isLocalP2pRelay(relay);
|
||||
if (shouldStart) {
|
||||
await startP2pRelay();
|
||||
}
|
||||
const endpoint = parseRelayEndpoint(relay);
|
||||
await waitForPort(endpoint.hostname, endpoint.port, {
|
||||
timeoutMs: Number(process.env.LIVESYNC_P2P_RELAY_READY_TIMEOUT_MS ?? "15000"),
|
||||
intervalMs: Number(process.env.LIVESYNC_P2P_RELAY_READY_INTERVAL_MS ?? "250"),
|
||||
connectTimeoutMs: Number(process.env.LIVESYNC_P2P_RELAY_CONNECT_TIMEOUT_MS ?? "1000"),
|
||||
});
|
||||
// Docker proxy accepts TCP connections instantly before the container's internal process is fully ready.
|
||||
// Wait an additional few seconds to ensure strfry is actually accepting WebSockets.
|
||||
if (shouldStart) {
|
||||
await sleep(3000);
|
||||
}
|
||||
return shouldStart;
|
||||
}
|
||||
|
||||
export async function stopLocalRelayIfStarted(started: boolean): Promise<void> {
|
||||
if (started) {
|
||||
await stopP2pRelay().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
export async function maybeStartCoturn(turnServers: string): Promise<boolean> {
|
||||
if (turnServers.includes("localhost") || turnServers.includes("127.0.0.1") || turnServers.includes("[::1]")) {
|
||||
await startCoturn();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function stopCoturnIfStarted(started: boolean): Promise<void> {
|
||||
if (started) {
|
||||
await stopCoturn().catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
export const REPOSITORY_ROOT = resolve(import.meta.dirname, "../../..");
|
||||
export const CLI_DIR = join(REPOSITORY_ROOT, "src", "apps", "cli");
|
||||
export const CLI_DIST = join(CLI_DIR, "dist", "index.cjs");
|
||||
@@ -0,0 +1,45 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { runProcess, spawnProcess } from "./process.ts";
|
||||
|
||||
test("subprocesses preserve arguments, environment overrides, and both output streams", async () => {
|
||||
const argument = "a path with spaces/日本語.txt";
|
||||
const result = await runProcess(process.execPath, {
|
||||
args: [
|
||||
"--eval",
|
||||
"process.stdout.write(JSON.stringify({arg:process.argv[1],value:process.env.LIVESYNC_PROCESS_TEST,inherited:process.env.PATH||process.env.Path}));process.stderr.write('diagnostic');process.exitCode=7",
|
||||
argument,
|
||||
],
|
||||
env: { LIVESYNC_PROCESS_TEST: "override" },
|
||||
});
|
||||
assert.equal(result.code, 7);
|
||||
assert.equal(result.success, false);
|
||||
const output = JSON.parse(new TextDecoder().decode(result.stdout));
|
||||
assert.equal(output.arg, argument);
|
||||
assert.equal(output.value, "override");
|
||||
assert.ok(output.inherited);
|
||||
assert.equal(new TextDecoder().decode(result.stderr), "diagnostic");
|
||||
});
|
||||
|
||||
test("piped stdin closes and large stdout and stderr drain before completion", async () => {
|
||||
const child = spawnProcess(process.execPath, {
|
||||
args: [
|
||||
"--eval",
|
||||
"let data='';process.stdin.setEncoding('utf8');process.stdin.on('data',v=>data+=v);process.stdin.on('end',()=>{process.stdout.write(data.repeat(100000));process.stderr.write('e'.repeat(100000));})",
|
||||
],
|
||||
stdin: "piped",
|
||||
});
|
||||
const stdout = new Response(child.stdout).text();
|
||||
const stderr = new Response(child.stderr).text();
|
||||
const writer = child.stdin.getWriter();
|
||||
await writer.write(new TextEncoder().encode("abc"));
|
||||
await writer.close();
|
||||
const [status, out, err] = await Promise.all([child.status, stdout, stderr]);
|
||||
assert.equal(status.success, true);
|
||||
assert.equal(out, "abc".repeat(100000));
|
||||
assert.equal(err, "e".repeat(100000));
|
||||
});
|
||||
|
||||
test("a missing executable rejects with its OS error", async () => {
|
||||
await assert.rejects(runProcess("livesync-missing-test-executable-847281"), { code: "ENOENT" });
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { constants } from "node:os";
|
||||
import { Readable, Writable } from "node:stream";
|
||||
|
||||
export type ProcessStatus = { code: number; success: boolean; signal: NodeJS.Signals | null };
|
||||
export type ProcessOutput = ProcessStatus & { stdout: Uint8Array; stderr: Uint8Array };
|
||||
export type ProcessOptions = {
|
||||
args?: string[];
|
||||
cwd?: string | URL;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
stdin?: "null" | "piped" | "inherit";
|
||||
stdout?: "piped" | "inherit";
|
||||
stderr?: "piped" | "inherit";
|
||||
};
|
||||
export type RunningProcess = {
|
||||
readonly pid: number;
|
||||
readonly stdin: WritableStream<Uint8Array>;
|
||||
readonly stdout: ReadableStream<Uint8Array>;
|
||||
readonly stderr: ReadableStream<Uint8Array>;
|
||||
readonly status: Promise<ProcessStatus>;
|
||||
kill(signal?: NodeJS.Signals): void;
|
||||
};
|
||||
|
||||
export function spawnProcess(command: string, options: ProcessOptions = {}): RunningProcess {
|
||||
const child = spawn(command, options.args ?? [], {
|
||||
cwd: options.cwd,
|
||||
env: { ...process.env, ...options.env },
|
||||
stdio: [
|
||||
options.stdin === "inherit" ? "inherit" : options.stdin === "piped" ? "pipe" : "ignore",
|
||||
options.stdout === "inherit" ? "inherit" : "pipe",
|
||||
options.stderr === "inherit" ? "inherit" : "pipe",
|
||||
],
|
||||
windowsHide: true,
|
||||
});
|
||||
const status = new Promise<ProcessStatus>((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) =>
|
||||
resolve({
|
||||
code: code ?? (signal ? 128 + (constants.signals[signal] ?? 0) : 1),
|
||||
success: code === 0 && signal === null,
|
||||
signal,
|
||||
})
|
||||
);
|
||||
});
|
||||
return {
|
||||
pid: child.pid ?? 0,
|
||||
get stdin() {
|
||||
if (!child.stdin) throw new Error("The child process stdin is not piped");
|
||||
return Writable.toWeb(child.stdin) as WritableStream<Uint8Array>;
|
||||
},
|
||||
get stdout() {
|
||||
if (!child.stdout) throw new Error("The child process stdout is not piped");
|
||||
return Readable.toWeb(child.stdout) as ReadableStream<Uint8Array>;
|
||||
},
|
||||
get stderr() {
|
||||
if (!child.stderr) throw new Error("The child process stderr is not piped");
|
||||
return Readable.toWeb(child.stderr) as ReadableStream<Uint8Array>;
|
||||
},
|
||||
status,
|
||||
kill(signal = "SIGTERM") {
|
||||
child.kill(signal);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function runProcess(command: string, options: ProcessOptions = {}): Promise<ProcessOutput> {
|
||||
const child = spawnProcess(command, options);
|
||||
const collect = async (stream: ReadableStream<Uint8Array>): Promise<Uint8Array> =>
|
||||
new Uint8Array(await new Response(stream).arrayBuffer());
|
||||
const [status, stdout, stderr] = await Promise.all([
|
||||
child.status,
|
||||
options.stdout === "inherit" ? new Uint8Array() : collect(child.stdout),
|
||||
options.stderr === "inherit" ? new Uint8Array() : collect(child.stderr),
|
||||
]);
|
||||
return { ...status, stdout, stderr };
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { runProcess } from "./process.ts";
|
||||
import { CLI_DIR, runCliOrFail } from "./cli.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Settings file initialisation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Generate a default settings file using the CLI's init-settings command. */
|
||||
export async function initSettingsFile(settingsFile: string): Promise<void> {
|
||||
await runCliOrFail("init-settings", "--force", settingsFile);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a full setup URI from a settings file via the Commonlib package API.
|
||||
* Mirrors the bash flow in test-setup-put-cat-linux.sh.
|
||||
*/
|
||||
export async function generateSetupUriFromSettings(
|
||||
settingsFile: string,
|
||||
setupPassphrase: string,
|
||||
preserveRemoteSettings = false
|
||||
): Promise<string> {
|
||||
const script = [
|
||||
"import { fs } from '@vrtmrz/livesync-commonlib/node';",
|
||||
"import { encodeSettingsToSetupURI } from '@vrtmrz/livesync-commonlib/compat/API/processSetting';",
|
||||
"(async () => {",
|
||||
" const settingsPath = process.env.SETTINGS_FILE;",
|
||||
" const passphrase = process.env.SETUP_PASSPHRASE;",
|
||||
" const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf-8'));",
|
||||
...(preserveRemoteSettings
|
||||
? []
|
||||
: [
|
||||
" settings.couchDB_DBNAME = 'setup-put-cat-db';",
|
||||
" settings.couchDB_URI = 'http://127.0.0.1:5999';",
|
||||
" settings.couchDB_USER = 'dummy';",
|
||||
" settings.couchDB_PASSWORD = 'dummy';",
|
||||
" settings.liveSync = false;",
|
||||
" settings.syncOnStart = false;",
|
||||
" settings.syncOnSave = false;",
|
||||
]),
|
||||
" const uri = await encodeSettingsToSetupURI(settings, passphrase);",
|
||||
" process.stdout.write(uri.trim());",
|
||||
"})();",
|
||||
].join("\n");
|
||||
|
||||
const { code, stdout, stderr } = await runProcess(process.execPath, {
|
||||
args: ["--input-type=module", "--eval", script],
|
||||
cwd: CLI_DIR,
|
||||
env: { SETTINGS_FILE: settingsFile, SETUP_PASSPHRASE: setupPassphrase },
|
||||
});
|
||||
const dec = new TextDecoder();
|
||||
if (code !== 0) {
|
||||
throw new Error(
|
||||
`Failed to generate setup URI (code ${code})\nstdout: ${dec.decode(stdout)}\nstderr: ${dec.decode(stderr)}`
|
||||
);
|
||||
}
|
||||
const uri = dec.decode(stdout).trim();
|
||||
if (!uri) throw new Error("Failed to generate setup URI: output is empty");
|
||||
return uri;
|
||||
}
|
||||
|
||||
/** Set isConfigured=true in a settings file (required for mirror / scan). */
|
||||
export async function markSettingsConfigured(settingsFile: string): Promise<void> {
|
||||
const data = JSON.parse(await readFile(settingsFile, "utf8"));
|
||||
data.isConfigured = true;
|
||||
await writeFile(settingsFile, JSON.stringify(data, null, 2));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CouchDB remote settings
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Apply CouchDB connection details to a settings file.
|
||||
* Mirrors cli_test_apply_couchdb_settings() from test-helpers.sh.
|
||||
*/
|
||||
export async function applyCouchdbSettings(
|
||||
settingsFile: string,
|
||||
couchdbUri: string,
|
||||
couchdbUser: string,
|
||||
couchdbPassword: string,
|
||||
couchdbDbname: string,
|
||||
liveSync = false
|
||||
): Promise<void> {
|
||||
const data = JSON.parse(await readFile(settingsFile, "utf8"));
|
||||
data.couchDB_URI = couchdbUri;
|
||||
data.couchDB_USER = couchdbUser;
|
||||
data.couchDB_PASSWORD = couchdbPassword;
|
||||
data.couchDB_DBNAME = couchdbDbname;
|
||||
if (liveSync) {
|
||||
data.liveSync = true;
|
||||
data.syncOnStart = false;
|
||||
data.syncOnSave = false;
|
||||
data.usePluginSync = false;
|
||||
}
|
||||
data.isConfigured = true;
|
||||
await writeFile(settingsFile, JSON.stringify(data, null, 2));
|
||||
}
|
||||
|
||||
export async function applyRemoteSyncSettings(
|
||||
settingsFile: string,
|
||||
options: {
|
||||
remoteType: "COUCHDB" | "MINIO";
|
||||
couchdbUri?: string;
|
||||
couchdbUser?: string;
|
||||
couchdbPassword?: string;
|
||||
couchdbDbname?: string;
|
||||
minioBucket?: string;
|
||||
minioEndpoint?: string;
|
||||
minioAccessKey?: string;
|
||||
minioSecretKey?: string;
|
||||
encrypt?: boolean;
|
||||
passphrase?: string;
|
||||
enableCompression?: boolean;
|
||||
usePathObfuscation?: boolean;
|
||||
}
|
||||
): Promise<void> {
|
||||
const data = JSON.parse(await readFile(settingsFile, "utf8"));
|
||||
|
||||
if (options.remoteType === "COUCHDB") {
|
||||
data.remoteType = "";
|
||||
data.couchDB_URI = options.couchdbUri;
|
||||
data.couchDB_USER = options.couchdbUser;
|
||||
data.couchDB_PASSWORD = options.couchdbPassword;
|
||||
data.couchDB_DBNAME = options.couchdbDbname;
|
||||
} else {
|
||||
data.remoteType = "MINIO";
|
||||
data.bucket = options.minioBucket;
|
||||
data.endpoint = options.minioEndpoint;
|
||||
data.accessKey = options.minioAccessKey;
|
||||
data.secretKey = options.minioSecretKey;
|
||||
data.region = "auto";
|
||||
data.forcePathStyle = true;
|
||||
}
|
||||
|
||||
data.liveSync = true;
|
||||
data.syncOnStart = false;
|
||||
data.syncOnSave = false;
|
||||
data.usePluginSync = false;
|
||||
data.encrypt = options.encrypt === true;
|
||||
data.passphrase = options.encrypt ? (options.passphrase ?? "") : "";
|
||||
if (options.enableCompression !== undefined) {
|
||||
data.enableCompression = options.enableCompression;
|
||||
}
|
||||
if (options.usePathObfuscation !== undefined) {
|
||||
data.usePathObfuscation = options.usePathObfuscation;
|
||||
}
|
||||
data.isConfigured = true;
|
||||
await writeFile(settingsFile, JSON.stringify(data, null, 2));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// P2P settings
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Apply P2P connection details to a settings file.
|
||||
* Mirrors cli_test_apply_p2p_settings() from test-helpers.sh.
|
||||
*/
|
||||
export async function applyP2pSettings(
|
||||
settingsFile: string,
|
||||
roomId: string,
|
||||
passphrase: string,
|
||||
appId = "self-hosted-livesync-cli-tests",
|
||||
relays = "ws://localhost:4000/",
|
||||
autoAccept = "~.*",
|
||||
turnServers = "turn:127.0.0.1:3478"
|
||||
): Promise<void> {
|
||||
const data = JSON.parse(await readFile(settingsFile, "utf8"));
|
||||
data.P2P_Enabled = true;
|
||||
data.P2P_AutoStart = false;
|
||||
data.P2P_AutoBroadcast = false;
|
||||
data.P2P_AppID = appId;
|
||||
data.P2P_roomID = roomId;
|
||||
data.P2P_passphrase = passphrase;
|
||||
data.P2P_relays = relays;
|
||||
data.P2P_AutoAcceptingPeers = autoAccept;
|
||||
data.P2P_AutoDenyingPeers = "";
|
||||
data.P2P_turnServers = turnServers;
|
||||
data.P2P_turnUsername = "testuser";
|
||||
data.P2P_turnCredential = "testpass";
|
||||
data.P2P_IsHeadless = true;
|
||||
data.isConfigured = true;
|
||||
await writeFile(settingsFile, JSON.stringify(data, null, 2));
|
||||
}
|
||||
|
||||
export async function applyP2pTestTweaks(settingsFile: string, deviceName: string, passphrase: string): Promise<void> {
|
||||
const data = JSON.parse(await readFile(settingsFile, "utf8"));
|
||||
data.remoteType = "ONLY_P2P";
|
||||
data.encrypt = true;
|
||||
data.passphrase = passphrase;
|
||||
data.usePathObfuscation = true;
|
||||
data.handleFilenameCaseSensitive = false;
|
||||
data.customChunkSize = 60;
|
||||
data.usePluginSyncV2 = true;
|
||||
data.P2P_DevicePeerName = deviceName;
|
||||
data.isConfigured = true;
|
||||
await writeFile(settingsFile, JSON.stringify(data, null, 2));
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
/** A temporary directory which is removed when its `await using` scope ends. */
|
||||
export class TempDir implements AsyncDisposable {
|
||||
readonly path: string;
|
||||
private constructor(path: string) {
|
||||
this.path = path;
|
||||
}
|
||||
static async create(prefix = "livesync-node-test"): Promise<TempDir> {
|
||||
return new TempDir(await mkdtemp(join(tmpdir(), `${prefix}.`)));
|
||||
}
|
||||
join(...parts: string[]): string {
|
||||
return join(this.path, ...parts);
|
||||
}
|
||||
async [Symbol.asyncDispose](): Promise<void> {
|
||||
await rm(this.path, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user