From df390ac4561d0d4e4118efde6fd5291dac34aa4b Mon Sep 17 00:00:00 2001 From: vorotamoroz Date: Fri, 22 May 2026 03:02:11 +0000 Subject: [PATCH] test: fix deno test helpers --- src/apps/cli/testdeno/helpers/cli.ts | 72 +++++++++++-- src/apps/cli/testdeno/helpers/dataset.ts | 123 +++++++++++++++++++++++ src/apps/cli/testdeno/helpers/docker.ts | 58 +++++++---- 3 files changed, 228 insertions(+), 25 deletions(-) create mode 100644 src/apps/cli/testdeno/helpers/dataset.ts diff --git a/src/apps/cli/testdeno/helpers/cli.ts b/src/apps/cli/testdeno/helpers/cli.ts index 8c78b52..d2279cb 100644 --- a/src/apps/cli/testdeno/helpers/cli.ts +++ b/src/apps/cli/testdeno/helpers/cli.ts @@ -39,27 +39,73 @@ function concatChunks(chunks: Uint8Array[]): Uint8Array { return out; } +function formatTeeCommand(args: string[]): string { + return ["node", CLI_DIST, ...args].map((part) => JSON.stringify(part)).join(" "); +} + +function createLineTeeWriter( + pid: number, + streamName: "stdout" | "stderr", + writer: (chunk: Uint8Array) => void +): { write: (chunk: Uint8Array) => void; close: () => void } { + const enc = new TextEncoder(); + const dec = new TextDecoder(); + let pending = ""; + let headerWritten = false; + const emitLine = (line: string) => { + if (!headerWritten) { + writer(enc.encode(`[CLI tee pid=${pid}:${streamName}]\n`)); + headerWritten = true; + } + writer(enc.encode(`[CLI tee pid=${pid}:${streamName}] ${line}\n`)); + }; + + const flush = (final = false) => { + let index = pending.indexOf("\n"); + while (index >= 0) { + const line = pending.slice(0, index).replace(/\r$/, ""); + pending = pending.slice(index + 1); + emitLine(line); + index = pending.indexOf("\n"); + } + if (final && pending.length > 0) { + emitLine(pending.replace(/\r$/, "")); + pending = ""; + } + }; + + return { + write(chunk: Uint8Array) { + pending += dec.decode(chunk, { stream: true }); + flush(false); + }, + close() { + pending += dec.decode(); + flush(true); + }, + }; +} + async function collectStream( stream: ReadableStream, - teeTarget: WritableStream | null + teeTarget: { write: (chunk: Uint8Array) => void; close: () => void } | null ): Promise { const reader = stream.getReader(); const chunks: Uint8Array[] = []; - const writer = teeTarget?.getWriter(); try { while (true) { const { done, value } = await reader.read(); if (done) break; if (value) { chunks.push(value); - if (writer) { - await writer.write(value); + if (teeTarget) { + teeTarget.write(value); } } } } finally { - if (writer) { - writer.releaseLock(); + if (teeTarget) { + teeTarget.close(); } reader.releaseLock(); } @@ -76,8 +122,18 @@ async function runNodeCommand(args: string[], stdinData?: Uint8Array): Promise Deno.stdout.writeSync(chunk)) : null + ); + const stderrPromise = collectStream( + child.stderr, + TEE_ENABLED ? createLineTeeWriter(child.pid, "stderr", (chunk) => Deno.stderr.writeSync(chunk)) : null + ); if (stdinData) { const w = child.stdin.getWriter(); diff --git a/src/apps/cli/testdeno/helpers/dataset.ts b/src/apps/cli/testdeno/helpers/dataset.ts new file mode 100644 index 0000000..ad9af47 --- /dev/null +++ b/src/apps/cli/testdeno/helpers/dataset.ts @@ -0,0 +1,123 @@ +export type DeterministicDatasetConfig = { + rootDir: string; + datasetDirName: string; + seed: string; + mdCount: number; + mdMinSizeBytes: number; + mdMaxSizeBytes: number; + binCount: number; + binSizeBytes: number; +}; + +export type DatasetEntry = { + kind: "md" | "bin"; + relativePath: string; + absolutePath: string; + size: number; +}; + +export type DeterministicDataset = { + rootDir: string; + datasetDirName: string; + seed: string; + entries: DatasetEntry[]; + totalFiles: number; + totalBytes: number; + mdCount: number; + binCount: number; +}; + +function fnv1a32(input: string): number { + let hash = 0x811c9dc5; + for (let i = 0; i < input.length; i++) { + hash ^= input.charCodeAt(i) & 0xff; + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} + +function createXorshift32(seed: number): () => number { + let state = seed >>> 0; + if (state === 0) { + state = 0x9e3779b9; + } + return () => { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return state >>> 0; + }; +} + +function createTextBytes(size: number, fileIndex: number, seed: string): Uint8Array { + const template = + `# Bench file ${fileIndex}\n` + + `seed: ${seed}\n` + + "lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n"; + + const templateBytes = new TextEncoder().encode(template); + const out = new Uint8Array(size); + for (let i = 0; i < size; i++) { + out[i] = templateBytes[i % templateBytes.length]; + } + return out; +} + +function toPath(rootDir: string, relativePath: string): string { + return `${rootDir}/${relativePath}`; +} + +export async function createDeterministicDataset(config: DeterministicDatasetConfig): Promise { + if (config.mdCount < 0 || config.binCount < 0) { + throw new Error("mdCount and binCount must be non-negative"); + } + if (config.mdMinSizeBytes <= 0 || config.mdMaxSizeBytes <= 0 || config.binSizeBytes <= 0) { + throw new Error("all size values must be positive"); + } + if (config.mdMinSizeBytes > config.mdMaxSizeBytes) { + throw new Error("mdMinSizeBytes must be <= mdMaxSizeBytes"); + } + + const datasetRoot = toPath(config.rootDir, config.datasetDirName); + const mdDir = `${datasetRoot}/md`; + const binDir = `${datasetRoot}/bin`; + await Deno.mkdir(mdDir, { recursive: true }); + await Deno.mkdir(binDir, { recursive: true }); + + const nextRandom = createXorshift32(fnv1a32(config.seed)); + const mdRange = config.mdMaxSizeBytes - config.mdMinSizeBytes + 1; + const entries: DatasetEntry[] = []; + + for (let index = 0; index < config.mdCount; index++) { + const size = config.mdMinSizeBytes + (nextRandom() % mdRange); + const relativePath = `${config.datasetDirName}/md/file-${String(index).padStart(4, "0")}.md`; + const absolutePath = toPath(config.rootDir, relativePath); + const body = createTextBytes(size, index, config.seed); + await Deno.writeFile(absolutePath, body); + entries.push({ kind: "md", relativePath, absolutePath, size }); + } + + for (let index = 0; index < config.binCount; index++) { + const size = config.binSizeBytes; + const relativePath = `${config.datasetDirName}/bin/file-${String(index).padStart(4, "0")}.bin`; + const absolutePath = toPath(config.rootDir, relativePath); + const body = new Uint8Array(size); + for (let i = 0; i < size; i++) { + body[i] = nextRandom() & 0xff; + } + await Deno.writeFile(absolutePath, body); + entries.push({ kind: "bin", relativePath, absolutePath, size }); + } + + const totalBytes = entries.reduce((sum, e) => sum + e.size, 0); + return { + rootDir: config.rootDir, + datasetDirName: config.datasetDirName, + seed: config.seed, + entries, + totalFiles: entries.length, + totalBytes, + mdCount: config.mdCount, + binCount: config.binCount, + }; +} diff --git a/src/apps/cli/testdeno/helpers/docker.ts b/src/apps/cli/testdeno/helpers/docker.ts index 5ecea1f..d04a1f9 100644 --- a/src/apps/cli/testdeno/helpers/docker.ts +++ b/src/apps/cli/testdeno/helpers/docker.ts @@ -27,29 +27,53 @@ function parseCommand(command: string): { bin: string; prefix: string[] } { return { bin: parts[0], prefix: parts.slice(1) }; } -async function runCommand(bin: string, args: string[]): Promise<{ code: number; stdout: string; stderr: string }> { - const cmd = new Deno.Command(bin, { - args, - stdin: "null", - stdout: "piped", - stderr: "piped", - }); +async function collectStream( + stream: ReadableStream, + teeTarget: ((chunk: Uint8Array) => void) | null +): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; try { - const { code, stdout, stderr } = await cmd.output(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + chunks.push(value); + if (teeTarget) { + teeTarget(value); + } + } + } finally { + reader.releaseLock(); + } + + const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; +} + +async function runCommand(bin: string, args: string[]): Promise<{ code: number; stdout: string; stderr: string }> { + try { + const child = new Deno.Command(bin, { + args, + stdin: "null", + stdout: "piped", + stderr: "piped", + }).spawn(); + const stdoutPromise = collectStream(child.stdout, DOCKER_TEE ? (chunk) => Deno.stdout.writeSync(chunk) : null); + const stderrPromise = collectStream(child.stderr, DOCKER_TEE ? (chunk) => Deno.stderr.writeSync(chunk) : null); + const [status, stdout, stderr] = await Promise.all([child.status, stdoutPromise, stderrPromise]); const dec = new TextDecoder(); const result = { - code, + code: status.code, stdout: dec.decode(stdout), stderr: dec.decode(stderr), }; - if (DOCKER_TEE) { - if (result.stdout.trim().length > 0) { - console.log(`[docker:${bin}] ${result.stdout.trimEnd()}`); - } - if (result.stderr.trim().length > 0) { - console.error(`[docker:${bin}] ${result.stderr.trimEnd()}`); - } - } return result; } catch (err) { if (err instanceof Deno.errors.NotFound) {