/** * orchestrator/reports.js — JSON + Markdown reports under cwd/tests/ * * File pattern (logical): test-name::hh:mm * On-disk: test-name__HH-mm.json / .md (Windows: no `:` in filenames) */ import { mkdir, writeFile } from 'node:fs/promises'; import { join, dirname } from 'node:path'; import os from 'node:os'; import { performance } from 'node:perf_hooks'; import { resourceUsage } from 'node:process'; const WIN_BAD = /[<>:"/\\|?*\x00-\x1f]/g; /** Strip characters invalid in Windows / POSIX filenames. */ export function sanitizeTestName(name) { const s = String(name).trim().replace(WIN_BAD, '_').replace(/\s+/g, '_'); return s || 'test'; } /** * @param {Date} [now] * @returns {{ hh: string, mm: string, label: string }} */ export function timeParts(now = new Date()) { const hh = String(now.getHours()).padStart(2, '0'); const mm = String(now.getMinutes()).padStart(2, '0'); return { hh, mm, label: `${hh}:${mm}` }; } /** * @param {string} testName * @param {string} ext — including dot, e.g. '.json' * @param {{ cwd?: string, now?: Date }} [options] */ export function reportFilePathWithExt(testName, ext, options = {}) { const cwd = options.cwd ?? process.cwd(); const now = options.now ?? new Date(); const base = sanitizeTestName(testName); const { hh, mm } = timeParts(now); const file = `${base}__${hh}-${mm}${ext}`; return join(cwd, 'tests', file); } export function reportFilePath(testName, options = {}) { return reportFilePathWithExt(testName, '.json', options); } export function reportMarkdownPath(testName, options = {}) { return reportFilePathWithExt(testName, '.md', options); } function formatBytes(n) { if (typeof n !== 'number' || Number.isNaN(n)) return String(n); const u = ['B', 'KB', 'MB', 'GB']; let i = 0; let x = n; while (x >= 1024 && i < u.length - 1) { x /= 1024; i++; } return `${x < 10 && i > 0 ? x.toFixed(1) : Math.round(x)} ${u[i]}`; } /** Snapshot of host / OS (cheap; call anytime). */ export function hostSnapshot() { const cpus = os.cpus(); const total = os.totalmem(); const free = os.freemem(); return { hostname: os.hostname(), platform: os.platform(), arch: os.arch(), release: os.release(), cpuCount: cpus.length, cpuModel: cpus[0]?.model?.trim() ?? '', totalMemBytes: total, freeMemBytes: free, usedMemBytes: total - free, loadAvg: os.loadavg(), osUptimeSec: os.uptime(), }; } /** * Call at test start; then call `.finalize()` at end for wall + CPU delta + memory. */ export function createMetricsCollector() { const cpu0 = process.cpuUsage(); const perf0 = performance.now(); const wall0 = Date.now(); return { hostSnapshot, finalize() { const cpu = process.cpuUsage(cpu0); const perf1 = performance.now(); let ru = null; try { ru = resourceUsage(); } catch { /* older runtimes */ } return { durationWallMs: Math.round((perf1 - perf0) * 1000) / 1000, durationClockMs: Date.now() - wall0, cpuUserUs: cpu.user, cpuSystemUs: cpu.system, cpuUserMs: cpu.user / 1000, cpuSystemMs: cpu.system / 1000, memory: process.memoryUsage(), resourceUsage: ru, pid: process.pid, node: process.version, processUptimeSec: process.uptime(), }; }, }; } /** * @param {Record} payload * @returns {string} */ export function renderMarkdownReport(payload) { const meta = payload.meta ?? {}; const m = payload.metrics ?? {}; const host = m.host ?? {}; const timing = m.timing ?? {}; const proc = m.process ?? {}; const tStart = timing.startedAt ?? payload.startedAt; const tEnd = timing.finishedAt ?? payload.finishedAt; const lines = []; lines.push(`# Test report: ${meta.displayName ?? meta.testName ?? 'run'}`); lines.push(''); lines.push('## Summary'); lines.push(''); lines.push(`| Key | Value |`); lines.push(`| --- | --- |`); lines.push(`| Result | ${payload.ok === true ? 'PASS' : payload.ok === false ? 'FAIL' : '—'} |`); if (payload.passed != null) lines.push(`| Assertions passed | ${payload.passed} |`); if (payload.failed != null) lines.push(`| Assertions failed | ${payload.failed} |`); if (payload.ipcLlm != null) lines.push(`| IPC LLM step | ${payload.ipcLlm ? 'enabled' : 'skipped'} |`); if (payload.ipcLlama != null) { lines.push(`| IPC llama :8888 step | ${payload.ipcLlama ? 'enabled' : 'skipped'} |`); } if (payload.ipcClassifierLlama != null) { lines.push( `| IPC classifier | ${payload.ipcClassifierLlama ? 'local llama :8888' : 'remote (KBOT_ROUTER / KBOT_IPC_MODEL)'} |` ); } lines.push(`| CWD | \`${String(meta.cwd ?? '').replace(/`/g, "'")}\` |`); lines.push(''); lines.push('## Timing'); lines.push(''); lines.push(`| Metric | Value |`); lines.push(`| --- | --- |`); if (tStart) lines.push(`| Started (ISO) | ${tStart} |`); if (tEnd) lines.push(`| Finished (ISO) | ${tEnd} |`); if (proc.durationWallMs != null) lines.push(`| Wall time (perf) | ${proc.durationWallMs} ms |`); if (proc.durationClockMs != null) lines.push(`| Wall time (clock) | ${proc.durationClockMs} ms |`); lines.push(''); lines.push('## Process (Node)'); lines.push(''); lines.push(`| Metric | Value |`); lines.push(`| --- | --- |`); if (proc.pid != null) lines.push(`| PID | ${proc.pid} |`); if (proc.node) lines.push(`| Node | ${proc.node} |`); if (proc.processUptimeSec != null) lines.push(`| process.uptime() | ${proc.processUptimeSec.toFixed(3)} s |`); if (proc.cpuUserMs != null && proc.cpuSystemMs != null) { lines.push(`| CPU user (process.cpuUsage Δ) | ${proc.cpuUserMs.toFixed(3)} ms (${proc.cpuUserUs ?? '—'} µs) |`); lines.push(`| CPU system (process.cpuUsage Δ) | ${proc.cpuSystemMs.toFixed(3)} ms (${proc.cpuSystemUs ?? '—'} µs) |`); } const ru = proc.resourceUsage; if (ru && typeof ru === 'object') { if (ru.userCPUTime != null) { lines.push(`| CPU user (resourceUsage) | ${(ru.userCPUTime / 1000).toFixed(3)} ms |`); } if (ru.systemCPUTime != null) { lines.push(`| CPU system (resourceUsage) | ${(ru.systemCPUTime / 1000).toFixed(3)} ms |`); } if (ru.maxRSS != null) { lines.push(`| Max RSS (resourceUsage) | ${formatBytes(ru.maxRSS * 1024)} |`); } } const mem = proc.memory; if (mem && typeof mem === 'object') { lines.push(`| RSS | ${formatBytes(mem.rss)} (${mem.rss} B) |`); lines.push(`| Heap used | ${formatBytes(mem.heapUsed)} |`); lines.push(`| Heap total | ${formatBytes(mem.heapTotal)} |`); lines.push(`| External | ${formatBytes(mem.external)} |`); if (mem.arrayBuffers != null) lines.push(`| Array buffers | ${formatBytes(mem.arrayBuffers)} |`); } lines.push(''); lines.push('## Host'); lines.push(''); lines.push(`| Metric | Value |`); lines.push(`| --- | --- |`); if (host.hostname) lines.push(`| Hostname | ${host.hostname} |`); if (host.platform) lines.push(`| OS | ${host.platform} ${host.release ?? ''} |`); if (host.arch) lines.push(`| Arch | ${host.arch} |`); if (host.cpuCount != null) lines.push(`| CPUs | ${host.cpuCount} |`); if (host.cpuModel) lines.push(`| CPU model | ${host.cpuModel} |`); if (host.totalMemBytes != null) { lines.push(`| RAM total | ${formatBytes(host.totalMemBytes)} |`); lines.push(`| RAM free | ${formatBytes(host.freeMemBytes)} |`); lines.push(`| RAM used | ${formatBytes(host.usedMemBytes)} |`); } if (host.loadAvg && host.loadAvg.length) { lines.push(`| Load avg (1/5/15) | ${host.loadAvg.map((x) => x.toFixed(2)).join(' / ')} |`); } if (host.osUptimeSec != null) lines.push(`| OS uptime | ${(host.osUptimeSec / 3600).toFixed(2)} h |`); lines.push(''); const kbotAi = payload.kbotAi; const hasKbotAiMeta = kbotAi && typeof kbotAi === 'object' && (kbotAi.routerStep != null || kbotAi.llamaStep != null); const hasClassifierLlm = payload.llm != null && typeof payload.llm === 'object'; if (hasKbotAiMeta || hasClassifierLlm) { lines.push('## LLM API (provider JSON)'); lines.push(''); lines.push( 'Fields from the chat completion response except assistant message bodies (`usage`, `model`, `id`, provider-specific).' ); lines.push(''); if (hasKbotAiMeta) { if (kbotAi.routerStep != null) { lines.push('### IPC step 6 — router / main kbot-ai'); lines.push(''); lines.push('```json'); lines.push(JSON.stringify(kbotAi.routerStep, null, 2)); lines.push('```'); lines.push(''); } if (kbotAi.llamaStep != null) { lines.push('### IPC step 7 — local llama :8888'); lines.push(''); lines.push('```json'); lines.push(JSON.stringify(kbotAi.llamaStep, null, 2)); lines.push('```'); lines.push(''); } } if (hasClassifierLlm) { lines.push('### Classifier — batched kbot-ai'); lines.push(''); lines.push('```json'); lines.push(JSON.stringify(payload.llm, null, 2)); lines.push('```'); lines.push(''); } } if (payload.anchor != null || (Array.isArray(payload.distances) && payload.distances.length > 0)) { lines.push('## Classifier batch'); lines.push(''); lines.push(`| Key | Value |`); lines.push(`| --- | --- |`); if (payload.anchor != null) lines.push(`| Anchor | ${payload.anchor} |`); if (payload.labelCount != null) lines.push(`| Label count | ${payload.labelCount} |`); if (payload.backend != null) lines.push(`| Backend | ${payload.backend} |`); const pe = payload.parseError; if (pe != null && String(pe).length) { lines.push(`| Parse | Failed: ${String(pe).replace(/\|/g, '\\|').slice(0, 500)}${String(pe).length > 500 ? '…' : ''} |`); } else { lines.push(`| Parse | OK |`); } lines.push(''); const sorted = Array.isArray(payload.byDistance) ? payload.byDistance : []; const preview = sorted.filter((r) => r && r.distance != null).slice(0, 12); if (preview.length > 0) { lines.push('### Nearest labels (by distance)'); lines.push(''); lines.push(`| Label | Distance |`); lines.push(`| --- | ---: |`); for (const row of preview) { const lab = String(row.label ?? '').replace(/\|/g, '\\|'); lines.push(`| ${lab} | ${row.distance} |`); } lines.push(''); } } if (payload.stress?.summary && typeof payload.stress.summary === 'object') { const s = payload.stress.summary; const w = s.wallMs; lines.push('## Classifier stress (batch repeats)'); lines.push(''); lines.push(`| Metric | Value |`); lines.push(`| --- | --- |`); lines.push(`| Requested runs | ${s.requestedRuns ?? '—'} |`); if (w && typeof w === 'object') { lines.push( `| Wall time (ms) | min ${w.min} · max ${w.max} · avg ${w.avg} · p50 ${w.p50} · p95 ${w.p95} |` ); } lines.push(`| Batch OK / fail | ${s.successCount ?? '—'} / ${s.failCount ?? '—'} |`); if (s.totalTokens > 0 || s.totalPromptTokens > 0 || s.totalCompletionTokens > 0) { lines.push( `| Σ tokens (prompt / completion / total) | ${s.totalPromptTokens} / ${s.totalCompletionTokens} / ${s.totalTokens} |` ); } lines.push(''); } if (payload.env && typeof payload.env === 'object') { lines.push('## Environment (selected)'); lines.push(''); lines.push(`| Variable | Value |`); lines.push(`| --- | --- |`); for (const [k, v] of Object.entries(payload.env)) { lines.push(`| \`${k}\` | ${v === null || v === undefined ? '—' : String(v)} |`); } lines.push(''); } if (payload.error) { lines.push('## Error'); lines.push(''); lines.push('```'); lines.push(String(payload.error)); lines.push('```'); lines.push(''); } lines.push('---'); lines.push(`*Written ${meta.writtenAt ?? new Date().toISOString()}*`); lines.push(''); return lines.join('\n'); } /** * Build metrics block for JSON + MD (host snapshot + process finalize). */ export function buildMetricsBundle(collector, startedAtIso, finishedAtIso) { const host = collector.hostSnapshot(); const processMetrics = collector.finalize(); return { timing: { startedAt: startedAtIso, finishedAt: finishedAtIso, }, host, process: processMetrics, }; } /** * @param {string} testName * @param {Record} data — merged into payload (meta + metrics added) * @param {{ cwd?: string, now?: Date, metrics?: object }} [options] * @returns {Promise<{ jsonPath: string, mdPath: string }>} */ export async function writeTestReports(testName, data, options = {}) { const cwd = options.cwd ?? process.cwd(); const now = options.now ?? new Date(); const jsonPath = reportFilePath(testName, { cwd, now }); const mdPath = reportMarkdownPath(testName, { cwd, now }); const { hh, mm, label } = timeParts(now); const base = sanitizeTestName(testName); const payload = { meta: { testName: base, displayName: `${base}::${label}`, cwd, writtenAt: now.toISOString(), jsonFile: jsonPath, mdFile: mdPath, }, ...data, }; await mkdir(dirname(jsonPath), { recursive: true }); await writeFile(jsonPath, JSON.stringify(payload, null, 2), 'utf8'); const md = renderMarkdownReport(payload); await writeFile(mdPath, md, 'utf8'); return { jsonPath, mdPath }; } /** @deprecated Prefer writeTestReports */ export async function writeJsonReport(testName, data, options = {}) { const { jsonPath } = await writeTestReports(testName, data, options); return jsonPath; }