273 lines
11 KiB
JavaScript
273 lines
11 KiB
JavaScript
/**
|
|
* orchestrator/test-ipc.mjs
|
|
*
|
|
* Integration test: spawn the C++ worker in UDS mode, exchange messages, verify responses.
|
|
*
|
|
* Run: npm run test:ipc
|
|
*
|
|
* Env:
|
|
* KBOT_IPC_LLM — real LLM step is on by default; set to 0 / false / no / off to skip (CI / offline).
|
|
* KBOT_ROUTER — router (default: openrouter; same defaults as C++ LLMClient / CLI)
|
|
* KBOT_IPC_MODEL — optional model id (e.g. openrouter slug); else C++ default for that router
|
|
* KBOT_IPC_PROMPT — custom prompt (default: capital of Germany; asserts "berlin" in reply)
|
|
* KBOT_IPC_LLM_LOG_MAX — max chars to print for LLM text (default: unlimited)
|
|
* KBOT_IPC_LLAMA — llama :8888 step on by default; set 0/false/no/off to skip
|
|
* KBOT_IPC_LLAMA_AUTOSTART — if 0, do not spawn scripts/run-7b.sh when :8888 is closed
|
|
* KBOT_LLAMA_* — KBOT_LLAMA_PORT, KBOT_LLAMA_BASE_URL, KBOT_LLAMA_MODEL, KBOT_LLAMA_START_TIMEOUT_MS
|
|
*
|
|
* Shared: presets.js, test-commons.js, reports.js
|
|
* Report: cwd/tests/test-ipc__HH-mm.{json,md} (see reports.js)
|
|
*/
|
|
|
|
import { spawn } from 'node:child_process';
|
|
import { dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import net from 'node:net';
|
|
import { existsSync, unlinkSync } from 'node:fs';
|
|
|
|
import {
|
|
distExePath,
|
|
platform,
|
|
uds,
|
|
timeouts,
|
|
kbotAiPayloadFromEnv,
|
|
kbotAiPayloadLlamaLocal,
|
|
usingDefaultGermanyPrompt,
|
|
ensureLlamaLocalServer,
|
|
} from './presets.js';
|
|
import {
|
|
createAssert,
|
|
payloadObj,
|
|
logKbotAiResponse,
|
|
ipcLlmEnabled,
|
|
ipcLlamaEnabled,
|
|
llamaAutostartEnabled,
|
|
createIpcClient,
|
|
pipeWorkerStderr,
|
|
} from './test-commons.js';
|
|
import {
|
|
createMetricsCollector,
|
|
buildMetricsBundle,
|
|
writeTestReports,
|
|
} from './reports.js';
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const EXE = distExePath(__dirname);
|
|
const stats = createAssert();
|
|
const { assert } = stats;
|
|
|
|
/** Set at run start for error reports */
|
|
let ipcRunStartedAt = null;
|
|
let ipcMetricsCollector = null;
|
|
|
|
async function run() {
|
|
ipcMetricsCollector = createMetricsCollector();
|
|
ipcRunStartedAt = new Date().toISOString();
|
|
console.log('\n🔧 IPC [UDS] Integration Tests\n');
|
|
|
|
if (!existsSync(EXE)) {
|
|
console.error(`❌ Binary not found at ${EXE}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
const CPP_UDS_ARG = uds.workerArg();
|
|
if (!platform.isWin && existsSync(CPP_UDS_ARG)) {
|
|
unlinkSync(CPP_UDS_ARG);
|
|
}
|
|
|
|
// ── 1. Spawn & ready ────────────────────────────────────────────────────
|
|
console.log('1. Spawn worker (UDS mode) and wait for ready signal');
|
|
const workerProc = spawn(EXE, ['worker', '--uds', CPP_UDS_ARG], { stdio: 'pipe' });
|
|
pipeWorkerStderr(workerProc);
|
|
|
|
let socket;
|
|
for (let i = 0; i < timeouts.connectAttempts; i++) {
|
|
try {
|
|
await new Promise((res, rej) => {
|
|
socket = net.connect(uds.connectOpts(CPP_UDS_ARG));
|
|
socket.once('connect', res);
|
|
socket.once('error', rej);
|
|
});
|
|
break;
|
|
} catch (e) {
|
|
if (i === timeouts.connectAttempts - 1) throw e;
|
|
await new Promise((r) => setTimeout(r, timeouts.connectRetryMs));
|
|
}
|
|
}
|
|
assert(true, 'Socket connected successfully');
|
|
|
|
const ipc = createIpcClient(socket);
|
|
ipc.attach();
|
|
|
|
const readyMsg = await ipc.readyPromise;
|
|
assert(readyMsg.type === 'ready', 'Worker sends ready message on startup');
|
|
|
|
// ── 2. Ping / Pong ─────────────────────────────────────────────────────
|
|
console.log('2. Ping → Pong');
|
|
const pong = await ipc.request({ type: 'ping' }, timeouts.ipcDefault);
|
|
assert(pong.type === 'pong', `Response type is "pong" (got "${pong.type}")`);
|
|
|
|
// ── 3. Job echo ─────────────────────────────────────────────────────────
|
|
console.log('3. Job → Job Result (echo payload)');
|
|
const payload = { action: 'resize', width: 1024, format: 'webp' };
|
|
const jobResult = await ipc.request({ type: 'job', payload }, timeouts.ipcDefault);
|
|
assert(jobResult.type === 'job_result', `Response type is "job_result" (got "${jobResult.type}")`);
|
|
assert(
|
|
jobResult.payload?.action === 'resize' && jobResult.payload?.width === 1024,
|
|
'Payload echoed back correctly'
|
|
);
|
|
|
|
// ── 4. Unknown type → error ─────────────────────────────────────────────
|
|
console.log('4. Unknown type → error response');
|
|
const errResp = await ipc.request({ type: 'nonsense' }, timeouts.ipcDefault);
|
|
assert(errResp.type === 'error', `Response type is "error" (got "${errResp.type}")`);
|
|
|
|
// ── 5. Multiple rapid requests ──────────────────────────────────────────
|
|
console.log('5. Multiple concurrent requests');
|
|
const promises = [];
|
|
for (let i = 0; i < 10; i++) {
|
|
promises.push(ipc.request({ type: 'ping', payload: { seq: i } }, timeouts.ipcDefault));
|
|
}
|
|
const results = await Promise.all(promises);
|
|
assert(results.length === 10, `All 10 responses received`);
|
|
assert(results.every((r) => r.type === 'pong'), 'All responses are pong');
|
|
|
|
// ── 6. kbot-ai — real LLM (optional via ipcLlmEnabled) ─────────────────
|
|
if (ipcLlmEnabled()) {
|
|
const aiPayload = kbotAiPayloadFromEnv();
|
|
const r = aiPayload.router;
|
|
console.log(`6. kbot-ai → real LLM (router=${r}, timeout 3m)`);
|
|
const live = await ipc.request(
|
|
{
|
|
type: 'kbot-ai',
|
|
payload: aiPayload,
|
|
},
|
|
timeouts.kbotAi
|
|
);
|
|
assert(live.type === 'job_result', `LLM response type job_result (got "${live.type}")`);
|
|
const lp = payloadObj(live);
|
|
assert(lp?.status === 'success', `payload status success (got "${lp?.status}")`);
|
|
assert(
|
|
typeof lp?.text === 'string' && lp.text.trim().length >= 3,
|
|
`assistant text present (length ${(lp?.text || '').length})`
|
|
);
|
|
if (usingDefaultGermanyPrompt()) {
|
|
assert(
|
|
/berlin/i.test(String(lp?.text || '')),
|
|
'assistant text mentions Berlin (capital of Germany)'
|
|
);
|
|
}
|
|
logKbotAiResponse('kbot-ai response', live);
|
|
} else {
|
|
console.log('6. kbot-ai — skipped (KBOT_IPC_LLM=0/false/no/off; default is to run live LLM)');
|
|
}
|
|
|
|
// ── 7. kbot-ai — llama local :8888 (optional; llama-basics parity) ───────
|
|
if (ipcLlamaEnabled()) {
|
|
console.log('7. kbot-ai → llama local runner (OpenAI :8888, presets.llama)');
|
|
let llamaReady = false;
|
|
try {
|
|
await ensureLlamaLocalServer({
|
|
autostart: llamaAutostartEnabled(),
|
|
startTimeoutMs: timeouts.llamaServerStart,
|
|
});
|
|
llamaReady = true;
|
|
} catch (e) {
|
|
console.error(` ❌ ${e?.message ?? e}`);
|
|
}
|
|
assert(llamaReady, 'llama-server listening on :8888 (or autostart run-7b.sh succeeded)');
|
|
|
|
if (llamaReady) {
|
|
const llamaPayload = kbotAiPayloadLlamaLocal();
|
|
const llamaRes = await ipc.request(
|
|
{ type: 'kbot-ai', payload: llamaPayload },
|
|
timeouts.llamaKbotAi
|
|
);
|
|
assert(llamaRes.type === 'job_result', `llama IPC response type job_result (got "${llamaRes.type}")`);
|
|
const llp = payloadObj(llamaRes);
|
|
assert(llp?.status === 'success', `llama payload status success (got "${llp?.status}")`);
|
|
assert(
|
|
typeof llp?.text === 'string' && llp.text.trim().length >= 1,
|
|
`llama assistant text present (length ${(llp?.text || '').length})`
|
|
);
|
|
assert(/\b8\b/.test(String(llp?.text || '')), 'llama arithmetic: reply mentions 8 (5+3)');
|
|
logKbotAiResponse('kbot-ai llama local', llamaRes);
|
|
}
|
|
} else {
|
|
console.log('7. kbot-ai llama local — skipped (KBOT_IPC_LLAMA=0; default is to run)');
|
|
}
|
|
|
|
// ── 8. Graceful shutdown ────────────────────────────────────────────────
|
|
console.log('8. Graceful shutdown');
|
|
const shutdownRes = await ipc.request({ type: 'shutdown' }, timeouts.ipcDefault);
|
|
assert(shutdownRes.type === 'shutdown_ack', `Shutdown acknowledged (got "${shutdownRes.type}")`);
|
|
|
|
await new Promise((r) => setTimeout(r, timeouts.postShutdownMs));
|
|
socket.destroy();
|
|
assert(workerProc.exitCode === 0, `Worker exited with code 0 (got ${workerProc.exitCode})`);
|
|
|
|
// ── Summary ─────────────────────────────────────────────────────────────
|
|
console.log(`\n────────────────────────────────`);
|
|
console.log(` Passed: ${stats.passed} Failed: ${stats.failed}`);
|
|
console.log(`────────────────────────────────\n`);
|
|
|
|
try {
|
|
const finishedAt = new Date().toISOString();
|
|
const { jsonPath, mdPath } = await writeTestReports(
|
|
'test-ipc',
|
|
{
|
|
startedAt: ipcRunStartedAt,
|
|
finishedAt,
|
|
passed: stats.passed,
|
|
failed: stats.failed,
|
|
ok: stats.failed === 0,
|
|
ipcLlm: ipcLlmEnabled(),
|
|
ipcLlama: ipcLlamaEnabled(),
|
|
env: {
|
|
KBOT_IPC_LLM: process.env.KBOT_IPC_LLM ?? null,
|
|
KBOT_IPC_LLAMA: process.env.KBOT_IPC_LLAMA ?? null,
|
|
KBOT_IPC_LLAMA_AUTOSTART: process.env.KBOT_IPC_LLAMA_AUTOSTART ?? null,
|
|
KBOT_ROUTER: process.env.KBOT_ROUTER ?? null,
|
|
KBOT_IPC_MODEL: process.env.KBOT_IPC_MODEL ?? null,
|
|
KBOT_IPC_PROMPT: process.env.KBOT_IPC_PROMPT ?? null,
|
|
KBOT_LLAMA_PORT: process.env.KBOT_LLAMA_PORT ?? null,
|
|
KBOT_LLAMA_BASE_URL: process.env.KBOT_LLAMA_BASE_URL ?? null,
|
|
},
|
|
metrics: buildMetricsBundle(ipcMetricsCollector, ipcRunStartedAt, finishedAt),
|
|
},
|
|
{ cwd: process.cwd() }
|
|
);
|
|
console.log(` 📄 Report JSON: ${jsonPath}`);
|
|
console.log(` 📄 Report MD: ${mdPath}\n`);
|
|
} catch (e) {
|
|
console.error(' ⚠️ Failed to write report:', e?.message ?? e);
|
|
}
|
|
|
|
process.exit(stats.failed > 0 ? 1 : 0);
|
|
}
|
|
|
|
run().catch(async (err) => {
|
|
console.error('Test runner error:', err);
|
|
try {
|
|
const finishedAt = new Date().toISOString();
|
|
const c = ipcMetricsCollector ?? createMetricsCollector();
|
|
const started = ipcRunStartedAt ?? finishedAt;
|
|
await writeTestReports(
|
|
'test-ipc',
|
|
{
|
|
startedAt: started,
|
|
finishedAt,
|
|
error: String(err?.stack ?? err),
|
|
passed: stats.passed,
|
|
failed: stats.failed,
|
|
ok: false,
|
|
metrics: buildMetricsBundle(c, started, finishedAt),
|
|
},
|
|
{ cwd: process.cwd() }
|
|
);
|
|
} catch (_) {
|
|
/* ignore */
|
|
}
|
|
process.exit(1);
|
|
});
|