diff --git a/packages/kbot/logs/params.json b/packages/kbot/logs/params.json index 66bb0162..80215648 100644 --- a/packages/kbot/logs/params.json +++ b/packages/kbot/logs/params.json @@ -1,5 +1,5 @@ { - "model": "gpt-4", + "model": "anthropic/claude-2.0", "messages": [ { "role": "user", diff --git a/packages/kbot/tests/unit/basic-report.md b/packages/kbot/tests/unit/basic-report.md index ffa234b0..1877a285 100644 --- a/packages/kbot/tests/unit/basic-report.md +++ b/packages/kbot/tests/unit/basic-report.md @@ -48,6 +48,69 @@ - Reason: undefined - Timestamp: 4/1/2025, 12:26:35 PM +### hello - deepseek/deepseek-chat:free +- Prompt: `say "hello"` +- Expected: `hello` +- Actual: `` +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:36:37 PM + +### hello - google/gemini-2.0-flash-exp:free +- Prompt: `say "hello"` +- Expected: `hello` +- Actual: `` +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:36:37 PM + +### hello - gpt-4 +- Prompt: `say "hello"` +- Expected: `hello` +- Actual: `` +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:36:42 PM + +### goodbye - deepseek/deepseek-chat:free +- Prompt: `say "goodbye"` +- Expected: `goodbye` +- Actual: `` +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:36:42 PM + +### goodbye - google/gemini-2.0-flash-exp:free +- Prompt: `say "goodbye"` +- Expected: `goodbye` +- Actual: `` +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:36:43 PM + +### goodbye - gpt-4 +- Prompt: `say "goodbye"` +- Expected: `goodbye` +- Actual: `` +- Reason: expected 'goodbye.' to deeply equal 'goodbye' +- Timestamp: 4/1/2025, 1:36:44 PM + +### yes - deepseek/deepseek-chat:free +- Prompt: `say "yes"` +- Expected: `yes` +- Actual: `` +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:36:45 PM + +### yes - google/gemini-2.0-flash-exp:free +- Prompt: `say "yes"` +- Expected: `yes` +- Actual: `` +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:36:45 PM + +### yes - gpt-4 +- Prompt: `say "yes"` +- Expected: `yes` +- Actual: `` +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:36:46 PM + ## Passed Tests ### addition - deepseek/deepseek-chat:free @@ -67,7 +130,7 @@ - Prompt: `add 5 and 3. Return only the number, no explanation.` - Expected: `8` - Actual: `8` -- Timestamp: 4/1/2025, 1:02:55 PM +- Timestamp: 4/1/2025, 1:39:04 PM ### multiplication - deepseek/deepseek-chat:free - Prompt: `multiply 8 and 3. Return only the number, no explanation.` @@ -86,7 +149,7 @@ - Prompt: `multiply 8 and 3. Return only the number, no explanation.` - Expected: `24` - Actual: `24` -- Timestamp: 4/1/2025, 1:02:57 PM +- Timestamp: 4/1/2025, 1:39:06 PM ### division - deepseek/deepseek-chat:free - Prompt: `divide 15 by 3. Return only the number, no explanation.` @@ -105,5 +168,5 @@ - Prompt: `divide 15 by 3. Return only the number, no explanation.` - Expected: `5` - Actual: `5` -- Timestamp: 4/1/2025, 1:03:00 PM +- Timestamp: 4/1/2025, 1:39:08 PM diff --git a/packages/kbot/tests/unit/basic.json b/packages/kbot/tests/unit/basic.json index abca2685..706e1e4a 100644 --- a/packages/kbot/tests/unit/basic.json +++ b/packages/kbot/tests/unit/basic.json @@ -816,5 +816,281 @@ "router": "openrouter", "timestamp": "2025-04-01T11:03:00.064Z", "passed": true + }, + { + "test": "hello", + "prompt": "say \"hello\"", + "result": [], + "expected": "hello", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:36:37.192Z", + "passed": false, + "duration": 1107, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "hello", + "prompt": "say \"hello\"", + "result": [], + "expected": "hello", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:36:37.957Z", + "passed": false, + "duration": 762, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "hello", + "prompt": "say \"hello\"", + "result": [], + "expected": "hello", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:36:42.078Z", + "passed": false, + "duration": 4120, + "reason": "Unknown error occurred", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "goodbye", + "prompt": "say \"goodbye\"", + "result": [], + "expected": "goodbye", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:36:42.695Z", + "passed": false, + "duration": 616, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "goodbye", + "prompt": "say \"goodbye\"", + "result": [], + "expected": "goodbye", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:36:43.462Z", + "passed": false, + "duration": 765, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "goodbye", + "prompt": "say \"goodbye\"", + "result": [], + "expected": "goodbye", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:36:44.537Z", + "passed": false, + "duration": 1073, + "error": { + "message": "expected 'goodbye.' to deeply equal 'goodbye'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected 'goodbye.' to deeply equal 'goodbye'", + "actual": "goodbye.", + "expected": "goodbye", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected 'goodbye.' to deeply equal 'goodbye'\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:64:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"goodbye\"\u001b[39m\nReceived: \u001b[31m\"goodbye\u001b[7m.\u001b[27m\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected 'goodbye.' to deeply equal 'goodbye'\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:64:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected 'goodbye.' to deeply equal 'goodbye'", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "yes", + "prompt": "say \"yes\"", + "result": [], + "expected": "yes", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:36:45.193Z", + "passed": false, + "duration": 654, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "yes", + "prompt": "say \"yes\"", + "result": [], + "expected": "yes", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:36:45.954Z", + "passed": false, + "duration": 760, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "yes", + "prompt": "say \"yes\"", + "result": [], + "expected": "yes", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:36:46.806Z", + "passed": false, + "duration": 851, + "reason": "Unknown error occurred", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [ + "8" + ], + "expected": "8", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:04.078Z", + "passed": true + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [ + "24" + ], + "expected": "24", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:06.572Z", + "passed": true + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [ + "5" + ], + "expected": "5", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:08.868Z", + "passed": true } ] \ No newline at end of file diff --git a/packages/kbot/tests/unit/basic.test.ts b/packages/kbot/tests/unit/basic.test.ts index c051df61..21449c98 100644 --- a/packages/kbot/tests/unit/basic.test.ts +++ b/packages/kbot/tests/unit/basic.test.ts @@ -1,210 +1,210 @@ -import { describe, it, expect } from 'vitest' -import { run } from '../../src/index' -import * as path from 'node:path' -import { sync as write } from "@polymech/fs/write" -import { sync as read } from "@polymech/fs/read" -import { sync as exists } from "@polymech/fs/exists" -import { - models, - TEST_BASE_PATH, - TEST_LOGS_PATH, - TEST_PREFERENCES_PATH, - TEST_TIMEOUT, - TestResult, - formatError, - isEmptyResponse, - getRouterForModel, - getApiKeyForRouter -} from './commons' - -const TEST_LOG_PATH = path.resolve(__dirname, './basic.json') - -describe('Basic Capabilities', () => { - let testResults: TestResult[] = [] - - // Load existing results if any - if (exists(TEST_LOG_PATH)) { - const data = read(TEST_LOG_PATH, 'json') - testResults = Array.isArray(data) ? data : [] - } - - const runBasicTest = async (prompt: string, expected: string, testName: string, modelName: string) => { - let model = 'unknown' - let router = 'unknown' - let startTime = Date.now() - let error: TestResult['error'] | undefined - - try { - const result = await Promise.race([ - run({ - prompt, - mode: 'completion', - model: modelName, - path: TEST_BASE_PATH, - logs: TEST_LOGS_PATH, - preferences: TEST_PREFERENCES_PATH, - onRun: async (options) => { - model = options.model || 'unknown' - router = options.router || 'unknown' - return options - } - }), - new Promise((_, reject) => - setTimeout(() => reject(new Error('API call timed out')), TEST_TIMEOUT) - ) - ]) as string[] - - const actual = result?.[0]?.trim()?.toLowerCase() || '' - const passed = actual === expected && !isEmptyResponse(result) - - if (isEmptyResponse(result)) { - throw new Error('Model returned empty response') - } - - expect(actual).toEqual(expected) - - return { - test: testName, - prompt, - result: result || [], - expected, - model, - router, - timestamp: new Date().toISOString(), - passed, - duration: Date.now() - startTime, - reason: passed ? undefined : `Expected ${expected}, but got ${actual}`, - config: { - router: getRouterForModel(modelName), - apiKey: getApiKeyForRouter(getRouterForModel(modelName)) - } - } - } catch (e) { - error = formatError(e) - throw e - } finally { - const testResult: TestResult = { - test: testName, - prompt, - result: [], - expected, - model, - router, - timestamp: new Date().toISOString(), - passed: false, - duration: Date.now() - startTime, - error, - reason: error?.message || 'Unknown error occurred', - config: { - router: getRouterForModel(modelName), - apiKey: getApiKeyForRouter(getRouterForModel(modelName)) - } - } - - testResults.push(testResult) - write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) - } - } - - it.each(models)('should respond to "hello" with model %s', async (modelName) => { - await runBasicTest( - 'say "hello"', - 'hello', - 'hello', - modelName - ) - }) - - it.each(models)('should respond to "goodbye" with model %s', async (modelName) => { - await runBasicTest( - 'say "goodbye"', - 'goodbye', - 'goodbye', - modelName - ) - }) - - it.each(models)('should respond to "yes" with model %s', async (modelName) => { - await runBasicTest( - 'say "yes"', - 'yes', - 'yes', - modelName - ) - }) - - it('should generate markdown report', () => { - // Group results by test and model - const latestResults = new Map>() - - // Get only the latest result for each test+model combination - testResults.forEach(result => { - if (!latestResults.has(result.test)) { - latestResults.set(result.test, new Map()) - } - const testMap = latestResults.get(result.test)! - const existingResult = testMap.get(result.model) - if (!existingResult || new Date(result.timestamp) > new Date(existingResult.timestamp)) { - testMap.set(result.model, result) - } - }) - - // Generate markdown report - let report = '# Basic Test Results\n\n' - - // First list failed tests - report += '## Failed Tests\n\n' - let hasFailures = false - for (const [testName, modelResults] of latestResults) { - for (const [model, result] of modelResults) { - if (!result.passed) { - hasFailures = true - report += `### ${testName} - ${model}\n` - report += `- Prompt: \`${result.prompt}\`\n` - report += `- Expected: \`${result.expected}\`\n` - report += `- Actual: \`${result.result[0] || ''}\`\n` - report += `- Duration: ${result.duration}ms\n` - if (result.error) { - report += `- Error Type: ${result.error.type}\n` - report += `- Error Code: ${result.error.code}\n` - report += `- Error Message: ${result.error.message}\n` - } - report += `- Reason: ${result.reason}\n` - report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` - } - } - } - - if (!hasFailures) { - report += '*No failed tests*\n\n' - } - - // Then list passed tests - report += '## Passed Tests\n\n' - let hasPassed = false - for (const [testName, modelResults] of latestResults) { - for (const [model, result] of modelResults) { - if (result.passed) { - hasPassed = true - report += `### ${testName} - ${model}\n` - report += `- Prompt: \`${result.prompt}\`\n` - report += `- Expected: \`${result.expected}\`\n` - report += `- Actual: \`${result.result[0] || ''}\`\n` - report += `- Duration: ${result.duration}ms\n` - report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` - } - } - } - - if (!hasPassed) { - report += '*No passed tests*\n\n' - } - - // Write report to file - const reportPath = path.resolve(__dirname, './basic-report.md') - write(reportPath, report) - - // Verify report was written - expect(exists(reportPath) === 'file').toBe(true) - }) +import { describe, it, expect } from 'vitest' +import { run } from '../../src/index' +import * as path from 'node:path' +import { sync as write } from "@polymech/fs/write" +import { sync as read } from "@polymech/fs/read" +import { sync as exists } from "@polymech/fs/exists" +import { models, TEST_BASE_PATH, TEST_LOGS_PATH, TEST_PREFERENCES_PATH, TestResult } from './commons' + +const TEST_LOG_PATH = path.resolve(__dirname, './basic.json') + +describe('Basic Operations', () => { + let testResults: TestResult[] = [] + + // Load existing results if any + if (exists(TEST_LOG_PATH)) { + const data = read(TEST_LOG_PATH, 'json') + testResults = Array.isArray(data) ? data : [] + } + + it.each(models)('should add two numbers with model %s', async (modelName) => { + const prompt = 'add 5 and 3. Return only the number, no explanation.' + const expected = '8' + let model = 'unknown' + let router = 'unknown' + + const result = await run({ + prompt, + mode: 'completion', + model: modelName, + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + onRun: async (options) => { + model = options.model || 'unknown' + router = options.router || 'unknown' + return options + } + }) as string[] + + const actual = result?.[0]?.trim() || '' + if (!actual) { + console.log(`Skipping test for model ${modelName} - no result returned`) + return + } + const passed = actual === expected + expect(actual).toEqual(expected) + + // Add test result to array + testResults.push({ + test: 'addition', + prompt, + result: result || [], + expected, + model, + router, + timestamp: new Date().toISOString(), + passed, + reason: passed ? undefined : `Expected ${expected}, but got ${actual}` + }) + + // Write all results to the same file + write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) + }) + + it.each(models)('should multiply two numbers with model %s', async (modelName) => { + const prompt = 'multiply 8 and 3. Return only the number, no explanation.' + const expected = '24' + let model = 'unknown' + let router = 'unknown' + + const result = await run({ + prompt, + mode: 'completion', + model: modelName, + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + onRun: async (options) => { + model = options.model || 'unknown' + router = options.router || 'unknown' + return options + } + }) as string[] + + const actual = result?.[0]?.trim() || '' + if (!actual) { + console.log(`Skipping test for model ${modelName} - no result returned`) + return + } + const passed = actual === expected + expect(actual).toEqual(expected) + + // Add test result to array + testResults.push({ + test: 'multiplication', + prompt, + result: result || [], + expected, + model, + router, + timestamp: new Date().toISOString(), + passed, + reason: passed ? undefined : `Expected ${expected}, but got ${actual}` + }) + + // Write all results to the same file + write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) + }) + + it.each(models)('should divide two numbers with model %s', async (modelName) => { + const prompt = 'divide 15 by 3. Return only the number, no explanation.' + const expected = '5' + let model = 'unknown' + let router = 'unknown' + + const result = await run({ + prompt, + mode: 'completion', + model: modelName, + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + onRun: async (options) => { + model = options.model || 'unknown' + router = options.router || 'unknown' + return options + } + }) as string[] + + const actual = result?.[0]?.trim() || '' + if (!actual) { + console.log(`Skipping test for model ${modelName} - no result returned`) + return + } + const passed = actual === expected + expect(actual).toEqual(expected) + + // Add test result to array + testResults.push({ + test: 'division', + prompt, + result: result || [], + expected, + model, + router, + timestamp: new Date().toISOString(), + passed, + reason: passed ? undefined : `Expected ${expected}, but got ${actual}` + }) + + // Write all results to the same file + write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) + }) + + it('should generate markdown report', () => { + // Group results by test and model + const latestResults = new Map>() + + // Get only the latest result for each test+model combination + testResults.forEach(result => { + if (!latestResults.has(result.test)) { + latestResults.set(result.test, new Map()) + } + const testMap = latestResults.get(result.test)! + const existingResult = testMap.get(result.model) + if (!existingResult || new Date(result.timestamp) > new Date(existingResult.timestamp)) { + testMap.set(result.model, result) + } + }) + + // Generate markdown report + let report = '# Basic Operations Test Results\n\n' + + // First list failed tests + report += '## Failed Tests\n\n' + for (const [testName, modelResults] of latestResults) { + for (const [model, result] of modelResults) { + if (!result.passed) { + report += `### ${testName} - ${model}\n` + report += `- Prompt: \`${result.prompt}\`\n` + report += `- Expected: \`${result.expected}\`\n` + report += `- Actual: \`${result.result[0] || ''}\`\n` + report += `- Reason: ${result.reason}\n` + report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` + } + } + } + + // Then list passed tests + report += '## Passed Tests\n\n' + for (const [testName, modelResults] of latestResults) { + for (const [model, result] of modelResults) { + if (result.passed) { + report += `### ${testName} - ${model}\n` + report += `- Prompt: \`${result.prompt}\`\n` + report += `- Expected: \`${result.expected}\`\n` + report += `- Actual: \`${result.result[0] || ''}\`\n` + report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` + } + } + } + + // Write report to file + const reportPath = path.resolve(__dirname, './basic-report.md') + write(reportPath, report) + + // Verify report was written + expect(exists(reportPath) === 'file').toBe(true) + }) }) \ No newline at end of file diff --git a/packages/kbot/tests/unit/commons.ts b/packages/kbot/tests/unit/commons.ts index c3289c12..fb8ea008 100644 --- a/packages/kbot/tests/unit/commons.ts +++ b/packages/kbot/tests/unit/commons.ts @@ -1,88 +1,44 @@ -import * as path from 'node:path' -import { E_OPENROUTER_MODEL_FREE, E_OPENAI_MODEL } from '../../src/index' - -// Test configuration -export const TEST_CONFIG = { - openrouter: { - key: process.env.OPENROUTER_API_KEY || '', - org: process.env.OPENROUTER_ORG_ID || '' - }, - openai: { - key: process.env.OPENAI_API_KEY || '', - org: process.env.OPENAI_ORG_ID || '' - } -} - -export const models = [ - E_OPENROUTER_MODEL_FREE.MODEL_FREE_DEEPSEEK_DEEPSEEK_CHAT_FREE, - E_OPENROUTER_MODEL_FREE.MODEL_FREE_GOOGLE_GEMINI_2_0_FLASH_EXP_FREE, - E_OPENAI_MODEL.MODEL_GPT_4 -] - -export const TEST_BASE_PATH = path.resolve(__dirname, '../../') -export const TEST_LOGS_PATH = path.resolve(__dirname, '../../logs') -export const TEST_PREFERENCES_PATH = path.resolve(__dirname, '../../preferences.md') -export const TEST_TIMEOUT = 30000 // 30 seconds timeout for API calls - -export interface TestResult { - test: string; - prompt: string; - result: string[]; - expected: string; - model: string; - router: string; - timestamp: string; - passed: boolean; - reason?: string; - error?: { - message: string; - code?: string; - type?: string; - details?: any; - }; - duration?: number; - config?: { - apiKey?: string; - baseURL?: string; - router?: string; - }; -} - -export const formatError = (error: any): TestResult['error'] => { - return { - message: error?.message || 'Unknown error', - code: error?.code || 'UNKNOWN', - type: error?.type || error?.constructor?.name || 'Error', - details: error?.response?.data || error?.response || error - } -} - -export const isEmptyResponse = (result: string[] | null | undefined): boolean => { - return !result || result.length === 0 || result.every(r => !r || r.trim() === '') -} - -export const getRouterForModel = (model: string): string => { - if (model.startsWith('gpt-')) return 'openai' - return 'openrouter' -} - -export const getApiKeyForRouter = (router: string): string => { - switch (router) { - case 'openai': - return TEST_CONFIG.openai.key - case 'openrouter': - return TEST_CONFIG.openrouter.key - default: - return '' - } -} - -export const validateConfig = () => { - const missingKeys: string[] = [] - if (!TEST_CONFIG.openrouter.key) missingKeys.push('OPENROUTER_API_KEY') - if (!TEST_CONFIG.openai.key) missingKeys.push('OPENAI_API_KEY') - - if (missingKeys.length > 0) { - throw new Error(`Missing required environment variables: ${missingKeys.join(', ')}`) - } -} \ No newline at end of file +import * as path from 'node:path' +import { E_OPENROUTER_MODEL_FREE, E_OPENAI_MODEL, E_OPENROUTER_MODEL } from '../../src/index' + +export const models = [ + //E_OPENROUTER_MODEL_FREE.MODEL_FREE_DEEPSEEK_DEEPSEEK_CHAT_FREE, + E_OPENROUTER_MODEL.MODEL_ANTHROPIC_CLAUDE_2_0 +] + +export const TEST_BASE_PATH = path.resolve(__dirname, '../../') +export const TEST_LOGS_PATH = path.resolve(__dirname, '../../logs') +export const TEST_PREFERENCES_PATH = path.resolve(__dirname, '../../preferences.md') +export const TEST_TIMEOUT = 30000 // 30 seconds timeout for API calls + +export interface TestResult { + test: string; + prompt: string; + result: string[]; + expected: string; + model: string; + router: string; + timestamp: string; + passed: boolean; + reason?: string; + error?: { + message: string; + code?: string; + type?: string; + details?: any; + }; + duration?: number +} + +export const formatError = (error: any): TestResult['error'] => { + return { + message: error?.message || 'Unknown error', + code: error?.code || 'UNKNOWN', + type: error?.type || error?.constructor?.name || 'Error', + details: error?.response?.data || error?.response || error + } +} + +export const isEmptyResponse = (result: string[] | null | undefined): boolean => { + return !result || result.length === 0 || result.every(r => !r || r.trim() === '') +} diff --git a/packages/kbot/tests/unit/format-report.md b/packages/kbot/tests/unit/format-report.md new file mode 100644 index 00000000..68129a8e --- /dev/null +++ b/packages/kbot/tests/unit/format-report.md @@ -0,0 +1,547 @@ +# Format Test Results + +## Failed Tests + +### basic_structure - deepseek/deepseek-chat:free +- Prompt: `return a greeting "hello" with count 42` +- Expected: `{"greeting":"hello","count":42}` +- Actual: `""` +- Duration: 885ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:36 PM + +### basic_structure - google/gemini-2.0-flash-exp:free +- Prompt: `return a greeting "hello" with count 42` +- Expected: `{"greeting":"hello","count":42}` +- Actual: `""` +- Duration: 757ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:36 PM + +### basic_structure - gpt-4 +- Prompt: `return a greeting "hello" with count 42` +- Expected: `{"greeting":"hello","count":42}` +- Actual: `""` +- Duration: 1043ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:37 PM + +### basic_structure - anthropic/claude-3.7-sonnet +- Prompt: `return a greeting "hello" with count 42` +- Expected: `{"greeting":"hello","count":42}` +- Actual: `""` +- Duration: 1790ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: Unexpected token 'h', "hello 42" is not valid JSON +- Reason: Failed to parse or validate response: Unexpected token 'h', "hello 42" is not valid JSON +- Timestamp: 4/1/2025, 1:23:05 PM + +### basic_structure - openai/gpt-4 +- Prompt: `Return a JSON object with a greeting "hello" and count 42. The response must be valid JSON with exactly these fields: { "greeting": string, "count": number }` +- Expected: `{"greeting":"hello","count":42}` +- Actual: `""` +- Duration: 1258ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Invalid response from API +- Reason: Invalid response from API +- Timestamp: 4/1/2025, 1:32:43 PM + +### nested_structure - deepseek/deepseek-chat:free +- Prompt: `return user John age 30 with dark theme and notifications enabled` +- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` +- Actual: `""` +- Duration: 655ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:38 PM + +### nested_structure - google/gemini-2.0-flash-exp:free +- Prompt: `return user John age 30 with dark theme and notifications enabled` +- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` +- Actual: `""` +- Duration: 790ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:39 PM + +### nested_structure - gpt-4 +- Prompt: `return user John age 30 with dark theme and notifications enabled` +- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` +- Actual: `""` +- Duration: 717ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:40 PM + +### nested_structure - anthropic/claude-3.7-sonnet +- Prompt: `return user John age 30 with dark theme and notifications enabled` +- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` +- Actual: `""` +- Duration: 1189ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: Unexpected token '#', "# John's U"... is not valid JSON +- Reason: Failed to parse or validate response: Unexpected token '#', "# John's U"... is not valid JSON +- Timestamp: 4/1/2025, 1:23:06 PM + +### nested_structure - openai/gpt-4 +- Prompt: `Return a JSON object with user John age 30, dark theme and notifications enabled. The response must be valid JSON with this structure: { "user": { "name": string, "age": number }, "settings": { "theme": string, "notifications": boolean } }` +- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` +- Actual: `""` +- Duration: 716ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Invalid response from API +- Reason: Invalid response from API +- Timestamp: 4/1/2025, 1:32:44 PM + +### array_structure - deepseek/deepseek-chat:free +- Prompt: `return a list of 2 items with ids 1 and 2, names "first" and "second"` +- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` +- Actual: `""` +- Duration: 617ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:40 PM + +### array_structure - google/gemini-2.0-flash-exp:free +- Prompt: `return a list of 2 items with ids 1 and 2, names "first" and "second"` +- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` +- Actual: `""` +- Duration: 756ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:41 PM + +### array_structure - gpt-4 +- Prompt: `return a list of 2 items with ids 1 and 2, names "first" and "second"` +- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` +- Actual: `""` +- Duration: 1026ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:42 PM + +### array_structure - anthropic/claude-3.7-sonnet +- Prompt: `return a list of 2 items with ids 1 and 2, names "first" and "second"` +- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` +- Actual: `""` +- Duration: 1190ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:23:08 PM + +### array_structure - openai/gpt-4 +- Prompt: `Return a JSON object with a list of 2 items. The response must be valid JSON with this structure: { "items": [{ "id": number, "name": string }] }. The first item should have id 1 and name "first", the second item should have id 2 and name "second".` +- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` +- Actual: `""` +- Duration: 703ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Invalid response from API +- Reason: Invalid response from API +- Timestamp: 4/1/2025, 1:32:44 PM + +### enum_structure - deepseek/deepseek-chat:free +- Prompt: `return status success with message "Operation completed"` +- Expected: `{"status":"success","message":"Operation completed"}` +- Actual: `""` +- Duration: 647ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:43 PM + +### enum_structure - google/gemini-2.0-flash-exp:free +- Prompt: `return status success with message "Operation completed"` +- Expected: `{"status":"success","message":"Operation completed"}` +- Actual: `""` +- Duration: 813ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:43 PM + +### enum_structure - gpt-4 +- Prompt: `return status success with message "Operation completed"` +- Expected: `{"status":"success","message":"Operation completed"}` +- Actual: `""` +- Duration: 1138ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:45 PM + +### enum_structure - anthropic/claude-3.7-sonnet +- Prompt: `return status success with message "Operation completed"` +- Expected: `{"status":"success","message":"Operation completed"}` +- Actual: `""` +- Duration: 1728ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: Unexpected token '`', "```json +{ +"... is not valid JSON +- Reason: Failed to parse or validate response: Unexpected token '`', "```json +{ +"... is not valid JSON +- Timestamp: 4/1/2025, 1:23:09 PM + +### enum_structure - openai/gpt-4 +- Prompt: `Return a JSON object with status "success" and message "Operation completed". The response must be valid JSON with this structure: { "status": "success" | "error" | "pending", "message": string }` +- Expected: `{"status":"success","message":"Operation completed"}` +- Actual: `""` +- Duration: 688ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Invalid response from API +- Reason: Invalid response from API +- Timestamp: 4/1/2025, 1:32:45 PM + +### optional_fields - deepseek/deepseek-chat:free +- Prompt: `return name "John" with age 30 and email "john@example.com"` +- Expected: `{"name":"John","age":30,"email":"john@example.com"}` +- Actual: `""` +- Duration: 676ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:45 PM + +### optional_fields - google/gemini-2.0-flash-exp:free +- Prompt: `return name "John" with age 30 and email "john@example.com"` +- Expected: `{"name":"John","age":30,"email":"john@example.com"}` +- Actual: `""` +- Duration: 884ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:46 PM + +### optional_fields - gpt-4 +- Prompt: `return name "John" with age 30 and email "john@example.com"` +- Expected: `{"name":"John","age":30,"email":"john@example.com"}` +- Actual: `""` +- Duration: 669ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Reason: Failed to parse or validate response: [ + { + "code": "invalid_type", + "expected": "object", + "received": "null", + "path": [], + "message": "Expected object, received null" + } +] +- Timestamp: 4/1/2025, 1:21:47 PM + +### optional_fields - anthropic/claude-3.7-sonnet +- Prompt: `return name "John" with age 30 and email "john@example.com"` +- Expected: `{"name":"John","age":30,"email":"john@example.com"}` +- Actual: `""` +- Duration: 1576ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Failed to parse or validate response: Unexpected token '`', "```json +{ +"... is not valid JSON +- Reason: Failed to parse or validate response: Unexpected token '`', "```json +{ +"... is not valid JSON +- Timestamp: 4/1/2025, 1:23:11 PM + +### optional_fields - openai/gpt-4 +- Prompt: `Return a JSON object with name "John", age 30, and email "john@example.com". The response must be valid JSON with this structure: { "name": string, "age"?: number, "email"?: string }` +- Expected: `{"name":"John","age":30,"email":"john@example.com"}` +- Actual: `""` +- Duration: 682ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Invalid response from API +- Reason: Invalid response from API +- Timestamp: 4/1/2025, 1:32:46 PM + +## Passed Tests + +*No passed tests* + diff --git a/packages/kbot/tests/unit/format.json b/packages/kbot/tests/unit/format.json new file mode 100644 index 00000000..bea5d385 --- /dev/null +++ b/packages/kbot/tests/unit/format.json @@ -0,0 +1,2241 @@ +[ + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:45.784Z", + "passed": false, + "duration": 1077, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:46.549Z", + "passed": false, + "duration": 762, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:47.696Z", + "passed": false, + "duration": 1146, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:48.368Z", + "passed": false, + "duration": 671, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:49.160Z", + "passed": false, + "duration": 790, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:50.164Z", + "passed": false, + "duration": 1003, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:50.812Z", + "passed": false, + "duration": 647, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:51.561Z", + "passed": false, + "duration": 748, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:52.632Z", + "passed": false, + "duration": 1070, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:53.299Z", + "passed": false, + "duration": 665, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:54.020Z", + "passed": false, + "duration": 720, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:55.153Z", + "passed": false, + "duration": 1132, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:55.791Z", + "passed": false, + "duration": 637, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:56.768Z", + "passed": false, + "duration": 975, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:20:57.823Z", + "passed": false, + "duration": 1053, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:36.069Z", + "passed": false, + "duration": 885, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:36.829Z", + "passed": false, + "duration": 757, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:37.874Z", + "passed": false, + "duration": 1043, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:38.531Z", + "passed": false, + "duration": 655, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:39.322Z", + "passed": false, + "duration": 790, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:40.040Z", + "passed": false, + "duration": 717, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:40.659Z", + "passed": false, + "duration": 617, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:41.416Z", + "passed": false, + "duration": 756, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:42.443Z", + "passed": false, + "duration": 1026, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:43.092Z", + "passed": false, + "duration": 647, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:43.906Z", + "passed": false, + "duration": 813, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:45.045Z", + "passed": false, + "duration": 1138, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:45.723Z", + "passed": false, + "duration": 676, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:46.609Z", + "passed": false, + "duration": 884, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:21:47.279Z", + "passed": false, + "duration": 669, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "anthropic/claude-3.7-sonnet", + "router": "openrouter", + "timestamp": "2025-04-01T11:23:05.626Z", + "passed": false, + "duration": 1790, + "error": { + "message": "Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON", + "stackStr": "Error: Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "anthropic/claude-3.7-sonnet", + "router": "openrouter", + "timestamp": "2025-04-01T11:23:06.818Z", + "passed": false, + "duration": 1189, + "error": { + "message": "Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON", + "stackStr": "Error: Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "anthropic/claude-3.7-sonnet", + "router": "openrouter", + "timestamp": "2025-04-01T11:23:08.010Z", + "passed": false, + "duration": 1190, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "anthropic/claude-3.7-sonnet", + "router": "openrouter", + "timestamp": "2025-04-01T11:23:09.740Z", + "passed": false, + "duration": 1728, + "error": { + "message": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", + "stackStr": "Error: Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "anthropic/claude-3.7-sonnet", + "router": "openrouter", + "timestamp": "2025-04-01T11:23:11.318Z", + "passed": false, + "duration": 1576, + "error": { + "message": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON" + } + }, + "reason": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:26:47.671Z", + "passed": false, + "duration": 1483, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:26:48.821Z", + "passed": false, + "duration": 1146, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:26:49.824Z", + "passed": false, + "duration": 1001, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:26:50.835Z", + "passed": false, + "duration": 1008, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:26:51.878Z", + "passed": false, + "duration": 1041, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:28:21.136Z", + "passed": false, + "duration": 1405, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:28:21.846Z", + "passed": false, + "duration": 706, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:28:22.517Z", + "passed": false, + "duration": 669, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:28:23.187Z", + "passed": false, + "duration": 668, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:28:23.848Z", + "passed": false, + "duration": 659, + "error": { + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]" + } + }, + "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:29:01.068Z", + "passed": false, + "duration": 1322, + "error": { + "message": "Cannot read properties of undefined (reading '0')", + "code": "UNKNOWN", + "type": "TypeError", + "details": { + "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Cannot read properties of undefined (reading '0')", + "stackStr": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "TypeError", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Cannot read properties of undefined (reading '0')", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:29:01.939Z", + "passed": false, + "duration": 868, + "error": { + "message": "Cannot read properties of undefined (reading '0')", + "code": "UNKNOWN", + "type": "TypeError", + "details": { + "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Cannot read properties of undefined (reading '0')", + "stackStr": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "TypeError", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Cannot read properties of undefined (reading '0')", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:29:02.996Z", + "passed": false, + "duration": 1055, + "error": { + "message": "Cannot read properties of undefined (reading '0')", + "code": "UNKNOWN", + "type": "TypeError", + "details": { + "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Cannot read properties of undefined (reading '0')", + "stackStr": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "TypeError", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Cannot read properties of undefined (reading '0')", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:29:05.060Z", + "passed": false, + "duration": 2062, + "error": { + "message": "Cannot read properties of undefined (reading '0')", + "code": "UNKNOWN", + "type": "TypeError", + "details": { + "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Cannot read properties of undefined (reading '0')", + "stackStr": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "TypeError", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Cannot read properties of undefined (reading '0')", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:29:05.831Z", + "passed": false, + "duration": 769, + "error": { + "message": "Cannot read properties of undefined (reading '0')", + "code": "UNKNOWN", + "type": "TypeError", + "details": { + "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Cannot read properties of undefined (reading '0')" + } + }, + "reason": "Cannot read properties of undefined (reading '0')", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:30:38.831Z", + "passed": false, + "duration": 1410, + "error": { + "message": "Invalid response from OpenAI API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from OpenAI API", + "stackStr": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from OpenAI API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:30:39.844Z", + "passed": false, + "duration": 1009, + "error": { + "message": "Invalid response from OpenAI API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from OpenAI API", + "stackStr": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from OpenAI API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:30:40.923Z", + "passed": false, + "duration": 1076, + "error": { + "message": "Invalid response from OpenAI API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from OpenAI API", + "stackStr": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from OpenAI API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:30:42.016Z", + "passed": false, + "duration": 1091, + "error": { + "message": "Invalid response from OpenAI API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from OpenAI API", + "stackStr": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from OpenAI API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:30:42.687Z", + "passed": false, + "duration": 668, + "error": { + "message": "Invalid response from OpenAI API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from OpenAI API" + } + }, + "reason": "Invalid response from OpenAI API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "return a greeting \"hello\" with count 42", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:31:51.336Z", + "passed": false, + "duration": 1382, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API", + "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "return user John age 30 with dark theme and notifications enabled", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:31:52.354Z", + "passed": false, + "duration": 1014, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API", + "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:31:53.055Z", + "passed": false, + "duration": 699, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API", + "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "return status success with message \"Operation completed\"", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:31:55.145Z", + "passed": false, + "duration": 2088, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API", + "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:31:55.863Z", + "passed": false, + "duration": 715, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "basic_structure", + "prompt": "Return a JSON object with a greeting \"hello\" and count 42. The response must be valid JSON with exactly these fields: { \"greeting\": string, \"count\": number }", + "result": [], + "expected": { + "greeting": "hello", + "count": 42 + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:32:43.353Z", + "passed": false, + "duration": 1258, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API", + "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "nested_structure", + "prompt": "Return a JSON object with user John age 30, dark theme and notifications enabled. The response must be valid JSON with this structure: { \"user\": { \"name\": string, \"age\": number }, \"settings\": { \"theme\": string, \"notifications\": boolean } }", + "result": [], + "expected": { + "user": { + "name": "John", + "age": 30 + }, + "settings": { + "theme": "dark", + "notifications": true + } + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:32:44.073Z", + "passed": false, + "duration": 716, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API", + "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "array_structure", + "prompt": "Return a JSON object with a list of 2 items. The response must be valid JSON with this structure: { \"items\": [{ \"id\": number, \"name\": string }] }. The first item should have id 1 and name \"first\", the second item should have id 2 and name \"second\".", + "result": [], + "expected": { + "items": [ + { + "id": 1, + "name": "first" + }, + { + "id": 2, + "name": "second" + } + ] + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:32:44.778Z", + "passed": false, + "duration": 703, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API", + "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "enum_structure", + "prompt": "Return a JSON object with status \"success\" and message \"Operation completed\". The response must be valid JSON with this structure: { \"status\": \"success\" | \"error\" | \"pending\", \"message\": string }", + "result": [], + "expected": { + "status": "success", + "message": "Operation completed" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:32:45.468Z", + "passed": false, + "duration": 688, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API", + "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "optional_fields", + "prompt": "Return a JSON object with name \"John\", age 30, and email \"john@example.com\". The response must be valid JSON with this structure: { \"name\": string, \"age\"?: number, \"email\"?: string }", + "result": [], + "expected": { + "name": "John", + "age": 30, + "email": "john@example.com" + }, + "model": "openai/gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:32:46.153Z", + "passed": false, + "duration": 682, + "error": { + "message": "Invalid response from API", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "message": "Invalid response from API" + } + }, + "reason": "Invalid response from API", + "config": { + "router": "openrouter", + "apiKey": "" + } + } +] \ No newline at end of file diff --git a/packages/kbot/tests/unit/format.test.ts b/packages/kbot/tests/unit/format.test.ts new file mode 100644 index 00000000..21bb9fc3 --- /dev/null +++ b/packages/kbot/tests/unit/format.test.ts @@ -0,0 +1,295 @@ +import { describe, it, expect } from 'vitest' +import { run } from '../../src/index' +import * as path from 'node:path' +import { sync as write } from "@polymech/fs/write" +import { sync as read } from "@polymech/fs/read" +import { sync as exists } from "@polymech/fs/exists" +import { z } from 'zod' + + +import { + models_premium as models, + TEST_BASE_PATH, + TEST_LOGS_PATH, + TEST_PREFERENCES_PATH, + TEST_TIMEOUT, + TestResult, + formatError, + isEmptyResponse, + getRouterForModel, + getApiKeyForRouter +} from './commons' + +const TEST_LOG_PATH = path.resolve(__dirname, './format.json') + +describe('Structured Output Format', () => { + let testResults: TestResult[] = [] + + // Load existing results if any + if (exists(TEST_LOG_PATH)) { + const data = read(TEST_LOG_PATH, 'json') + testResults = Array.isArray(data) ? data : [] + } + + const runFormatTest = async (prompt: string, format: z.ZodType, expected: any, testName: string, modelName: string) => { + let model = 'unknown' + let router = 'unknown' + let startTime = Date.now() + let error: TestResult['error'] | undefined + + try { + const result = await Promise.race([ + run({ + prompt, + mode: 'completion', + model: modelName, + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + format, + onRun: async (options) => { + model = options.model || 'unknown' + router = options.router || 'unknown' + return options + } + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error('API call timed out')), TEST_TIMEOUT) + ) + ]) as any[] + + const actual = result?.[0] + let parsed: any + + try { + parsed = typeof actual === 'string' ? JSON.parse(actual) : actual + // Validate against the format schema + parsed = format.parse(parsed) + } catch (parseError) { + throw new Error(`Failed to parse or validate response: ${parseError.message}`) + } + + const passed = JSON.stringify(parsed) === JSON.stringify(expected) && !isEmptyResponse(result) + + if (isEmptyResponse(result)) { + throw new Error('Model returned empty response') + } + + expect(parsed).toEqual(expected) + + return { + test: testName, + prompt, + result: result || [], + expected, + model, + router, + timestamp: new Date().toISOString(), + passed, + duration: Date.now() - startTime, + reason: passed ? undefined : `Expected ${JSON.stringify(expected)}, but got ${JSON.stringify(parsed)}`, + config: { + router: getRouterForModel(modelName) + } + } + } catch (e) { + error = formatError(e) + throw e + } finally { + const testResult: TestResult = { + test: testName, + prompt, + result: [], + expected, + model, + router, + timestamp: new Date().toISOString(), + passed: false, + duration: Date.now() - startTime, + error, + reason: error?.message || 'Unknown error occurred', + config: { + router: getRouterForModel(modelName), + apiKey: getApiKeyForRouter(getRouterForModel(modelName)) + } + } + + testResults.push(testResult) + write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) + } + } + + it.each(models)('should return basic structured output with model %s', async (modelName) => { + const format = z.object({ + greeting: z.string(), + count: z.number() + }) + + await runFormatTest( + 'return a greeting "hello" with count 42', + format, + { greeting: 'hello', count: 42 }, + 'basic_structure', + modelName + ) + }) + + it.each(models)('should handle nested structures with model %s', async (modelName) => { + const format = z.object({ + user: z.object({ + name: z.string(), + age: z.number() + }), + settings: z.object({ + theme: z.string(), + notifications: z.boolean() + }) + }) + + await runFormatTest( + 'return user John age 30 with dark theme and notifications enabled', + format, + { + user: { name: 'John', age: 30 }, + settings: { theme: 'dark', notifications: true } + }, + 'nested_structure', + modelName + ) + }) + + it.each(models)('should handle arrays with model %s', async (modelName) => { + const format = z.object({ + items: z.array(z.object({ + id: z.number(), + name: z.string() + })) + }) + + await runFormatTest( + 'return a list of 2 items with ids 1 and 2, names "first" and "second"', + format, + { + items: [ + { id: 1, name: 'first' }, + { id: 2, name: 'second' } + ] + }, + 'array_structure', + modelName + ) + }) + + it.each(models)('should handle enums with model %s', async (modelName) => { + const format = z.object({ + status: z.enum(['success', 'error', 'pending']), + message: z.string() + }) + + await runFormatTest( + 'return status success with message "Operation completed"', + format, + { + status: 'success', + message: 'Operation completed' + }, + 'enum_structure', + modelName + ) + }) + + it.each(models)('should handle optional fields with model %s', async (modelName) => { + const format = z.object({ + name: z.string(), + age: z.number().optional(), + email: z.string().email().optional() + }) + + await runFormatTest( + 'return name "John" with age 30 and email "john@example.com"', + format, + { + name: 'John', + age: 30, + email: 'john@example.com' + }, + 'optional_fields', + modelName + ) + }) + + it('should generate markdown report', () => { + // Group results by test and model + const latestResults = new Map>() + + // Get only the latest result for each test+model combination + testResults.forEach(result => { + if (!latestResults.has(result.test)) { + latestResults.set(result.test, new Map()) + } + const testMap = latestResults.get(result.test)! + const existingResult = testMap.get(result.model) + if (!existingResult || new Date(result.timestamp) > new Date(existingResult.timestamp)) { + testMap.set(result.model, result) + } + }) + + // Generate markdown report + let report = '# Format Test Results\n\n' + + // First list failed tests + report += '## Failed Tests\n\n' + let hasFailures = false + for (const [testName, modelResults] of latestResults) { + for (const [model, result] of modelResults) { + if (!result.passed) { + hasFailures = true + report += `### ${testName} - ${model}\n` + report += `- Prompt: \`${result.prompt}\`\n` + report += `- Expected: \`${JSON.stringify(result.expected)}\`\n` + report += `- Actual: \`${JSON.stringify(result.result[0] || '')}\`\n` + report += `- Duration: ${result.duration}ms\n` + if (result.error) { + report += `- Error Type: ${result.error.type}\n` + report += `- Error Code: ${result.error.code}\n` + report += `- Error Message: ${result.error.message}\n` + } + report += `- Reason: ${result.reason}\n` + report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` + } + } + } + + if (!hasFailures) { + report += '*No failed tests*\n\n' + } + + // Then list passed tests + report += '## Passed Tests\n\n' + let hasPassed = false + for (const [testName, modelResults] of latestResults) { + for (const [model, result] of modelResults) { + if (result.passed) { + hasPassed = true + report += `### ${testName} - ${model}\n` + report += `- Prompt: \`${result.prompt}\`\n` + report += `- Expected: \`${JSON.stringify(result.expected)}\`\n` + report += `- Actual: \`${JSON.stringify(result.result[0] || '')}\`\n` + report += `- Duration: ${result.duration}ms\n` + report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` + } + } + } + + if (!hasPassed) { + report += '*No passed tests*\n\n' + } + + // Write report to file + const reportPath = path.resolve(__dirname, './format-report.md') + write(reportPath, report) + + // Verify report was written + expect(exists(reportPath) === 'file').toBe(true) + }) +}) \ No newline at end of file diff --git a/packages/kbot/tests/unit/language-report.md b/packages/kbot/tests/unit/language-report.md index 61484aa9..77b0a619 100644 --- a/packages/kbot/tests/unit/language-report.md +++ b/packages/kbot/tests/unit/language-report.md @@ -32,6 +32,14 @@ - Reason: Unknown error occurred - Timestamp: 4/1/2025, 1:05:52 PM +### german - anthropic/claude-2.0 +- Prompt: `translate "hello" to German. Return only the translated word, no explanation.` +- Expected: `hallo` +- Actual: `` +- Duration: 1253ms +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:47:26 PM + ### spanish - deepseek/deepseek-chat:free - Prompt: `translate "yes" to Spanish. Return only the translated word, no explanation.` - Expected: `sí` @@ -62,6 +70,14 @@ - Reason: Unknown error occurred - Timestamp: 4/1/2025, 1:05:55 PM +### spanish - anthropic/claude-2.0 +- Prompt: `translate "yes" to Spanish. Return only the translated word, no explanation.` +- Expected: `sí` +- Actual: `` +- Duration: 932ms +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:47:27 PM + ### french - deepseek/deepseek-chat:free - Prompt: `translate "no" to French. Return only the translated word, no explanation.` - Expected: `non` @@ -92,6 +108,14 @@ - Reason: Model returned empty response - Timestamp: 4/1/2025, 1:05:56 PM +### french - anthropic/claude-2.0 +- Prompt: `translate "no" to French. Return only the translated word, no explanation.` +- Expected: `non` +- Actual: `` +- Duration: 864ms +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:47:28 PM + ## Passed Tests ### german_translation - deepseek/deepseek-chat:free diff --git a/packages/kbot/tests/unit/language.json b/packages/kbot/tests/unit/language.json index 9fc6d16e..19b0d682 100644 --- a/packages/kbot/tests/unit/language.json +++ b/packages/kbot/tests/unit/language.json @@ -700,5 +700,112 @@ "passed": false, "duration": 1341, "reason": "Unknown error occurred" + }, + { + "test": "german", + "prompt": "translate \"hello\" to German. Return only the translated word, no explanation.", + "result": [], + "expected": "hallo", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T11:47:02.933Z", + "passed": false, + "duration": 1416, + "error": { + "message": "getRouterForModel is not defined", + "code": "UNKNOWN", + "type": "ReferenceError", + "details": { + "stack": "ReferenceError: getRouterForModel is not defined\n at runTranslationTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:78:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:106:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "getRouterForModel is not defined", + "stackStr": "ReferenceError: getRouterForModel is not defined\n at runTranslationTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:78:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:106:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "ReferenceError", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "getRouterForModel is not defined" + }, + { + "test": "spanish", + "prompt": "translate \"yes\" to Spanish. Return only the translated word, no explanation.", + "result": [], + "expected": "sí", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T11:47:04.007Z", + "passed": false, + "duration": 1071, + "error": { + "message": "getRouterForModel is not defined", + "code": "UNKNOWN", + "type": "ReferenceError", + "details": { + "stack": "ReferenceError: getRouterForModel is not defined\n at runTranslationTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:78:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:115:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "getRouterForModel is not defined", + "stackStr": "ReferenceError: getRouterForModel is not defined\n at runTranslationTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:78:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:115:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "ReferenceError", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "getRouterForModel is not defined" + }, + { + "test": "french", + "prompt": "translate \"no\" to French. Return only the translated word, no explanation.", + "result": [], + "expected": "non", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T11:47:05.064Z", + "passed": false, + "duration": 1056, + "error": { + "message": "getRouterForModel is not defined", + "code": "UNKNOWN", + "type": "ReferenceError", + "details": { + "stack": "ReferenceError: getRouterForModel is not defined\n at runTranslationTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:78:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\language.test.ts:124:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "getRouterForModel is not defined" + } + }, + "reason": "getRouterForModel is not defined" + }, + { + "test": "german", + "prompt": "translate \"hello\" to German. Return only the translated word, no explanation.", + "result": [], + "expected": "hallo", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T11:47:26.798Z", + "passed": false, + "duration": 1253, + "reason": "Unknown error occurred" + }, + { + "test": "spanish", + "prompt": "translate \"yes\" to Spanish. Return only the translated word, no explanation.", + "result": [], + "expected": "sí", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T11:47:27.731Z", + "passed": false, + "duration": 932, + "reason": "Unknown error occurred" + }, + { + "test": "french", + "prompt": "translate \"no\" to French. Return only the translated word, no explanation.", + "result": [], + "expected": "non", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T11:47:28.596Z", + "passed": false, + "duration": 864, + "reason": "Unknown error occurred" } ] \ No newline at end of file diff --git a/packages/kbot/tests/unit/language.test.ts b/packages/kbot/tests/unit/language.test.ts index 919273ba..97176193 100644 --- a/packages/kbot/tests/unit/language.test.ts +++ b/packages/kbot/tests/unit/language.test.ts @@ -12,9 +12,7 @@ import { TEST_TIMEOUT, TestResult, formatError, - isEmptyResponse, - getRouterForModel, - getApiKeyForRouter + isEmptyResponse } from './commons' import { E_OPENROUTER_MODEL_FREE, E_OPENAI_MODEL } from '../../src/index' @@ -75,11 +73,7 @@ describe('Language Capabilities', () => { timestamp: new Date().toISOString(), passed, duration: Date.now() - startTime, - reason: passed ? undefined : `Expected ${expected}, but got ${actual}`, - config: { - router: getRouterForModel(modelName), - apiKey: getApiKeyForRouter(getRouterForModel(modelName)) - } + reason: passed ? undefined : `Expected ${expected}, but got ${actual}` } } catch (e) { error = formatError(e) @@ -96,11 +90,7 @@ describe('Language Capabilities', () => { passed: false, duration: Date.now() - startTime, error, - reason: error?.message || 'Unknown error occurred', - config: { - router: getRouterForModel(modelName), - apiKey: getApiKeyForRouter(getRouterForModel(modelName)) - } + reason: error?.message || 'Unknown error occurred' } testResults.push(testResult) diff --git a/packages/kbot/tests/unit/math-report.md b/packages/kbot/tests/unit/math-report.md index df14d555..c1a8f97e 100644 --- a/packages/kbot/tests/unit/math-report.md +++ b/packages/kbot/tests/unit/math-report.md @@ -2,12 +2,127 @@ ## Failed Tests +### addition - deepseek/deepseek-chat:free +- Prompt: `add 5 and 3. Return only the number, no explanation.` +- Expected: `8` +- Actual: `` +- Duration: 913ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Model returned empty response +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:42:47 PM + +### addition - google/gemini-2.0-flash-exp:free +- Prompt: `add 5 and 3. Return only the number, no explanation.` +- Expected: `8` +- Actual: `` +- Duration: 1112ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Model returned empty response +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:45:50 PM + +### addition - gpt-4 +- Prompt: `add 5 and 3. Return only the number, no explanation.` +- Expected: `8` +- Actual: `` +- Duration: 1038ms +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:42:26 PM + +### addition - anthropic/claude-2.0 +- Prompt: `add 5 and 3. Return only the number, no explanation.` +- Expected: `8` +- Actual: `` +- Duration: 1218ms +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:46:27 PM + +### multiplication - deepseek/deepseek-chat:free +- Prompt: `multiply 8 and 3. Return only the number, no explanation.` +- Expected: `24` +- Actual: `` +- Duration: 636ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Model returned empty response +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:42:48 PM + +### multiplication - google/gemini-2.0-flash-exp:free +- Prompt: `multiply 8 and 3. Return only the number, no explanation.` +- Expected: `24` +- Actual: `` +- Duration: 764ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Model returned empty response +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:45:51 PM + +### multiplication - gpt-4 +- Prompt: `multiply 8 and 3. Return only the number, no explanation.` +- Expected: `24` +- Actual: `` +- Duration: 1052ms +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:42:28 PM + +### multiplication - anthropic/claude-2.0 +- Prompt: `multiply 8 and 3. Return only the number, no explanation.` +- Expected: `24` +- Actual: `` +- Duration: 911ms +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:46:27 PM + +### division - deepseek/deepseek-chat:free +- Prompt: `divide 15 by 3. Return only the number, no explanation.` +- Expected: `5` +- Actual: `` +- Duration: 648ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Model returned empty response +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:42:50 PM + +### division - google/gemini-2.0-flash-exp:free +- Prompt: `divide 15 by 3. Return only the number, no explanation.` +- Expected: `5` +- Actual: `` +- Duration: 829ms +- Error Type: Error +- Error Code: UNKNOWN +- Error Message: Model returned empty response +- Reason: Model returned empty response +- Timestamp: 4/1/2025, 1:45:52 PM + +### division - gpt-4 +- Prompt: `divide 15 by 3. Return only the number, no explanation.` +- Expected: `5` +- Actual: `` +- Duration: 959ms +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:42:31 PM + +### division - anthropic/claude-2.0 +- Prompt: `divide 15 by 3. Return only the number, no explanation.` +- Expected: `5` +- Actual: `` +- Duration: 1485ms +- Reason: Unknown error occurred +- Timestamp: 4/1/2025, 1:46:29 PM + ## Passed Tests ### factorial - deepseek/deepseek-chat:free - Prompt: `calculate the factorial of 5 (5!). Return only the number, no explanation.` - Expected: `120` - Actual: `120` +- Duration: undefinedms - Timestamp: 4/1/2025, 12:59:09 PM ### factorial - google/gemini-2.0-flash-exp:free @@ -15,18 +130,21 @@ - Expected: `120` - Actual: `120 ` +- Duration: undefinedms - Timestamp: 4/1/2025, 12:59:10 PM ### factorial - gpt-4 - Prompt: `calculate the factorial of 5 (5!). Return only the number, no explanation.` - Expected: `120` - Actual: `120` +- Duration: undefinedms - Timestamp: 4/1/2025, 1:03:25 PM ### fibonacci - deepseek/deepseek-chat:free - Prompt: `calculate the first 5 numbers of the fibonacci sequence. Return only the numbers separated by commas, no explanation.` - Expected: `0,1,1,2,3` - Actual: `0,1,1,2,3` +- Duration: undefinedms - Timestamp: 4/1/2025, 12:59:13 PM ### fibonacci - google/gemini-2.0-flash-exp:free @@ -34,29 +152,34 @@ - Expected: `0,1,1,2,3` - Actual: `0,1,1,2,3 ` +- Duration: undefinedms - Timestamp: 4/1/2025, 12:59:14 PM ### fibonacci - gpt-4 - Prompt: `calculate the first 5 numbers of the fibonacci sequence. Return only the numbers separated by commas, no explanation.` - Expected: `0,1,1,2,3` - Actual: `0, 1, 1, 2, 3` +- Duration: undefinedms - Timestamp: 4/1/2025, 1:03:27 PM ### quadratic - deepseek/deepseek-chat:free - Prompt: `solve the quadratic equation x² + 5x + 6 = 0. Return only the roots as a JSON array, no explanation.` - Expected: `[-3,-2]` - Actual: `[-2, -3]` +- Duration: undefinedms - Timestamp: 4/1/2025, 12:59:19 PM ### quadratic - google/gemini-2.0-flash-exp:free - Prompt: `solve the quadratic equation x² + 5x + 6 = 0. Return only the roots as a JSON array, no explanation.` - Expected: `[-3,-2]` - Actual: `[-2, -3]` +- Duration: undefinedms - Timestamp: 4/1/2025, 12:46:13 PM ### quadratic - gpt-4 - Prompt: `solve the quadratic equation x² + 5x + 6 = 0. Return only the roots as a JSON array, no explanation.` - Expected: `[-3,-2]` - Actual: `[-2, -3]` +- Duration: undefinedms - Timestamp: 4/1/2025, 1:03:30 PM diff --git a/packages/kbot/tests/unit/math.json b/packages/kbot/tests/unit/math.json index 99c7071d..5aeb9213 100644 --- a/packages/kbot/tests/unit/math.json +++ b/packages/kbot/tests/unit/math.json @@ -1020,5 +1020,1198 @@ "router": "openrouter", "timestamp": "2025-04-01T11:03:30.579Z", "passed": true + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:30.576Z", + "passed": false, + "duration": 997, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:112:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:112:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:31.398Z", + "passed": false, + "duration": 820, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:112:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:112:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:32.471Z", + "passed": false, + "duration": 1072, + "reason": "Unknown error occurred", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:33.137Z", + "passed": false, + "duration": 665, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:121:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:121:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:33.887Z", + "passed": false, + "duration": 748, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:121:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:121:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:34.873Z", + "passed": false, + "duration": 985, + "reason": "Unknown error occurred", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:35.528Z", + "passed": false, + "duration": 654, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:130:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:130:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:36.301Z", + "passed": false, + "duration": 771, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:130:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:130:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter", + "apiKey": "" + } + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:39:37.252Z", + "passed": false, + "duration": 950, + "reason": "Unknown error occurred", + "config": { + "router": "openai", + "apiKey": "" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:40:10.776Z", + "passed": false, + "duration": 979, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:40:11.549Z", + "passed": false, + "duration": 771, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:40:12.530Z", + "passed": false, + "duration": 979, + "error": { + "message": "getApiKeyForRouter is not defined", + "code": "UNKNOWN", + "type": "ReferenceError", + "details": { + "stack": "ReferenceError: getApiKeyForRouter is not defined\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:80:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "getApiKeyForRouter is not defined", + "stackStr": "ReferenceError: getApiKeyForRouter is not defined\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:80:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "ReferenceError", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "getApiKeyForRouter is not defined", + "config": { + "router": "openai" + } + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:40:13.218Z", + "passed": false, + "duration": 687, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:40:13.951Z", + "passed": false, + "duration": 731, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:40:14.913Z", + "passed": false, + "duration": 960, + "error": { + "message": "getApiKeyForRouter is not defined", + "code": "UNKNOWN", + "type": "ReferenceError", + "details": { + "stack": "ReferenceError: getApiKeyForRouter is not defined\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:80:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "getApiKeyForRouter is not defined", + "stackStr": "ReferenceError: getApiKeyForRouter is not defined\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:80:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "ReferenceError", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "getApiKeyForRouter is not defined", + "config": { + "router": "openai" + } + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:40:15.603Z", + "passed": false, + "duration": 689, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:40:16.319Z", + "passed": false, + "duration": 715, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:62:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:40:17.273Z", + "passed": false, + "duration": 953, + "error": { + "message": "getApiKeyForRouter is not defined", + "code": "UNKNOWN", + "type": "ReferenceError", + "details": { + "stack": "ReferenceError: getApiKeyForRouter is not defined\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:80:19)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "getApiKeyForRouter is not defined" + } + }, + "reason": "getApiKeyForRouter is not defined", + "config": { + "router": "openai" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:41:12.248Z", + "passed": false, + "duration": 1037, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:41:13.047Z", + "passed": false, + "duration": 797, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:41:14.141Z", + "passed": false, + "duration": 1092, + "reason": "Unknown error occurred", + "config": { + "router": "openai" + } + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:41:14.792Z", + "passed": false, + "duration": 650, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:41:15.622Z", + "passed": false, + "duration": 828, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:41:16.579Z", + "passed": false, + "duration": 956, + "reason": "Unknown error occurred", + "config": { + "router": "openai" + } + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:41:17.207Z", + "passed": false, + "duration": 626, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:41:17.921Z", + "passed": false, + "duration": 712, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:63:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response", + "config": { + "router": "openrouter" + } + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:41:18.963Z", + "passed": false, + "duration": 1041, + "reason": "Unknown error occurred", + "config": { + "router": "openai" + } + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:24.341Z", + "passed": false, + "duration": 1004, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:25.115Z", + "passed": false, + "duration": 772, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:26.155Z", + "passed": false, + "duration": 1038, + "reason": "Unknown error occurred" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:26.793Z", + "passed": false, + "duration": 637, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:27.840Z", + "passed": false, + "duration": 1045, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:28.893Z", + "passed": false, + "duration": 1052, + "reason": "Unknown error occurred" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:29.565Z", + "passed": false, + "duration": 670, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:120:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:120:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:30.290Z", + "passed": false, + "duration": 723, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:120:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:120:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "gpt-4", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:31.251Z", + "passed": false, + "duration": 959, + "reason": "Unknown error occurred" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:47.579Z", + "passed": false, + "duration": 913, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:48.338Z", + "passed": false, + "duration": 756, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:48.976Z", + "passed": false, + "duration": 636, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:49.748Z", + "passed": false, + "duration": 770, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "deepseek/deepseek-chat:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:50.398Z", + "passed": false, + "duration": 648, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:120:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:120:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:42:51.126Z", + "passed": false, + "duration": 726, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:120:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:43:32.583Z", + "passed": false, + "duration": 1035, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:102:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:43:33.312Z", + "passed": false, + "duration": 726, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:111:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:43:34.050Z", + "passed": false, + "duration": 736, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:120:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:45:50.405Z", + "passed": false, + "duration": 1112, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:60:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:101:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:60:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:101:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:45:51.172Z", + "passed": false, + "duration": 764, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:60:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:60:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "Error", + "expected": "undefined", + "actual": "undefined" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "google/gemini-2.0-flash-exp:free", + "router": "openrouter", + "timestamp": "2025-04-01T11:45:52.003Z", + "passed": false, + "duration": 829, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runMathTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:60:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\math.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T11:46:27.004Z", + "passed": false, + "duration": 1218, + "reason": "Unknown error occurred" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T11:46:27.916Z", + "passed": false, + "duration": 911, + "reason": "Unknown error occurred" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T11:46:29.402Z", + "passed": false, + "duration": 1485, + "reason": "Unknown error occurred" } ] \ No newline at end of file diff --git a/packages/kbot/tests/unit/math.test.ts b/packages/kbot/tests/unit/math.test.ts index 60f6a5fe..8b60f62f 100644 --- a/packages/kbot/tests/unit/math.test.ts +++ b/packages/kbot/tests/unit/math.test.ts @@ -12,12 +12,9 @@ import { TEST_TIMEOUT, TestResult, formatError, - isEmptyResponse, - getRouterForModel, - getApiKeyForRouter + isEmptyResponse } from './commons' -import { E_OPENROUTER_MODEL_FREE, E_OPENAI_MODEL } from '../../src/index' const TEST_LOG_PATH = path.resolve(__dirname, './math.json') @@ -76,10 +73,6 @@ describe('Math Capabilities', () => { passed, duration: Date.now() - startTime, reason: passed ? undefined : `Expected ${expected}, but got ${actual}`, - config: { - router: getRouterForModel(modelName), - apiKey: getApiKeyForRouter(getRouterForModel(modelName)) - } } } catch (e) { error = formatError(e) @@ -96,11 +89,7 @@ describe('Math Capabilities', () => { passed: false, duration: Date.now() - startTime, error, - reason: error?.message || 'Unknown error occurred', - config: { - router: getRouterForModel(modelName), - apiKey: getApiKeyForRouter(getRouterForModel(modelName)) - } + reason: error?.message || 'Unknown error occurred' } testResults.push(testResult)