diff --git a/packages/kbot/logs/params.json b/packages/kbot/logs/params.json index cee79f77..15527994 100644 --- a/packages/kbot/logs/params.json +++ b/packages/kbot/logs/params.json @@ -1,5 +1,5 @@ { - "model": "gpt-4o", + "model": "google/gemini-2.0-flash-lite-001", "messages": [ { "role": "user", diff --git a/packages/kbot/src/zod_schema.ts b/packages/kbot/src/zod_schema.ts index c9bbf52a..a02ae247 100644 --- a/packages/kbot/src/zod_schema.ts +++ b/packages/kbot/src/zod_schema.ts @@ -31,7 +31,6 @@ export const EMode = { CUSTOM: 'custom' } as const - export const EType = z.enum([ EMode.COMPLETION, EMode.TOOLS, @@ -291,14 +290,11 @@ export const OptionsSchema = (opts?: any): any => { .passthrough() .describe('IKBotOptions') } - - export const types = () => { generate_interfaces([OptionsSchema()], 'src/zod_types.ts') generate_interfaces([OptionsSchema()], path.resolve(resolve('../ai-tools/src/types_kbot.ts'))) schemas() } - export const schemas = () => { write([OptionsSchema()], 'schema.json', 'kbot', {}) writeFS('schema_ui.json', schemaMap.getUISchema()) diff --git a/packages/kbot/tests/unit/format-report.md b/packages/kbot/tests/unit/format-report.md index 9bd03afe..6cef7145 100644 --- a/packages/kbot/tests/unit/format-report.md +++ b/packages/kbot/tests/unit/format-report.md @@ -2,189 +2,11 @@ ## Failed Tests -### json-schema-file-format - anthropic/claude-3.5-sonnet -- Prompt: `Create a user profile with name John Doe, age 30, and tags ["developer", "javascript"]. Return only the JSON object, no explanation.` -- Expected: `{"name":"John Doe","age":30,"tags":["developer","javascript"]}` -- Actual: `` -- Duration: 1690ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '{\n "name": "John Doe",\n "age": 30…' to deeply equal '{"name":"John Doe","age":30,"tags":["…' -- Error Details: expected '{\n "name": "John Doe",\n "age": 30…' to deeply equal '{"name":"John Doe","age":30,"tags":["…' -- Reason: expected '{\n "name": "John Doe",\n "age": 30…' to deeply equal '{"name":"John Doe","age":30,"tags":["…' -- Timestamp: 4/2/2025, 12:23:43 AM - -### json-schema-file-format - qwen/qwq-32b -- Prompt: `Create a user profile with name John Doe, age 30, and tags ["developer", "javascript"]. Return only the JSON object, no explanation.` -- Expected: `{"name":"John Doe","age":30,"tags":["developer","javascript"]}` -- Actual: `` -- Duration: 3426ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '"John Doe",\n "age": 30,\n "tags": …' to deeply equal '{"name":"John Doe","age":30,"tags":["…' -- Error Details: expected '"John Doe",\n "age": 30,\n "tags": …' to deeply equal '{"name":"John Doe","age":30,"tags":["…' -- Reason: expected '"John Doe",\n "age": 30,\n "tags": …' to deeply equal '{"name":"John Doe","age":30,"tags":["…' -- Timestamp: 4/2/2025, 12:23:47 AM - -### json-schema-object-format - anthropic/claude-3.5-sonnet -- Prompt: `Create a user profile with name Jane Smith, age 25, and tags ["designer", "ui"]. Return only the JSON object, no explanation.` -- Expected: `{"name":"Jane Smith","age":25,"tags":["designer","ui"]}` -- Actual: `` -- Duration: 1918ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '{\n "name": "Jane Smith",\n "age": …' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' -- Error Details: expected '{\n "name": "Jane Smith",\n "age": …' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' -- Reason: expected '{\n "name": "Jane Smith",\n "age": …' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' -- Timestamp: 4/2/2025, 12:23:49 AM - -### json-schema-object-format - qwen/qwq-32b -- Prompt: `Create a user profile with name Jane Smith, age 25, and tags ["designer", "ui"]. Return only the JSON object, no explanation.` -- Expected: `{"name":"Jane Smith","age":25,"tags":["designer","ui"]}` -- Actual: `` -- Duration: 9789ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '{"name": "Jane Smith", "age": 25, "ta…' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' -- Error Details: expected '{"name": "Jane Smith", "age": 25, "ta…' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' -- Reason: expected '{"name": "Jane Smith", "age": 25, "ta…' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' -- Timestamp: 4/2/2025, 12:23:58 AM - -### zod-string-format - anthropic/claude-3.5-sonnet -- Prompt: `Generate a valid email address for a business domain. Return only the email, no explanation.` -- Expected: `john.doe@company.com` -- Actual: `` -- Duration: 1347ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com' -- Error Details: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com' -- Reason: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com' -- Timestamp: 4/2/2025, 12:24:00 AM - -### zod-string-format - qwen/qwq-32b -- Prompt: `Generate a valid email address for a business domain. Return only the email, no explanation.` -- Expected: `john.doe@company.com` -- Actual: `` -- Duration: 13704ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected 'info@techstart.com' to deeply equal 'john.doe@company.com' -- Error Details: expected 'info@techstart.com' to deeply equal 'john.doe@company.com' -- Reason: expected 'info@techstart.com' to deeply equal 'john.doe@company.com' -- Timestamp: 4/2/2025, 12:24:13 AM - -### zod-string-format - gpt-4o -- Prompt: `Generate a valid email address for a business domain. Return only the email, no explanation.` -- Expected: `john.doe@company.com` -- Actual: `` -- Duration: 1794ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com' -- Error Details: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com' -- Reason: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com' -- Timestamp: 4/2/2025, 12:32:20 AM - -### zod-number-format - anthropic/claude-3.5-sonnet -- Prompt: `Generate a random age between 18 and 65. Return only the number, no explanation.` -- Expected: `25` -- Actual: `` -- Duration: 1376ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '42' to deeply equal '25' -- Error Details: expected '42' to deeply equal '25' -- Reason: expected '42' to deeply equal '25' -- Timestamp: 4/2/2025, 12:24:11 AM - -### zod-number-format - gpt-4o -- Prompt: `Generate a random age between 18 and 65. Return only the number, no explanation.` -- Expected: `25` -- Actual: `` -- Duration: 2399ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '39' to deeply equal '25' -- Error Details: expected '39' to deeply equal '25' -- Reason: expected '39' to deeply equal '25' -- Timestamp: 4/2/2025, 12:32:23 AM - -### zod-array-format - anthropic/claude-3.5-sonnet -- Prompt: `Generate a list of 3 programming languages. Return only the array, no explanation.` -- Expected: `["JavaScript","Python","Java"]` -- Actual: `` -- Duration: 1009ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '["Python", "Java", "JavaScript"]' to deeply equal '["JavaScript","Python","Java"]' -- Error Details: expected '["Python", "Java", "JavaScript"]' to deeply equal '["JavaScript","Python","Java"]' -- Reason: expected '["Python", "Java", "JavaScript"]' to deeply equal '["JavaScript","Python","Java"]' -- Timestamp: 4/2/2025, 12:24:22 AM - -### zod-array-format - qwen/qwq-32b -- Prompt: `Generate a list of 3 programming languages. Return only the array, no explanation.` -- Expected: `["JavaScript","Python","Java"]` -- Actual: `` -- Duration: 4147ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '["Python", "JavaScript", "Java"]' to deeply equal '["JavaScript","Python","Java"]' -- Error Details: expected '["Python", "JavaScript", "Java"]' to deeply equal '["JavaScript","Python","Java"]' -- Reason: expected '["Python", "JavaScript", "Java"]' to deeply equal '["JavaScript","Python","Java"]' -- Timestamp: 4/2/2025, 12:24:26 AM - -### zod-array-format - gpt-4o -- Prompt: `Generate a list of 3 programming languages. Return only the array, no explanation.` -- Expected: `["JavaScript","Python","Java"]` -- Actual: `` -- Duration: 693ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '["Python","JavaScript","Java"]' to deeply equal '["JavaScript","Python","Java"]' -- Error Details: expected '["Python","JavaScript","Java"]' to deeply equal '["JavaScript","Python","Java"]' -- Reason: expected '["Python","JavaScript","Java"]' to deeply equal '["JavaScript","Python","Java"]' -- Timestamp: 4/2/2025, 12:32:23 AM - -### invalid-format - anthropic/claude-3.5-sonnet -- Prompt: `Generate a random number.` -- Expected: `Invalid format option` -- Actual: `` -- Duration: 1026ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '73' to deeply equal 'Invalid format option' -- Error Details: expected '73' to deeply equal 'Invalid format option' -- Reason: expected '73' to deeply equal 'Invalid format option' -- Timestamp: 4/2/2025, 12:24:27 AM - -### invalid-format - qwen/qwq-32b -- Prompt: `Generate a random number.` -- Expected: `Invalid format option` -- Actual: `` -- Duration: 7614ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '72' to deeply equal 'Invalid format option' -- Error Details: expected '72' to deeply equal 'Invalid format option' -- Reason: expected '72' to deeply equal 'Invalid format option' -- Timestamp: 4/2/2025, 12:24:35 AM - -### invalid-format - gpt-4o -- Prompt: `Generate a random number.` -- Expected: `Invalid format option` -- Actual: `` -- Duration: 826ms -- Error Type: AssertionError -- Error Code: UNKNOWN -- Error Message: expected '786984' to deeply equal 'Invalid format option' -- Error Details: expected '786984' to deeply equal 'Invalid format option' -- Reason: expected '786984' to deeply equal 'Invalid format option' -- Timestamp: 4/2/2025, 12:32:24 AM +*No failed tests* ## Passed Tests -### json-schema-file-format - gpt-4o +### json-schema-file-format - google/gemini-2.0-flash-lite-001 - Prompt: `Create a user profile with name John Doe, age 30, and tags ["developer", "javascript"]. Return only the JSON object, no explanation.` - Expected: `{"name":"John Doe","age":30,"tags":["developer","javascript"]}` - Actual: ````json @@ -193,11 +15,12 @@ "age": 30, "tags": ["developer", "javascript"] } -```` -- Duration: 960ms -- Timestamp: 4/2/2025, 12:38:51 AM +``` +` +- Duration: 1122ms +- Timestamp: 4/2/2025, 9:03:33 AM -### json-schema-object-format - gpt-4o +### json-schema-object-format - google/gemini-2.0-flash-lite-001 - Prompt: `Create a user profile with the following details: - Name: Jane Smith - Age: 25 @@ -234,14 +57,15 @@ "language": "English" } } -```` -- Duration: 1505ms -- Timestamp: 4/2/2025, 12:38:52 AM +``` +` +- Duration: 1289ms +- Timestamp: 4/2/2025, 9:03:34 AM ## Summary -- Total Tests: 51 -- Passed: 13 -- Failed: 38 -- Success Rate: 25.49% +- Total Tests: 2 +- Passed: 2 +- Failed: 0 +- Success Rate: 100.00% diff --git a/packages/kbot/tests/unit/format.json b/packages/kbot/tests/unit/format.json index c9ab7d09..4e154acb 100644 --- a/packages/kbot/tests/unit/format.json +++ b/packages/kbot/tests/unit/format.json @@ -2,1236 +2,27 @@ { "test": "json-schema-file-format", "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [], + "result": [ + "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```\n" + ], "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "anthropic/claude-3.5-sonnet", + "model": "google/gemini-2.0-flash-lite-001", "router": "openrouter", - "timestamp": "2025-04-01T22:23:43.742Z", - "passed": false, - "duration": 1690, - "error": { - "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "actual": "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}", - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "qwen/qwq-32b", - "router": "openrouter", - "timestamp": "2025-04-01T22:23:47.173Z", - "passed": false, - "duration": 3426, - "error": { - "message": "expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "actual": "\"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}", - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\"name\":\u001b[27m\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ \"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with name Jane Smith, age 25, and tags [\"designer\", \"ui\"]. Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", - "model": "anthropic/claude-3.5-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T22:23:49.093Z", - "passed": false, - "duration": 1918, - "error": { - "message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", - "actual": "{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"tags\": [\"designer\", \"ui\"]\n}", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"designer\",\u001b[7m \u001b[27m\"ui\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'" - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with name Jane Smith, age 25, and tags [\"designer\", \"ui\"]. Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", - "model": "qwen/qwq-32b", - "router": "openrouter", - "timestamp": "2025-04-01T22:23:58.884Z", - "passed": false, - "duration": 9789, - "error": { - "message": "expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", - "actual": "{\"name\": \"Jane Smith\", \"age\": 25, \"tags\": [\"designer\", \"ui\"]}", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}\"\u001b[39m\nReceived: \u001b[31m\"{\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"designer\",\u001b[7m \u001b[27m\"ui\"]}\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'" - }, - { - "test": "zod-string-format", - "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", - "result": [], - "expected": "john.doe@company.com", - "model": "anthropic/claude-3.5-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T22:24:00.233Z", - "passed": false, - "duration": 1347, - "error": { - "message": "expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'", - "actual": "sales@companyplus.com", - "expected": "john.doe@company.com", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:174:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"\u001b[7mjohn.doe\u001b[27m@company.com\"\u001b[39m\nReceived: \u001b[31m\"\u001b[7msales\u001b[27m@company\u001b[7mplus\u001b[27m.com\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:174:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'" - }, - { - "test": "zod-number-format", - "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", - "result": [], - "expected": "25", - "model": "anthropic/claude-3.5-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T22:24:11.615Z", - "passed": false, - "duration": 1376, - "error": { - "message": "expected '42' to deeply equal '25'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '42' to deeply equal '25'", - "actual": "42", - "expected": "25", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '42' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:190:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"2\u001b[7m5\u001b[27m\"\u001b[39m\nReceived: \u001b[31m\"\u001b[7m4\u001b[27m2\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '42' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:190:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '42' to deeply equal '25'" - }, - { - "test": "zod-string-format", - "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", - "result": [], - "expected": "john.doe@company.com", - "model": "qwen/qwq-32b", - "router": "openrouter", - "timestamp": "2025-04-01T22:24:13.939Z", - "passed": false, - "duration": 13704, - "error": { - "message": "expected 'info@techstart.com' to deeply equal 'john.doe@company.com'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected 'info@techstart.com' to deeply equal 'john.doe@company.com'", - "actual": "info@techstart.com", - "expected": "john.doe@company.com", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected 'info@techstart.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:174:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5" - } - }, - "reason": "expected 'info@techstart.com' to deeply equal 'john.doe@company.com'" - }, - { - "test": "zod-array-format", - "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", - "result": [], - "expected": "[\"JavaScript\",\"Python\",\"Java\"]", - "model": "anthropic/claude-3.5-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T22:24:22.636Z", - "passed": false, - "duration": 1009, - "error": { - "message": "expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", - "actual": "[\"Python\", \"Java\", \"JavaScript\"]", - "expected": "[\"JavaScript\",\"Python\",\"Java\"]", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"[\"\u001b[7mJavaScript\",\"Python\u001b[27m\",\"Java\"]\"\u001b[39m\nReceived: \u001b[31m\"[\"\u001b[7mPython\", \"Java\u001b[27m\",\u001b[7m \u001b[27m\"Java\u001b[7mScript\u001b[27m\"]\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'" - }, - { - "test": "zod-array-format", - "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", - "result": [], - "expected": "[\"JavaScript\",\"Python\",\"Java\"]", - "model": "qwen/qwq-32b", - "router": "openrouter", - "timestamp": "2025-04-01T22:24:26.785Z", - "passed": false, - "duration": 4147, - "error": { - "message": "expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", - "actual": "[\"Python\", \"JavaScript\", \"Java\"]", - "expected": "[\"JavaScript\",\"Python\",\"Java\"]", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"[\"JavaScript\u001b[7m\",\"Python\u001b[27m\",\"Java\"]\"\u001b[39m\nReceived: \u001b[31m\"[\"\u001b[7mPython\", \"\u001b[27mJavaScript\",\u001b[7m \u001b[27m\"Java\"]\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'" - }, - { - "test": "invalid-format", - "prompt": "Generate a random number.", - "result": [], - "expected": "Invalid format option", - "model": "anthropic/claude-3.5-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T22:24:27.813Z", - "passed": false, - "duration": 1026, - "error": { - "message": "expected '73' to deeply equal 'Invalid format option'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '73' to deeply equal 'Invalid format option'", - "actual": "73", - "expected": "Invalid format option", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '73' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:222:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"Invalid format option\"\u001b[39m\nReceived: \u001b[31m\"73\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '73' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:222:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '73' to deeply equal 'Invalid format option'" - }, - { - "test": "invalid-format", - "prompt": "Generate a random number.", - "result": [], - "expected": "Invalid format option", - "model": "qwen/qwq-32b", - "router": "openrouter", - "timestamp": "2025-04-01T22:24:35.428Z", - "passed": false, - "duration": 7614, - "error": { - "message": "expected '72' to deeply equal 'Invalid format option'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '72' to deeply equal 'Invalid format option'", - "actual": "72", - "expected": "Invalid format option", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '72' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:222:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '72' to deeply equal 'Invalid format option'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:28:54.288Z", - "passed": false, - "duration": 1146, - "error": { - "message": "expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "actual": "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```", - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:140:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]\u001b[7m}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:140:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with name Jane Smith, age 25, and tags [\"designer\", \"ui\"]. Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:28:55.427Z", - "passed": false, - "duration": 1134, - "error": { - "message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", - "actual": "{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"tags\": [\"designer\", \"ui\"]\n}", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:160:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"designer\",\u001b[7m \u001b[27m\"ui\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:160:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'" - }, - { - "test": "zod-string-format", - "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", - "result": [], - "expected": "john.doe@company.com", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:28:55.754Z", - "passed": false, - "duration": 326, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:176:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:176:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response" - }, - { - "test": "zod-number-format", - "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", - "result": [], - "expected": "25", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:28:56.051Z", - "passed": false, - "duration": 295, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:192:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:192:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response" - }, - { - "test": "zod-array-format", - "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", - "result": [], - "expected": "[\"JavaScript\",\"Python\",\"Java\"]", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:28:56.385Z", - "passed": false, - "duration": 332, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response" - }, - { - "test": "invalid-format", - "prompt": "Generate a random number.", - "result": [], - "expected": "Invalid format option", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:28:56.983Z", - "passed": false, - "duration": 596, - "error": { - "message": "expected '83' to deeply equal 'Invalid format option'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '83' to deeply equal 'Invalid format option'", - "actual": "83", - "expected": "Invalid format option", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '83' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:224:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '83' to deeply equal 'Invalid format option'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:07.138Z", - "passed": false, - "duration": 980, - "error": { - "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "actual": "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}", - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:08.771Z", - "passed": false, - "duration": 1628, - "error": { - "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "actual": "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": \"123 Main St, New York, US, 10001\",\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": true,\n \"language\": \"English\"\n }\n}\n```", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\u001b[7m{\"street\":\u001b[27m\"123 Main St\u001b[7m\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"\u001b[27m10001\"\u001b[7m}\u001b[27m,\"preferences\":\u001b[7m{\u001b[27m\"theme\":\"light\",\"notifications\":true,\"language\":\u001b[7m\"en\"}}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"email\":\u001b[7m \u001b[27m\"jane.smith@company.com\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"designer\"],\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"address\":\u001b[7m \u001b[27m\"123 Main St\u001b[7m, New York, US, \u001b[27m10001\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"preferences\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"theme\":\u001b[7m \u001b[27m\"light\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"notifications\":\u001b[7m \u001b[27mtrue,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"language\":\u001b[7m \"English\"\u001b[27m\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" - }, - { - "test": "zod-string-format", - "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", - "result": [], - "expected": "john.doe@company.com", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:09.057Z", - "passed": false, - "duration": 282, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response" - }, - { - "test": "zod-number-format", - "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", - "result": [], - "expected": "25", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:09.356Z", - "passed": false, - "duration": 297, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:275:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response" - } - }, - "reason": "Model returned empty response" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:31.573Z", - "passed": false, - "duration": 1074, - "error": { - "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", - "actual": "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}", - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:33.577Z", - "passed": false, - "duration": 1999, - "error": { - "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "actual": "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": {\n \"street\": \"123 Main St\",\n \"city\": \"New York\",\n \"country\": \"US\",\n \"zipcode\": \"10001\"\n },\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": \"enabled\",\n \"language\": \"English\"\n }\n}\n```", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\u001b[7m{\u001b[27m\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zip\u001b[7mC\u001b[27mode\":\"10001\"\u001b[7m},\u001b[27m\"preferences\":\u001b[7m{\u001b[27m\"theme\":\"light\",\"notifications\":\u001b[7mtrue,\"language\":\"en\"}}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"email\":\u001b[7m \u001b[27m\"jane.smith@company.com\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"designer\"],\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"address\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"street\":\u001b[7m \u001b[27m\"123 Main St\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"city\":\u001b[7m \u001b[27m\"New York\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"country\":\u001b[7m \u001b[27m\"US\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"zip\u001b[7mc\u001b[27mode\":\u001b[7m \u001b[27m\"10001\"\u001b[39m\n\u001b[31m+ },\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"preferences\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"theme\":\u001b[7m \u001b[27m\"light\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"notifications\":\u001b[7m \"enabled\",\u001b[27m\u001b[39m\n\u001b[31m+ \"language\": \"English\"\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" - }, - { - "test": "zod-string-format", - "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", - "result": [], - "expected": "john.doe@company.com", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:33.906Z", - "passed": false, - "duration": 324, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response" - }, - { - "test": "zod-number-format", - "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", - "result": [], - "expected": "25", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:34.272Z", - "passed": false, - "duration": 363, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:275:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:275:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response" - }, - { - "test": "zod-array-format", - "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", - "result": [], - "expected": "[\"JavaScript\",\"Python\",\"Java\"]", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:34.613Z", - "passed": false, - "duration": 339, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:291:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:291:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response" - }, - { - "test": "invalid-format", - "prompt": "Generate a random number.", - "result": [], - "expected": "Invalid format option", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:31:35.189Z", - "passed": false, - "duration": 574, - "error": { - "message": "expected '847293' to deeply equal 'Invalid format option'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '847293' to deeply equal 'Invalid format option'", - "actual": "847293", - "expected": "Invalid format option", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '847293' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:307:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '847293' to deeply equal 'Invalid format option'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:32:12.404Z", + "timestamp": "2025-04-02T07:03:33.038Z", "passed": true, - "duration": 1098 - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:32:19.059Z", - "passed": false, - "duration": 6653, - "error": { - "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "actual": "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": {\n \"street\": \"123 Main St\",\n \"city\": \"New York\",\n \"country\": \"US\",\n \"postal_code\": \"10001\"\n },\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": true,\n \"language\": \"English\"\n }\n}\n```", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:254:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\u001b[7m{\u001b[27m\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\u001b[7m\"zipC\u001b[27mode\":\"10001\"\u001b[7m},\u001b[27m\"preferences\":\u001b[7m{\u001b[27m\"theme\":\"light\",\"notifications\":true,\"language\":\u001b[7m\"en\"}}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"email\":\u001b[7m \u001b[27m\"jane.smith@company.com\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"designer\"],\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"address\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"street\":\u001b[7m \u001b[27m\"123 Main St\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"city\":\u001b[7m \u001b[27m\"New York\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"country\":\u001b[7m \u001b[27m\"US\",\u001b[39m\n\u001b[31m+ \u001b[7m \"postal_c\u001b[27mode\":\u001b[7m \u001b[27m\"10001\"\u001b[39m\n\u001b[31m+ },\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"preferences\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"theme\":\u001b[7m \u001b[27m\"light\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"notifications\":\u001b[7m \u001b[27mtrue,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"language\":\u001b[7m \"English\"\u001b[27m\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:254:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" - }, - { - "test": "zod-string-format", - "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", - "result": [], - "expected": "john.doe@company.com", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:32:20.861Z", - "passed": false, - "duration": 1794, - "error": { - "message": "expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'", - "actual": "contact@businessdomain.com", - "expected": "john.doe@company.com", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:270:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"\u001b[7mjohn.doe@company\u001b[27m.com\"\u001b[39m\nReceived: \u001b[31m\"\u001b[7mcontact@businessdomain\u001b[27m.com\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:270:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'" - }, - { - "test": "zod-number-format", - "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", - "result": [], - "expected": "25", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:32:23.263Z", - "passed": false, - "duration": 2399, - "error": { - "message": "expected '39' to deeply equal '25'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '39' to deeply equal '25'", - "actual": "39", - "expected": "25", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '39' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:295:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"25\"\u001b[39m\nReceived: \u001b[31m\"39\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '39' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:295:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '39' to deeply equal '25'" - }, - { - "test": "zod-array-format", - "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", - "result": [], - "expected": "[\"JavaScript\",\"Python\",\"Java\"]", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:32:23.958Z", - "passed": false, - "duration": 693, - "error": { - "message": "expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", - "actual": "[\"Python\",\"JavaScript\",\"Java\"]", - "expected": "[\"JavaScript\",\"Python\",\"Java\"]", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:321:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"[\"JavaScript\u001b[7m\",\"Python\u001b[27m\",\"Java\"]\"\u001b[39m\nReceived: \u001b[31m\"[\"\u001b[7mPython\",\"\u001b[27mJavaScript\",\"Java\"]\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:321:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'" - }, - { - "test": "invalid-format", - "prompt": "Generate a random number.", - "result": [], - "expected": "Invalid format option", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:32:24.787Z", - "passed": false, - "duration": 826, - "error": { - "message": "expected '786984' to deeply equal 'Invalid format option'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '786984' to deeply equal 'Invalid format option'", - "actual": "786984", - "expected": "Invalid format option", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '786984' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:350:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '786984' to deeply equal 'Invalid format option'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:33:05.518Z", - "passed": true, - "duration": 1027 - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:33:08.138Z", - "passed": false, - "duration": 2618, - "error": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "actual": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postal_code\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:184:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:271:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:33:41.869Z", - "passed": true, - "duration": 2353 - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:34:16.181Z", - "passed": true, - "duration": 1098 - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:34:28.660Z", - "passed": true, - "duration": 995 - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:35:05.732Z", - "passed": true, - "duration": 1101 - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:35:37.746Z", - "passed": true, - "duration": 1187 - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:36:19.070Z", - "passed": true, - "duration": 1238 - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postalCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:36:20.664Z", - "passed": false, - "duration": 1592, - "error": { - "message": "expected '{\"Name\":\"Jane Smith\",\"Age\":25,\"Email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\"Name\":\"Jane Smith\",\"Age\":25,\"Email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "actual": "{\"Name\":\"Jane Smith\",\"Age\":25,\"Email\":\"jane.smith@company.com\",\"Tags\":[\"developer\",\"designer\"],\"Address\":{\"Street\":\"123 Main St\",\"City\":\"New York\",\"Country\":\"US\",\"PostalCode\":\"10001\"},\"Preferences\":{\"Theme\":\"light\",\"Notifications\":\"enabled\",\"Language\":\"English\"}}", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postalCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected '{\"Name\":\"Jane Smith\",\"Age\":25,\"Email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:184:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:271:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"{\"\u001b[7mn\u001b[27mame\":\"Jane Smith\",\"\u001b[7ma\u001b[27mge\":25,\"\u001b[7me\u001b[27mmail\":\"jane.smith@company.com\",\"\u001b[7mt\u001b[27mags\":[\"developer\",\"designer\"],\"\u001b[7ma\u001b[27mddress\":{\"\u001b[7ms\u001b[27mtreet\":\"123 Main St\",\"\u001b[7mc\u001b[27mity\":\"New York\",\"\u001b[7mc\u001b[27mountry\":\"US\",\"\u001b[7mp\u001b[27mostalCode\":\"10001\"},\"\u001b[7mp\u001b[27mreferences\":{\"\u001b[7mt\u001b[27mheme\":\"light\",\"\u001b[7mn\u001b[27motifications\":\"enabled\",\"\u001b[7ml\u001b[27manguage\":\"English\"}}\"\u001b[39m\nReceived: \u001b[31m\"{\"\u001b[7mN\u001b[27mame\":\"Jane Smith\",\"\u001b[7mA\u001b[27mge\":25,\"\u001b[7mE\u001b[27mmail\":\"jane.smith@company.com\",\"\u001b[7mT\u001b[27mags\":[\"developer\",\"designer\"],\"\u001b[7mA\u001b[27mddress\":{\"\u001b[7mS\u001b[27mtreet\":\"123 Main St\",\"\u001b[7mC\u001b[27mity\":\"New York\",\"\u001b[7mC\u001b[27mountry\":\"US\",\"\u001b[7mP\u001b[27mostalCode\":\"10001\"},\"\u001b[7mP\u001b[27mreferences\":{\"\u001b[7mT\u001b[27mheme\":\"light\",\"\u001b[7mN\u001b[27motifications\":\"enabled\",\"\u001b[7mL\u001b[27manguage\":\"English\"}}\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\"Name\":\"Jane Smith\",\"Age\":25,\"Email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:184:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:271:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\"Name\":\"Jane Smith\",\"Age\":25,\"Email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postalCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:36:23.463Z", - "passed": false, - "duration": 2794, - "error": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "actual": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\"123 Main St, New York, US, 10001\",\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postalCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:184:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:271:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:37:16.453Z", - "passed": true, - "duration": 2477 - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postalCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:37:19.610Z", - "passed": false, - "duration": 3155, - "error": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "actual": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postalCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"English\"}}", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postalCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:183:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:270:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\n \"developer\",\n \"javascript\"\n ]\n}\n```" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:38:14.271Z", - "passed": true, - "duration": 1276 - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postalCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:38:15.883Z", - "passed": false, - "duration": 1609, - "error": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "actual": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postal_code\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postalCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:185:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:272:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:38:32.055Z", - "passed": true, - "duration": 1134 - }, - { - "test": "json-schema-object-format", - "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", - "result": [], - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postal_code\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:38:33.304Z", - "passed": false, - "duration": 1247, - "error": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", - "actual": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\"123 Main St, New York, US, 10001\",\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postal_code\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "showDiff": true, - "operator": "strictEqual", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:185:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:272:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" - }, - { - "test": "json-schema-file-format", - "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", - "result": [ - "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" - ], - "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:38:51.385Z", - "passed": true, - "duration": 960 + "duration": 1122 }, { "test": "json-schema-object-format", "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: {\n \"street\": \"123 Main St\",\n \"city\": \"New York\",\n \"country\": \"US\",\n \"postal_code\": \"10001\"\n }\n - Preferences: {\n \"theme\": \"light\",\n \"notifications\": \"enabled\",\n \"language\": \"English\"\n }\n Return only the JSON object, no explanation.", "result": [ - "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": {\n \"street\": \"123 Main St\",\n \"city\": \"New York\",\n \"country\": \"US\",\n \"postal_code\": \"10001\"\n },\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": \"enabled\",\n \"language\": \"English\"\n }\n}\n```" + "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": {\n \"street\": \"123 Main St\",\n \"city\": \"New York\",\n \"country\": \"US\",\n \"postal_code\": \"10001\"\n },\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": \"enabled\",\n \"language\": \"English\"\n }\n}\n```\n" ], "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postal_code\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", - "model": "gpt-4o", - "router": "openai", - "timestamp": "2025-04-01T22:38:52.892Z", + "model": "google/gemini-2.0-flash-lite-001", + "router": "openrouter", + "timestamp": "2025-04-02T07:03:34.328Z", "passed": true, - "duration": 1505 + "duration": 1289 } ] \ No newline at end of file diff --git a/packages/kbot/tests/unit/format.test.ts b/packages/kbot/tests/unit/format.test.ts index 59906af7..c7185017 100644 --- a/packages/kbot/tests/unit/format.test.ts +++ b/packages/kbot/tests/unit/format.test.ts @@ -17,8 +17,8 @@ import { const TEST_LOG_PATH = path.resolve(__dirname, './format.json') const TEST_SCHEMA_PATH = path.resolve(__dirname, './test-schema.json') -const TEST_MODEL = 'gpt-4o' -const TEST_ROUTER = 'openai' +const TEST_MODEL = 'google/gemini-2.0-flash-lite-001' +const TEST_ROUTER = 'openrouter' // Sample JSON Schema for testing const testJsonSchema = { @@ -138,12 +138,6 @@ const hasValidArrayLength = (arr: any[], length: number) => { describe('Format Options', () => { let testResults: TestResult[] = [] - // Load existing results if any - if (exists(TEST_LOG_PATH)) { - const data = read(TEST_LOG_PATH, 'json') - testResults = Array.isArray(data) ? data : [] - } - const runFormatTest = async (prompt: string, expected: string, testName: string, modelName: string, options: any = {}) => { let model = TEST_MODEL let router = TEST_ROUTER @@ -178,8 +172,8 @@ describe('Format Options', () => { } const actual = result?.[0]?.trim() || '' - const normalizedActual = normalizeJson(actual) - const normalizedExpected = normalizeJson(expected) + const normalizedActual = normalizeJson(actual).toLowerCase() + const normalizedExpected = normalizeJson(expected).toLowerCase() const passed = normalizedActual === normalizedExpected expect(normalizedActual).toEqual(normalizedExpected)