995 lines
128 KiB
JSON
995 lines
128 KiB
JSON
[
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "anthropic/claude-3.5-sonnet",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:23:43.742Z",
|
|
"passed": false,
|
|
"duration": 1690,
|
|
"error": {
|
|
"message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"actual": "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}",
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'"
|
|
},
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "qwen/qwq-32b",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:23:47.173Z",
|
|
"passed": false,
|
|
"duration": 3426,
|
|
"error": {
|
|
"message": "expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"actual": "\"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}",
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\"name\":\u001b[27m\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ \"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'"
|
|
},
|
|
{
|
|
"test": "json-schema-object-format",
|
|
"prompt": "Create a user profile with name Jane Smith, age 25, and tags [\"designer\", \"ui\"]. Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}",
|
|
"model": "anthropic/claude-3.5-sonnet",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:23:49.093Z",
|
|
"passed": false,
|
|
"duration": 1918,
|
|
"error": {
|
|
"message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'",
|
|
"actual": "{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"tags\": [\"designer\", \"ui\"]\n}",
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"designer\",\u001b[7m \u001b[27m\"ui\"]\u001b[39m\n\u001b[31m+ }\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'"
|
|
},
|
|
{
|
|
"test": "json-schema-object-format",
|
|
"prompt": "Create a user profile with name Jane Smith, age 25, and tags [\"designer\", \"ui\"]. Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}",
|
|
"model": "qwen/qwq-32b",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:23:58.884Z",
|
|
"passed": false,
|
|
"duration": 9789,
|
|
"error": {
|
|
"message": "expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'",
|
|
"actual": "{\"name\": \"Jane Smith\", \"age\": 25, \"tags\": [\"designer\", \"ui\"]}",
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "Expected: \u001b[32m\"{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}\"\u001b[39m\nReceived: \u001b[31m\"{\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"designer\",\u001b[7m \u001b[27m\"ui\"]}\"\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'"
|
|
},
|
|
{
|
|
"test": "zod-string-format",
|
|
"prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.",
|
|
"result": [],
|
|
"expected": "john.doe@company.com",
|
|
"model": "anthropic/claude-3.5-sonnet",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:24:00.233Z",
|
|
"passed": false,
|
|
"duration": 1347,
|
|
"error": {
|
|
"message": "expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'",
|
|
"actual": "sales@companyplus.com",
|
|
"expected": "john.doe@company.com",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:174:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "Expected: \u001b[32m\"\u001b[7mjohn.doe\u001b[27m@company.com\"\u001b[39m\nReceived: \u001b[31m\"\u001b[7msales\u001b[27m@company\u001b[7mplus\u001b[27m.com\"\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:174:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'"
|
|
},
|
|
{
|
|
"test": "zod-number-format",
|
|
"prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.",
|
|
"result": [],
|
|
"expected": "25",
|
|
"model": "anthropic/claude-3.5-sonnet",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:24:11.615Z",
|
|
"passed": false,
|
|
"duration": 1376,
|
|
"error": {
|
|
"message": "expected '42' to deeply equal '25'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '42' to deeply equal '25'",
|
|
"actual": "42",
|
|
"expected": "25",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '42' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:190:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "Expected: \u001b[32m\"2\u001b[7m5\u001b[27m\"\u001b[39m\nReceived: \u001b[31m\"\u001b[7m4\u001b[27m2\"\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '42' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:190:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '42' to deeply equal '25'"
|
|
},
|
|
{
|
|
"test": "zod-string-format",
|
|
"prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.",
|
|
"result": [],
|
|
"expected": "john.doe@company.com",
|
|
"model": "qwen/qwq-32b",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:24:13.939Z",
|
|
"passed": false,
|
|
"duration": 13704,
|
|
"error": {
|
|
"message": "expected 'info@techstart.com' to deeply equal 'john.doe@company.com'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected 'info@techstart.com' to deeply equal 'john.doe@company.com'",
|
|
"actual": "info@techstart.com",
|
|
"expected": "john.doe@company.com",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected 'info@techstart.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:174:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5"
|
|
}
|
|
},
|
|
"reason": "expected 'info@techstart.com' to deeply equal 'john.doe@company.com'"
|
|
},
|
|
{
|
|
"test": "zod-array-format",
|
|
"prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.",
|
|
"result": [],
|
|
"expected": "[\"JavaScript\",\"Python\",\"Java\"]",
|
|
"model": "anthropic/claude-3.5-sonnet",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:24:22.636Z",
|
|
"passed": false,
|
|
"duration": 1009,
|
|
"error": {
|
|
"message": "expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'",
|
|
"actual": "[\"Python\", \"Java\", \"JavaScript\"]",
|
|
"expected": "[\"JavaScript\",\"Python\",\"Java\"]",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "Expected: \u001b[32m\"[\"\u001b[7mJavaScript\",\"Python\u001b[27m\",\"Java\"]\"\u001b[39m\nReceived: \u001b[31m\"[\"\u001b[7mPython\", \"Java\u001b[27m\",\u001b[7m \u001b[27m\"Java\u001b[7mScript\u001b[27m\"]\"\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'"
|
|
},
|
|
{
|
|
"test": "zod-array-format",
|
|
"prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.",
|
|
"result": [],
|
|
"expected": "[\"JavaScript\",\"Python\",\"Java\"]",
|
|
"model": "qwen/qwq-32b",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:24:26.785Z",
|
|
"passed": false,
|
|
"duration": 4147,
|
|
"error": {
|
|
"message": "expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'",
|
|
"actual": "[\"Python\", \"JavaScript\", \"Java\"]",
|
|
"expected": "[\"JavaScript\",\"Python\",\"Java\"]",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "Expected: \u001b[32m\"[\"JavaScript\u001b[7m\",\"Python\u001b[27m\",\"Java\"]\"\u001b[39m\nReceived: \u001b[31m\"[\"\u001b[7mPython\", \"\u001b[27mJavaScript\",\u001b[7m \u001b[27m\"Java\"]\"\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'"
|
|
},
|
|
{
|
|
"test": "invalid-format",
|
|
"prompt": "Generate a random number.",
|
|
"result": [],
|
|
"expected": "Invalid format option",
|
|
"model": "anthropic/claude-3.5-sonnet",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:24:27.813Z",
|
|
"passed": false,
|
|
"duration": 1026,
|
|
"error": {
|
|
"message": "expected '73' to deeply equal 'Invalid format option'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '73' to deeply equal 'Invalid format option'",
|
|
"actual": "73",
|
|
"expected": "Invalid format option",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '73' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:222:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "Expected: \u001b[32m\"Invalid format option\"\u001b[39m\nReceived: \u001b[31m\"73\"\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '73' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:222:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '73' to deeply equal 'Invalid format option'"
|
|
},
|
|
{
|
|
"test": "invalid-format",
|
|
"prompt": "Generate a random number.",
|
|
"result": [],
|
|
"expected": "Invalid format option",
|
|
"model": "qwen/qwq-32b",
|
|
"router": "openrouter",
|
|
"timestamp": "2025-04-01T22:24:35.428Z",
|
|
"passed": false,
|
|
"duration": 7614,
|
|
"error": {
|
|
"message": "expected '72' to deeply equal 'Invalid format option'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '72' to deeply equal 'Invalid format option'",
|
|
"actual": "72",
|
|
"expected": "Invalid format option",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '72' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:222:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '72' to deeply equal 'Invalid format option'"
|
|
},
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:28:54.288Z",
|
|
"passed": false,
|
|
"duration": 1146,
|
|
"error": {
|
|
"message": "expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"actual": "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```",
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:140:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]\u001b[7m}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:140:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'"
|
|
},
|
|
{
|
|
"test": "json-schema-object-format",
|
|
"prompt": "Create a user profile with name Jane Smith, age 25, and tags [\"designer\", \"ui\"]. Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:28:55.427Z",
|
|
"passed": false,
|
|
"duration": 1134,
|
|
"error": {
|
|
"message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'",
|
|
"actual": "{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"tags\": [\"designer\", \"ui\"]\n}",
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:160:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"designer\",\u001b[7m \u001b[27m\"ui\"]\u001b[39m\n\u001b[31m+ }\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:160:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'"
|
|
},
|
|
{
|
|
"test": "zod-string-format",
|
|
"prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.",
|
|
"result": [],
|
|
"expected": "john.doe@company.com",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:28:55.754Z",
|
|
"passed": false,
|
|
"duration": 326,
|
|
"error": {
|
|
"message": "Model returned empty response",
|
|
"code": "UNKNOWN",
|
|
"type": "Error",
|
|
"details": {
|
|
"stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:176:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"message": "Model returned empty response",
|
|
"stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:176:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "Error",
|
|
"expected": "undefined",
|
|
"actual": "undefined"
|
|
}
|
|
},
|
|
"reason": "Model returned empty response"
|
|
},
|
|
{
|
|
"test": "zod-number-format",
|
|
"prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.",
|
|
"result": [],
|
|
"expected": "25",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:28:56.051Z",
|
|
"passed": false,
|
|
"duration": 295,
|
|
"error": {
|
|
"message": "Model returned empty response",
|
|
"code": "UNKNOWN",
|
|
"type": "Error",
|
|
"details": {
|
|
"stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:192:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"message": "Model returned empty response",
|
|
"stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:192:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "Error",
|
|
"expected": "undefined",
|
|
"actual": "undefined"
|
|
}
|
|
},
|
|
"reason": "Model returned empty response"
|
|
},
|
|
{
|
|
"test": "zod-array-format",
|
|
"prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.",
|
|
"result": [],
|
|
"expected": "[\"JavaScript\",\"Python\",\"Java\"]",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:28:56.385Z",
|
|
"passed": false,
|
|
"duration": 332,
|
|
"error": {
|
|
"message": "Model returned empty response",
|
|
"code": "UNKNOWN",
|
|
"type": "Error",
|
|
"details": {
|
|
"stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"message": "Model returned empty response",
|
|
"stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "Error",
|
|
"expected": "undefined",
|
|
"actual": "undefined"
|
|
}
|
|
},
|
|
"reason": "Model returned empty response"
|
|
},
|
|
{
|
|
"test": "invalid-format",
|
|
"prompt": "Generate a random number.",
|
|
"result": [],
|
|
"expected": "Invalid format option",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:28:56.983Z",
|
|
"passed": false,
|
|
"duration": 596,
|
|
"error": {
|
|
"message": "expected '83' to deeply equal 'Invalid format option'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '83' to deeply equal 'Invalid format option'",
|
|
"actual": "83",
|
|
"expected": "Invalid format option",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '83' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:224:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '83' to deeply equal 'Invalid format option'"
|
|
},
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:07.138Z",
|
|
"passed": false,
|
|
"duration": 980,
|
|
"error": {
|
|
"message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"actual": "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}",
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'"
|
|
},
|
|
{
|
|
"test": "json-schema-object-format",
|
|
"prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:08.771Z",
|
|
"passed": false,
|
|
"duration": 1628,
|
|
"error": {
|
|
"message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'",
|
|
"actual": "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": \"123 Main St, New York, US, 10001\",\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": true,\n \"language\": \"English\"\n }\n}\n```",
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\u001b[7m{\"street\":\u001b[27m\"123 Main St\u001b[7m\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"\u001b[27m10001\"\u001b[7m}\u001b[27m,\"preferences\":\u001b[7m{\u001b[27m\"theme\":\"light\",\"notifications\":true,\"language\":\u001b[7m\"en\"}}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"email\":\u001b[7m \u001b[27m\"jane.smith@company.com\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"designer\"],\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"address\":\u001b[7m \u001b[27m\"123 Main St\u001b[7m, New York, US, \u001b[27m10001\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"preferences\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"theme\":\u001b[7m \u001b[27m\"light\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"notifications\":\u001b[7m \u001b[27mtrue,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"language\":\u001b[7m \"English\"\u001b[27m\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'"
|
|
},
|
|
{
|
|
"test": "zod-string-format",
|
|
"prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.",
|
|
"result": [],
|
|
"expected": "john.doe@company.com",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:09.057Z",
|
|
"passed": false,
|
|
"duration": 282,
|
|
"error": {
|
|
"message": "Model returned empty response",
|
|
"code": "UNKNOWN",
|
|
"type": "Error",
|
|
"details": {
|
|
"stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"message": "Model returned empty response",
|
|
"stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "Error",
|
|
"expected": "undefined",
|
|
"actual": "undefined"
|
|
}
|
|
},
|
|
"reason": "Model returned empty response"
|
|
},
|
|
{
|
|
"test": "zod-number-format",
|
|
"prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.",
|
|
"result": [],
|
|
"expected": "25",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:09.356Z",
|
|
"passed": false,
|
|
"duration": 297,
|
|
"error": {
|
|
"message": "Model returned empty response",
|
|
"code": "UNKNOWN",
|
|
"type": "Error",
|
|
"details": {
|
|
"stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:275:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"message": "Model returned empty response"
|
|
}
|
|
},
|
|
"reason": "Model returned empty response"
|
|
},
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:31.573Z",
|
|
"passed": false,
|
|
"duration": 1074,
|
|
"error": {
|
|
"message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'",
|
|
"actual": "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}",
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'"
|
|
},
|
|
{
|
|
"test": "json-schema-object-format",
|
|
"prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:33.577Z",
|
|
"passed": false,
|
|
"duration": 1999,
|
|
"error": {
|
|
"message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'",
|
|
"actual": "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": {\n \"street\": \"123 Main St\",\n \"city\": \"New York\",\n \"country\": \"US\",\n \"zipcode\": \"10001\"\n },\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": \"enabled\",\n \"language\": \"English\"\n }\n}\n```",
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\u001b[7m{\u001b[27m\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zip\u001b[7mC\u001b[27mode\":\"10001\"\u001b[7m},\u001b[27m\"preferences\":\u001b[7m{\u001b[27m\"theme\":\"light\",\"notifications\":\u001b[7mtrue,\"language\":\"en\"}}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"email\":\u001b[7m \u001b[27m\"jane.smith@company.com\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"designer\"],\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"address\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"street\":\u001b[7m \u001b[27m\"123 Main St\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"city\":\u001b[7m \u001b[27m\"New York\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"country\":\u001b[7m \u001b[27m\"US\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"zip\u001b[7mc\u001b[27mode\":\u001b[7m \u001b[27m\"10001\"\u001b[39m\n\u001b[31m+ },\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"preferences\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"theme\":\u001b[7m \u001b[27m\"light\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"notifications\":\u001b[7m \"enabled\",\u001b[27m\u001b[39m\n\u001b[31m+ \"language\": \"English\"\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'"
|
|
},
|
|
{
|
|
"test": "zod-string-format",
|
|
"prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.",
|
|
"result": [],
|
|
"expected": "john.doe@company.com",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:33.906Z",
|
|
"passed": false,
|
|
"duration": 324,
|
|
"error": {
|
|
"message": "Model returned empty response",
|
|
"code": "UNKNOWN",
|
|
"type": "Error",
|
|
"details": {
|
|
"stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"message": "Model returned empty response",
|
|
"stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "Error",
|
|
"expected": "undefined",
|
|
"actual": "undefined"
|
|
}
|
|
},
|
|
"reason": "Model returned empty response"
|
|
},
|
|
{
|
|
"test": "zod-number-format",
|
|
"prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.",
|
|
"result": [],
|
|
"expected": "25",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:34.272Z",
|
|
"passed": false,
|
|
"duration": 363,
|
|
"error": {
|
|
"message": "Model returned empty response",
|
|
"code": "UNKNOWN",
|
|
"type": "Error",
|
|
"details": {
|
|
"stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:275:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"message": "Model returned empty response",
|
|
"stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:275:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "Error",
|
|
"expected": "undefined",
|
|
"actual": "undefined"
|
|
}
|
|
},
|
|
"reason": "Model returned empty response"
|
|
},
|
|
{
|
|
"test": "zod-array-format",
|
|
"prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.",
|
|
"result": [],
|
|
"expected": "[\"JavaScript\",\"Python\",\"Java\"]",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:34.613Z",
|
|
"passed": false,
|
|
"duration": 339,
|
|
"error": {
|
|
"message": "Model returned empty response",
|
|
"code": "UNKNOWN",
|
|
"type": "Error",
|
|
"details": {
|
|
"stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:291:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"message": "Model returned empty response",
|
|
"stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:291:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "Error",
|
|
"expected": "undefined",
|
|
"actual": "undefined"
|
|
}
|
|
},
|
|
"reason": "Model returned empty response"
|
|
},
|
|
{
|
|
"test": "invalid-format",
|
|
"prompt": "Generate a random number.",
|
|
"result": [],
|
|
"expected": "Invalid format option",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:31:35.189Z",
|
|
"passed": false,
|
|
"duration": 574,
|
|
"error": {
|
|
"message": "expected '847293' to deeply equal 'Invalid format option'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '847293' to deeply equal 'Invalid format option'",
|
|
"actual": "847293",
|
|
"expected": "Invalid format option",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '847293' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:307:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '847293' to deeply equal 'Invalid format option'"
|
|
},
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [
|
|
"{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}"
|
|
],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:32:12.404Z",
|
|
"passed": true,
|
|
"duration": 1098
|
|
},
|
|
{
|
|
"test": "json-schema-object-format",
|
|
"prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:32:19.059Z",
|
|
"passed": false,
|
|
"duration": 6653,
|
|
"error": {
|
|
"message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'",
|
|
"actual": "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": {\n \"street\": \"123 Main St\",\n \"city\": \"New York\",\n \"country\": \"US\",\n \"postal_code\": \"10001\"\n },\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": true,\n \"language\": \"English\"\n }\n}\n```",
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:254:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\u001b[7m{\u001b[27m\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\u001b[7m\"zipC\u001b[27mode\":\"10001\"\u001b[7m},\u001b[27m\"preferences\":\u001b[7m{\u001b[27m\"theme\":\"light\",\"notifications\":true,\"language\":\u001b[7m\"en\"}}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"email\":\u001b[7m \u001b[27m\"jane.smith@company.com\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"designer\"],\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"address\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"street\":\u001b[7m \u001b[27m\"123 Main St\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"city\":\u001b[7m \u001b[27m\"New York\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"country\":\u001b[7m \u001b[27m\"US\",\u001b[39m\n\u001b[31m+ \u001b[7m \"postal_c\u001b[27mode\":\u001b[7m \u001b[27m\"10001\"\u001b[39m\n\u001b[31m+ },\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"preferences\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"theme\":\u001b[7m \u001b[27m\"light\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"notifications\":\u001b[7m \u001b[27mtrue,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"language\":\u001b[7m \"English\"\u001b[27m\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:254:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'"
|
|
},
|
|
{
|
|
"test": "zod-string-format",
|
|
"prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.",
|
|
"result": [],
|
|
"expected": "john.doe@company.com",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:32:20.861Z",
|
|
"passed": false,
|
|
"duration": 1794,
|
|
"error": {
|
|
"message": "expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'",
|
|
"actual": "contact@businessdomain.com",
|
|
"expected": "john.doe@company.com",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:270:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "Expected: \u001b[32m\"\u001b[7mjohn.doe@company\u001b[27m.com\"\u001b[39m\nReceived: \u001b[31m\"\u001b[7mcontact@businessdomain\u001b[27m.com\"\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:270:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'"
|
|
},
|
|
{
|
|
"test": "zod-number-format",
|
|
"prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.",
|
|
"result": [],
|
|
"expected": "25",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:32:23.263Z",
|
|
"passed": false,
|
|
"duration": 2399,
|
|
"error": {
|
|
"message": "expected '39' to deeply equal '25'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '39' to deeply equal '25'",
|
|
"actual": "39",
|
|
"expected": "25",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '39' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:295:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "Expected: \u001b[32m\"25\"\u001b[39m\nReceived: \u001b[31m\"39\"\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '39' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:295:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '39' to deeply equal '25'"
|
|
},
|
|
{
|
|
"test": "zod-array-format",
|
|
"prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.",
|
|
"result": [],
|
|
"expected": "[\"JavaScript\",\"Python\",\"Java\"]",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:32:23.958Z",
|
|
"passed": false,
|
|
"duration": 693,
|
|
"error": {
|
|
"message": "expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'",
|
|
"actual": "[\"Python\",\"JavaScript\",\"Java\"]",
|
|
"expected": "[\"JavaScript\",\"Python\",\"Java\"]",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"stackStr": "AssertionError: expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:321:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11",
|
|
"nameStr": "AssertionError",
|
|
"diff": "Expected: \u001b[32m\"[\"JavaScript\u001b[7m\",\"Python\u001b[27m\",\"Java\"]\"\u001b[39m\nReceived: \u001b[31m\"[\"\u001b[7mPython\",\"\u001b[27mJavaScript\",\"Java\"]\"\u001b[39m",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:321:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'"
|
|
},
|
|
{
|
|
"test": "invalid-format",
|
|
"prompt": "Generate a random number.",
|
|
"result": [],
|
|
"expected": "Invalid format option",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:32:24.787Z",
|
|
"passed": false,
|
|
"duration": 826,
|
|
"error": {
|
|
"message": "expected '786984' to deeply equal 'Invalid format option'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '786984' to deeply equal 'Invalid format option'",
|
|
"actual": "786984",
|
|
"expected": "Invalid format option",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '786984' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:350:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '786984' to deeply equal 'Invalid format option'"
|
|
},
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [
|
|
"```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```"
|
|
],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:33:05.518Z",
|
|
"passed": true,
|
|
"duration": 1027
|
|
},
|
|
{
|
|
"test": "json-schema-object-format",
|
|
"prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.",
|
|
"result": [],
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:33:08.138Z",
|
|
"passed": false,
|
|
"duration": 2618,
|
|
"error": {
|
|
"message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'",
|
|
"code": "UNKNOWN",
|
|
"type": "AssertionError",
|
|
"details": {
|
|
"message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'",
|
|
"actual": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postal_code\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}",
|
|
"expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}",
|
|
"showDiff": true,
|
|
"operator": "strictEqual",
|
|
"name": "AssertionError",
|
|
"ok": false,
|
|
"stack": "AssertionError: expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:184:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:271:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11"
|
|
}
|
|
},
|
|
"reason": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'"
|
|
},
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [
|
|
"{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}"
|
|
],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:33:41.869Z",
|
|
"passed": true,
|
|
"duration": 2353
|
|
},
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [
|
|
"{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}"
|
|
],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:34:16.181Z",
|
|
"passed": true,
|
|
"duration": 1098
|
|
},
|
|
{
|
|
"test": "json-schema-file-format",
|
|
"prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.",
|
|
"result": [
|
|
"```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```"
|
|
],
|
|
"expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}",
|
|
"model": "gpt-4o",
|
|
"router": "openai",
|
|
"timestamp": "2025-04-01T22:34:28.660Z",
|
|
"passed": true,
|
|
"duration": 995
|
|
}
|
|
] |