From 20cb6861af11a80af4d0b761afcda46fd42e83f3 Mon Sep 17 00:00:00 2001 From: babayaga Date: Wed, 2 Apr 2025 00:34:47 +0200 Subject: [PATCH] tests:response_format --- .../kbot/dist-in/commands/run-completion.js | 3 +- packages/kbot/dist-in/models/index.js | 6 +- packages/kbot/logs/params.json | 4 +- packages/kbot/package.json | 1 + packages/kbot/src/models/index.ts | 6 +- packages/kbot/tests/unit/basic-report.md | 167 +- packages/kbot/tests/unit/basic.json | 1072 +------ packages/kbot/tests/unit/basic.test.ts | 268 +- packages/kbot/tests/unit/commons.ts | 3 +- packages/kbot/tests/unit/format-report.md | 696 ++-- packages/kbot/tests/unit/format.json | 2794 +++++------------ packages/kbot/tests/unit/format.test.ts | 801 +++-- packages/kbot/tests/unit/language-report.md | 12 +- packages/kbot/tests/unit/language.json | 36 + packages/kbot/tests/unit/math-report.md | 12 +- packages/kbot/tests/unit/math.json | 36 + packages/kbot/tests/unit/test-schema.json | 113 + 17 files changed, 1865 insertions(+), 4165 deletions(-) create mode 100644 packages/kbot/tests/unit/test-schema.json diff --git a/packages/kbot/dist-in/commands/run-completion.js b/packages/kbot/dist-in/commands/run-completion.js index 7f285bda..5af94839 100644 --- a/packages/kbot/dist-in/commands/run-completion.js +++ b/packages/kbot/dist-in/commands/run-completion.js @@ -35,6 +35,7 @@ export const runCompletion = async (client, params, options) => { logger.info('Dry run - skipping API call'); return false; } + // await client.beta.chat.completions.parse const completion = await client.chat.completions.create({ model: options.model, messages: params.messages, @@ -44,4 +45,4 @@ export const runCompletion = async (client, params, options) => { result = await onCompletion(result, options); return result; }; -//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVuLWNvbXBsZXRpb24uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvY29tbWFuZHMvcnVuLWNvbXBsZXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLFFBQVEsQ0FBQTtBQUMvQixPQUFPLEVBQUUsY0FBYyxFQUFFLE1BQU0saUJBQWlCLENBQUE7QUFDaEQsT0FBTyxLQUFLLElBQUksTUFBTSxXQUFXLENBQUE7QUFDakMsT0FBTyxFQUFFLElBQUksSUFBSSxLQUFLLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQTtBQUNsRCxPQUFPLEVBQUUsT0FBTyxFQUFFLE1BQU0sbUJBQW1CLENBQUE7QUFJM0MsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLGFBQWEsQ0FBQTtBQUNwQyxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sb0JBQW9CLENBQUE7QUFDakQsT0FBTyxFQUFFLFlBQVksRUFBVSxNQUFNLGVBQWUsQ0FBQTtBQUNwRCxPQUFPLEVBQUUsU0FBUyxFQUFFLE1BQU0saUJBQWlCLENBQUE7QUFHM0MsTUFBTSxDQUFDLE1BQU0sWUFBWSxHQUFHLEtBQUssRUFBRSxTQUFjLEVBQUUsRUFBRSxPQUFrQixFQUFFLEVBQUU7SUFDekUsTUFBTSxHQUFHLFlBQVksQ0FBQyxNQUFNLEVBQUUsT0FBTyxDQUFDLE9BQW1CLElBQUksRUFBRSxDQUFDLENBQUE7SUFDaEUsTUFBTSxJQUFJLEdBQUcsU0FBUyxDQUFDLE9BQU8sQ0FBQyxDQUFBO0lBQy9CLElBQUksT0FBTyxDQUFDLEdBQUcsRUFBRSxDQUFDO1FBQ2hCLE1BQU0sT0FBTyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxHQUFHLEVBQUUsS0FBSyxFQUFFO1lBQ3ZELEdBQUcsSUFBSTtZQUNQLEtBQUssRUFBRSxJQUFJLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxJQUFJO1lBQ3JDLE1BQU0sRUFBRSxPQUFPLENBQUMsTUFBTTtTQUN2QixDQUFDLENBQUMsQ0FBQTtRQUNILEtBQUssQ0FBQyxPQUFPLEVBQUUsTUFBTSxDQUFDLENBQUE7UUFDdEIsTUFBTSxDQUFDLEtBQUssQ0FBQyw4QkFBOEIsT0FBTyxNQUFNLE9BQU8sQ0FBQyxHQUFHLEVBQUUsQ0FBQyxDQUFBO0lBQ3hFLENBQUM7U0FBTSxDQUFDO1FBQ04sTUFBTSxDQUFDLEdBQUcsQ0FBQyxjQUFjLENBQUM7WUFDeEIsS0FBSyxFQUFFLEtBQUs7U0FDYixDQUFDLENBQUMsQ0FBQTtRQUNILE1BQU0sT0FBTyxHQUFXLE1BQU0sQ0FBQyxNQUFNLENBQVcsQ0FBQztRQUNqRCxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQTtJQUMvQixDQUFDO0lBQ0QsWUFBWSxDQUFDLE9BQU8sQ0FBQyxDQUFBO0lBQ3JCLGtCQUFrQjtJQUNsQixPQUFPLE1BQU0sQ0FBQTtBQUNmLENBQUMsQ0FBQTtBQUVELE1BQU0sQ0FBQyxNQUFNLGFBQWEsR0FBRyxLQUFLLEVBQUUsTUFBYyxFQUFFLE1BQVcsRUFBRSxPQUFrQixFQUFFLEVBQUU7SUFDckYsSUFBSSxPQUFPLENBQUMsR0FBRyxFQUFFLENBQUM7UUFDaEIsTUFBTSxDQUFDLElBQUksQ0FBQyw2QkFBNkIsQ0FBQyxDQUFBO1FBQzFDLE9BQU8sS0FBSyxDQUFBO0lBQ2QsQ0FBQztJQUNELE1BQU0sVUFBVSxHQUFHLE1BQU0sTUFBTSxDQUFDLElBQUksQ0FBQyxXQUFXLENBQUMsTUFBTSxDQUFDO1FBQ3RELEtBQUssRUFBRSxPQUFPLENBQUMsS0FBSztRQUNwQixRQUFRLEVBQUUsTUFBTSxDQUFDLFFBQVE7UUFDekIsZUFBZSxFQUFFLE9BQU8sQ0FBQyxNQUFhO0tBQ3ZDLENBQUMsQ0FBQTtJQUNGLElBQUksTUFBTSxHQUFHLFVBQVUsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQTtJQUNsRCxNQUFNLEdBQUcsTUFBTSxZQUFZLENBQUMsTUFBTSxFQUFFLE9BQU8sQ0FBQyxDQUFBO0lBQzVDLE9BQU8sTUFBTSxDQUFBO0FBQ2YsQ0FBQyxDQUFBIn0= \ No newline at end of file +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVuLWNvbXBsZXRpb24uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvY29tbWFuZHMvcnVuLWNvbXBsZXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLFFBQVEsQ0FBQTtBQUMvQixPQUFPLEVBQUUsY0FBYyxFQUFFLE1BQU0saUJBQWlCLENBQUE7QUFDaEQsT0FBTyxLQUFLLElBQUksTUFBTSxXQUFXLENBQUE7QUFDakMsT0FBTyxFQUFFLElBQUksSUFBSSxLQUFLLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQTtBQUNsRCxPQUFPLEVBQUUsT0FBTyxFQUFFLE1BQU0sbUJBQW1CLENBQUE7QUFJM0MsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLGFBQWEsQ0FBQTtBQUNwQyxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sb0JBQW9CLENBQUE7QUFDakQsT0FBTyxFQUFFLFlBQVksRUFBVSxNQUFNLGVBQWUsQ0FBQTtBQUNwRCxPQUFPLEVBQUUsU0FBUyxFQUFFLE1BQU0saUJBQWlCLENBQUE7QUFHM0MsTUFBTSxDQUFDLE1BQU0sWUFBWSxHQUFHLEtBQUssRUFBRSxTQUFjLEVBQUUsRUFBRSxPQUFrQixFQUFFLEVBQUU7SUFDekUsTUFBTSxHQUFHLFlBQVksQ0FBQyxNQUFNLEVBQUUsT0FBTyxDQUFDLE9BQW1CLElBQUksRUFBRSxDQUFDLENBQUE7SUFDaEUsTUFBTSxJQUFJLEdBQUcsU0FBUyxDQUFDLE9BQU8sQ0FBQyxDQUFBO0lBQy9CLElBQUksT0FBTyxDQUFDLEdBQUcsRUFBRSxDQUFDO1FBQ2hCLE1BQU0sT0FBTyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxHQUFHLEVBQUUsS0FBSyxFQUFFO1lBQ3ZELEdBQUcsSUFBSTtZQUNQLEtBQUssRUFBRSxJQUFJLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxJQUFJO1lBQ3JDLE1BQU0sRUFBRSxPQUFPLENBQUMsTUFBTTtTQUN2QixDQUFDLENBQUMsQ0FBQTtRQUNILEtBQUssQ0FBQyxPQUFPLEVBQUUsTUFBTSxDQUFDLENBQUE7UUFDdEIsTUFBTSxDQUFDLEtBQUssQ0FBQyw4QkFBOEIsT0FBTyxNQUFNLE9BQU8sQ0FBQyxHQUFHLEVBQUUsQ0FBQyxDQUFBO0lBQ3hFLENBQUM7U0FBTSxDQUFDO1FBQ04sTUFBTSxDQUFDLEdBQUcsQ0FBQyxjQUFjLENBQUM7WUFDeEIsS0FBSyxFQUFFLEtBQUs7U0FDYixDQUFDLENBQUMsQ0FBQTtRQUNILE1BQU0sT0FBTyxHQUFXLE1BQU0sQ0FBQyxNQUFNLENBQVcsQ0FBQztRQUNqRCxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQTtJQUMvQixDQUFDO0lBQ0QsWUFBWSxDQUFDLE9BQU8sQ0FBQyxDQUFBO0lBQ3JCLGtCQUFrQjtJQUNsQixPQUFPLE1BQU0sQ0FBQTtBQUNmLENBQUMsQ0FBQTtBQUVELE1BQU0sQ0FBQyxNQUFNLGFBQWEsR0FBRyxLQUFLLEVBQUUsTUFBYyxFQUFFLE1BQVcsRUFBRSxPQUFrQixFQUFFLEVBQUU7SUFDckYsSUFBSSxPQUFPLENBQUMsR0FBRyxFQUFFLENBQUM7UUFDaEIsTUFBTSxDQUFDLElBQUksQ0FBQyw2QkFBNkIsQ0FBQyxDQUFBO1FBQzFDLE9BQU8sS0FBSyxDQUFBO0lBQ2QsQ0FBQztJQUNELDJDQUEyQztJQUMzQyxNQUFNLFVBQVUsR0FBRyxNQUFNLE1BQU0sQ0FBQyxJQUFJLENBQUMsV0FBVyxDQUFDLE1BQU0sQ0FBQztRQUN0RCxLQUFLLEVBQUUsT0FBTyxDQUFDLEtBQUs7UUFDcEIsUUFBUSxFQUFFLE1BQU0sQ0FBQyxRQUFRO1FBQ3pCLGVBQWUsRUFBRSxPQUFPLENBQUMsTUFBYTtLQUN2QyxDQUFDLENBQUE7SUFDRixJQUFJLE1BQU0sR0FBRyxVQUFVLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUE7SUFDbEQsTUFBTSxHQUFHLE1BQU0sWUFBWSxDQUFDLE1BQU0sRUFBRSxPQUFPLENBQUMsQ0FBQTtJQUM1QyxPQUFPLE1BQU0sQ0FBQTtBQUNmLENBQUMsQ0FBQSJ9 \ No newline at end of file diff --git a/packages/kbot/dist-in/models/index.js b/packages/kbot/dist-in/models/index.js index b1699a32..c8edaa09 100644 --- a/packages/kbot/dist-in/models/index.js +++ b/packages/kbot/dist-in/models/index.js @@ -47,7 +47,7 @@ export const models = () => { if (!exists(openRouterPath)) { fetchOpenRouterModels(); } - if (exists(openRouterPath)) { + else { const modelData = read(openRouterPath, 'json'); models.push(chalk.magenta.bold('\n OpenRouter models:\n')); models.push(...listOpenRouterModelsAsStrings(modelData.models)); @@ -73,7 +73,7 @@ export const all = () => { if (!exists(openRouterPath)) { fetchOpenRouterModels(); } - if (exists(openRouterPath)) { + else { const modelData = read(openRouterPath, 'json'); models = models.concat(modelData.models); } @@ -89,4 +89,4 @@ export const all = () => { } return models; }; -//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvbW9kZWxzL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxNQUFNLE9BQU8sQ0FBQTtBQUN6QixPQUFPLEtBQUssSUFBSSxNQUFNLFdBQVcsQ0FBQTtBQUNqQyxPQUFPLEVBQUUsSUFBSSxJQUFJLElBQUksRUFBRSxNQUFNLG1CQUFtQixDQUFBO0FBQ2hELE9BQU8sRUFBRSxJQUFJLElBQUksTUFBTSxFQUFFLE1BQU0scUJBQXFCLENBQUE7QUFFcEQsT0FBTyxFQUFFLE1BQU0sRUFBRSxXQUFXLEVBQUUsTUFBTSxhQUFhLENBQUE7QUFDakQsT0FBTyxFQUFFLFVBQVUsSUFBSSxxQkFBcUIsRUFBMEMscUJBQXFCLEVBQUUsbUJBQW1CLElBQUksNkJBQTZCLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQTtBQUMxTCxPQUFPLEVBQUUsVUFBVSxJQUFJLGlCQUFpQixFQUFzQyxtQkFBbUIsSUFBSSx5QkFBeUIsRUFBRSxNQUFNLGFBQWEsQ0FBQTtBQUVuSixPQUFPLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQTtBQUN2RCxPQUFPLEVBQUUsY0FBYyxFQUFFLE1BQU0sbUJBQW1CLENBQUE7QUFFbEQsT0FBTyxFQUFFLE1BQU0sSUFBSSxZQUFZLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQTtBQUMxRCxPQUFPLEVBQUUsTUFBTSxJQUFJLGdCQUFnQixFQUFFLE1BQU0sdUJBQXVCLENBQUE7QUFFbEUsTUFBTSxDQUFDLE1BQU0sV0FBVyxHQUFHLEdBQUcsRUFBRTtJQUU5QixJQUFJLFNBQVMsR0FBRyxnQkFBZ0IsQ0FBQTtJQUNoQyxJQUFJLFVBQVUsR0FBRyxZQUFZLENBQUE7SUFDN0IsSUFBSSxlQUFlLEdBQUc7UUFDcEI7WUFDRSxJQUFJLEVBQUUsZUFBZTtZQUNyQixNQUFNLEVBQUUsZUFBZTtTQUN4QjtRQUNEO1lBQ0UsSUFBSSxFQUFFLG1CQUFtQjtZQUN6QixNQUFNLEVBQUUsbUJBQW1CO1NBQzVCO0tBQ0YsQ0FBQTtJQUVELE1BQU0sZ0JBQWdCLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxXQUFXLEVBQUUsRUFBRSxhQUFhLENBQUMsQ0FBQTtJQUNuRSxJQUFJLE1BQU0sQ0FBQyxnQkFBZ0IsQ0FBQyxFQUFFLENBQUM7UUFDN0IsVUFBVSxHQUFHLElBQUksQ0FBQyxnQkFBZ0IsRUFBRSxNQUFNLENBQVEsQ0FBQTtJQUNwRCxDQUFDO0lBRUQsTUFBTSxnQkFBZ0IsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLFdBQVcsRUFBRSxFQUFFLGlCQUFpQixDQUFDLENBQUE7SUFDdkUsSUFBSSxNQUFNLENBQUMsZ0JBQWdCLENBQUMsRUFBRSxDQUFDO1FBQzdCLFNBQVMsR0FBRyxJQUFJLENBQUMsZ0JBQWdCLEVBQUUsTUFBTSxDQUFRLENBQUE7SUFDbkQsQ0FBQztJQUNELE1BQU0sTUFBTSxHQUFhLEVBQUUsQ0FBQTtJQUMzQixNQUFNLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLHlCQUF5QixDQUFDLENBQUMsQ0FBQTtJQUMxRCxNQUFNLENBQUMsSUFBSSxDQUFDLEdBQUcsNkJBQTZCLENBQUMsU0FBZ0IsQ0FBQyxDQUFDLENBQUE7SUFFL0QsTUFBTSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxxQkFBcUIsQ0FBQyxDQUFDLENBQUE7SUFDdEQsTUFBTSxDQUFDLElBQUksQ0FBQyxHQUFHLHlCQUF5QixDQUFDLFVBQWlCLENBQUMsQ0FBQyxDQUFBO0lBQzVELE1BQU0sQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLENBQUE7SUFFdEIsTUFBTSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyx1QkFBdUIsQ0FBQyxDQUFDLENBQUE7SUFDeEQsTUFBTSxDQUFDLElBQUksQ0FBQyxHQUFHLHlCQUF5QixDQUFDLGVBQXNCLENBQUMsQ0FBQyxDQUFBO0lBQ2pFLE1BQU0sQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLENBQUE7SUFDdEIsT0FBTyxNQUFNLENBQUE7QUFDZixDQUFDLENBQUE7QUFFRCxNQUFNLENBQUMsTUFBTSxNQUFNLEdBQUcsR0FBRyxFQUFFO0lBQ3pCLE1BQU0sTUFBTSxHQUFhLEVBQUUsQ0FBQTtJQUMzQixNQUFNLGNBQWMsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLHFCQUFxQixDQUFDLENBQUE7SUFDMUQsSUFBSSxDQUFDLE1BQU0sQ0FBQyxjQUFjLENBQUMsRUFBRSxDQUFDO1FBQzVCLHFCQUFxQixFQUFFLENBQUE7SUFDekIsQ0FBQztJQUNELElBQUksTUFBTSxDQUFDLGNBQWMsQ0FBQyxFQUFFLENBQUM7UUFDM0IsTUFBTSxTQUFTLEdBQTJCLElBQUksQ0FBQyxjQUFjLEVBQUUsTUFBTSxDQUEyQixDQUFBO1FBQ2hHLE1BQU0sQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMseUJBQXlCLENBQUMsQ0FBQyxDQUFBO1FBQzFELE1BQU0sQ0FBQyxJQUFJLENBQUMsR0FBRyw2QkFBNkIsQ0FBQyxTQUFTLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQTtJQUNqRSxDQUFDO0lBQ0QsTUFBTSxDQUFDLEtBQUssQ0FBQywyQkFBMkIsRUFBRSxpQkFBaUIsQ0FBQyxDQUFBO0lBRTVELE1BQU0sVUFBVSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsaUJBQWlCLENBQUMsQ0FBQTtJQUNsRCxNQUFNLE1BQU0sR0FBRyxjQUFjLEVBQVMsQ0FBQTtJQUN0QyxJQUFJLENBQUMsTUFBTSxDQUFDLFVBQVUsQ0FBQyxJQUFJLE1BQU0sRUFBRSxNQUFNLEVBQUUsR0FBRyxFQUFFLENBQUM7UUFDL0MsaUJBQWlCLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQTtJQUN0QyxDQUFDO0lBRUQsSUFBSSxNQUFNLENBQUMsVUFBVSxDQUFDLEVBQUUsQ0FBQztRQUN2QixNQUFNLFNBQVMsR0FBdUIsSUFBSSxDQUFDLFVBQVUsRUFBRSxNQUFNLENBQXVCLENBQUE7UUFDcEYsTUFBTSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxxQkFBcUIsQ0FBQyxDQUFDLENBQUE7UUFDdEQsTUFBTSxDQUFDLElBQUksQ0FBQyxHQUFHLHlCQUF5QixDQUFDLFNBQVMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFBO0lBQzdELENBQUM7SUFDRCxNQUFNLENBQUMsS0FBSyxDQUFDLHVCQUF1QixFQUFFLGlCQUFpQixDQUFDLENBQUE7SUFDeEQsTUFBTSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQTtJQUN0QixPQUFPLE1BQU0sQ0FBQTtBQUNmLENBQUMsQ0FBQTtBQUVELE1BQU0sQ0FBQyxNQUFNLEdBQUcsR0FBRyxHQUFHLEVBQUU7SUFDdEIsSUFBSSxNQUFNLEdBQVUsRUFBRSxDQUFBO0lBQ3RCLE1BQU0sY0FBYyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMscUJBQXFCLENBQUMsQ0FBQTtJQUMxRCxJQUFJLENBQUMsTUFBTSxDQUFDLGNBQWMsQ0FBQyxFQUFFLENBQUM7UUFDNUIscUJBQXFCLEVBQUUsQ0FBQTtJQUN6QixDQUFDO0lBQ0QsSUFBSSxNQUFNLENBQUMsY0FBYyxDQUFDLEVBQUUsQ0FBQztRQUMzQixNQUFNLFNBQVMsR0FBMkIsSUFBSSxDQUFDLGNBQWMsRUFBRSxNQUFNLENBQTJCLENBQUE7UUFDaEcsTUFBTSxHQUFHLE1BQU0sQ0FBQyxNQUFNLENBQUMsU0FBUyxDQUFDLE1BQU0sQ0FBQyxDQUFBO0lBQzFDLENBQUM7SUFDRCxNQUFNLFVBQVUsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLGlCQUFpQixDQUFDLENBQUE7SUFDbEQsTUFBTSxNQUFNLEdBQUcsY0FBYyxFQUFTLENBQUE7SUFDdEMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxVQUFVLENBQUMsSUFBSSxNQUFNLEVBQUUsTUFBTSxFQUFFLEdBQUcsRUFBRSxDQUFDO1FBQy9DLGlCQUFpQixDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUE7SUFDdEMsQ0FBQztJQUVELElBQUksTUFBTSxDQUFDLFVBQVUsQ0FBQyxFQUFFLENBQUM7UUFDdkIsTUFBTSxTQUFTLEdBQXVCLElBQUksQ0FBQyxVQUFVLEVBQUUsTUFBTSxDQUF1QixDQUFBO1FBQ3BGLE1BQU0sQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMscUJBQXFCLENBQUMsQ0FBQyxDQUFBO1FBQ3RELE1BQU0sR0FBRyxNQUFNLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxNQUFNLENBQUMsQ0FBQTtJQUMxQyxDQUFDO0lBQ0QsT0FBTyxNQUFNLENBQUE7QUFDZixDQUFDLENBQUEifQ== \ No newline at end of file +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvbW9kZWxzL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxNQUFNLE9BQU8sQ0FBQTtBQUN6QixPQUFPLEtBQUssSUFBSSxNQUFNLFdBQVcsQ0FBQTtBQUNqQyxPQUFPLEVBQUUsSUFBSSxJQUFJLElBQUksRUFBRSxNQUFNLG1CQUFtQixDQUFBO0FBQ2hELE9BQU8sRUFBRSxJQUFJLElBQUksTUFBTSxFQUFFLE1BQU0scUJBQXFCLENBQUE7QUFFcEQsT0FBTyxFQUFFLE1BQU0sRUFBRSxXQUFXLEVBQUUsTUFBTSxhQUFhLENBQUE7QUFDakQsT0FBTyxFQUFFLFVBQVUsSUFBSSxxQkFBcUIsRUFBMEMscUJBQXFCLEVBQUUsbUJBQW1CLElBQUksNkJBQTZCLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQTtBQUMxTCxPQUFPLEVBQUUsVUFBVSxJQUFJLGlCQUFpQixFQUFzQyxtQkFBbUIsSUFBSSx5QkFBeUIsRUFBRSxNQUFNLGFBQWEsQ0FBQTtBQUVuSixPQUFPLEVBQUUsaUJBQWlCLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQTtBQUN2RCxPQUFPLEVBQUUsY0FBYyxFQUFFLE1BQU0sbUJBQW1CLENBQUE7QUFFbEQsT0FBTyxFQUFFLE1BQU0sSUFBSSxZQUFZLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQTtBQUMxRCxPQUFPLEVBQUUsTUFBTSxJQUFJLGdCQUFnQixFQUFFLE1BQU0sdUJBQXVCLENBQUE7QUFFbEUsTUFBTSxDQUFDLE1BQU0sV0FBVyxHQUFHLEdBQUcsRUFBRTtJQUU5QixJQUFJLFNBQVMsR0FBRyxnQkFBZ0IsQ0FBQTtJQUNoQyxJQUFJLFVBQVUsR0FBRyxZQUFZLENBQUE7SUFDN0IsSUFBSSxlQUFlLEdBQUc7UUFDcEI7WUFDRSxJQUFJLEVBQUUsZUFBZTtZQUNyQixNQUFNLEVBQUUsZUFBZTtTQUN4QjtRQUNEO1lBQ0UsSUFBSSxFQUFFLG1CQUFtQjtZQUN6QixNQUFNLEVBQUUsbUJBQW1CO1NBQzVCO0tBQ0YsQ0FBQTtJQUVELE1BQU0sZ0JBQWdCLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxXQUFXLEVBQUUsRUFBRSxhQUFhLENBQUMsQ0FBQTtJQUNuRSxJQUFJLE1BQU0sQ0FBQyxnQkFBZ0IsQ0FBQyxFQUFFLENBQUM7UUFDN0IsVUFBVSxHQUFHLElBQUksQ0FBQyxnQkFBZ0IsRUFBRSxNQUFNLENBQVEsQ0FBQTtJQUNwRCxDQUFDO0lBRUQsTUFBTSxnQkFBZ0IsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLFdBQVcsRUFBRSxFQUFFLGlCQUFpQixDQUFDLENBQUE7SUFDdkUsSUFBSSxNQUFNLENBQUMsZ0JBQWdCLENBQUMsRUFBRSxDQUFDO1FBQzdCLFNBQVMsR0FBRyxJQUFJLENBQUMsZ0JBQWdCLEVBQUUsTUFBTSxDQUFRLENBQUE7SUFDbkQsQ0FBQztJQUNELE1BQU0sTUFBTSxHQUFhLEVBQUUsQ0FBQTtJQUMzQixNQUFNLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLHlCQUF5QixDQUFDLENBQUMsQ0FBQTtJQUMxRCxNQUFNLENBQUMsSUFBSSxDQUFDLEdBQUcsNkJBQTZCLENBQUMsU0FBZ0IsQ0FBQyxDQUFDLENBQUE7SUFFL0QsTUFBTSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxxQkFBcUIsQ0FBQyxDQUFDLENBQUE7SUFDdEQsTUFBTSxDQUFDLElBQUksQ0FBQyxHQUFHLHlCQUF5QixDQUFDLFVBQWlCLENBQUMsQ0FBQyxDQUFBO0lBQzVELE1BQU0sQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLENBQUE7SUFFdEIsTUFBTSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyx1QkFBdUIsQ0FBQyxDQUFDLENBQUE7SUFDeEQsTUFBTSxDQUFDLElBQUksQ0FBQyxHQUFHLHlCQUF5QixDQUFDLGVBQXNCLENBQUMsQ0FBQyxDQUFBO0lBQ2pFLE1BQU0sQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLENBQUE7SUFDdEIsT0FBTyxNQUFNLENBQUE7QUFDZixDQUFDLENBQUE7QUFFRCxNQUFNLENBQUMsTUFBTSxNQUFNLEdBQUcsR0FBRyxFQUFFO0lBQ3pCLE1BQU0sTUFBTSxHQUFhLEVBQUUsQ0FBQTtJQUMzQixNQUFNLGNBQWMsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLHFCQUFxQixDQUFDLENBQUE7SUFDMUQsSUFBSSxDQUFDLE1BQU0sQ0FBQyxjQUFjLENBQUMsRUFBRSxDQUFDO1FBQzVCLHFCQUFxQixFQUFFLENBQUE7SUFDekIsQ0FBQztTQUFJLENBQUM7UUFDSixNQUFNLFNBQVMsR0FBMkIsSUFBSSxDQUFDLGNBQWMsRUFBRSxNQUFNLENBQTJCLENBQUE7UUFDaEcsTUFBTSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyx5QkFBeUIsQ0FBQyxDQUFDLENBQUE7UUFDMUQsTUFBTSxDQUFDLElBQUksQ0FBQyxHQUFHLDZCQUE2QixDQUFDLFNBQVMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFBO0lBQ2pFLENBQUM7SUFDRCxNQUFNLENBQUMsS0FBSyxDQUFDLDJCQUEyQixFQUFFLGlCQUFpQixDQUFDLENBQUE7SUFFNUQsTUFBTSxVQUFVLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxpQkFBaUIsQ0FBQyxDQUFBO0lBQ2xELE1BQU0sTUFBTSxHQUFHLGNBQWMsRUFBUyxDQUFBO0lBQ3RDLElBQUksQ0FBQyxNQUFNLENBQUMsVUFBVSxDQUFDLElBQUksTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLEVBQUUsQ0FBQztRQUMvQyxpQkFBaUIsQ0FBQyxNQUFNLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFBO0lBQ3RDLENBQUM7SUFFRCxJQUFJLE1BQU0sQ0FBQyxVQUFVLENBQUMsRUFBRSxDQUFDO1FBQ3ZCLE1BQU0sU0FBUyxHQUF1QixJQUFJLENBQUMsVUFBVSxFQUFFLE1BQU0sQ0FBdUIsQ0FBQTtRQUNwRixNQUFNLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLHFCQUFxQixDQUFDLENBQUMsQ0FBQTtRQUN0RCxNQUFNLENBQUMsSUFBSSxDQUFDLEdBQUcseUJBQXlCLENBQUMsU0FBUyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUE7SUFDN0QsQ0FBQztJQUNELE1BQU0sQ0FBQyxLQUFLLENBQUMsdUJBQXVCLEVBQUUsaUJBQWlCLENBQUMsQ0FBQTtJQUN4RCxNQUFNLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxDQUFBO0lBQ3RCLE9BQU8sTUFBTSxDQUFBO0FBQ2YsQ0FBQyxDQUFBO0FBRUQsTUFBTSxDQUFDLE1BQU0sR0FBRyxHQUFHLEdBQUcsRUFBRTtJQUN0QixJQUFJLE1BQU0sR0FBVSxFQUFFLENBQUE7SUFDdEIsTUFBTSxjQUFjLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxxQkFBcUIsQ0FBQyxDQUFBO0lBQzFELElBQUksQ0FBQyxNQUFNLENBQUMsY0FBYyxDQUFDLEVBQUUsQ0FBQztRQUM1QixxQkFBcUIsRUFBRSxDQUFBO0lBQ3pCLENBQUM7U0FBSSxDQUFDO1FBQ0osTUFBTSxTQUFTLEdBQTJCLElBQUksQ0FBQyxjQUFjLEVBQUUsTUFBTSxDQUEyQixDQUFBO1FBQ2hHLE1BQU0sR0FBRyxNQUFNLENBQUMsTUFBTSxDQUFDLFNBQVMsQ0FBQyxNQUFNLENBQUMsQ0FBQTtJQUMxQyxDQUFDO0lBQ0QsTUFBTSxVQUFVLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxpQkFBaUIsQ0FBQyxDQUFBO0lBQ2xELE1BQU0sTUFBTSxHQUFHLGNBQWMsRUFBUyxDQUFBO0lBQ3RDLElBQUksQ0FBQyxNQUFNLENBQUMsVUFBVSxDQUFDLElBQUksTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLEVBQUUsQ0FBQztRQUMvQyxpQkFBaUIsQ0FBQyxNQUFNLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFBO0lBQ3RDLENBQUM7SUFFRCxJQUFJLE1BQU0sQ0FBQyxVQUFVLENBQUMsRUFBRSxDQUFDO1FBQ3ZCLE1BQU0sU0FBUyxHQUF1QixJQUFJLENBQUMsVUFBVSxFQUFFLE1BQU0sQ0FBdUIsQ0FBQTtRQUNwRixNQUFNLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLHFCQUFxQixDQUFDLENBQUMsQ0FBQTtRQUN0RCxNQUFNLEdBQUcsTUFBTSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsTUFBTSxDQUFDLENBQUE7SUFDMUMsQ0FBQztJQUNELE9BQU8sTUFBTSxDQUFBO0FBQ2YsQ0FBQyxDQUFBIn0= \ No newline at end of file diff --git a/packages/kbot/logs/params.json b/packages/kbot/logs/params.json index 80215648..4e9cabb1 100644 --- a/packages/kbot/logs/params.json +++ b/packages/kbot/logs/params.json @@ -1,9 +1,9 @@ { - "model": "anthropic/claude-2.0", + "model": "gpt-4o", "messages": [ { "role": "user", - "content": "translate \"no\" to French. Return only the translated word, no explanation." + "content": "Generate a random number." }, { "role": "user", diff --git a/packages/kbot/package.json b/packages/kbot/package.json index 34475111..286829e5 100644 --- a/packages/kbot/package.json +++ b/packages/kbot/package.json @@ -19,6 +19,7 @@ "test": "vitest run", "test:basic": "vitest run tests/unit/basic.test.ts", "test:math": "vitest run tests/unit/math.test.ts", + "test:format": "vitest run tests/unit/format.test.ts", "test:language": "vitest run tests/unit/language.test.ts", "test2:watch": "vitest", "test2:coverage": "vitest run --coverage", diff --git a/packages/kbot/src/models/index.ts b/packages/kbot/src/models/index.ts index a14ccaf9..cb1000c5 100644 --- a/packages/kbot/src/models/index.ts +++ b/packages/kbot/src/models/index.ts @@ -56,8 +56,7 @@ export const models = () => { const openRouterPath = path.resolve(OPENROUTER_CACHE_PATH) if (!exists(openRouterPath)) { fetchOpenRouterModels() - } - if (exists(openRouterPath)) { + }else{ const modelData: OpenRouterCachedModels = read(openRouterPath, 'json') as OpenRouterCachedModels models.push(chalk.magenta.bold('\n OpenRouter models:\n')) models.push(...listOpenRouterModelsAsStrings(modelData.models)) @@ -85,8 +84,7 @@ export const all = () => { const openRouterPath = path.resolve(OPENROUTER_CACHE_PATH) if (!exists(openRouterPath)) { fetchOpenRouterModels() - } - if (exists(openRouterPath)) { + }else{ const modelData: OpenRouterCachedModels = read(openRouterPath, 'json') as OpenRouterCachedModels models = models.concat(modelData.models) } diff --git a/packages/kbot/tests/unit/basic-report.md b/packages/kbot/tests/unit/basic-report.md index 1877a285..d94a4807 100644 --- a/packages/kbot/tests/unit/basic-report.md +++ b/packages/kbot/tests/unit/basic-report.md @@ -2,171 +2,56 @@ ## Failed Tests -### basic_arithmetic - deepseek/deepseek-chat:free -- Prompt: `return the result of 2+2, dont comment` -- Expected: `undefined` -- Actual: `4` -- Reason: undefined -- Timestamp: 4/1/2025, 12:26:30 PM - -### basic_arithmetic - google/gemini-2.0-flash-exp:free -- Prompt: `return the result of 2+2, dont comment` -- Expected: `undefined` -- Actual: `4 -` -- Reason: undefined -- Timestamp: 4/1/2025, 12:26:31 PM - -### basic_arithmetic - gpt-4 -- Prompt: `return the result of 2+2, dont comment` -- Expected: `undefined` -- Actual: `4` -- Reason: undefined -- Timestamp: 4/1/2025, 12:26:32 PM - -### json_structure - deepseek/deepseek-chat:free -- Prompt: `return a JSON object with two fields: "name" as "test" and "value" as 42. Return only the JSON, no other text.` -- Expected: `undefined` -- Actual: `{"name":"test","value":42}` -- Reason: undefined -- Timestamp: 4/1/2025, 12:26:33 PM - -### json_structure - gpt-4 -- Prompt: `return a JSON object with two fields: "name" as "test" and "value" as 42. Return only the JSON, no other text.` -- Expected: `undefined` -- Actual: `{"name": "test", "value": 42}` -- Reason: undefined -- Timestamp: 4/1/2025, 12:26:36 PM - -### json_structure - google/gemini-2.0-flash-exp:free -- Prompt: `return a JSON object with two fields: "name" as "test" and "value" as 42. Return only the JSON, no other text.` -- Expected: `undefined` -- Actual: `{ - "name": "test", - "value": 42 -}` -- Reason: undefined -- Timestamp: 4/1/2025, 12:26:35 PM - -### hello - deepseek/deepseek-chat:free -- Prompt: `say "hello"` -- Expected: `hello` -- Actual: `` -- Reason: Model returned empty response -- Timestamp: 4/1/2025, 1:36:37 PM - -### hello - google/gemini-2.0-flash-exp:free -- Prompt: `say "hello"` -- Expected: `hello` -- Actual: `` -- Reason: Model returned empty response -- Timestamp: 4/1/2025, 1:36:37 PM - -### hello - gpt-4 -- Prompt: `say "hello"` -- Expected: `hello` -- Actual: `` -- Reason: Unknown error occurred -- Timestamp: 4/1/2025, 1:36:42 PM - -### goodbye - deepseek/deepseek-chat:free -- Prompt: `say "goodbye"` -- Expected: `goodbye` -- Actual: `` -- Reason: Model returned empty response -- Timestamp: 4/1/2025, 1:36:42 PM - -### goodbye - google/gemini-2.0-flash-exp:free -- Prompt: `say "goodbye"` -- Expected: `goodbye` -- Actual: `` -- Reason: Model returned empty response -- Timestamp: 4/1/2025, 1:36:43 PM - -### goodbye - gpt-4 -- Prompt: `say "goodbye"` -- Expected: `goodbye` -- Actual: `` -- Reason: expected 'goodbye.' to deeply equal 'goodbye' -- Timestamp: 4/1/2025, 1:36:44 PM - -### yes - deepseek/deepseek-chat:free -- Prompt: `say "yes"` -- Expected: `yes` -- Actual: `` -- Reason: Model returned empty response -- Timestamp: 4/1/2025, 1:36:45 PM - -### yes - google/gemini-2.0-flash-exp:free -- Prompt: `say "yes"` -- Expected: `yes` -- Actual: `` -- Reason: Model returned empty response -- Timestamp: 4/1/2025, 1:36:45 PM - -### yes - gpt-4 -- Prompt: `say "yes"` -- Expected: `yes` -- Actual: `` -- Reason: Unknown error occurred -- Timestamp: 4/1/2025, 1:36:46 PM +*No failed tests* ## Passed Tests -### addition - deepseek/deepseek-chat:free +### addition - anthropic/claude-3.5-sonnet - Prompt: `add 5 and 3. Return only the number, no explanation.` - Expected: `8` - Actual: `8` -- Timestamp: 4/1/2025, 12:59:06 PM +- Duration: 1551ms +- Timestamp: 4/2/2025, 12:17:39 AM -### addition - google/gemini-2.0-flash-exp:free -- Prompt: `add 5 and 3. Return only the number, no explanation.` -- Expected: `8` -- Actual: `8 -` -- Timestamp: 4/1/2025, 12:59:08 PM - -### addition - gpt-4 +### addition - qwen/qwq-32b - Prompt: `add 5 and 3. Return only the number, no explanation.` - Expected: `8` - Actual: `8` -- Timestamp: 4/1/2025, 1:39:04 PM +- Duration: 3621ms +- Timestamp: 4/2/2025, 12:17:42 AM -### multiplication - deepseek/deepseek-chat:free +### multiplication - anthropic/claude-3.5-sonnet - Prompt: `multiply 8 and 3. Return only the number, no explanation.` - Expected: `24` - Actual: `24` -- Timestamp: 4/1/2025, 12:59:13 PM +- Duration: 873ms +- Timestamp: 4/2/2025, 12:17:43 AM -### multiplication - google/gemini-2.0-flash-exp:free -- Prompt: `multiply 8 and 3. Return only the number, no explanation.` -- Expected: `24` -- Actual: `24 -` -- Timestamp: 4/1/2025, 12:59:15 PM - -### multiplication - gpt-4 +### multiplication - qwen/qwq-32b - Prompt: `multiply 8 and 3. Return only the number, no explanation.` - Expected: `24` - Actual: `24` -- Timestamp: 4/1/2025, 1:39:06 PM +- Duration: 3472ms +- Timestamp: 4/2/2025, 12:17:47 AM -### division - deepseek/deepseek-chat:free +### division - anthropic/claude-3.5-sonnet - Prompt: `divide 15 by 3. Return only the number, no explanation.` - Expected: `5` - Actual: `5` -- Timestamp: 4/1/2025, 12:59:18 PM +- Duration: 1183ms +- Timestamp: 4/2/2025, 12:17:48 AM -### division - google/gemini-2.0-flash-exp:free -- Prompt: `divide 15 by 3. Return only the number, no explanation.` -- Expected: `5` -- Actual: `5 -` -- Timestamp: 4/1/2025, 12:56:09 PM - -### division - gpt-4 +### division - qwen/qwq-32b - Prompt: `divide 15 by 3. Return only the number, no explanation.` - Expected: `5` - Actual: `5` -- Timestamp: 4/1/2025, 1:39:08 PM +- Duration: 4841ms +- Timestamp: 4/2/2025, 12:17:53 AM + +## Summary + +- Total Tests: 6 +- Passed: 6 +- Failed: 0 +- Success Rate: 100.00% diff --git a/packages/kbot/tests/unit/basic.json b/packages/kbot/tests/unit/basic.json index 706e1e4a..cd6be239 100644 --- a/packages/kbot/tests/unit/basic.json +++ b/packages/kbot/tests/unit/basic.json @@ -1,233 +1,16 @@ [ { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", "result": [ - "4" + "8" ], - "model": "deepseek/deepseek-chat:free", + "expected": "8", + "model": "anthropic/claude-3.5-sonnet", "router": "openrouter", - "timestamp": "2025-04-01T10:20:31.673Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4\n" - ], - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:20:33.287Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4" - ], - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:20:34.328Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4" - ], - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:21:49.790Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4\n" - ], - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:21:51.896Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4" - ], - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:21:52.849Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4" - ], - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:24:35.975Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4\n" - ], - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:24:37.455Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4" - ], - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:24:38.366Z" - }, - { - "test": "json_structure", - "prompt": "return a JSON object with two fields: \"name\" as \"test\" and \"value\" as 42. Return only the JSON, no other text.", - "result": [ - "{\"name\": \"test\", \"value\": 42}" - ], - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:24:39.480Z" - }, - { - "test": "json_structure", - "prompt": "return a JSON object with two fields: \"name\" as \"test\" and \"value\" as 42. Return only the JSON, no other text.", - "result": [ - "{\"name\": \"test\", \"value\": 42}" - ], - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:24:42.228Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4" - ], - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:25:06.304Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4\n" - ], - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:25:07.392Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4" - ], - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:25:08.429Z" - }, - { - "test": "json_structure", - "prompt": "return a JSON object with two fields: \"name\" as \"test\" and \"value\" as 42. Return only the JSON, no other text.", - "result": [ - "```json\n{\"name\":\"test\",\"value\":42}\n```" - ], - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:25:09.627Z" - }, - { - "test": "json_structure", - "prompt": "return a JSON object with two fields: \"name\" as \"test\" and \"value\" as 42. Return only the JSON, no other text.", - "result": [ - "```json\n{\n \"name\": \"test\",\n \"value\": 42\n}\n```\n" - ], - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:25:10.720Z" - }, - { - "test": "json_structure", - "prompt": "return a JSON object with two fields: \"name\" as \"test\" and \"value\" as 42. Return only the JSON, no other text.", - "result": [ - "{\"name\": \"test\", \"value\": 42}" - ], - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:25:12.099Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4" - ], - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:26:30.128Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4\n" - ], - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:26:31.316Z" - }, - { - "test": "basic_arithmetic", - "prompt": "return the result of 2+2, dont comment", - "result": [ - "4" - ], - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:26:32.279Z" - }, - { - "test": "json_structure", - "prompt": "return a JSON object with two fields: \"name\" as \"test\" and \"value\" as 42. Return only the JSON, no other text.", - "result": [ - "{\"name\":\"test\",\"value\":42}" - ], - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:26:33.459Z" - }, - { - "test": "json_structure", - "prompt": "return a JSON object with two fields: \"name\" as \"test\" and \"value\" as 42. Return only the JSON, no other text.", - "result": [ - "{\n \"name\": \"test\",\n \"value\": 42\n}" - ], - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:26:35.012Z" - }, - { - "test": "json_structure", - "prompt": "return a JSON object with two fields: \"name\" as \"test\" and \"value\" as 42. Return only the JSON, no other text.", - "result": [ - "{\"name\": \"test\", \"value\": 42}" - ], - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:26:36.239Z" + "timestamp": "2025-04-01T22:17:39.340Z", + "passed": true, + "duration": 1551 }, { "test": "addition", @@ -236,382 +19,11 @@ "8" ], "expected": "8", - "model": "deepseek/deepseek-chat:free", + "model": "qwen/qwq-32b", "router": "openrouter", - "timestamp": "2025-04-01T10:46:03.882Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8\n" - ], - "expected": "8", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:46:05.343Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:46:06.365Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:46:07.564Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24\n" - ], - "expected": "24", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:46:09.013Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:46:09.968Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:46:10.981Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5\n" - ], - "expected": "5", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:46:12.829Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:46:13.957Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:53:47.732Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8\n" - ], - "expected": "8", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:53:50.075Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:53:50.977Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:53:52.835Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24\n" - ], - "expected": "24", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:53:54.875Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:53:55.866Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:53:56.950Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:53:59.317Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:01.223Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8\n" - ], - "expected": "8", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:02.717Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:03.867Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:04.864Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24\n" - ], - "expected": "24", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:05.973Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:07.075Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:08.103Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5\n" - ], - "expected": "5", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:09.298Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:10.178Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:32.641Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 4 and 6. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:35.038Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:56:37.567Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:59:06.742Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8\n" - ], - "expected": "8", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:59:08.889Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:59:09.871Z", - "passed": true + "timestamp": "2025-04-01T22:17:42.962Z", + "passed": true, + "duration": 3621 }, { "test": "multiplication", @@ -620,22 +32,11 @@ "24" ], "expected": "24", - "model": "deepseek/deepseek-chat:free", + "model": "anthropic/claude-3.5-sonnet", "router": "openrouter", - "timestamp": "2025-04-01T10:59:13.627Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 8 and 3. Return only the number, no explanation.", - "result": [ - "24\n" - ], - "expected": "24", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T10:59:15.963Z", - "passed": true + "timestamp": "2025-04-01T22:17:43.836Z", + "passed": true, + "duration": 873 }, { "test": "multiplication", @@ -644,10 +45,11 @@ "24" ], "expected": "24", - "model": "gpt-4", + "model": "qwen/qwq-32b", "router": "openrouter", - "timestamp": "2025-04-01T10:59:17.124Z", - "passed": true + "timestamp": "2025-04-01T22:17:47.309Z", + "passed": true, + "duration": 3472 }, { "test": "division", @@ -656,10 +58,11 @@ "5" ], "expected": "5", - "model": "deepseek/deepseek-chat:free", + "model": "anthropic/claude-3.5-sonnet", "router": "openrouter", - "timestamp": "2025-04-01T10:59:18.527Z", - "passed": true + "timestamp": "2025-04-01T22:17:48.493Z", + "passed": true, + "duration": 1183 }, { "test": "division", @@ -668,429 +71,10 @@ "5" ], "expected": "5", - "model": "gpt-4", + "model": "qwen/qwq-32b", "router": "openrouter", - "timestamp": "2025-04-01T10:59:20.298Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T10:59:58.244Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 8 and 3. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:00:00.620Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:00:03.143Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:00:10.210Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 8 and 3. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:00:12.798Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:00:15.604Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:00:35.256Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 8 and 3. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:00:37.681Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:00:40.556Z", - "passed": true - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:02:55.210Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 8 and 3. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:02:57.579Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:03:00.064Z", - "passed": true - }, - { - "test": "hello", - "prompt": "say \"hello\"", - "result": [], - "expected": "hello", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:36:37.192Z", - "passed": false, - "duration": 1107, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "hello", - "prompt": "say \"hello\"", - "result": [], - "expected": "hello", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:36:37.957Z", - "passed": false, - "duration": 762, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:110:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "hello", - "prompt": "say \"hello\"", - "result": [], - "expected": "hello", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:36:42.078Z", - "passed": false, - "duration": 4120, - "reason": "Unknown error occurred", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "goodbye", - "prompt": "say \"goodbye\"", - "result": [], - "expected": "goodbye", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:36:42.695Z", - "passed": false, - "duration": 616, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "goodbye", - "prompt": "say \"goodbye\"", - "result": [], - "expected": "goodbye", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:36:43.462Z", - "passed": false, - "duration": 765, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "goodbye", - "prompt": "say \"goodbye\"", - "result": [], - "expected": "goodbye", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:36:44.537Z", - "passed": false, - "duration": 1073, - "error": { - "message": "expected 'goodbye.' to deeply equal 'goodbye'", - "code": "UNKNOWN", - "type": "AssertionError", - "details": { - "message": "expected 'goodbye.' to deeply equal 'goodbye'", - "actual": "goodbye.", - "expected": "goodbye", - "showDiff": true, - "operator": "strictEqual", - "stackStr": "AssertionError: expected 'goodbye.' to deeply equal 'goodbye'\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:64:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "AssertionError", - "diff": "Expected: \u001b[32m\"goodbye\"\u001b[39m\nReceived: \u001b[31m\"goodbye\u001b[7m.\u001b[27m\"\u001b[39m", - "name": "AssertionError", - "ok": false, - "stack": "AssertionError: expected 'goodbye.' to deeply equal 'goodbye'\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:64:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:119:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" - } - }, - "reason": "expected 'goodbye.' to deeply equal 'goodbye'", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "yes", - "prompt": "say \"yes\"", - "result": [], - "expected": "yes", - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:36:45.193Z", - "passed": false, - "duration": 654, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "yes", - "prompt": "say \"yes\"", - "result": [], - "expected": "yes", - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:36:45.954Z", - "passed": false, - "duration": 760, - "error": { - "message": "Model returned empty response", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Model returned empty response", - "stackStr": "Error: Model returned empty response\n at runBasicTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:61:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\basic.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Model returned empty response", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "yes", - "prompt": "say \"yes\"", - "result": [], - "expected": "yes", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:36:46.806Z", - "passed": false, - "duration": 851, - "reason": "Unknown error occurred", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "addition", - "prompt": "add 5 and 3. Return only the number, no explanation.", - "result": [ - "8" - ], - "expected": "8", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:39:04.078Z", - "passed": true - }, - { - "test": "multiplication", - "prompt": "multiply 8 and 3. Return only the number, no explanation.", - "result": [ - "24" - ], - "expected": "24", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:39:06.572Z", - "passed": true - }, - { - "test": "division", - "prompt": "divide 15 by 3. Return only the number, no explanation.", - "result": [ - "5" - ], - "expected": "5", - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:39:08.868Z", - "passed": true + "timestamp": "2025-04-01T22:17:53.335Z", + "passed": true, + "duration": 4841 } ] \ No newline at end of file diff --git a/packages/kbot/tests/unit/basic.test.ts b/packages/kbot/tests/unit/basic.test.ts index 21449c98..dd1b3eb7 100644 --- a/packages/kbot/tests/unit/basic.test.ts +++ b/packages/kbot/tests/unit/basic.test.ts @@ -4,153 +4,121 @@ import * as path from 'node:path' import { sync as write } from "@polymech/fs/write" import { sync as read } from "@polymech/fs/read" import { sync as exists } from "@polymech/fs/exists" -import { models, TEST_BASE_PATH, TEST_LOGS_PATH, TEST_PREFERENCES_PATH, TestResult } from './commons' +import { + models, + TEST_BASE_PATH, + TEST_LOGS_PATH, + TEST_PREFERENCES_PATH, + TEST_TIMEOUT, + TestResult, + formatError, + isEmptyResponse +} from './commons' const TEST_LOG_PATH = path.resolve(__dirname, './basic.json') describe('Basic Operations', () => { let testResults: TestResult[] = [] - // Load existing results if any - if (exists(TEST_LOG_PATH)) { - const data = read(TEST_LOG_PATH, 'json') - testResults = Array.isArray(data) ? data : [] + + const runBasicTest = async (prompt: string, expected: string, testName: string, modelName: string) => { + let model = 'unknown' + let router = 'unknown' + let startTime = Date.now() + let error: TestResult['error'] | undefined + let testResult: TestResult | undefined + + try { + const result = await Promise.race([ + run({ + prompt, + mode: 'completion', + model: modelName, + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + onRun: async (options) => { + model = options.model || 'unknown' + router = options.router || 'unknown' + return options + } + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error('API call timed out')), TEST_TIMEOUT) + ) + ]) as string[] + + if (isEmptyResponse(result)) { + throw new Error('Model returned empty response') + } + + const actual = result?.[0]?.trim()?.toLowerCase() || '' + const passed = actual === expected + + expect(actual).toEqual(expected) + + testResult = { + test: testName, + prompt, + result: result || [], + expected, + model, + router, + timestamp: new Date().toISOString(), + passed, + duration: Date.now() - startTime, + reason: passed ? undefined : `Expected ${expected}, but got ${actual}`, + } + } catch (e) { + error = formatError(e) + testResult = { + test: testName, + prompt, + result: [], + expected, + model, + router, + timestamp: new Date().toISOString(), + passed: false, + duration: Date.now() - startTime, + error, + reason: error?.message || 'Unknown error occurred' + } + throw e + } finally { + if (testResult) { + testResults.push(testResult) + write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) + } + } } it.each(models)('should add two numbers with model %s', async (modelName) => { - const prompt = 'add 5 and 3. Return only the number, no explanation.' - const expected = '8' - let model = 'unknown' - let router = 'unknown' - - const result = await run({ - prompt, - mode: 'completion', - model: modelName, - path: TEST_BASE_PATH, - logs: TEST_LOGS_PATH, - preferences: TEST_PREFERENCES_PATH, - onRun: async (options) => { - model = options.model || 'unknown' - router = options.router || 'unknown' - return options - } - }) as string[] - - const actual = result?.[0]?.trim() || '' - if (!actual) { - console.log(`Skipping test for model ${modelName} - no result returned`) - return - } - const passed = actual === expected - expect(actual).toEqual(expected) - - // Add test result to array - testResults.push({ - test: 'addition', - prompt, - result: result || [], - expected, - model, - router, - timestamp: new Date().toISOString(), - passed, - reason: passed ? undefined : `Expected ${expected}, but got ${actual}` - }) - - // Write all results to the same file - write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) - }) + await runBasicTest( + 'add 5 and 3. Return only the number, no explanation.', + '8', + 'addition', + modelName + ) + }, { timeout: 10000 }) it.each(models)('should multiply two numbers with model %s', async (modelName) => { - const prompt = 'multiply 8 and 3. Return only the number, no explanation.' - const expected = '24' - let model = 'unknown' - let router = 'unknown' - - const result = await run({ - prompt, - mode: 'completion', - model: modelName, - path: TEST_BASE_PATH, - logs: TEST_LOGS_PATH, - preferences: TEST_PREFERENCES_PATH, - onRun: async (options) => { - model = options.model || 'unknown' - router = options.router || 'unknown' - return options - } - }) as string[] - - const actual = result?.[0]?.trim() || '' - if (!actual) { - console.log(`Skipping test for model ${modelName} - no result returned`) - return - } - const passed = actual === expected - expect(actual).toEqual(expected) - - // Add test result to array - testResults.push({ - test: 'multiplication', - prompt, - result: result || [], - expected, - model, - router, - timestamp: new Date().toISOString(), - passed, - reason: passed ? undefined : `Expected ${expected}, but got ${actual}` - }) - - // Write all results to the same file - write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) - }) + await runBasicTest( + 'multiply 8 and 3. Return only the number, no explanation.', + '24', + 'multiplication', + modelName + ) + }, { timeout: 10000 }) it.each(models)('should divide two numbers with model %s', async (modelName) => { - const prompt = 'divide 15 by 3. Return only the number, no explanation.' - const expected = '5' - let model = 'unknown' - let router = 'unknown' - - const result = await run({ - prompt, - mode: 'completion', - model: modelName, - path: TEST_BASE_PATH, - logs: TEST_LOGS_PATH, - preferences: TEST_PREFERENCES_PATH, - onRun: async (options) => { - model = options.model || 'unknown' - router = options.router || 'unknown' - return options - } - }) as string[] - - const actual = result?.[0]?.trim() || '' - if (!actual) { - console.log(`Skipping test for model ${modelName} - no result returned`) - return - } - const passed = actual === expected - expect(actual).toEqual(expected) - - // Add test result to array - testResults.push({ - test: 'division', - prompt, - result: result || [], - expected, - model, - router, - timestamp: new Date().toISOString(), - passed, - reason: passed ? undefined : `Expected ${expected}, but got ${actual}` - }) - - // Write all results to the same file - write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) - }) + await runBasicTest( + 'divide 15 by 3. Return only the number, no explanation.', + '5', + 'division', + modelName + ) + }, { timeout: 10000 }) it('should generate markdown report', () => { // Group results by test and model @@ -173,32 +141,64 @@ describe('Basic Operations', () => { // First list failed tests report += '## Failed Tests\n\n' + let hasFailures = false for (const [testName, modelResults] of latestResults) { for (const [model, result] of modelResults) { if (!result.passed) { + hasFailures = true report += `### ${testName} - ${model}\n` report += `- Prompt: \`${result.prompt}\`\n` report += `- Expected: \`${result.expected}\`\n` report += `- Actual: \`${result.result[0] || ''}\`\n` + report += `- Duration: ${result.duration}ms\n` + if (result.error) { + report += `- Error Type: ${result.error.type}\n` + report += `- Error Code: ${result.error.code}\n` + report += `- Error Message: ${result.error.message}\n` + if (result.error.details?.message) { + report += `- Error Details: ${result.error.details.message}\n` + } + } report += `- Reason: ${result.reason}\n` report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` } } } + + if (!hasFailures) { + report += '*No failed tests*\n\n' + } // Then list passed tests report += '## Passed Tests\n\n' + let hasPassed = false for (const [testName, modelResults] of latestResults) { for (const [model, result] of modelResults) { if (result.passed) { + hasPassed = true report += `### ${testName} - ${model}\n` report += `- Prompt: \`${result.prompt}\`\n` report += `- Expected: \`${result.expected}\`\n` report += `- Actual: \`${result.result[0] || ''}\`\n` + report += `- Duration: ${result.duration}ms\n` report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` } } } + + if (!hasPassed) { + report += '*No passed tests*\n\n' + } + + // Add summary section + report += '## Summary\n\n' + const totalTests = testResults.length + const passedTests = testResults.filter(r => r.passed).length + const failedTests = totalTests - passedTests + report += `- Total Tests: ${totalTests}\n` + report += `- Passed: ${passedTests}\n` + report += `- Failed: ${failedTests}\n` + report += `- Success Rate: ${((passedTests / totalTests) * 100).toFixed(2)}%\n\n` // Write report to file const reportPath = path.resolve(__dirname, './basic-report.md') diff --git a/packages/kbot/tests/unit/commons.ts b/packages/kbot/tests/unit/commons.ts index fb8ea008..7e084fad 100644 --- a/packages/kbot/tests/unit/commons.ts +++ b/packages/kbot/tests/unit/commons.ts @@ -3,7 +3,8 @@ import { E_OPENROUTER_MODEL_FREE, E_OPENAI_MODEL, E_OPENROUTER_MODEL } from '../ export const models = [ //E_OPENROUTER_MODEL_FREE.MODEL_FREE_DEEPSEEK_DEEPSEEK_CHAT_FREE, - E_OPENROUTER_MODEL.MODEL_ANTHROPIC_CLAUDE_2_0 + E_OPENROUTER_MODEL.MODEL_ANTHROPIC_CLAUDE_3_5_SONNET, + E_OPENROUTER_MODEL.MODEL_QWEN_QWQ_32B ] export const TEST_BASE_PATH = path.resolve(__dirname, '../../') diff --git a/packages/kbot/tests/unit/format-report.md b/packages/kbot/tests/unit/format-report.md index 68129a8e..3e968672 100644 --- a/packages/kbot/tests/unit/format-report.md +++ b/packages/kbot/tests/unit/format-report.md @@ -2,546 +2,224 @@ ## Failed Tests -### basic_structure - deepseek/deepseek-chat:free -- Prompt: `return a greeting "hello" with count 42` -- Expected: `{"greeting":"hello","count":42}` -- Actual: `""` -- Duration: 885ms -- Error Type: Error +### json-schema-file-format - anthropic/claude-3.5-sonnet +- Prompt: `Create a user profile with name John Doe, age 30, and tags ["developer", "javascript"]. Return only the JSON object, no explanation.` +- Expected: `{"name":"John Doe","age":30,"tags":["developer","javascript"]}` +- Actual: `` +- Duration: 1690ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:36 PM +- Error Message: expected '{\n "name": "John Doe",\n "age": 30…' to deeply equal '{"name":"John Doe","age":30,"tags":["…' +- Error Details: expected '{\n "name": "John Doe",\n "age": 30…' to deeply equal '{"name":"John Doe","age":30,"tags":["…' +- Reason: expected '{\n "name": "John Doe",\n "age": 30…' to deeply equal '{"name":"John Doe","age":30,"tags":["…' +- Timestamp: 4/2/2025, 12:23:43 AM -### basic_structure - google/gemini-2.0-flash-exp:free -- Prompt: `return a greeting "hello" with count 42` -- Expected: `{"greeting":"hello","count":42}` -- Actual: `""` -- Duration: 757ms -- Error Type: Error +### json-schema-file-format - qwen/qwq-32b +- Prompt: `Create a user profile with name John Doe, age 30, and tags ["developer", "javascript"]. Return only the JSON object, no explanation.` +- Expected: `{"name":"John Doe","age":30,"tags":["developer","javascript"]}` +- Actual: `` +- Duration: 3426ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:36 PM +- Error Message: expected '"John Doe",\n "age": 30,\n "tags": …' to deeply equal '{"name":"John Doe","age":30,"tags":["…' +- Error Details: expected '"John Doe",\n "age": 30,\n "tags": …' to deeply equal '{"name":"John Doe","age":30,"tags":["…' +- Reason: expected '"John Doe",\n "age": 30,\n "tags": …' to deeply equal '{"name":"John Doe","age":30,"tags":["…' +- Timestamp: 4/2/2025, 12:23:47 AM -### basic_structure - gpt-4 -- Prompt: `return a greeting "hello" with count 42` -- Expected: `{"greeting":"hello","count":42}` -- Actual: `""` -- Duration: 1043ms -- Error Type: Error +### json-schema-object-format - anthropic/claude-3.5-sonnet +- Prompt: `Create a user profile with name Jane Smith, age 25, and tags ["designer", "ui"]. Return only the JSON object, no explanation.` +- Expected: `{"name":"Jane Smith","age":25,"tags":["designer","ui"]}` +- Actual: `` +- Duration: 1918ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:37 PM +- Error Message: expected '{\n "name": "Jane Smith",\n "age": …' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' +- Error Details: expected '{\n "name": "Jane Smith",\n "age": …' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' +- Reason: expected '{\n "name": "Jane Smith",\n "age": …' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' +- Timestamp: 4/2/2025, 12:23:49 AM -### basic_structure - anthropic/claude-3.7-sonnet -- Prompt: `return a greeting "hello" with count 42` -- Expected: `{"greeting":"hello","count":42}` -- Actual: `""` -- Duration: 1790ms -- Error Type: Error +### json-schema-object-format - qwen/qwq-32b +- Prompt: `Create a user profile with name Jane Smith, age 25, and tags ["designer", "ui"]. Return only the JSON object, no explanation.` +- Expected: `{"name":"Jane Smith","age":25,"tags":["designer","ui"]}` +- Actual: `` +- Duration: 9789ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: Unexpected token 'h', "hello 42" is not valid JSON -- Reason: Failed to parse or validate response: Unexpected token 'h', "hello 42" is not valid JSON -- Timestamp: 4/1/2025, 1:23:05 PM +- Error Message: expected '{"name": "Jane Smith", "age": 25, "ta…' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' +- Error Details: expected '{"name": "Jane Smith", "age": 25, "ta…' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' +- Reason: expected '{"name": "Jane Smith", "age": 25, "ta…' to deeply equal '{"name":"Jane Smith","age":25,"tags":…' +- Timestamp: 4/2/2025, 12:23:58 AM -### basic_structure - openai/gpt-4 -- Prompt: `Return a JSON object with a greeting "hello" and count 42. The response must be valid JSON with exactly these fields: { "greeting": string, "count": number }` -- Expected: `{"greeting":"hello","count":42}` -- Actual: `""` -- Duration: 1258ms -- Error Type: Error +### json-schema-object-format - gpt-4o +- Prompt: `Create a user profile with the following details: + - Name: Jane Smith + - Age: 25 + - Email: jane.smith@company.com + - Tags: ["developer", "designer"] + - Address: 123 Main St, New York, US, 10001 + - Preferences: light theme, notifications enabled, English language + Return only the JSON object, no explanation.` +- Expected: `{"name":"Jane Smith","age":25,"email":"jane.smith@company.com","tags":["developer","designer"],"address":{"street":"123 Main St","city":"New York","country":"US","zipCode":"10001"},"preferences":{"theme":"light","notifications":true,"language":"en"}}` +- Actual: `` +- Duration: 2618ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Invalid response from API -- Reason: Invalid response from API -- Timestamp: 4/1/2025, 1:32:43 PM +- Error Message: expected '{"name":"Jane Smith","age":25,"email"…' to deeply equal '{"name":"Jane Smith","age":25,"email"…' +- Error Details: expected '{"name":"Jane Smith","age":25,"email"…' to deeply equal '{"name":"Jane Smith","age":25,"email"…' +- Reason: expected '{"name":"Jane Smith","age":25,"email"…' to deeply equal '{"name":"Jane Smith","age":25,"email"…' +- Timestamp: 4/2/2025, 12:33:08 AM -### nested_structure - deepseek/deepseek-chat:free -- Prompt: `return user John age 30 with dark theme and notifications enabled` -- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` -- Actual: `""` -- Duration: 655ms -- Error Type: Error +### zod-string-format - anthropic/claude-3.5-sonnet +- Prompt: `Generate a valid email address for a business domain. Return only the email, no explanation.` +- Expected: `john.doe@company.com` +- Actual: `` +- Duration: 1347ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:38 PM +- Error Message: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com' +- Error Details: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com' +- Reason: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com' +- Timestamp: 4/2/2025, 12:24:00 AM -### nested_structure - google/gemini-2.0-flash-exp:free -- Prompt: `return user John age 30 with dark theme and notifications enabled` -- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` -- Actual: `""` -- Duration: 790ms -- Error Type: Error +### zod-string-format - qwen/qwq-32b +- Prompt: `Generate a valid email address for a business domain. Return only the email, no explanation.` +- Expected: `john.doe@company.com` +- Actual: `` +- Duration: 13704ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:39 PM +- Error Message: expected 'info@techstart.com' to deeply equal 'john.doe@company.com' +- Error Details: expected 'info@techstart.com' to deeply equal 'john.doe@company.com' +- Reason: expected 'info@techstart.com' to deeply equal 'john.doe@company.com' +- Timestamp: 4/2/2025, 12:24:13 AM -### nested_structure - gpt-4 -- Prompt: `return user John age 30 with dark theme and notifications enabled` -- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` -- Actual: `""` -- Duration: 717ms -- Error Type: Error +### zod-string-format - gpt-4o +- Prompt: `Generate a valid email address for a business domain. Return only the email, no explanation.` +- Expected: `john.doe@company.com` +- Actual: `` +- Duration: 1794ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:40 PM +- Error Message: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com' +- Error Details: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com' +- Reason: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com' +- Timestamp: 4/2/2025, 12:32:20 AM -### nested_structure - anthropic/claude-3.7-sonnet -- Prompt: `return user John age 30 with dark theme and notifications enabled` -- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` -- Actual: `""` -- Duration: 1189ms -- Error Type: Error +### zod-number-format - anthropic/claude-3.5-sonnet +- Prompt: `Generate a random age between 18 and 65. Return only the number, no explanation.` +- Expected: `25` +- Actual: `` +- Duration: 1376ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: Unexpected token '#', "# John's U"... is not valid JSON -- Reason: Failed to parse or validate response: Unexpected token '#', "# John's U"... is not valid JSON -- Timestamp: 4/1/2025, 1:23:06 PM +- Error Message: expected '42' to deeply equal '25' +- Error Details: expected '42' to deeply equal '25' +- Reason: expected '42' to deeply equal '25' +- Timestamp: 4/2/2025, 12:24:11 AM -### nested_structure - openai/gpt-4 -- Prompt: `Return a JSON object with user John age 30, dark theme and notifications enabled. The response must be valid JSON with this structure: { "user": { "name": string, "age": number }, "settings": { "theme": string, "notifications": boolean } }` -- Expected: `{"user":{"name":"John","age":30},"settings":{"theme":"dark","notifications":true}}` -- Actual: `""` -- Duration: 716ms -- Error Type: Error +### zod-number-format - gpt-4o +- Prompt: `Generate a random age between 18 and 65. Return only the number, no explanation.` +- Expected: `25` +- Actual: `` +- Duration: 2399ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Invalid response from API -- Reason: Invalid response from API -- Timestamp: 4/1/2025, 1:32:44 PM +- Error Message: expected '39' to deeply equal '25' +- Error Details: expected '39' to deeply equal '25' +- Reason: expected '39' to deeply equal '25' +- Timestamp: 4/2/2025, 12:32:23 AM -### array_structure - deepseek/deepseek-chat:free -- Prompt: `return a list of 2 items with ids 1 and 2, names "first" and "second"` -- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` -- Actual: `""` -- Duration: 617ms -- Error Type: Error +### zod-array-format - anthropic/claude-3.5-sonnet +- Prompt: `Generate a list of 3 programming languages. Return only the array, no explanation.` +- Expected: `["JavaScript","Python","Java"]` +- Actual: `` +- Duration: 1009ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:40 PM +- Error Message: expected '["Python", "Java", "JavaScript"]' to deeply equal '["JavaScript","Python","Java"]' +- Error Details: expected '["Python", "Java", "JavaScript"]' to deeply equal '["JavaScript","Python","Java"]' +- Reason: expected '["Python", "Java", "JavaScript"]' to deeply equal '["JavaScript","Python","Java"]' +- Timestamp: 4/2/2025, 12:24:22 AM -### array_structure - google/gemini-2.0-flash-exp:free -- Prompt: `return a list of 2 items with ids 1 and 2, names "first" and "second"` -- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` -- Actual: `""` -- Duration: 756ms -- Error Type: Error +### zod-array-format - qwen/qwq-32b +- Prompt: `Generate a list of 3 programming languages. Return only the array, no explanation.` +- Expected: `["JavaScript","Python","Java"]` +- Actual: `` +- Duration: 4147ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:41 PM +- Error Message: expected '["Python", "JavaScript", "Java"]' to deeply equal '["JavaScript","Python","Java"]' +- Error Details: expected '["Python", "JavaScript", "Java"]' to deeply equal '["JavaScript","Python","Java"]' +- Reason: expected '["Python", "JavaScript", "Java"]' to deeply equal '["JavaScript","Python","Java"]' +- Timestamp: 4/2/2025, 12:24:26 AM -### array_structure - gpt-4 -- Prompt: `return a list of 2 items with ids 1 and 2, names "first" and "second"` -- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` -- Actual: `""` +### zod-array-format - gpt-4o +- Prompt: `Generate a list of 3 programming languages. Return only the array, no explanation.` +- Expected: `["JavaScript","Python","Java"]` +- Actual: `` +- Duration: 693ms +- Error Type: AssertionError +- Error Code: UNKNOWN +- Error Message: expected '["Python","JavaScript","Java"]' to deeply equal '["JavaScript","Python","Java"]' +- Error Details: expected '["Python","JavaScript","Java"]' to deeply equal '["JavaScript","Python","Java"]' +- Reason: expected '["Python","JavaScript","Java"]' to deeply equal '["JavaScript","Python","Java"]' +- Timestamp: 4/2/2025, 12:32:23 AM + +### invalid-format - anthropic/claude-3.5-sonnet +- Prompt: `Generate a random number.` +- Expected: `Invalid format option` +- Actual: `` - Duration: 1026ms -- Error Type: Error +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:42 PM +- Error Message: expected '73' to deeply equal 'Invalid format option' +- Error Details: expected '73' to deeply equal 'Invalid format option' +- Reason: expected '73' to deeply equal 'Invalid format option' +- Timestamp: 4/2/2025, 12:24:27 AM -### array_structure - anthropic/claude-3.7-sonnet -- Prompt: `return a list of 2 items with ids 1 and 2, names "first" and "second"` -- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` -- Actual: `""` -- Duration: 1190ms -- Error Type: Error +### invalid-format - qwen/qwq-32b +- Prompt: `Generate a random number.` +- Expected: `Invalid format option` +- Actual: `` +- Duration: 7614ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:23:08 PM +- Error Message: expected '72' to deeply equal 'Invalid format option' +- Error Details: expected '72' to deeply equal 'Invalid format option' +- Reason: expected '72' to deeply equal 'Invalid format option' +- Timestamp: 4/2/2025, 12:24:35 AM -### array_structure - openai/gpt-4 -- Prompt: `Return a JSON object with a list of 2 items. The response must be valid JSON with this structure: { "items": [{ "id": number, "name": string }] }. The first item should have id 1 and name "first", the second item should have id 2 and name "second".` -- Expected: `{"items":[{"id":1,"name":"first"},{"id":2,"name":"second"}]}` -- Actual: `""` -- Duration: 703ms -- Error Type: Error +### invalid-format - gpt-4o +- Prompt: `Generate a random number.` +- Expected: `Invalid format option` +- Actual: `` +- Duration: 826ms +- Error Type: AssertionError - Error Code: UNKNOWN -- Error Message: Invalid response from API -- Reason: Invalid response from API -- Timestamp: 4/1/2025, 1:32:44 PM - -### enum_structure - deepseek/deepseek-chat:free -- Prompt: `return status success with message "Operation completed"` -- Expected: `{"status":"success","message":"Operation completed"}` -- Actual: `""` -- Duration: 647ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:43 PM - -### enum_structure - google/gemini-2.0-flash-exp:free -- Prompt: `return status success with message "Operation completed"` -- Expected: `{"status":"success","message":"Operation completed"}` -- Actual: `""` -- Duration: 813ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:43 PM - -### enum_structure - gpt-4 -- Prompt: `return status success with message "Operation completed"` -- Expected: `{"status":"success","message":"Operation completed"}` -- Actual: `""` -- Duration: 1138ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:45 PM - -### enum_structure - anthropic/claude-3.7-sonnet -- Prompt: `return status success with message "Operation completed"` -- Expected: `{"status":"success","message":"Operation completed"}` -- Actual: `""` -- Duration: 1728ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: Unexpected token '`', "```json -{ -"... is not valid JSON -- Reason: Failed to parse or validate response: Unexpected token '`', "```json -{ -"... is not valid JSON -- Timestamp: 4/1/2025, 1:23:09 PM - -### enum_structure - openai/gpt-4 -- Prompt: `Return a JSON object with status "success" and message "Operation completed". The response must be valid JSON with this structure: { "status": "success" | "error" | "pending", "message": string }` -- Expected: `{"status":"success","message":"Operation completed"}` -- Actual: `""` -- Duration: 688ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Invalid response from API -- Reason: Invalid response from API -- Timestamp: 4/1/2025, 1:32:45 PM - -### optional_fields - deepseek/deepseek-chat:free -- Prompt: `return name "John" with age 30 and email "john@example.com"` -- Expected: `{"name":"John","age":30,"email":"john@example.com"}` -- Actual: `""` -- Duration: 676ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:45 PM - -### optional_fields - google/gemini-2.0-flash-exp:free -- Prompt: `return name "John" with age 30 and email "john@example.com"` -- Expected: `{"name":"John","age":30,"email":"john@example.com"}` -- Actual: `""` -- Duration: 884ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:46 PM - -### optional_fields - gpt-4 -- Prompt: `return name "John" with age 30 and email "john@example.com"` -- Expected: `{"name":"John","age":30,"email":"john@example.com"}` -- Actual: `""` -- Duration: 669ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Reason: Failed to parse or validate response: [ - { - "code": "invalid_type", - "expected": "object", - "received": "null", - "path": [], - "message": "Expected object, received null" - } -] -- Timestamp: 4/1/2025, 1:21:47 PM - -### optional_fields - anthropic/claude-3.7-sonnet -- Prompt: `return name "John" with age 30 and email "john@example.com"` -- Expected: `{"name":"John","age":30,"email":"john@example.com"}` -- Actual: `""` -- Duration: 1576ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Failed to parse or validate response: Unexpected token '`', "```json -{ -"... is not valid JSON -- Reason: Failed to parse or validate response: Unexpected token '`', "```json -{ -"... is not valid JSON -- Timestamp: 4/1/2025, 1:23:11 PM - -### optional_fields - openai/gpt-4 -- Prompt: `Return a JSON object with name "John", age 30, and email "john@example.com". The response must be valid JSON with this structure: { "name": string, "age"?: number, "email"?: string }` -- Expected: `{"name":"John","age":30,"email":"john@example.com"}` -- Actual: `""` -- Duration: 682ms -- Error Type: Error -- Error Code: UNKNOWN -- Error Message: Invalid response from API -- Reason: Invalid response from API -- Timestamp: 4/1/2025, 1:32:46 PM +- Error Message: expected '786984' to deeply equal 'Invalid format option' +- Error Details: expected '786984' to deeply equal 'Invalid format option' +- Reason: expected '786984' to deeply equal 'Invalid format option' +- Timestamp: 4/2/2025, 12:32:24 AM ## Passed Tests -*No passed tests* +### json-schema-file-format - gpt-4o +- Prompt: `Create a user profile with name John Doe, age 30, and tags ["developer", "javascript"]. Return only the JSON object, no explanation.` +- Expected: `{"name":"John Doe","age":30,"tags":["developer","javascript"]}` +- Actual: ````json +{ + "name": "John Doe", + "age": 30, + "tags": ["developer", "javascript"] +} +```` +- Duration: 995ms +- Timestamp: 4/2/2025, 12:34:28 AM + +## Summary + +- Total Tests: 38 +- Passed: 5 +- Failed: 33 +- Success Rate: 13.16% diff --git a/packages/kbot/tests/unit/format.json b/packages/kbot/tests/unit/format.json index bea5d385..b444081e 100644 --- a/packages/kbot/tests/unit/format.json +++ b/packages/kbot/tests/unit/format.json @@ -1,2241 +1,995 @@ [ { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "deepseek/deepseek-chat:free", + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "anthropic/claude-3.5-sonnet", "router": "openrouter", - "timestamp": "2025-04-01T11:20:45.784Z", + "timestamp": "2025-04-01T22:23:43.742Z", "passed": false, - "duration": 1077, + "duration": 1690, "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", + "actual": "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}", + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" }, { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "google/gemini-2.0-flash-exp:free", + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "qwen/qwq-32b", "router": "openrouter", - "timestamp": "2025-04-01T11:20:46.549Z", + "timestamp": "2025-04-01T22:23:47.173Z", "passed": false, - "duration": 762, + "duration": 3426, "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "message": "expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", + "actual": "\"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}", + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\"name\":\u001b[27m\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ \"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:138:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected '\"John Doe\",\\n \"age\": 30,\\n \"tags\": …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" }, { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", + "test": "json-schema-object-format", + "prompt": "Create a user profile with name Jane Smith, age 25, and tags [\"designer\", \"ui\"]. Return only the JSON object, no explanation.", "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "gpt-4", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", + "model": "anthropic/claude-3.5-sonnet", "router": "openrouter", - "timestamp": "2025-04-01T11:20:47.696Z", + "timestamp": "2025-04-01T22:23:49.093Z", "passed": false, - "duration": 1146, + "duration": 1918, "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", + "actual": "{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"tags\": [\"designer\", \"ui\"]\n}", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"designer\",\u001b[7m \u001b[27m\"ui\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } + "reason": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'" }, { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", + "test": "json-schema-object-format", + "prompt": "Create a user profile with name Jane Smith, age 25, and tags [\"designer\", \"ui\"]. Return only the JSON object, no explanation.", "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "deepseek/deepseek-chat:free", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", + "model": "qwen/qwq-32b", "router": "openrouter", - "timestamp": "2025-04-01T11:20:48.368Z", + "timestamp": "2025-04-01T22:23:58.884Z", "passed": false, - "duration": 671, + "duration": 9789, "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "message": "expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", + "actual": "{\"name\": \"Jane Smith\", \"age\": 25, \"tags\": [\"designer\", \"ui\"]}", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}\"\u001b[39m\nReceived: \u001b[31m\"{\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"designer\",\u001b[7m \u001b[27m\"ui\"]}\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:158:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected '{\"name\": \"Jane Smith\", \"age\": 25, \"ta…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'" }, { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", + "test": "zod-string-format", + "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "google/gemini-2.0-flash-exp:free", + "expected": "john.doe@company.com", + "model": "anthropic/claude-3.5-sonnet", "router": "openrouter", - "timestamp": "2025-04-01T11:20:49.160Z", + "timestamp": "2025-04-01T22:24:00.233Z", "passed": false, - "duration": 790, + "duration": 1347, "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "message": "expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'", + "actual": "sales@companyplus.com", + "expected": "john.doe@company.com", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:174:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"\u001b[7mjohn.doe\u001b[27m@company.com\"\u001b[39m\nReceived: \u001b[31m\"\u001b[7msales\u001b[27m@company\u001b[7mplus\u001b[27m.com\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:174:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected 'sales@companyplus.com' to deeply equal 'john.doe@company.com'" }, { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", + "test": "zod-number-format", + "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "gpt-4", + "expected": "25", + "model": "anthropic/claude-3.5-sonnet", "router": "openrouter", - "timestamp": "2025-04-01T11:20:50.164Z", + "timestamp": "2025-04-01T22:24:11.615Z", "passed": false, - "duration": 1003, + "duration": 1376, "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "message": "expected '42' to deeply equal '25'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected '42' to deeply equal '25'", + "actual": "42", + "expected": "25", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '42' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:190:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"2\u001b[7m5\u001b[27m\"\u001b[39m\nReceived: \u001b[31m\"\u001b[7m4\u001b[27m2\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '42' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:190:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } + "reason": "expected '42' to deeply equal '25'" }, { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "test": "zod-string-format", + "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "deepseek/deepseek-chat:free", + "expected": "john.doe@company.com", + "model": "qwen/qwq-32b", "router": "openrouter", - "timestamp": "2025-04-01T11:20:50.812Z", + "timestamp": "2025-04-01T22:24:13.939Z", "passed": false, - "duration": 647, + "duration": 13704, "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", + "message": "expected 'info@techstart.com' to deeply equal 'john.doe@company.com'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected 'info@techstart.com' to deeply equal 'john.doe@company.com'", + "actual": "info@techstart.com", + "expected": "john.doe@company.com", + "showDiff": true, + "operator": "strictEqual", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected 'info@techstart.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:174:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5" } }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected 'info@techstart.com' to deeply equal 'john.doe@company.com'" }, { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "test": "zod-array-format", + "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "google/gemini-2.0-flash-exp:free", + "expected": "[\"JavaScript\",\"Python\",\"Java\"]", + "model": "anthropic/claude-3.5-sonnet", "router": "openrouter", - "timestamp": "2025-04-01T11:20:51.561Z", - "passed": false, - "duration": 748, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", - "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:20:52.632Z", - "passed": false, - "duration": 1070, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:20:53.299Z", - "passed": false, - "duration": 665, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:20:54.020Z", - "passed": false, - "duration": 720, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:20:55.153Z", - "passed": false, - "duration": 1132, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:20:55.791Z", - "passed": false, - "duration": 637, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:20:56.768Z", - "passed": false, - "duration": 975, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:20:57.823Z", - "passed": false, - "duration": 1053, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", - "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:36.069Z", - "passed": false, - "duration": 885, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", - "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:36.829Z", - "passed": false, - "duration": 757, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", - "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:37.874Z", - "passed": false, - "duration": 1043, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", - "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:38.531Z", - "passed": false, - "duration": 655, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", - "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:39.322Z", - "passed": false, - "duration": 790, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", - "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:40.040Z", - "passed": false, - "duration": 717, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", - "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:40.659Z", - "passed": false, - "duration": 617, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", - "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:41.416Z", - "passed": false, - "duration": 756, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", - "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:42.443Z", - "passed": false, - "duration": 1026, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:43.092Z", - "passed": false, - "duration": 647, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:43.906Z", - "passed": false, - "duration": 813, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:45.045Z", - "passed": false, - "duration": 1138, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "deepseek/deepseek-chat:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:45.723Z", - "passed": false, - "duration": 676, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "google/gemini-2.0-flash-exp:free", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:46.609Z", - "passed": false, - "duration": 884, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:21:47.279Z", - "passed": false, - "duration": 669, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openai", - "apiKey": "" - } - }, - { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", - "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "anthropic/claude-3.7-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T11:23:05.626Z", - "passed": false, - "duration": 1790, - "error": { - "message": "Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON", - "stackStr": "Error: Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:127:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: Unexpected token 'h', \"hello 42\" is not valid JSON", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", - "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "anthropic/claude-3.7-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T11:23:06.818Z", - "passed": false, - "duration": 1189, - "error": { - "message": "Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON", - "stackStr": "Error: Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:148:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: Unexpected token '#', \"# John's U\"... is not valid JSON", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", - "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "anthropic/claude-3.7-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T11:23:08.010Z", - "passed": false, - "duration": 1190, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:168:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "anthropic/claude-3.7-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T11:23:09.740Z", - "passed": false, - "duration": 1728, - "error": { - "message": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", - "stackStr": "Error: Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:188:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "anthropic/claude-3.7-sonnet", - "router": "openrouter", - "timestamp": "2025-04-01T11:23:11.318Z", - "passed": false, - "duration": 1576, - "error": { - "message": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:67:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:207:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON" - } - }, - "reason": "Failed to parse or validate response: Unexpected token '`', \"```json\n{\n\"... is not valid JSON", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", - "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:26:47.671Z", - "passed": false, - "duration": 1483, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", - "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:26:48.821Z", - "passed": false, - "duration": 1146, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", - "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:26:49.824Z", - "passed": false, - "duration": 1001, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:26:50.835Z", - "passed": false, - "duration": 1008, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:26:51.878Z", - "passed": false, - "duration": 1041, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", - "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:28:21.136Z", - "passed": false, - "duration": 1405, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", - "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:28:21.846Z", - "passed": false, - "duration": 706, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", - "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:28:22.517Z", - "passed": false, - "duration": 669, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:28:23.187Z", - "passed": false, - "duration": 668, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "stackStr": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:28:23.848Z", - "passed": false, - "duration": 659, - "error": { - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:68:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", - "message": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]" - } - }, - "reason": "Failed to parse or validate response: [\n {\n \"code\": \"invalid_type\",\n \"expected\": \"object\",\n \"received\": \"null\",\n \"path\": [],\n \"message\": \"Expected object, received null\"\n }\n]", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", - "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:29:01.068Z", - "passed": false, - "duration": 1322, - "error": { - "message": "Cannot read properties of undefined (reading '0')", - "code": "UNKNOWN", - "type": "TypeError", - "details": { - "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Cannot read properties of undefined (reading '0')", - "stackStr": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "nameStr": "TypeError", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Cannot read properties of undefined (reading '0')", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", - "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:29:01.939Z", - "passed": false, - "duration": 868, - "error": { - "message": "Cannot read properties of undefined (reading '0')", - "code": "UNKNOWN", - "type": "TypeError", - "details": { - "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Cannot read properties of undefined (reading '0')", - "stackStr": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "nameStr": "TypeError", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Cannot read properties of undefined (reading '0')", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", - "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:29:02.996Z", - "passed": false, - "duration": 1055, - "error": { - "message": "Cannot read properties of undefined (reading '0')", - "code": "UNKNOWN", - "type": "TypeError", - "details": { - "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Cannot read properties of undefined (reading '0')", - "stackStr": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "nameStr": "TypeError", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Cannot read properties of undefined (reading '0')", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", - "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:29:05.060Z", - "passed": false, - "duration": 2062, - "error": { - "message": "Cannot read properties of undefined (reading '0')", - "code": "UNKNOWN", - "type": "TypeError", - "details": { - "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Cannot read properties of undefined (reading '0')", - "stackStr": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "nameStr": "TypeError", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Cannot read properties of undefined (reading '0')", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", - "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:29:05.831Z", - "passed": false, - "duration": 769, - "error": { - "message": "Cannot read properties of undefined (reading '0')", - "code": "UNKNOWN", - "type": "TypeError", - "details": { - "stack": "TypeError: Cannot read properties of undefined (reading '0')\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:59:27)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:94:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:215:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Cannot read properties of undefined (reading '0')" - } - }, - "reason": "Cannot read properties of undefined (reading '0')", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", - "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:30:38.831Z", - "passed": false, - "duration": 1410, - "error": { - "message": "Invalid response from OpenAI API", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from OpenAI API", - "stackStr": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Invalid response from OpenAI API", - "config": { - "router": "openrouter", - "apiKey": "" - } - }, - { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", - "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:30:39.844Z", + "timestamp": "2025-04-01T22:24:22.636Z", "passed": false, "duration": 1009, "error": { - "message": "Invalid response from OpenAI API", + "message": "expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", + "actual": "[\"Python\", \"Java\", \"JavaScript\"]", + "expected": "[\"JavaScript\",\"Python\",\"Java\"]", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"[\"\u001b[7mJavaScript\",\"Python\u001b[27m\",\"Java\"]\"\u001b[39m\nReceived: \u001b[31m\"[\"\u001b[7mPython\", \"Java\u001b[27m\",\u001b[7m \u001b[27m\"Java\u001b[7mScript\u001b[27m\"]\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '[\"Python\", \"Java\", \"JavaScript\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'" + }, + { + "test": "zod-array-format", + "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", + "result": [], + "expected": "[\"JavaScript\",\"Python\",\"Java\"]", + "model": "qwen/qwq-32b", + "router": "openrouter", + "timestamp": "2025-04-01T22:24:26.785Z", + "passed": false, + "duration": 4147, + "error": { + "message": "expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", + "actual": "[\"Python\", \"JavaScript\", \"Java\"]", + "expected": "[\"JavaScript\",\"Python\",\"Java\"]", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"[\"JavaScript\u001b[7m\",\"Python\u001b[27m\",\"Java\"]\"\u001b[39m\nReceived: \u001b[31m\"[\"\u001b[7mPython\", \"\u001b[27mJavaScript\",\u001b[7m \u001b[27m\"Java\"]\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:206:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '[\"Python\", \"JavaScript\", \"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'" + }, + { + "test": "invalid-format", + "prompt": "Generate a random number.", + "result": [], + "expected": "Invalid format option", + "model": "anthropic/claude-3.5-sonnet", + "router": "openrouter", + "timestamp": "2025-04-01T22:24:27.813Z", + "passed": false, + "duration": 1026, + "error": { + "message": "expected '73' to deeply equal 'Invalid format option'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '73' to deeply equal 'Invalid format option'", + "actual": "73", + "expected": "Invalid format option", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '73' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:222:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"Invalid format option\"\u001b[39m\nReceived: \u001b[31m\"73\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '73' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:222:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '73' to deeply equal 'Invalid format option'" + }, + { + "test": "invalid-format", + "prompt": "Generate a random number.", + "result": [], + "expected": "Invalid format option", + "model": "qwen/qwq-32b", + "router": "openrouter", + "timestamp": "2025-04-01T22:24:35.428Z", + "passed": false, + "duration": 7614, + "error": { + "message": "expected '72' to deeply equal 'Invalid format option'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '72' to deeply equal 'Invalid format option'", + "actual": "72", + "expected": "Invalid format option", + "showDiff": true, + "operator": "strictEqual", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '72' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:91:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.each.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:222:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '72' to deeply equal 'Invalid format option'" + }, + { + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", + "result": [], + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:28:54.288Z", + "passed": false, + "duration": 1146, + "error": { + "message": "expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", + "actual": "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```", + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:140:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]\u001b[7m}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:140:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '```json\\n{\\n \"name\": \"John Doe\",\\n …' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" + }, + { + "test": "json-schema-object-format", + "prompt": "Create a user profile with name Jane Smith, age 25, and tags [\"designer\", \"ui\"]. Return only the JSON object, no explanation.", + "result": [], + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:28:55.427Z", + "passed": false, + "duration": 1134, + "error": { + "message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'", + "actual": "{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"tags\": [\"designer\", \"ui\"]\n}", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:160:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"Jane Smith\",\"age\":25,\"tags\":[\"designer\",\"ui\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"designer\",\u001b[7m \u001b[27m\"ui\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:160:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '{\\n \"name\": \"Jane Smith\",\\n \"age\": …' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"tags\":…'" + }, + { + "test": "zod-string-format", + "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", + "result": [], + "expected": "john.doe@company.com", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:28:55.754Z", + "passed": false, + "duration": 326, + "error": { + "message": "Model returned empty response", "code": "UNKNOWN", "type": "Error", "details": { - "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from OpenAI API", - "stackStr": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:176:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:176:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", "nameStr": "Error", "expected": "undefined", "actual": "undefined" } }, - "reason": "Invalid response from OpenAI API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "Model returned empty response" }, { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "test": "zod-number-format", + "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:30:40.923Z", + "expected": "25", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:28:56.051Z", "passed": false, - "duration": 1076, + "duration": 295, "error": { - "message": "Invalid response from OpenAI API", + "message": "Model returned empty response", "code": "UNKNOWN", "type": "Error", "details": { - "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from OpenAI API", - "stackStr": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:192:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:192:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", "nameStr": "Error", "expected": "undefined", "actual": "undefined" } }, - "reason": "Invalid response from OpenAI API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "Model returned empty response" }, { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", + "test": "zod-array-format", + "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:30:42.016Z", + "expected": "[\"JavaScript\",\"Python\",\"Java\"]", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:28:56.385Z", "passed": false, - "duration": 1091, + "duration": 332, "error": { - "message": "Invalid response from OpenAI API", + "message": "Model returned empty response", "code": "UNKNOWN", "type": "Error", "details": { - "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from OpenAI API", - "stackStr": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:87:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", "nameStr": "Error", "expected": "undefined", "actual": "undefined" } }, - "reason": "Invalid response from OpenAI API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "Model returned empty response" }, { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "test": "invalid-format", + "prompt": "Generate a random number.", "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:30:42.687Z", + "expected": "Invalid format option", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:28:56.983Z", "passed": false, - "duration": 668, + "duration": 596, "error": { - "message": "Invalid response from OpenAI API", + "message": "expected '83' to deeply equal 'Invalid format option'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Invalid response from OpenAI API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:52:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from OpenAI API" + "message": "expected '83' to deeply equal 'Invalid format option'", + "actual": "83", + "expected": "Invalid format option", + "showDiff": true, + "operator": "strictEqual", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '83' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:93:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:224:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Invalid response from OpenAI API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected '83' to deeply equal 'Invalid format option'" }, { - "test": "basic_structure", - "prompt": "return a greeting \"hello\" with count 42", + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:31:51.336Z", + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:07.138Z", "passed": false, - "duration": 1382, + "duration": 980, "error": { - "message": "Invalid response from API", + "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", + "actual": "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}", + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" + }, + { + "test": "json-schema-object-format", + "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", + "result": [], + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:08.771Z", + "passed": false, + "duration": 1628, + "error": { + "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", + "actual": "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": \"123 Main St, New York, US, 10001\",\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": true,\n \"language\": \"English\"\n }\n}\n```", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\u001b[7m{\"street\":\u001b[27m\"123 Main St\u001b[7m\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"\u001b[27m10001\"\u001b[7m}\u001b[27m,\"preferences\":\u001b[7m{\u001b[27m\"theme\":\"light\",\"notifications\":true,\"language\":\u001b[7m\"en\"}}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"email\":\u001b[7m \u001b[27m\"jane.smith@company.com\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"designer\"],\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"address\":\u001b[7m \u001b[27m\"123 Main St\u001b[7m, New York, US, \u001b[27m10001\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"preferences\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"theme\":\u001b[7m \u001b[27m\"light\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"notifications\":\u001b[7m \u001b[27mtrue,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"language\":\u001b[7m \"English\"\u001b[27m\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '```json\\n{\\n \"name\": \"Jane Smith\",\\n…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" + }, + { + "test": "zod-string-format", + "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", + "result": [], + "expected": "john.doe@company.com", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:09.057Z", + "passed": false, + "duration": 282, + "error": { + "message": "Model returned empty response", "code": "UNKNOWN", "type": "Error", "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API", - "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", "nameStr": "Error", "expected": "undefined", "actual": "undefined" } }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "Model returned empty response" }, { - "test": "nested_structure", - "prompt": "return user John age 30 with dark theme and notifications enabled", + "test": "zod-number-format", + "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:31:52.354Z", + "expected": "25", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:09.356Z", "passed": false, - "duration": 1014, + "duration": 297, "error": { - "message": "Invalid response from API", + "message": "Model returned empty response", "code": "UNKNOWN", "type": "Error", "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API", - "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:275:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response" + } + }, + "reason": "Model returned empty response" + }, + { + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", + "result": [], + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:31.573Z", + "passed": false, + "duration": 1074, + "error": { + "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'", + "actual": "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}", + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- {\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"John Doe\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m30,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"javascript\"]\u001b[39m\n\u001b[31m+ }\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:203:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '{\\n \"name\": \"John Doe\",\\n \"age\": 30…' to deeply equal '{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"…'" + }, + { + "test": "json-schema-object-format", + "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", + "result": [], + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:33.577Z", + "passed": false, + "duration": 1999, + "error": { + "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", + "actual": "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": {\n \"street\": \"123 Main St\",\n \"city\": \"New York\",\n \"country\": \"US\",\n \"zipcode\": \"10001\"\n },\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": \"enabled\",\n \"language\": \"English\"\n }\n}\n```", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\u001b[7m{\u001b[27m\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zip\u001b[7mC\u001b[27mode\":\"10001\"\u001b[7m},\u001b[27m\"preferences\":\u001b[7m{\u001b[27m\"theme\":\"light\",\"notifications\":\u001b[7mtrue,\"language\":\"en\"}}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"email\":\u001b[7m \u001b[27m\"jane.smith@company.com\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"designer\"],\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"address\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"street\":\u001b[7m \u001b[27m\"123 Main St\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"city\":\u001b[7m \u001b[27m\"New York\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"country\":\u001b[7m \u001b[27m\"US\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"zip\u001b[7mc\u001b[27mode\":\u001b[7m \u001b[27m\"10001\"\u001b[39m\n\u001b[31m+ },\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"preferences\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"theme\":\u001b[7m \u001b[27m\"light\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"notifications\":\u001b[7m \"enabled\",\u001b[27m\u001b[39m\n\u001b[31m+ \"language\": \"English\"\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:243:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" + }, + { + "test": "zod-string-format", + "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", + "result": [], + "expected": "john.doe@company.com", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:33.906Z", + "passed": false, + "duration": 324, + "error": { + "message": "Model returned empty response", + "code": "UNKNOWN", + "type": "Error", + "details": { + "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:259:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", "nameStr": "Error", "expected": "undefined", "actual": "undefined" } }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "Model returned empty response" }, { - "test": "array_structure", - "prompt": "return a list of 2 items with ids 1 and 2, names \"first\" and \"second\"", + "test": "zod-number-format", + "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:31:53.055Z", + "expected": "25", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:34.272Z", "passed": false, - "duration": 699, + "duration": 363, "error": { - "message": "Invalid response from API", + "message": "Model returned empty response", "code": "UNKNOWN", "type": "Error", "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API", - "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:275:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:275:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", "nameStr": "Error", "expected": "undefined", "actual": "undefined" } }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "Model returned empty response" }, { - "test": "enum_structure", - "prompt": "return status success with message \"Operation completed\"", + "test": "zod-array-format", + "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:31:55.145Z", + "expected": "[\"JavaScript\",\"Python\",\"Java\"]", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:34.613Z", "passed": false, - "duration": 2088, + "duration": 339, "error": { - "message": "Invalid response from API", + "message": "Model returned empty response", "code": "UNKNOWN", "type": "Error", "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API", - "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", + "stack": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:291:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "message": "Model returned empty response", + "stackStr": "Error: Model returned empty response\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:150:15)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:291:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", "nameStr": "Error", "expected": "undefined", "actual": "undefined" } }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "Model returned empty response" }, { - "test": "optional_fields", - "prompt": "return name \"John\" with age 30 and email \"john@example.com\"", + "test": "invalid-format", + "prompt": "Generate a random number.", "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:31:55.863Z", + "expected": "Invalid format option", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:31:35.189Z", "passed": false, - "duration": 715, + "duration": 574, "error": { - "message": "Invalid response from API", + "message": "expected '847293' to deeply equal 'Invalid format option'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API" + "message": "expected '847293' to deeply equal 'Invalid format option'", + "actual": "847293", + "expected": "Invalid format option", + "showDiff": true, + "operator": "strictEqual", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '847293' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:156:22)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:307:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected '847293' to deeply equal 'Invalid format option'" }, { - "test": "basic_structure", - "prompt": "Return a JSON object with a greeting \"hello\" and count 42. The response must be valid JSON with exactly these fields: { \"greeting\": string, \"count\": number }", - "result": [], - "expected": { - "greeting": "hello", - "count": 42 - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:32:43.353Z", - "passed": false, - "duration": 1258, - "error": { - "message": "Invalid response from API", - "code": "UNKNOWN", - "type": "Error", - "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API", - "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:128:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" - } - }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", + "result": [ + "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}" + ], + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:32:12.404Z", + "passed": true, + "duration": 1098 }, { - "test": "nested_structure", - "prompt": "Return a JSON object with user John age 30, dark theme and notifications enabled. The response must be valid JSON with this structure: { \"user\": { \"name\": string, \"age\": number }, \"settings\": { \"theme\": string, \"notifications\": boolean } }", + "test": "json-schema-object-format", + "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", "result": [], - "expected": { - "user": { - "name": "John", - "age": 30 - }, - "settings": { - "theme": "dark", - "notifications": true - } - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:32:44.073Z", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:32:19.059Z", "passed": false, - "duration": 716, + "duration": 6653, "error": { - "message": "Invalid response from API", + "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API", - "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:149:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", + "actual": "```json\n{\n \"name\": \"Jane Smith\",\n \"age\": 25,\n \"email\": \"jane.smith@company.com\",\n \"tags\": [\"developer\", \"designer\"],\n \"address\": {\n \"street\": \"123 Main St\",\n \"city\": \"New York\",\n \"country\": \"US\",\n \"postal_code\": \"10001\"\n },\n \"preferences\": {\n \"theme\": \"light\",\n \"notifications\": true,\n \"language\": \"English\"\n }\n}\n```", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:254:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "\u001b[32m- Expected\u001b[39m\n\u001b[31m+ Received\u001b[39m\n\n\u001b[32m- \u001b[7m{\u001b[27m\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":\u001b[7m{\u001b[27m\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\u001b[7m\"zipC\u001b[27mode\":\"10001\"\u001b[7m},\u001b[27m\"preferences\":\u001b[7m{\u001b[27m\"theme\":\"light\",\"notifications\":true,\"language\":\u001b[7m\"en\"}}\u001b[27m\u001b[39m\n\u001b[31m+ ```json\u001b[39m\n\u001b[31m+ {\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"name\":\u001b[7m \u001b[27m\"Jane Smith\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"age\":\u001b[7m \u001b[27m25,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"email\":\u001b[7m \u001b[27m\"jane.smith@company.com\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"tags\":\u001b[7m \u001b[27m[\"developer\",\u001b[7m \u001b[27m\"designer\"],\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"address\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"street\":\u001b[7m \u001b[27m\"123 Main St\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"city\":\u001b[7m \u001b[27m\"New York\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"country\":\u001b[7m \u001b[27m\"US\",\u001b[39m\n\u001b[31m+ \u001b[7m \"postal_c\u001b[27mode\":\u001b[7m \u001b[27m\"10001\"\u001b[39m\n\u001b[31m+ },\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"preferences\":\u001b[7m {\u001b[27m\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"theme\":\u001b[7m \u001b[27m\"light\",\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"notifications\":\u001b[7m \u001b[27mtrue,\u001b[39m\n\u001b[31m+ \u001b[7m \u001b[27m\"language\":\u001b[7m \"English\"\u001b[27m\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ }\u001b[39m\n\u001b[31m+ ```\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:254:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected '```json\\n{\\n \"name\": \"Jane Smith\",…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" }, { - "test": "array_structure", - "prompt": "Return a JSON object with a list of 2 items. The response must be valid JSON with this structure: { \"items\": [{ \"id\": number, \"name\": string }] }. The first item should have id 1 and name \"first\", the second item should have id 2 and name \"second\".", + "test": "zod-string-format", + "prompt": "Generate a valid email address for a business domain. Return only the email, no explanation.", "result": [], - "expected": { - "items": [ - { - "id": 1, - "name": "first" - }, - { - "id": 2, - "name": "second" - } - ] - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:32:44.778Z", + "expected": "john.doe@company.com", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:32:20.861Z", "passed": false, - "duration": 703, + "duration": 1794, "error": { - "message": "Invalid response from API", + "message": "expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API", - "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:169:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'", + "actual": "contact@businessdomain.com", + "expected": "john.doe@company.com", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:270:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"\u001b[7mjohn.doe@company\u001b[27m.com\"\u001b[39m\nReceived: \u001b[31m\"\u001b[7mcontact@businessdomain\u001b[27m.com\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:270:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected 'contact@businessdomain.com' to deeply equal 'john.doe@company.com'" }, { - "test": "enum_structure", - "prompt": "Return a JSON object with status \"success\" and message \"Operation completed\". The response must be valid JSON with this structure: { \"status\": \"success\" | \"error\" | \"pending\", \"message\": string }", + "test": "zod-number-format", + "prompt": "Generate a random age between 18 and 65. Return only the number, no explanation.", "result": [], - "expected": { - "status": "success", - "message": "Operation completed" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:32:45.468Z", + "expected": "25", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:32:23.263Z", "passed": false, - "duration": 688, + "duration": 2399, "error": { - "message": "Invalid response from API", + "message": "expected '39' to deeply equal '25'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API", - "stackStr": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:189:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "nameStr": "Error", - "expected": "undefined", - "actual": "undefined" + "message": "expected '39' to deeply equal '25'", + "actual": "39", + "expected": "25", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '39' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:295:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"25\"\u001b[39m\nReceived: \u001b[31m\"39\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '39' to deeply equal '25'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:295:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected '39' to deeply equal '25'" }, { - "test": "optional_fields", - "prompt": "Return a JSON object with name \"John\", age 30, and email \"john@example.com\". The response must be valid JSON with this structure: { \"name\": string, \"age\"?: number, \"email\"?: string }", + "test": "zod-array-format", + "prompt": "Generate a list of 3 programming languages. Return only the array, no explanation.", "result": [], - "expected": { - "name": "John", - "age": 30, - "email": "john@example.com" - }, - "model": "openai/gpt-4", - "router": "openrouter", - "timestamp": "2025-04-01T11:32:46.153Z", + "expected": "[\"JavaScript\",\"Python\",\"Java\"]", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:32:23.958Z", "passed": false, - "duration": 682, + "duration": 693, "error": { - "message": "Invalid response from API", + "message": "expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", "code": "UNKNOWN", - "type": "Error", + "type": "AssertionError", "details": { - "stack": "Error: Invalid response from API\n at Module.runCompletion (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run-completion.ts:54:11)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at processRun (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:104:15)\n at Module.run (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\src\\commands\\run.ts:225:14)\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:40:22)\n at C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:208:5\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)", - "message": "Invalid response from API" + "message": "expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'", + "actual": "[\"Python\",\"JavaScript\",\"Java\"]", + "expected": "[\"JavaScript\",\"Python\",\"Java\"]", + "showDiff": true, + "operator": "strictEqual", + "stackStr": "AssertionError: expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:321:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11", + "nameStr": "AssertionError", + "diff": "Expected: \u001b[32m\"[\"JavaScript\u001b[7m\",\"Python\u001b[27m\",\"Java\"]\"\u001b[39m\nReceived: \u001b[31m\"[\"\u001b[7mPython\",\"\u001b[27mJavaScript\",\"Java\"]\"\u001b[39m", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:321:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" } }, - "reason": "Invalid response from API", - "config": { - "router": "openrouter", - "apiKey": "" - } + "reason": "expected '[\"Python\",\"JavaScript\",\"Java\"]' to deeply equal '[\"JavaScript\",\"Python\",\"Java\"]'" + }, + { + "test": "invalid-format", + "prompt": "Generate a random number.", + "result": [], + "expected": "Invalid format option", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:32:24.787Z", + "passed": false, + "duration": 826, + "error": { + "message": "expected '786984' to deeply equal 'Invalid format option'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '786984' to deeply equal 'Invalid format option'", + "actual": "786984", + "expected": "Invalid format option", + "showDiff": true, + "operator": "strictEqual", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '786984' to deeply equal 'Invalid format option'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:167:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:350:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '786984' to deeply equal 'Invalid format option'" + }, + { + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", + "result": [ + "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" + ], + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:33:05.518Z", + "passed": true, + "duration": 1027 + }, + { + "test": "json-schema-object-format", + "prompt": "Create a user profile with the following details:\n - Name: Jane Smith\n - Age: 25\n - Email: jane.smith@company.com\n - Tags: [\"developer\", \"designer\"]\n - Address: 123 Main St, New York, US, 10001\n - Preferences: light theme, notifications enabled, English language\n Return only the JSON object, no explanation.", + "result": [], + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:33:08.138Z", + "passed": false, + "duration": 2618, + "error": { + "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", + "code": "UNKNOWN", + "type": "AssertionError", + "details": { + "message": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'", + "actual": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"postal_code\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":\"enabled\",\"language\":\"English\"}}", + "expected": "{\"name\":\"Jane Smith\",\"age\":25,\"email\":\"jane.smith@company.com\",\"tags\":[\"developer\",\"designer\"],\"address\":{\"street\":\"123 Main St\",\"city\":\"New York\",\"country\":\"US\",\"zipCode\":\"10001\"},\"preferences\":{\"theme\":\"light\",\"notifications\":true,\"language\":\"en\"}}", + "showDiff": true, + "operator": "strictEqual", + "name": "AssertionError", + "ok": false, + "stack": "AssertionError: expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'\n at runFormatTest (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:184:32)\n at processTicksAndRejections (node:internal/process/task_queues:105:5)\n at __vite_ssr_import_0__.it.timeout (C:\\Users\\zx\\Desktop\\polymech\\polymech-mono\\packages\\kbot\\tests\\unit\\format.test.ts:271:5)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:533:5\n at runTest (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1056:11)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runSuite (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1205:15)\n at runFiles (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1262:5)\n at startTests (file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/@vitest/runner/dist/index.js:1271:3)\n at file:///C:/Users/zx/Desktop/polymech/polymech-mono/packages/kbot/node_modules/vitest/dist/chunks/runBaseTests.3qpJUEJM.js:126:11" + } + }, + "reason": "expected '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…' to deeply equal '{\"name\":\"Jane Smith\",\"age\":25,\"email\"…'" + }, + { + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", + "result": [ + "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}" + ], + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:33:41.869Z", + "passed": true, + "duration": 2353 + }, + { + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", + "result": [ + "{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}" + ], + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:34:16.181Z", + "passed": true, + "duration": 1098 + }, + { + "test": "json-schema-file-format", + "prompt": "Create a user profile with name John Doe, age 30, and tags [\"developer\", \"javascript\"]. Return only the JSON object, no explanation.", + "result": [ + "```json\n{\n \"name\": \"John Doe\",\n \"age\": 30,\n \"tags\": [\"developer\", \"javascript\"]\n}\n```" + ], + "expected": "{\"name\":\"John Doe\",\"age\":30,\"tags\":[\"developer\",\"javascript\"]}", + "model": "gpt-4o", + "router": "openai", + "timestamp": "2025-04-01T22:34:28.660Z", + "passed": true, + "duration": 995 } ] \ No newline at end of file diff --git a/packages/kbot/tests/unit/format.test.ts b/packages/kbot/tests/unit/format.test.ts index 21bb9fc3..22294d09 100644 --- a/packages/kbot/tests/unit/format.test.ts +++ b/packages/kbot/tests/unit/format.test.ts @@ -1,295 +1,508 @@ -import { describe, it, expect } from 'vitest' -import { run } from '../../src/index' -import * as path from 'node:path' -import { sync as write } from "@polymech/fs/write" -import { sync as read } from "@polymech/fs/read" -import { sync as exists } from "@polymech/fs/exists" -import { z } from 'zod' - - -import { - models_premium as models, - TEST_BASE_PATH, - TEST_LOGS_PATH, - TEST_PREFERENCES_PATH, - TEST_TIMEOUT, - TestResult, - formatError, - isEmptyResponse, - getRouterForModel, - getApiKeyForRouter -} from './commons' - -const TEST_LOG_PATH = path.resolve(__dirname, './format.json') - -describe('Structured Output Format', () => { - let testResults: TestResult[] = [] - - // Load existing results if any - if (exists(TEST_LOG_PATH)) { - const data = read(TEST_LOG_PATH, 'json') - testResults = Array.isArray(data) ? data : [] - } - - const runFormatTest = async (prompt: string, format: z.ZodType, expected: any, testName: string, modelName: string) => { - let model = 'unknown' - let router = 'unknown' - let startTime = Date.now() - let error: TestResult['error'] | undefined - - try { - const result = await Promise.race([ - run({ - prompt, - mode: 'completion', - model: modelName, - path: TEST_BASE_PATH, - logs: TEST_LOGS_PATH, - preferences: TEST_PREFERENCES_PATH, - format, - onRun: async (options) => { - model = options.model || 'unknown' - router = options.router || 'unknown' - return options - } - }), - new Promise((_, reject) => - setTimeout(() => reject(new Error('API call timed out')), TEST_TIMEOUT) - ) - ]) as any[] - - const actual = result?.[0] - let parsed: any - - try { - parsed = typeof actual === 'string' ? JSON.parse(actual) : actual - // Validate against the format schema - parsed = format.parse(parsed) - } catch (parseError) { - throw new Error(`Failed to parse or validate response: ${parseError.message}`) - } - - const passed = JSON.stringify(parsed) === JSON.stringify(expected) && !isEmptyResponse(result) - - if (isEmptyResponse(result)) { - throw new Error('Model returned empty response') - } - - expect(parsed).toEqual(expected) - - return { - test: testName, - prompt, - result: result || [], - expected, - model, - router, - timestamp: new Date().toISOString(), - passed, - duration: Date.now() - startTime, - reason: passed ? undefined : `Expected ${JSON.stringify(expected)}, but got ${JSON.stringify(parsed)}`, - config: { - router: getRouterForModel(modelName) - } - } - } catch (e) { - error = formatError(e) - throw e - } finally { - const testResult: TestResult = { - test: testName, - prompt, - result: [], - expected, - model, - router, - timestamp: new Date().toISOString(), - passed: false, - duration: Date.now() - startTime, - error, - reason: error?.message || 'Unknown error occurred', - config: { - router: getRouterForModel(modelName), - apiKey: getApiKeyForRouter(getRouterForModel(modelName)) - } - } - - testResults.push(testResult) - write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) - } - } - - it.each(models)('should return basic structured output with model %s', async (modelName) => { - const format = z.object({ - greeting: z.string(), - count: z.number() - }) - - await runFormatTest( - 'return a greeting "hello" with count 42', - format, - { greeting: 'hello', count: 42 }, - 'basic_structure', - modelName - ) - }) - - it.each(models)('should handle nested structures with model %s', async (modelName) => { - const format = z.object({ - user: z.object({ - name: z.string(), - age: z.number() - }), - settings: z.object({ - theme: z.string(), - notifications: z.boolean() - }) - }) - - await runFormatTest( - 'return user John age 30 with dark theme and notifications enabled', - format, - { - user: { name: 'John', age: 30 }, - settings: { theme: 'dark', notifications: true } - }, - 'nested_structure', - modelName - ) - }) - - it.each(models)('should handle arrays with model %s', async (modelName) => { - const format = z.object({ - items: z.array(z.object({ - id: z.number(), - name: z.string() - })) - }) - - await runFormatTest( - 'return a list of 2 items with ids 1 and 2, names "first" and "second"', - format, - { - items: [ - { id: 1, name: 'first' }, - { id: 2, name: 'second' } - ] - }, - 'array_structure', - modelName - ) - }) - - it.each(models)('should handle enums with model %s', async (modelName) => { - const format = z.object({ - status: z.enum(['success', 'error', 'pending']), - message: z.string() - }) - - await runFormatTest( - 'return status success with message "Operation completed"', - format, - { - status: 'success', - message: 'Operation completed' - }, - 'enum_structure', - modelName - ) - }) - - it.each(models)('should handle optional fields with model %s', async (modelName) => { - const format = z.object({ - name: z.string(), - age: z.number().optional(), - email: z.string().email().optional() - }) - - await runFormatTest( - 'return name "John" with age 30 and email "john@example.com"', - format, - { - name: 'John', - age: 30, - email: 'john@example.com' - }, - 'optional_fields', - modelName - ) - }) - - it('should generate markdown report', () => { - // Group results by test and model - const latestResults = new Map>() - - // Get only the latest result for each test+model combination - testResults.forEach(result => { - if (!latestResults.has(result.test)) { - latestResults.set(result.test, new Map()) - } - const testMap = latestResults.get(result.test)! - const existingResult = testMap.get(result.model) - if (!existingResult || new Date(result.timestamp) > new Date(existingResult.timestamp)) { - testMap.set(result.model, result) - } - }) - - // Generate markdown report - let report = '# Format Test Results\n\n' - - // First list failed tests - report += '## Failed Tests\n\n' - let hasFailures = false - for (const [testName, modelResults] of latestResults) { - for (const [model, result] of modelResults) { - if (!result.passed) { - hasFailures = true - report += `### ${testName} - ${model}\n` - report += `- Prompt: \`${result.prompt}\`\n` - report += `- Expected: \`${JSON.stringify(result.expected)}\`\n` - report += `- Actual: \`${JSON.stringify(result.result[0] || '')}\`\n` - report += `- Duration: ${result.duration}ms\n` - if (result.error) { - report += `- Error Type: ${result.error.type}\n` - report += `- Error Code: ${result.error.code}\n` - report += `- Error Message: ${result.error.message}\n` - } - report += `- Reason: ${result.reason}\n` - report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` - } - } - } - - if (!hasFailures) { - report += '*No failed tests*\n\n' - } - - // Then list passed tests - report += '## Passed Tests\n\n' - let hasPassed = false - for (const [testName, modelResults] of latestResults) { - for (const [model, result] of modelResults) { - if (result.passed) { - hasPassed = true - report += `### ${testName} - ${model}\n` - report += `- Prompt: \`${result.prompt}\`\n` - report += `- Expected: \`${JSON.stringify(result.expected)}\`\n` - report += `- Actual: \`${JSON.stringify(result.result[0] || '')}\`\n` - report += `- Duration: ${result.duration}ms\n` - report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` - } - } - } - - if (!hasPassed) { - report += '*No passed tests*\n\n' - } - - // Write report to file - const reportPath = path.resolve(__dirname, './format-report.md') - write(reportPath, report) - - // Verify report was written - expect(exists(reportPath) === 'file').toBe(true) - }) +import { describe, it, expect } from 'vitest' +import { run } from '../../src/index' +import * as path from 'node:path' +import { sync as write } from "@polymech/fs/write" +import { sync as read } from "@polymech/fs/read" +import { sync as exists } from "@polymech/fs/exists" +import { z } from 'zod' +import { + models, + TEST_BASE_PATH, + TEST_LOGS_PATH, + TEST_PREFERENCES_PATH, + TEST_TIMEOUT, + TestResult, + formatError, + isEmptyResponse +} from './commons' + +const TEST_LOG_PATH = path.resolve(__dirname, './format.json') +const TEST_SCHEMA_PATH = path.resolve(__dirname, './test-schema.json') + +// Sample JSON Schema for testing +const testJsonSchema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/user-profile.schema.json", + "title": "User Profile", + "description": "A user profile containing name, age, and tags", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "User's full name", + "minLength": 1, + "pattern": "^[A-Za-z\\s]+$" + }, + "age": { + "type": "number", + "description": "User's age in years", + "minimum": 0, + "maximum": 150 + }, + "email": { + "type": "string", + "description": "User's email address", + "format": "email" + }, + "tags": { + "type": "array", + "description": "List of user's tags", + "items": { + "type": "string", + "enum": ["developer", "designer", "manager", "admin", "user"] + }, + "minItems": 1, + "maxItems": 5, + "uniqueItems": true + }, + "address": { + "type": "object", + "description": "User's address", + "properties": { + "street": { + "type": "string", + "description": "Street address" + }, + "city": { + "type": "string", + "description": "City name" + }, + "country": { + "type": "string", + "description": "Country name", + "enum": ["US", "UK", "CA", "AU"] + }, + "zipCode": { + "type": "string", + "description": "ZIP/Postal code", + "pattern": "^[0-9]{5}(-[0-9]{4})?$" + } + }, + "required": ["street", "city", "country"] + }, + "preferences": { + "type": "object", + "description": "User preferences", + "properties": { + "theme": { + "type": "string", + "enum": ["light", "dark", "system"], + "default": "system" + }, + "notifications": { + "type": "boolean", + "default": true + }, + "language": { + "type": "string", + "enum": ["en", "es", "fr", "de", "ja"], + "default": "en" + } + } + } + }, + "required": ["name", "age", "email"], + "additionalProperties": false +} + +// Write test schema to file +write(TEST_SCHEMA_PATH, JSON.stringify(testJsonSchema, null, 2)) + +// Helper function to normalize JSON strings +const normalizeJson = (json: string) => { + try { + // Remove markdown code block if present + const cleanJson = json.replace(/```json\n|\n```/g, '').trim() + return JSON.stringify(JSON.parse(cleanJson)) + } catch { + return json + } +} + +// Helper function to validate email +const isValidEmail = (email: string) => { + return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email) +} + +// Helper function to validate number in range +const isNumberInRange = (num: number, min: number, max: number) => { + return num >= min && num <= max +} + +// Helper function to validate array length +const hasValidArrayLength = (arr: any[], length: number) => { + return Array.isArray(arr) && arr.length === length && arr.every(item => typeof item === 'string') +} + +describe('Format Options', () => { + let testResults: TestResult[] = [] + + // Load existing results if any + if (exists(TEST_LOG_PATH)) { + const data = read(TEST_LOG_PATH, 'json') + testResults = Array.isArray(data) ? data : [] + } + + const runFormatTest = async (prompt: string, expected: string, testName: string, modelName: string, options: any = {}) => { + let model = 'gpt-4o' + let router = 'openai' + let startTime = Date.now() + let error: TestResult['error'] | undefined + let testResult: TestResult | undefined + + try { + const result = await Promise.race([ + run({ + prompt, + mode: 'completion', + model: 'gpt-4o', + router: 'openai', + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + ...options, + onRun: async (options) => { + model = options.model || 'unknown' + router = options.router || 'unknown' + return options + } + }), + new Promise((_, reject) => + setTimeout(() => reject(new Error('API call timed out')), TEST_TIMEOUT) + ) + ]) as string[] + + if (isEmptyResponse(result)) { + throw new Error('Model returned empty response') + } + + const actual = result?.[0]?.trim() || '' + const normalizedActual = normalizeJson(actual) + const normalizedExpected = normalizeJson(expected) + const passed = normalizedActual === normalizedExpected + + expect(normalizedActual).toEqual(normalizedExpected) + + testResult = { + test: testName, + prompt, + result: result || [], + expected, + model, + router, + timestamp: new Date().toISOString(), + passed, + duration: Date.now() - startTime, + reason: passed ? undefined : `Expected ${normalizedExpected}, but got ${normalizedActual}`, + } + } catch (e) { + error = formatError(e) + testResult = { + test: testName, + prompt, + result: [], + expected, + model, + router, + timestamp: new Date().toISOString(), + passed: false, + duration: Date.now() - startTime, + error, + reason: error?.message || 'Unknown error occurred' + } + throw e + } finally { + if (testResult) { + testResults.push(testResult) + write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2)) + } + } + } + + // Test JSON Schema format using file path + it('should format response according to JSON Schema file', async () => { + const prompt = 'Create a user profile with name John Doe, age 30, and tags ["developer", "javascript"]. Return only the JSON object, no explanation.' + const expected = JSON.stringify({ + name: "John Doe", + age: 30, + tags: ["developer", "javascript"] + }) + + await runFormatTest( + prompt, + expected, + 'json-schema-file-format', + 'gpt-4o', + { + format: TEST_SCHEMA_PATH + } + ) + }, { timeout: 10000 }) + + // Test JSON Schema format using schema object + it('should format response according to JSON Schema object', async () => { + const prompt = `Create a user profile with the following details: + - Name: Jane Smith + - Age: 25 + - Email: jane.smith@company.com + - Tags: ["developer", "designer"] + - Address: 123 Main St, New York, US, 10001 + - Preferences: light theme, notifications enabled, English language + Return only the JSON object, no explanation.` + + try { + const result = await run({ + prompt, + mode: 'completion', + model: 'gpt-4o', + router: 'openai', + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + format: testJsonSchema + }) as string[] + + const response = JSON.parse(normalizeJson(result?.[0] || '{}')) + + // Validate required fields + expect(response.name).toBe('Jane Smith') + expect(response.age).toBe(25) + expect(response.email).toBe('jane.smith@company.com') + + // Validate tags + expect(Array.isArray(response.tags)).toBe(true) + expect(response.tags).toContain('developer') + expect(response.tags).toContain('designer') + + // Validate address + expect(response.address.street).toBe('123 Main St') + expect(response.address.city).toBe('New York') + expect(response.address.country).toBe('US') + expect(response.address.zipCode || response.address.postal_code).toMatch(/^[0-9]{5}$/) + + // Validate preferences + expect(response.preferences.theme).toBe('light') + expect(['true', true, 'enabled'].includes(response.preferences.notifications)).toBe(true) + expect(['en', 'English'].includes(response.preferences.language)).toBe(true) + } catch (e) { + throw e + } + }, { timeout: 10000 }) + + // Test Zod Schema format with string + it('should format response according to Zod string schema', async () => { + const prompt = 'Generate a valid email address for a business domain. Return only the email, no explanation.' + + try { + const result = await run({ + prompt, + mode: 'completion', + model: 'gpt-4o', + router: 'openai', + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + format: { + type: "object", + properties: { + email: { + type: "string", + format: "email" + } + }, + required: ["email"] + } + }) as string[] + + const email = result?.[0]?.trim() || '' + expect(isValidEmail(email)).toBe(true) + } catch (e) { + throw e + } + }, { timeout: 10000 }) + + // Test Zod Schema format with number + it('should format response according to Zod number schema', async () => { + const prompt = 'Generate a random age between 18 and 65. Return only the number, no explanation.' + + try { + const result = await run({ + prompt, + mode: 'completion', + model: 'gpt-4o', + router: 'openai', + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + format: { + type: "object", + properties: { + age: { + type: "number", + minimum: 18, + maximum: 65 + } + }, + required: ["age"] + } + }) as string[] + + const age = parseInt(result?.[0]?.trim() || '0', 10) + expect(isNumberInRange(age, 18, 65)).toBe(true) + } catch (e) { + throw e + } + }, { timeout: 10000 }) + + // Test Zod Schema format with array + it('should format response according to Zod array schema', async () => { + const prompt = 'Generate a list of 3 programming languages. Return only the array, no explanation.' + + try { + const result = await run({ + prompt, + mode: 'completion', + model: 'gpt-4o', + router: 'openai', + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + format: { + type: "object", + properties: { + languages: { + type: "array", + items: { + type: "string" + }, + minItems: 3, + maxItems: 3 + } + }, + required: ["languages"] + } + }) as string[] + + const languages = JSON.parse(result?.[0]?.trim() || '[]') + expect(hasValidArrayLength(languages, 3)).toBe(true) + } catch (e) { + throw e + } + }, { timeout: 10000 }) + + // Test invalid format option + it('should handle invalid format option', async () => { + const prompt = 'Generate a random number.' + + try { + await run({ + prompt, + mode: 'completion', + model: 'gpt-4o', + router: 'openai', + path: TEST_BASE_PATH, + logs: TEST_LOGS_PATH, + preferences: TEST_PREFERENCES_PATH, + format: { + type: "invalid", + properties: {} + } + }) + // If we get here, the format validation didn't work + throw new Error('Expected format validation to fail') + } catch (e: any) { + // The error should be about invalid format/schema + if (!e.message.match(/invalid|Invalid|schema|Schema/)) { + throw new Error(`Unexpected error: ${e.message}`) + } + } + }, { timeout: 10000 }) + + it('should generate markdown report', () => { + // Group results by test and model + const latestResults = new Map>() + + // Get only the latest result for each test+model combination + testResults.forEach(result => { + if (!latestResults.has(result.test)) { + latestResults.set(result.test, new Map()) + } + const testMap = latestResults.get(result.test)! + const existingResult = testMap.get(result.model) + if (!existingResult || new Date(result.timestamp) > new Date(existingResult.timestamp)) { + testMap.set(result.model, result) + } + }) + + // Generate markdown report + let report = '# Format Test Results\n\n' + + // First list failed tests + report += '## Failed Tests\n\n' + let hasFailures = false + for (const [testName, modelResults] of latestResults) { + for (const [model, result] of modelResults) { + if (!result.passed) { + hasFailures = true + report += `### ${testName} - ${model}\n` + report += `- Prompt: \`${result.prompt}\`\n` + report += `- Expected: \`${result.expected}\`\n` + report += `- Actual: \`${result.result[0] || ''}\`\n` + report += `- Duration: ${result.duration}ms\n` + if (result.error) { + report += `- Error Type: ${result.error.type}\n` + report += `- Error Code: ${result.error.code}\n` + report += `- Error Message: ${result.error.message}\n` + if (result.error.details?.message) { + report += `- Error Details: ${result.error.details.message}\n` + } + } + report += `- Reason: ${result.reason}\n` + report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` + } + } + } + + if (!hasFailures) { + report += '*No failed tests*\n\n' + } + + // Then list passed tests + report += '## Passed Tests\n\n' + let hasPassed = false + for (const [testName, modelResults] of latestResults) { + for (const [model, result] of modelResults) { + if (result.passed) { + hasPassed = true + report += `### ${testName} - ${model}\n` + report += `- Prompt: \`${result.prompt}\`\n` + report += `- Expected: \`${result.expected}\`\n` + report += `- Actual: \`${result.result[0] || ''}\`\n` + report += `- Duration: ${result.duration}ms\n` + report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n` + } + } + } + + if (!hasPassed) { + report += '*No passed tests*\n\n' + } + + // Add summary section + report += '## Summary\n\n' + const totalTests = testResults.length + const passedTests = testResults.filter(r => r.passed).length + const failedTests = totalTests - passedTests + report += `- Total Tests: ${totalTests}\n` + report += `- Passed: ${passedTests}\n` + report += `- Failed: ${failedTests}\n` + report += `- Success Rate: ${((passedTests / totalTests) * 100).toFixed(2)}%\n\n` + + // Write report to file + const reportPath = path.resolve(__dirname, './format-report.md') + write(reportPath, report) + + // Verify report was written + expect(exists(reportPath) === 'file').toBe(true) + }) }) \ No newline at end of file diff --git a/packages/kbot/tests/unit/language-report.md b/packages/kbot/tests/unit/language-report.md index 77b0a619..e0891e6e 100644 --- a/packages/kbot/tests/unit/language-report.md +++ b/packages/kbot/tests/unit/language-report.md @@ -36,9 +36,9 @@ - Prompt: `translate "hello" to German. Return only the translated word, no explanation.` - Expected: `hallo` - Actual: `` -- Duration: 1253ms +- Duration: 1192ms - Reason: Unknown error occurred -- Timestamp: 4/1/2025, 1:47:26 PM +- Timestamp: 4/1/2025, 11:53:55 PM ### spanish - deepseek/deepseek-chat:free - Prompt: `translate "yes" to Spanish. Return only the translated word, no explanation.` @@ -74,9 +74,9 @@ - Prompt: `translate "yes" to Spanish. Return only the translated word, no explanation.` - Expected: `sí` - Actual: `` -- Duration: 932ms +- Duration: 1193ms - Reason: Unknown error occurred -- Timestamp: 4/1/2025, 1:47:27 PM +- Timestamp: 4/1/2025, 11:53:56 PM ### french - deepseek/deepseek-chat:free - Prompt: `translate "no" to French. Return only the translated word, no explanation.` @@ -112,9 +112,9 @@ - Prompt: `translate "no" to French. Return only the translated word, no explanation.` - Expected: `non` - Actual: `` -- Duration: 864ms +- Duration: 968ms - Reason: Unknown error occurred -- Timestamp: 4/1/2025, 1:47:28 PM +- Timestamp: 4/1/2025, 11:53:57 PM ## Passed Tests diff --git a/packages/kbot/tests/unit/language.json b/packages/kbot/tests/unit/language.json index 19b0d682..df969918 100644 --- a/packages/kbot/tests/unit/language.json +++ b/packages/kbot/tests/unit/language.json @@ -807,5 +807,41 @@ "passed": false, "duration": 864, "reason": "Unknown error occurred" + }, + { + "test": "german", + "prompt": "translate \"hello\" to German. Return only the translated word, no explanation.", + "result": [], + "expected": "hallo", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T21:53:55.163Z", + "passed": false, + "duration": 1192, + "reason": "Unknown error occurred" + }, + { + "test": "spanish", + "prompt": "translate \"yes\" to Spanish. Return only the translated word, no explanation.", + "result": [], + "expected": "sí", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T21:53:56.357Z", + "passed": false, + "duration": 1193, + "reason": "Unknown error occurred" + }, + { + "test": "french", + "prompt": "translate \"no\" to French. Return only the translated word, no explanation.", + "result": [], + "expected": "non", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T21:53:57.326Z", + "passed": false, + "duration": 968, + "reason": "Unknown error occurred" } ] \ No newline at end of file diff --git a/packages/kbot/tests/unit/math-report.md b/packages/kbot/tests/unit/math-report.md index c1a8f97e..1b8710e1 100644 --- a/packages/kbot/tests/unit/math-report.md +++ b/packages/kbot/tests/unit/math-report.md @@ -36,9 +36,9 @@ - Prompt: `add 5 and 3. Return only the number, no explanation.` - Expected: `8` - Actual: `` -- Duration: 1218ms +- Duration: 1992ms - Reason: Unknown error occurred -- Timestamp: 4/1/2025, 1:46:27 PM +- Timestamp: 4/1/2025, 11:53:40 PM ### multiplication - deepseek/deepseek-chat:free - Prompt: `multiply 8 and 3. Return only the number, no explanation.` @@ -74,9 +74,9 @@ - Prompt: `multiply 8 and 3. Return only the number, no explanation.` - Expected: `24` - Actual: `` -- Duration: 911ms +- Duration: 1078ms - Reason: Unknown error occurred -- Timestamp: 4/1/2025, 1:46:27 PM +- Timestamp: 4/1/2025, 11:53:41 PM ### division - deepseek/deepseek-chat:free - Prompt: `divide 15 by 3. Return only the number, no explanation.` @@ -112,9 +112,9 @@ - Prompt: `divide 15 by 3. Return only the number, no explanation.` - Expected: `5` - Actual: `` -- Duration: 1485ms +- Duration: 940ms - Reason: Unknown error occurred -- Timestamp: 4/1/2025, 1:46:29 PM +- Timestamp: 4/1/2025, 11:53:42 PM ## Passed Tests diff --git a/packages/kbot/tests/unit/math.json b/packages/kbot/tests/unit/math.json index 5aeb9213..8c7d26df 100644 --- a/packages/kbot/tests/unit/math.json +++ b/packages/kbot/tests/unit/math.json @@ -2213,5 +2213,41 @@ "passed": false, "duration": 1485, "reason": "Unknown error occurred" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [], + "expected": "8", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T21:53:40.351Z", + "passed": false, + "duration": 1992, + "reason": "Unknown error occurred" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [], + "expected": "24", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T21:53:41.431Z", + "passed": false, + "duration": 1078, + "reason": "Unknown error occurred" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [], + "expected": "5", + "model": "anthropic/claude-2.0", + "router": "openrouter", + "timestamp": "2025-04-01T21:53:42.372Z", + "passed": false, + "duration": 940, + "reason": "Unknown error occurred" } ] \ No newline at end of file diff --git a/packages/kbot/tests/unit/test-schema.json b/packages/kbot/tests/unit/test-schema.json new file mode 100644 index 00000000..5780a5a4 --- /dev/null +++ b/packages/kbot/tests/unit/test-schema.json @@ -0,0 +1,113 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/user-profile.schema.json", + "title": "User Profile", + "description": "A user profile containing name, age, and tags", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "User's full name", + "minLength": 1, + "pattern": "^[A-Za-z\\s]+$" + }, + "age": { + "type": "number", + "description": "User's age in years", + "minimum": 0, + "maximum": 150 + }, + "email": { + "type": "string", + "description": "User's email address", + "format": "email" + }, + "tags": { + "type": "array", + "description": "List of user's tags", + "items": { + "type": "string", + "enum": [ + "developer", + "designer", + "manager", + "admin", + "user" + ] + }, + "minItems": 1, + "maxItems": 5, + "uniqueItems": true + }, + "address": { + "type": "object", + "description": "User's address", + "properties": { + "street": { + "type": "string", + "description": "Street address" + }, + "city": { + "type": "string", + "description": "City name" + }, + "country": { + "type": "string", + "description": "Country name", + "enum": [ + "US", + "UK", + "CA", + "AU" + ] + }, + "zipCode": { + "type": "string", + "description": "ZIP/Postal code", + "pattern": "^[0-9]{5}(-[0-9]{4})?$" + } + }, + "required": [ + "street", + "city", + "country" + ] + }, + "preferences": { + "type": "object", + "description": "User preferences", + "properties": { + "theme": { + "type": "string", + "enum": [ + "light", + "dark", + "system" + ], + "default": "system" + }, + "notifications": { + "type": "boolean", + "default": true + }, + "language": { + "type": "string", + "enum": [ + "en", + "es", + "fr", + "de", + "ja" + ], + "default": "en" + } + } + } + }, + "required": [ + "name", + "age", + "email" + ], + "additionalProperties": false +} \ No newline at end of file