From 1535cb754af1254b3220df9c58dacfa7ec014736 Mon Sep 17 00:00:00 2001 From: babayaga Date: Fri, 6 Jun 2025 00:35:53 +0200 Subject: [PATCH] fix:response format for tests --- packages/kbot/.vscode/launch.json | 20 +- packages/kbot/dist-in/commands/run.js | 4 +- packages/kbot/logs/params.json | 2 +- packages/kbot/package-lock.json | 9 + packages/kbot/package.json | 1 + packages/kbot/src/commands/run.ts | 2 +- packages/kbot/tests/unit/commons.ts | 60 +- packages/kbot/tests/unit/files.test.ts | 84 +- packages/kbot/tests/unit/reports/all.json | 2222 ++++++++++++++++- packages/kbot/tests/unit/reports/basic.json | 197 +- packages/kbot/tests/unit/reports/basic.md | 102 +- packages/kbot/tests/unit/reports/files.json | 1773 ++++++++++++- packages/kbot/tests/unit/reports/files.md | 45 +- .../kbot/tests/unit/reports/language.json | 256 +- packages/kbot/tests/unit/reports/language.md | 167 +- 15 files changed, 4661 insertions(+), 283 deletions(-) diff --git a/packages/kbot/.vscode/launch.json b/packages/kbot/.vscode/launch.json index 0f93b9f8..df231b2a 100644 --- a/packages/kbot/.vscode/launch.json +++ b/packages/kbot/.vscode/launch.json @@ -4,6 +4,23 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Vitest: Debug Open File", + "program": "${workspaceFolder}/node_modules/vitest/vitest.mjs", + "args": [ + "run", + "${relativeFile}" + ], + "skipFiles": [ + "/**" + ], + "console": "integratedTerminal", + "sourceMaps": true, + "smartStep": true, + "internalConsoleOptions": "neverOpen" + }, { "type": "node", "request": "launch", @@ -768,5 +785,6 @@ "console": "integratedTerminal", "outputCapture": "std" } - ] + ], + "compounds": [] } \ No newline at end of file diff --git a/packages/kbot/dist-in/commands/run.js b/packages/kbot/dist-in/commands/run.js index 3084ec5d..a674cb4c 100644 --- a/packages/kbot/dist-in/commands/run.js +++ b/packages/kbot/dist-in/commands/run.js @@ -70,6 +70,7 @@ export const complete_options = async (opts) => { options.client = client; options.variables = { ...options.variables, ...variables(options) }; options.collector = collector(options, client); + options.format = opts.format; options.onRun = options.onRun || (async (options) => options); return options; } @@ -141,7 +142,6 @@ export const complete_params = async (options, messages) => { if (options.mode === E_Mode.TOOLS || options.mode === E_Mode.ASSISTANT) { params.tools = await loadTools(options); params.tool_choice = 'auto'; - //params.parallel_tool_calls = false } return params; }; @@ -349,4 +349,4 @@ export const run = async (opts) => { } return ret; }; -//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVuLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2NvbW1hbmRzL3J1bi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssSUFBSSxNQUFNLFdBQVcsQ0FBQTtBQUNqQyxPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sWUFBWSxDQUFBO0FBQ25DLE9BQU8sRUFBRSxHQUFHLElBQUksVUFBVSxFQUFFLE1BQU0sY0FBYyxDQUFBO0FBRWhELE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSxNQUFNLENBQUE7QUFDL0IsT0FBTyxFQUFFLElBQUksSUFBSSxHQUFHLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQTtBQUM5QyxPQUFPLEVBQUUsSUFBSSxJQUFJLE1BQU0sRUFBRSxNQUFNLHFCQUFxQixDQUFBO0FBQ3BELE9BQU8sRUFBRSxJQUFJLElBQUksS0FBSyxFQUFFLE1BQU0sb0JBQW9CLENBQUE7QUFDbEQsT0FBTyxFQUFFLElBQUksSUFBSSxJQUFJLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQTtBQUNoRCxPQUFPLEVBQUUsYUFBYSxFQUFFLE9BQU8sRUFBRSxNQUFNLEVBQUUsVUFBVSxFQUFFLE1BQU0sbUJBQW1CLENBQUE7QUFDOUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQTtBQUs3RCxPQUFPLEVBQUUsTUFBTSxFQUFXLE1BQU0sT0FBTyxDQUFBO0FBQ3ZDLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSxjQUFjLENBQUE7QUFDM0MsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLGtCQUFrQixDQUFBO0FBQ2hELE9BQU8sRUFBRSxHQUFHLEVBQUUsTUFBTSxjQUFjLENBQUE7QUFDbEMsT0FBTyxFQUFFLFFBQVEsRUFBRSxNQUFNLFlBQVksQ0FBQTtBQUNyQyxPQUFPLEVBQUUsT0FBTyxFQUFFLE1BQU0sbUJBQW1CLENBQUE7QUFDM0MsT0FBTyxFQUFFLFNBQVMsRUFBRSxNQUFNLGlCQUFpQixDQUFBO0FBQzNDLE9BQU8sRUFBRSxJQUFJLElBQUksV0FBVyxFQUFFLE1BQU0sZUFBZSxDQUFBO0FBQ25ELE9BQU8sRUFBRSxJQUFJLElBQUksU0FBUyxFQUFFLE1BQU0sYUFBYSxDQUFBO0FBQy9DLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxFQUFFLE1BQU0sY0FBYyxDQUFBO0FBQ2xELE9BQU8sRUFBRSxTQUFTLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQTtBQUMzQyxPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sa0JBQWtCLENBQUE7QUFFekMsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLHFCQUFxQixDQUFBO0FBQ25ELE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQTtBQUN6QyxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sb0JBQW9CLENBQUE7QUFDakQsT0FBTyxFQUFFLEdBQUcsRUFBRSxNQUFNLG9CQUFvQixDQUFBO0FBQ3hDLE9BQU8sRUFBRSxTQUFTLEVBQUUsTUFBTSxhQUFhLENBQUE7QUFJdkMsTUFBTSxDQUFDLE1BQU0sWUFBWSxHQUFHLENBQUMsS0FBZSxFQUFZLEVBQUU7SUFDeEQsTUFBTSxnQkFBZ0IsR0FBRyxDQUFDLFFBQWdCLEVBQUUsRUFBRTtRQUMxQyxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDLFFBQVEsQ0FBQyxDQUFDLElBQUksQ0FBQztRQUMzQyxNQUFNLEtBQUssR0FBRyxRQUFRLENBQUMsS0FBSyxDQUFDLGVBQWUsQ0FBQyxDQUFDLENBQUMsd0JBQXdCO1FBQ3ZFLE1BQU0sT0FBTyxHQUFHLEtBQUssQ0FBQyxDQUFDLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDO1FBQ3JELE1BQU0sUUFBUSxHQUFHLEtBQUssQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxRQUFRLENBQUMsQ0FBQyxvQkFBb0I7UUFFbEUsT0FBTyxFQUFFLE9BQU8sRUFBRSxRQUFRLEVBQUUsQ0FBQztJQUNqQyxDQUFDLENBQUE7SUFDRCxPQUFPLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxFQUFFLEVBQUU7UUFDdkIsTUFBTSxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsUUFBUSxFQUFFLEtBQUssRUFBRSxHQUFHLGdCQUFnQixDQUFDLENBQUMsQ0FBQyxDQUFBO1FBQzlELE1BQU0sRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLFFBQVEsRUFBRSxLQUFLLEVBQUUsR0FBRyxnQkFBZ0IsQ0FBQyxDQUFDLENBQUMsQ0FBQTtRQUM5RCxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7WUFDL0IsT0FBTyxJQUFJLEdBQUcsSUFBSSxJQUFJLEtBQUssQ0FBQyxhQUFhLENBQUMsS0FBSyxFQUFFLFNBQVMsRUFBRSxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsV0FBVyxFQUFFLE1BQU0sRUFBRSxDQUFDLENBQUE7UUFDdkcsQ0FBQztRQUNELE9BQU8sS0FBSyxDQUFDLGFBQWEsQ0FBQyxLQUFLLEVBQUUsU0FBUyxFQUFFLEVBQUUsT0FBTyxFQUFFLElBQUksRUFBRSxXQUFXLEVBQUUsTUFBTSxFQUFFLENBQUMsQ0FBQTtJQUN4RixDQUFDLENBQUMsQ0FBQTtBQUNKLENBQUMsQ0FBQTtBQUlEOzs7OztHQUtHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sZ0JBQWdCLEdBQUcsS0FBSyxFQUFFLElBQWUsRUFBNkIsRUFBRTtJQUNuRixJQUFJLENBQUMsTUFBTSxHQUFHLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQTtJQUU3QixNQUFNLE1BQU0sR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxNQUFNLElBQUksSUFBSSxDQUFDLElBQUksSUFBSSxHQUFHLENBQUMsQ0FBQTtJQUM1RCxJQUFJLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUM7UUFDcEIsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFBO0lBQ2IsQ0FBQztJQUVELElBQUksQ0FBQyxPQUFPLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQTtJQUNwQyxJQUFJLENBQUMsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLENBQUE7SUFDOUMsSUFBSSxDQUFDLE9BQU8sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFBO0lBQ3BDLElBQUksQ0FBQyxPQUFPLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxPQUFPLElBQUksRUFBRSxDQUFDLENBQUE7SUFDMUMsSUFBSSxDQUFDLFNBQVMsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxNQUFNLFdBQVcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFBO0lBRTVELElBQUksQ0FBQztRQUNILE1BQU0sTUFBTSxHQUFHLGFBQWEsRUFBRSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUMzQyxNQUFNLE9BQU8sR0FBRyxNQUFtQixDQUFDO1FBRXBDLE1BQU0sTUFBTSxHQUFHLFlBQVksQ0FBQyxPQUFPLENBQUMsQ0FBQTtRQUNwQyxJQUFJLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDWixJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyx5QkFBeUIsQ0FBQyxDQUFBO1lBQzVDLE9BQU8sSUFBSSxDQUFBO1FBQ2IsQ0FBQztRQUVELE9BQU8sQ0FBQyxNQUFNLEdBQUcsTUFBTSxDQUFBO1FBQ3ZCLE9BQU8sQ0FBQyxTQUFTLEdBQUcsRUFBRSxHQUFHLE9BQU8sQ0FBQyxTQUFTLEVBQUUsR0FBRyxTQUFTLENBQUMsT0FBTyxDQUFDLEVBQUUsQ0FBQTtRQUNuRSxPQUFPLENBQUMsU0FBUyxHQUFHLFNBQVMsQ0FBQyxPQUFPLEVBQUUsTUFBTSxDQUFDLENBQUE7UUFDOUMsT0FBTyxDQUFDLEtBQUssR0FBRyxPQUFPLENBQUMsS0FBSyxJQUFJLENBQUMsS0FBSyxFQUFFLE9BQU8sRUFBRSxFQUFFLENBQUMsT0FBTyxDQUFDLENBQUE7UUFFN0QsT0FBTyxPQUFPLENBQUE7SUFDaEIsQ0FBQztJQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7UUFDZixJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQywwQkFBMEIsRUFBRSxLQUFLLENBQUMsT0FBTyxFQUFFLEtBQUssQ0FBQyxNQUFNLEVBQUUsS0FBSyxDQUFDLEtBQUssQ0FBQyxDQUFBO1FBQ3ZGLE9BQU8sSUFBSSxDQUFBO0lBQ2IsQ0FBQztBQUNILENBQUMsQ0FBQTtBQUVEOzs7Ozs7R0FNRztBQUNILE1BQU0sQ0FBQyxNQUFNLGlCQUFpQixHQUFHLEtBQUssRUFDcEMsSUFBZSxFQUNmLE9BQWtCLEVBSWpCLEVBQUU7SUFDSCxJQUFJLFlBQVksR0FBc0MsRUFBRSxDQUFBO0lBRXhELE1BQU0sYUFBYSxHQUFHLE1BQU0sTUFBTSxDQUFDLElBQUksQ0FBQyxDQUFBO0lBQ3hDLElBQUksQ0FBQyxhQUFhLEVBQUUsT0FBTyxFQUFFLENBQUM7UUFDNUIsT0FBTyxFQUFFLFFBQVEsRUFBRSxFQUFFLEVBQUUsS0FBSyxFQUFFLEVBQUUsRUFBRSxDQUFBO0lBQ3BDLENBQUM7SUFDRCxZQUFZLENBQUMsSUFBSSxDQUFDLGFBQTJDLENBQUMsQ0FBQTtJQUM5RCxZQUFZLENBQUMsSUFBSSxDQUFDLENBQUMsTUFBTSxXQUFXLENBQUMsSUFBSSxDQUFDLENBQStCLENBQUMsQ0FBQTtJQUUxRSxNQUFNLFdBQVcsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxJQUFJLElBQUksR0FBRyxDQUFDLENBQUE7SUFDckQsSUFBSSxXQUFXLEdBQUcsTUFBTSxHQUFHLENBQUMsV0FBVyxFQUFFLE9BQU8sQ0FBQyxPQUFPLEVBQUUsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFBO0lBRXhFLE1BQU0scUJBQXFCLEdBQWlDLFdBQVcsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUU7UUFDOUUsSUFBSSxDQUFDLENBQUMsSUFBSSxJQUFJLENBQUMsQ0FBQyxPQUFPLElBQUksT0FBTyxDQUFDLENBQUMsT0FBTyxLQUFLLFFBQVEsRUFBRSxDQUFDO1lBQ3pELE1BQU0sUUFBUSxHQUFHLE1BQU0sQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUE7WUFDL0IsdUdBQXVHO1lBQ3ZHLElBQUksUUFBUSxJQUFJLENBQUMsUUFBUSxDQUFDLFVBQVUsQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLGtCQUFrQixFQUFFLGlCQUFpQixFQUFFLHdCQUF3QixFQUFFLHdCQUF3QixDQUFDLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxDQUFDLEVBQUUsQ0FBQztnQkFDakssSUFBSSxPQUFPLENBQUMsSUFBSSxLQUFLLE1BQU0sRUFBRSxDQUFDO29CQUM1QixNQUFNLFlBQVksR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxJQUFJLElBQUksR0FBRyxFQUFFLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQTtvQkFDOUQsTUFBTSxZQUFZLEdBQUcsQ0FBQyxDQUFDLElBQUksQ0FBQSxDQUFDLHFEQUFxRDtvQkFDakYsTUFBTSx1QkFBdUIsR0FBRyxVQUFVLEVBQUUsQ0FBQTtvQkFDNUMsTUFBTSxVQUFVLEdBQUc7O1FBRXJCLFlBQVk7aUJBQ0gsWUFBWTtPQUN0Qix1QkFBdUI7Ozs7Q0FJN0IsQ0FBQTtvQkFDUyxPQUFPLEVBQUUsR0FBRyxDQUFDLEVBQUUsT0FBTyxFQUFFLFVBQVUsR0FBRyxDQUFDLENBQUMsT0FBTyxFQUFFLElBQUksRUFBRSxNQUFNLEVBQUUsQ0FBQTtnQkFDaEUsQ0FBQztxQkFBTSxDQUFDO29CQUNOLE9BQU8sRUFBRSxHQUFHLENBQUMsRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLENBQUE7Z0JBQy9CLENBQUM7WUFDSCxDQUFDO1FBQ0gsQ0FBQztRQUNELE9BQU8sRUFBRSxHQUFHLENBQUMsRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLENBQUE7SUFDL0IsQ0FBQyxDQUFDLENBQUE7SUFFRixZQUFZLEdBQUcsQ0FBQyxHQUFHLFlBQVksRUFBRSxHQUFHLHFCQUFxQixDQUFDLENBQUE7SUFDMUQsT0FBTyxFQUFFLFFBQVEsRUFBRSxZQUFZLEVBQUUsS0FBSyxFQUFFLFdBQVcsRUFBRSxDQUFBO0FBQ3ZELENBQUMsQ0FBQTtBQUVEOzs7Ozs7R0FNRztBQUNILE1BQU0sQ0FBQyxNQUFNLGVBQWUsR0FBRyxLQUFLLEVBQ2xDLE9BQWtCLEVBQ2xCLFFBQTJDLEVBQ0csRUFBRTtJQUNoRCxNQUFNLE1BQU0sR0FBRztRQUNiLEtBQUssRUFBRSxPQUFPLENBQUMsS0FBSztRQUNwQixRQUFRO1FBQ1IsS0FBSyxFQUFFLEVBQUU7S0FDNkIsQ0FBQTtJQUV4QyxJQUFJLE9BQU8sQ0FBQyxJQUFJLEtBQUssTUFBTSxDQUFDLEtBQUssSUFBSSxPQUFPLENBQUMsSUFBSSxLQUFLLE1BQU0sQ0FBQyxTQUFTLEVBQUUsQ0FBQztRQUN2RSxNQUFNLENBQUMsS0FBSyxHQUFHLE1BQU0sU0FBUyxDQUFDLE9BQU8sQ0FBQyxDQUFBO1FBQ3ZDLE1BQU0sQ0FBQyxXQUFXLEdBQUcsTUFBTSxDQUFBO1FBQzNCLG9DQUFvQztJQUN0QyxDQUFDO0lBRUQsT0FBTyxNQUFNLENBQUE7QUFDZixDQUFDLENBQUE7QUFFRDs7Ozs7OztHQU9HO0FBQ0gsTUFBTSxDQUFDLE1BQU0sZUFBZSxHQUFHLEtBQUssRUFDbEMsT0FBa0IsRUFDbEIsTUFBVyxFQUNYLE1BQTJDLEVBQ2hCLEVBQUU7SUFDN0IsSUFBSSxHQUFHLEdBQVEsSUFBSSxDQUFBO0lBRW5CLElBQUksQ0FBQztRQUNILFFBQVEsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDO1lBQ3JCLEtBQUssTUFBTSxDQUFDLFVBQVU7Z0JBQ3BCLEdBQUcsR0FBRyxNQUFNLGFBQWEsQ0FBQyxNQUFNLEVBQUUsTUFBTSxFQUFFLE9BQU8sQ0FBQyxDQUFBO2dCQUNsRCxNQUFLO1lBRVAsS0FBSyxNQUFNLENBQUMsS0FBSztnQkFDZixHQUFHLEdBQUcsTUFBTSxRQUFRLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxPQUFPLENBQUMsQ0FBQTtnQkFDN0MsTUFBSztZQUVQLEtBQUssTUFBTSxDQUFDLFNBQVM7Z0JBQ25CLEdBQUcsR0FBRyxNQUFNLFlBQVksQ0FBQyxNQUFNLEVBQUUsTUFBTSxFQUFFLE9BQU8sQ0FBQyxDQUFBO2dCQUNqRCxNQUFLO1lBRVA7Z0JBQ0UsTUFBTSxJQUFJLEtBQUssQ0FBQyxxQkFBcUIsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUE7UUFDeEQsQ0FBQztJQUNILENBQUM7SUFBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1FBQ1gsT0FBTyxDQUFDLE1BQU0sRUFBRSxLQUFLLENBQUMsaUJBQWlCLE9BQU8sQ0FBQyxJQUFJLFVBQVUsQ0FBQyxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUE7SUFDM0UsQ0FBQztJQUVELE9BQU8sR0FBRyxDQUFBO0FBQ1osQ0FBQyxDQUFBO0FBRUQ7Ozs7O0dBS0c7QUFDSCxNQUFNLENBQUMsTUFBTSxVQUFVLEdBQUcsS0FBSyxFQUFFLElBQWUsRUFBNkIsRUFBRTtJQUM3RSxNQUFNLGNBQWMsR0FBRyxDQUFDLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFBO0lBQ3hDLElBQUksT0FBTyxHQUFHLE1BQU0sZ0JBQWdCLENBQUMsSUFBSSxDQUFDLENBQUE7SUFDMUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ2IsT0FBTyxJQUFJLENBQUE7SUFDYixDQUFDO0lBRUQsTUFBTSxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQTtJQUU3QixNQUFNLEVBQUUsUUFBUSxFQUFFLEtBQUssRUFBRSxHQUFHLE1BQU0saUJBQWlCLENBQUMsSUFBSSxFQUFFLE9BQU8sQ0FBQyxDQUFBO0lBQ2xFLElBQUcsY0FBYyxDQUFDLE1BQU0sR0FBRyxDQUFDLElBQUksS0FBSyxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQUUsQ0FBQztRQUNuRCxPQUFPLEVBQUUsQ0FBQTtJQUNYLENBQUM7SUFDRCxJQUFJLFFBQVEsQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDMUIsT0FBTyxFQUFFLENBQUE7SUFDWCxDQUFDO0lBQ0QsTUFBTSxNQUFNLEdBQUcsTUFBTSxlQUFlLENBQUMsT0FBTyxFQUFFLFFBQVEsQ0FBQyxDQUFBO0lBRXZELE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxJQUFJLElBQUksUUFBUSxDQUFDLENBQUMsQ0FBQTtJQUMzRCxNQUFNLFVBQVUsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLE1BQU0sRUFBRSxhQUFhLENBQUMsQ0FBQTtJQUNuRCxLQUFLLENBQUMsVUFBVSxFQUFFLElBQUksQ0FBQyxTQUFTLENBQUMsRUFBRSxHQUFHLE1BQU0sRUFBRSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFBO0lBQ3pELElBQUksSUFBSSxDQUFDLElBQUksS0FBSyxNQUFNLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDL0IsT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsWUFBWSxPQUFPLENBQUMsSUFBSSxNQUFNLE9BQU8sQ0FBQyxLQUFLLE1BQU0sT0FBTyxDQUFDLE1BQU0sT0FBTyxLQUFLLENBQUMsTUFBTSx1QkFBdUIsSUFBSSxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUMsSUFBSSxJQUFJLEdBQUcsQ0FBQyxTQUFTLE9BQU8sQ0FBQyxPQUFPLEVBQUUsRUFBRSxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxFQUFFLE1BQU0sQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsR0FBRyxDQUFDLENBQUMsUUFBUSxDQUFDLElBQUksTUFBTSxDQUFDLENBQUMsUUFBUSxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUMsQ0FBQTtJQUM3UixDQUFDO1NBQU0sQ0FBQztRQUNOLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLFlBQVksT0FBTyxDQUFDLElBQUksTUFBTSxPQUFPLENBQUMsS0FBSyxNQUFNLE9BQU8sQ0FBQyxNQUFNLE9BQU8sS0FBSyxDQUFDLE1BQU0sdUJBQXVCLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLElBQUksSUFBSSxHQUFHLENBQUMsU0FBUyxPQUFPLENBQUMsT0FBTyxFQUFFLEVBQUUsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFBO0lBQ3BOLENBQUM7SUFDRCxPQUFPLEdBQUcsTUFBTSxPQUFPLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxJQUFJLE9BQU8sQ0FBQTtJQUNqRCxNQUFNLEdBQUcsR0FBRyxNQUFNLGVBQWUsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sQ0FBQyxDQUFBO0lBQzFELE9BQU8sR0FBRyxDQUFBO0FBQ1osQ0FBQyxDQUFBO0FBRUQ7Ozs7Ozs7OztHQVNHO0FBQ0gsU0FBUyxZQUFZLENBQUMsS0FBYTtJQUNqQywrQkFBK0I7SUFDL0IsSUFBSSxRQUFRLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztRQUNwQixPQUFPLENBQUMsS0FBSyxDQUFDLENBQUE7SUFDaEIsQ0FBQztJQUVELCtDQUErQztJQUMvQyxJQUFJLE1BQU0sQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztRQUN6QyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUE7SUFDaEIsQ0FBQztJQUVELDBDQUEwQztJQUMxQyxJQUFJLFFBQVEsQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDO1FBQ3BCLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQTtJQUNoQixDQUFDO0lBRUQsNkZBQTZGO0lBQzdGLE1BQU0sY0FBYyxHQUFHLGlDQUFpQyxDQUFDO0lBRXpELE1BQU0sWUFBWSxHQUFhLEVBQUUsQ0FBQztJQUNsQyxJQUFJLEtBQTZCLENBQUM7SUFFbEMsdUNBQXVDO0lBQ3ZDLE9BQU8sQ0FBQyxLQUFLLEdBQUcsY0FBYyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQyxLQUFLLElBQUksRUFBRSxDQUFDO1FBQ3JELFlBQVksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ2pDLENBQUM7SUFFRCw4QkFBOEI7SUFDOUIsSUFBSSxDQUFDLFlBQVksQ0FBQyxNQUFNLEVBQUUsQ0FBQztRQUN6QixPQUFPLEVBQUUsQ0FBQztJQUNaLENBQUM7SUFFRCxnREFBZ0Q7SUFDaEQsTUFBTSxLQUFLLEdBQWEsRUFBRSxDQUFDO0lBQzNCLEtBQUssSUFBSSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsR0FBRyxZQUFZLENBQUMsTUFBTSxFQUFFLENBQUMsRUFBRSxFQUFFLENBQUM7UUFDN0MsTUFBTSxLQUFLLEdBQUcsWUFBWSxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQzlCLE1BQU0sR0FBRyxHQUFHLENBQUMsR0FBRyxZQUFZLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsWUFBWSxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLE1BQU0sQ0FBQztRQUM3RSxNQUFNLElBQUksR0FBRyxLQUFLLENBQUMsU0FBUyxDQUFDLEtBQUssRUFBRSxHQUFHLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUNoRCxJQUFJLElBQUksRUFBRSxDQUFDO1lBQ1QsS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNuQixDQUFDO0lBQ0gsQ0FBQztJQUVELE9BQU8sS0FBSyxDQUFDO0FBQ2YsQ0FBQztBQUVELFNBQVMsYUFBYSxDQUFJLE1BQWE7SUFDckMsT0FBTyxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUMsV0FBVyxFQUFFLE9BQU8sRUFBRSxFQUFFO1FBQzVDLE9BQU8sV0FBVyxDQUFDLE1BQU0sQ0FBQyxPQUFPLENBQUMsQ0FBQztJQUNyQyxDQUFDLEVBQUUsRUFBUyxDQUFDLENBQUM7QUFDaEIsQ0FBQztBQUVELE1BQU0sQ0FBQyxNQUFNLEdBQUcsR0FBRyxLQUFLLEVBQUUsSUFBZSxFQUErQixFQUFFO0lBQ3hFLE1BQU0sR0FBRyxHQUF1QixFQUFFLENBQUE7SUFDbEMsTUFBTSxNQUFNLEdBQUcsSUFBSSxNQUFNLENBQVUsRUFBRSxRQUFRLEVBQUUsSUFBSSxDQUFDLFFBQVEsSUFBSSxDQUFDLEVBQUUsQ0FBQyxDQUFBO0lBQ3BFLElBQUksQ0FBQyxNQUFNLEdBQUcsTUFBTSxDQUFBO0lBRXBCLElBQUksSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ2pCLElBQUksUUFBUSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQzNCLElBQUksQ0FBQyxPQUFPLEdBQUcsQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUE7UUFDL0IsQ0FBQztRQUNELElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQzFCLDZDQUE2QztZQUM3QyxNQUFNLGVBQWUsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDLElBQVksRUFBRSxFQUFFLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxJQUFJLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFBO1lBQy9GLDRCQUE0QjtZQUM1QixNQUFNLFdBQVcsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDLElBQVksRUFBRSxFQUFFLENBQUMsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQTtZQUM3RixNQUFNLGNBQWMsR0FBRyxhQUFhLENBQUMsV0FBVyxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFBO1lBQ25FLCtDQUErQztZQUMvQyxJQUFJLENBQUMsT0FBTyxHQUFHLENBQUMsR0FBRyxlQUFlLEVBQUUsR0FBRyxjQUFjLENBQUMsQ0FBQTtRQUN4RCxDQUFDO0lBQ0gsQ0FBQztTQUFNLENBQUM7UUFDTixJQUFJLENBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQTtJQUNuQixDQUFDO0lBRUQsaURBQWlEO0lBQ2pELElBQUksSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ2pCLElBQUksUUFBUSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQzNCLElBQUksQ0FBQyxPQUFPLEdBQUcsQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUE7UUFDL0IsQ0FBQztRQUNELElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQzFCLE1BQU0sZUFBZSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUMsSUFBWSxFQUFFLEVBQUUsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7WUFDaEcsTUFBTSxXQUFXLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBQyxJQUFZLEVBQUUsRUFBRSxDQUFDLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7WUFDOUYsTUFBTSxjQUFjLEdBQUcsYUFBYSxDQUFDLFdBQVcsQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUMsQ0FBQztZQUNwRSxJQUFJLENBQUMsT0FBTyxHQUFHLENBQUMsR0FBRyxlQUFlLEVBQUUsR0FBRyxjQUFjLENBQUMsQ0FBQztRQUN6RCxDQUFDO0lBQ0gsQ0FBQztTQUFNLENBQUM7UUFDTixJQUFJLENBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQTtJQUNuQixDQUFDO0lBRUQsSUFBSSxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDZCxJQUFJLEtBQUssR0FBYSxFQUFFLENBQUE7UUFDeEIsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7WUFDdkIsS0FBSyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUE7UUFDbkIsQ0FBQzthQUFNLElBQUksTUFBTSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxNQUFNLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLElBQUksQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDLEdBQUcsS0FBSyxPQUFPLEVBQUUsQ0FBQztZQUMzRixLQUFLLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLEVBQUUsTUFBTSxDQUFPLElBQUksRUFBRSxDQUFBO1FBQzdDLENBQUM7YUFBTSxJQUFJLFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQztZQUMvQixNQUFNLElBQUksR0FBRyxVQUFVLENBQUMsYUFBYSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUMsS0FBSyxFQUFDO2dCQUM1RSxRQUFRLEVBQUUsSUFBSTthQUNmLENBQUMsQ0FBQTtZQUNGLEtBQUssR0FBRyxZQUFZLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFBO1FBQ2xDLENBQUM7YUFBTSxJQUFJLE1BQU0sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksTUFBTSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDO1lBQ2xELEtBQUssR0FBRyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQTtRQUNyQixDQUFDO2FBQU0sSUFBSSxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7WUFDL0IsS0FBSyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFBO1FBQzlCLENBQUM7UUFDRCxJQUFJLEtBQUssQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUFFLENBQUM7WUFDdkIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsNkJBQTZCLElBQUksQ0FBQyxJQUFJLGFBQWEsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDLENBQUE7WUFDaEYsT0FBTyxHQUFHLENBQUE7UUFDWixDQUFDO1FBQ0QsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsY0FBYyxLQUFLLENBQUMsTUFBTSwyQkFBMkIsSUFBSSxDQUFDLElBQUksS0FBSyxDQUFDLENBQUE7UUFDckYsTUFBTSxPQUFPLEdBQUcsR0FBRyxFQUFFLENBQUE7UUFDckIsS0FBSyxNQUFNLElBQUksSUFBSSxLQUFLLEVBQUUsQ0FBQztZQUN6QixNQUFNLFFBQVEsR0FBRztnQkFDZixHQUFHLElBQUk7Z0JBQ1AsSUFBSSxFQUFFLElBQUk7Z0JBQ1YsU0FBUyxFQUFFLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRTthQUMxQixDQUFBO1lBQ0QsTUFBTSxLQUFLLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLEtBQUssSUFBSSxDQUFDLENBQUE7WUFDOUMsSUFBSSxLQUFLLEVBQUUsQ0FBQztnQkFDVixRQUFRLENBQUMsS0FBSyxHQUFHLElBQUksQ0FBQTtZQUN2QixDQUFDO1lBRUQsSUFBSSwyQkFBMkIsR0FBRyxDQUFDLGFBQWEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDO1lBRXhELFFBQVEsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxHQUFHLElBQUksQ0FBQyxPQUFPLEVBQUUsR0FBRywyQkFBMkIsQ0FBQyxDQUFDO1lBRXJFLE1BQU0sTUFBTSxHQUFHLE1BQU0sVUFBVSxDQUFDLFFBQVEsQ0FBQyxDQUFBO1lBQ3pDLElBQUksTUFBTSxLQUFLLFNBQVMsRUFBRSxDQUFDO2dCQUN6QixHQUFHLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFBO1lBQ2xCLENBQUM7UUFDSCxDQUFDO0lBQ0gsQ0FBQztTQUFNLENBQUM7UUFDTixNQUFNLE1BQU0sR0FBRyxNQUFNLFVBQVUsQ0FBQyxJQUFJLENBQUMsQ0FBQTtRQUNyQyxJQUFJLE1BQU0sS0FBSyxTQUFTLEVBQUUsQ0FBQztZQUN6QixHQUFHLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFBO1FBQ2xCLENBQUM7SUFDSCxDQUFDO0lBQ0QsT0FBTyxHQUFHLENBQUE7QUFDWixDQUFDLENBQUEifQ== \ No newline at end of file +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVuLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2NvbW1hbmRzL3J1bi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssSUFBSSxNQUFNLFdBQVcsQ0FBQTtBQUNqQyxPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sWUFBWSxDQUFBO0FBQ25DLE9BQU8sRUFBRSxHQUFHLElBQUksVUFBVSxFQUFFLE1BQU0sY0FBYyxDQUFBO0FBRWhELE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSxNQUFNLENBQUE7QUFDL0IsT0FBTyxFQUFFLElBQUksSUFBSSxHQUFHLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQTtBQUM5QyxPQUFPLEVBQUUsSUFBSSxJQUFJLE1BQU0sRUFBRSxNQUFNLHFCQUFxQixDQUFBO0FBQ3BELE9BQU8sRUFBRSxJQUFJLElBQUksS0FBSyxFQUFFLE1BQU0sb0JBQW9CLENBQUE7QUFDbEQsT0FBTyxFQUFFLElBQUksSUFBSSxJQUFJLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQTtBQUNoRCxPQUFPLEVBQUUsYUFBYSxFQUFFLE9BQU8sRUFBRSxNQUFNLEVBQUUsVUFBVSxFQUFFLE1BQU0sbUJBQW1CLENBQUE7QUFDOUUsT0FBTyxFQUFFLE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQTtBQUs3RCxPQUFPLEVBQUUsTUFBTSxFQUFXLE1BQU0sT0FBTyxDQUFBO0FBQ3ZDLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSxjQUFjLENBQUE7QUFDM0MsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLGtCQUFrQixDQUFBO0FBQ2hELE9BQU8sRUFBRSxHQUFHLEVBQUUsTUFBTSxjQUFjLENBQUE7QUFDbEMsT0FBTyxFQUFFLFFBQVEsRUFBRSxNQUFNLFlBQVksQ0FBQTtBQUNyQyxPQUFPLEVBQUUsT0FBTyxFQUFFLE1BQU0sbUJBQW1CLENBQUE7QUFDM0MsT0FBTyxFQUFFLFNBQVMsRUFBRSxNQUFNLGlCQUFpQixDQUFBO0FBQzNDLE9BQU8sRUFBRSxJQUFJLElBQUksV0FBVyxFQUFFLE1BQU0sZUFBZSxDQUFBO0FBQ25ELE9BQU8sRUFBRSxJQUFJLElBQUksU0FBUyxFQUFFLE1BQU0sYUFBYSxDQUFBO0FBQy9DLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxFQUFFLE1BQU0sY0FBYyxDQUFBO0FBQ2xELE9BQU8sRUFBRSxTQUFTLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQTtBQUMzQyxPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sa0JBQWtCLENBQUE7QUFFekMsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLHFCQUFxQixDQUFBO0FBQ25ELE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQTtBQUN6QyxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sb0JBQW9CLENBQUE7QUFDakQsT0FBTyxFQUFFLEdBQUcsRUFBRSxNQUFNLG9CQUFvQixDQUFBO0FBQ3hDLE9BQU8sRUFBRSxTQUFTLEVBQUUsTUFBTSxhQUFhLENBQUE7QUFJdkMsTUFBTSxDQUFDLE1BQU0sWUFBWSxHQUFHLENBQUMsS0FBZSxFQUFZLEVBQUU7SUFDeEQsTUFBTSxnQkFBZ0IsR0FBRyxDQUFDLFFBQWdCLEVBQUUsRUFBRTtRQUMxQyxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDLFFBQVEsQ0FBQyxDQUFDLElBQUksQ0FBQztRQUMzQyxNQUFNLEtBQUssR0FBRyxRQUFRLENBQUMsS0FBSyxDQUFDLGVBQWUsQ0FBQyxDQUFDLENBQUMsd0JBQXdCO1FBQ3ZFLE1BQU0sT0FBTyxHQUFHLEtBQUssQ0FBQyxDQUFDLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDO1FBQ3JELE1BQU0sUUFBUSxHQUFHLEtBQUssQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxRQUFRLENBQUMsQ0FBQyxvQkFBb0I7UUFFbEUsT0FBTyxFQUFFLE9BQU8sRUFBRSxRQUFRLEVBQUUsQ0FBQztJQUNqQyxDQUFDLENBQUE7SUFDRCxPQUFPLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxFQUFFLEVBQUU7UUFDdkIsTUFBTSxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsUUFBUSxFQUFFLEtBQUssRUFBRSxHQUFHLGdCQUFnQixDQUFDLENBQUMsQ0FBQyxDQUFBO1FBQzlELE1BQU0sRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLFFBQVEsRUFBRSxLQUFLLEVBQUUsR0FBRyxnQkFBZ0IsQ0FBQyxDQUFDLENBQUMsQ0FBQTtRQUM5RCxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7WUFDL0IsT0FBTyxJQUFJLEdBQUcsSUFBSSxJQUFJLEtBQUssQ0FBQyxhQUFhLENBQUMsS0FBSyxFQUFFLFNBQVMsRUFBRSxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsV0FBVyxFQUFFLE1BQU0sRUFBRSxDQUFDLENBQUE7UUFDdkcsQ0FBQztRQUNELE9BQU8sS0FBSyxDQUFDLGFBQWEsQ0FBQyxLQUFLLEVBQUUsU0FBUyxFQUFFLEVBQUUsT0FBTyxFQUFFLElBQUksRUFBRSxXQUFXLEVBQUUsTUFBTSxFQUFFLENBQUMsQ0FBQTtJQUN4RixDQUFDLENBQUMsQ0FBQTtBQUNKLENBQUMsQ0FBQTtBQUlEOzs7OztHQUtHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sZ0JBQWdCLEdBQUcsS0FBSyxFQUFFLElBQWUsRUFBNkIsRUFBRTtJQUNuRixJQUFJLENBQUMsTUFBTSxHQUFHLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQTtJQUU3QixNQUFNLE1BQU0sR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxNQUFNLElBQUksSUFBSSxDQUFDLElBQUksSUFBSSxHQUFHLENBQUMsQ0FBQTtJQUM1RCxJQUFJLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUM7UUFDcEIsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFBO0lBQ2IsQ0FBQztJQUVELElBQUksQ0FBQyxPQUFPLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQTtJQUNwQyxJQUFJLENBQUMsWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLENBQUE7SUFDOUMsSUFBSSxDQUFDLE9BQU8sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFBO0lBQ3BDLElBQUksQ0FBQyxPQUFPLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxPQUFPLElBQUksRUFBRSxDQUFDLENBQUE7SUFDMUMsSUFBSSxDQUFDLFNBQVMsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxNQUFNLFdBQVcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFBO0lBRTVELElBQUksQ0FBQztRQUNILE1BQU0sTUFBTSxHQUFHLGFBQWEsRUFBRSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUMzQyxNQUFNLE9BQU8sR0FBRyxNQUFtQixDQUFDO1FBRXBDLE1BQU0sTUFBTSxHQUFHLFlBQVksQ0FBQyxPQUFPLENBQUMsQ0FBQTtRQUNwQyxJQUFJLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDWixJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyx5QkFBeUIsQ0FBQyxDQUFBO1lBQzVDLE9BQU8sSUFBSSxDQUFBO1FBQ2IsQ0FBQztRQUVELE9BQU8sQ0FBQyxNQUFNLEdBQUcsTUFBTSxDQUFBO1FBQ3ZCLE9BQU8sQ0FBQyxTQUFTLEdBQUcsRUFBRSxHQUFHLE9BQU8sQ0FBQyxTQUFTLEVBQUUsR0FBRyxTQUFTLENBQUMsT0FBTyxDQUFDLEVBQUUsQ0FBQTtRQUNuRSxPQUFPLENBQUMsU0FBUyxHQUFHLFNBQVMsQ0FBQyxPQUFPLEVBQUUsTUFBTSxDQUFDLENBQUE7UUFDOUMsT0FBTyxDQUFDLE1BQU0sR0FBRyxJQUFJLENBQUMsTUFBTSxDQUFBO1FBQzVCLE9BQU8sQ0FBQyxLQUFLLEdBQUcsT0FBTyxDQUFDLEtBQUssSUFBSSxDQUFDLEtBQUssRUFBRSxPQUFPLEVBQUUsRUFBRSxDQUFDLE9BQU8sQ0FBQyxDQUFBO1FBRTdELE9BQU8sT0FBTyxDQUFBO0lBQ2hCLENBQUM7SUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1FBQ2YsSUFBSSxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsMEJBQTBCLEVBQUUsS0FBSyxDQUFDLE9BQU8sRUFBRSxLQUFLLENBQUMsTUFBTSxFQUFFLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQTtRQUN2RixPQUFPLElBQUksQ0FBQTtJQUNiLENBQUM7QUFDSCxDQUFDLENBQUE7QUFFRDs7Ozs7O0dBTUc7QUFDSCxNQUFNLENBQUMsTUFBTSxpQkFBaUIsR0FBRyxLQUFLLEVBQ3BDLElBQWUsRUFDZixPQUFrQixFQUlqQixFQUFFO0lBQ0gsSUFBSSxZQUFZLEdBQXNDLEVBQUUsQ0FBQTtJQUV4RCxNQUFNLGFBQWEsR0FBRyxNQUFNLE1BQU0sQ0FBQyxJQUFJLENBQUMsQ0FBQTtJQUN4QyxJQUFJLENBQUMsYUFBYSxFQUFFLE9BQU8sRUFBRSxDQUFDO1FBQzVCLE9BQU8sRUFBRSxRQUFRLEVBQUUsRUFBRSxFQUFFLEtBQUssRUFBRSxFQUFFLEVBQUUsQ0FBQTtJQUNwQyxDQUFDO0lBQ0QsWUFBWSxDQUFDLElBQUksQ0FBQyxhQUEyQyxDQUFDLENBQUE7SUFDOUQsWUFBWSxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQU0sV0FBVyxDQUFDLElBQUksQ0FBQyxDQUErQixDQUFDLENBQUE7SUFFMUUsTUFBTSxXQUFXLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUMsSUFBSSxJQUFJLEdBQUcsQ0FBQyxDQUFBO0lBQ3JELElBQUksV0FBVyxHQUFHLE1BQU0sR0FBRyxDQUFDLFdBQVcsRUFBRSxPQUFPLENBQUMsT0FBTyxFQUFFLE9BQU8sQ0FBQyxJQUFJLEVBQUUsQ0FBQTtJQUV4RSxNQUFNLHFCQUFxQixHQUFpQyxXQUFXLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFO1FBQzlFLElBQUksQ0FBQyxDQUFDLElBQUksSUFBSSxDQUFDLENBQUMsT0FBTyxJQUFJLE9BQU8sQ0FBQyxDQUFDLE9BQU8sS0FBSyxRQUFRLEVBQUUsQ0FBQztZQUN6RCxNQUFNLFFBQVEsR0FBRyxNQUFNLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxDQUFBO1lBQy9CLHVHQUF1RztZQUN2RyxJQUFJLFFBQVEsSUFBSSxDQUFDLFFBQVEsQ0FBQyxVQUFVLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxrQkFBa0IsRUFBRSxpQkFBaUIsRUFBRSx3QkFBd0IsRUFBRSx3QkFBd0IsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLENBQUM7Z0JBQ2pLLElBQUksT0FBTyxDQUFDLElBQUksS0FBSyxNQUFNLEVBQUUsQ0FBQztvQkFDNUIsTUFBTSxZQUFZLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUMsSUFBSSxJQUFJLEdBQUcsRUFBRSxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUE7b0JBQzlELE1BQU0sWUFBWSxHQUFHLENBQUMsQ0FBQyxJQUFJLENBQUEsQ0FBQyxxREFBcUQ7b0JBQ2pGLE1BQU0sdUJBQXVCLEdBQUcsVUFBVSxFQUFFLENBQUE7b0JBQzVDLE1BQU0sVUFBVSxHQUFHOztRQUVyQixZQUFZO2lCQUNILFlBQVk7T0FDdEIsdUJBQXVCOzs7O0NBSTdCLENBQUE7b0JBQ1MsT0FBTyxFQUFFLEdBQUcsQ0FBQyxFQUFFLE9BQU8sRUFBRSxVQUFVLEdBQUcsQ0FBQyxDQUFDLE9BQU8sRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLENBQUE7Z0JBQ2hFLENBQUM7cUJBQU0sQ0FBQztvQkFDTixPQUFPLEVBQUUsR0FBRyxDQUFDLEVBQUUsSUFBSSxFQUFFLE1BQU0sRUFBRSxDQUFBO2dCQUMvQixDQUFDO1lBQ0gsQ0FBQztRQUNILENBQUM7UUFDRCxPQUFPLEVBQUUsR0FBRyxDQUFDLEVBQUUsSUFBSSxFQUFFLE1BQU0sRUFBRSxDQUFBO0lBQy9CLENBQUMsQ0FBQyxDQUFBO0lBRUYsWUFBWSxHQUFHLENBQUMsR0FBRyxZQUFZLEVBQUUsR0FBRyxxQkFBcUIsQ0FBQyxDQUFBO0lBQzFELE9BQU8sRUFBRSxRQUFRLEVBQUUsWUFBWSxFQUFFLEtBQUssRUFBRSxXQUFXLEVBQUUsQ0FBQTtBQUN2RCxDQUFDLENBQUE7QUFFRDs7Ozs7O0dBTUc7QUFDSCxNQUFNLENBQUMsTUFBTSxlQUFlLEdBQUcsS0FBSyxFQUNsQyxPQUFrQixFQUNsQixRQUEyQyxFQUNHLEVBQUU7SUFDaEQsTUFBTSxNQUFNLEdBQUc7UUFDYixLQUFLLEVBQUUsT0FBTyxDQUFDLEtBQUs7UUFDcEIsUUFBUTtRQUNSLEtBQUssRUFBRSxFQUFFO0tBQzZCLENBQUE7SUFFeEMsSUFBSSxPQUFPLENBQUMsSUFBSSxLQUFLLE1BQU0sQ0FBQyxLQUFLLElBQUksT0FBTyxDQUFDLElBQUksS0FBSyxNQUFNLENBQUMsU0FBUyxFQUFFLENBQUM7UUFDdkUsTUFBTSxDQUFDLEtBQUssR0FBRyxNQUFNLFNBQVMsQ0FBQyxPQUFPLENBQUMsQ0FBQTtRQUN2QyxNQUFNLENBQUMsV0FBVyxHQUFHLE1BQU0sQ0FBQTtJQUM3QixDQUFDO0lBRUQsT0FBTyxNQUFNLENBQUE7QUFDZixDQUFDLENBQUE7QUFFRDs7Ozs7OztHQU9HO0FBQ0gsTUFBTSxDQUFDLE1BQU0sZUFBZSxHQUFHLEtBQUssRUFDbEMsT0FBa0IsRUFDbEIsTUFBVyxFQUNYLE1BQTJDLEVBQ2hCLEVBQUU7SUFDN0IsSUFBSSxHQUFHLEdBQVEsSUFBSSxDQUFBO0lBRW5CLElBQUksQ0FBQztRQUNILFFBQVEsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDO1lBQ3JCLEtBQUssTUFBTSxDQUFDLFVBQVU7Z0JBQ3BCLEdBQUcsR0FBRyxNQUFNLGFBQWEsQ0FBQyxNQUFNLEVBQUUsTUFBTSxFQUFFLE9BQU8sQ0FBQyxDQUFBO2dCQUNsRCxNQUFLO1lBRVAsS0FBSyxNQUFNLENBQUMsS0FBSztnQkFDZixHQUFHLEdBQUcsTUFBTSxRQUFRLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxPQUFPLENBQUMsQ0FBQTtnQkFDN0MsTUFBSztZQUVQLEtBQUssTUFBTSxDQUFDLFNBQVM7Z0JBQ25CLEdBQUcsR0FBRyxNQUFNLFlBQVksQ0FBQyxNQUFNLEVBQUUsTUFBTSxFQUFFLE9BQU8sQ0FBQyxDQUFBO2dCQUNqRCxNQUFLO1lBRVA7Z0JBQ0UsTUFBTSxJQUFJLEtBQUssQ0FBQyxxQkFBcUIsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUE7UUFDeEQsQ0FBQztJQUNILENBQUM7SUFBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1FBQ1gsT0FBTyxDQUFDLE1BQU0sRUFBRSxLQUFLLENBQUMsaUJBQWlCLE9BQU8sQ0FBQyxJQUFJLFVBQVUsQ0FBQyxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUE7SUFDM0UsQ0FBQztJQUVELE9BQU8sR0FBRyxDQUFBO0FBQ1osQ0FBQyxDQUFBO0FBRUQ7Ozs7O0dBS0c7QUFDSCxNQUFNLENBQUMsTUFBTSxVQUFVLEdBQUcsS0FBSyxFQUFFLElBQWUsRUFBNkIsRUFBRTtJQUM3RSxNQUFNLGNBQWMsR0FBRyxDQUFDLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFBO0lBQ3hDLElBQUksT0FBTyxHQUFHLE1BQU0sZ0JBQWdCLENBQUMsSUFBSSxDQUFDLENBQUE7SUFDMUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ2IsT0FBTyxJQUFJLENBQUE7SUFDYixDQUFDO0lBRUQsTUFBTSxNQUFNLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQTtJQUU3QixNQUFNLEVBQUUsUUFBUSxFQUFFLEtBQUssRUFBRSxHQUFHLE1BQU0saUJBQWlCLENBQUMsSUFBSSxFQUFFLE9BQU8sQ0FBQyxDQUFBO0lBQ2xFLElBQUcsY0FBYyxDQUFDLE1BQU0sR0FBRyxDQUFDLElBQUksS0FBSyxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQUUsQ0FBQztRQUNuRCxPQUFPLEVBQUUsQ0FBQTtJQUNYLENBQUM7SUFDRCxJQUFJLFFBQVEsQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUFFLENBQUM7UUFDMUIsT0FBTyxFQUFFLENBQUE7SUFDWCxDQUFDO0lBQ0QsTUFBTSxNQUFNLEdBQUcsTUFBTSxlQUFlLENBQUMsT0FBTyxFQUFFLFFBQVEsQ0FBQyxDQUFBO0lBRXZELE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxJQUFJLElBQUksUUFBUSxDQUFDLENBQUMsQ0FBQTtJQUMzRCxNQUFNLFVBQVUsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLE1BQU0sRUFBRSxhQUFhLENBQUMsQ0FBQTtJQUNuRCxLQUFLLENBQUMsVUFBVSxFQUFFLElBQUksQ0FBQyxTQUFTLENBQUMsRUFBRSxHQUFHLE1BQU0sRUFBRSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFBO0lBQ3pELElBQUksSUFBSSxDQUFDLElBQUksS0FBSyxNQUFNLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDL0IsT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsWUFBWSxPQUFPLENBQUMsSUFBSSxNQUFNLE9BQU8sQ0FBQyxLQUFLLE1BQU0sT0FBTyxDQUFDLE1BQU0sT0FBTyxLQUFLLENBQUMsTUFBTSx1QkFBdUIsSUFBSSxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUMsSUFBSSxJQUFJLEdBQUcsQ0FBQyxTQUFTLE9BQU8sQ0FBQyxPQUFPLEVBQUUsRUFBRSxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxFQUFFLE1BQU0sQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsR0FBRyxDQUFDLENBQUMsUUFBUSxDQUFDLElBQUksTUFBTSxDQUFDLENBQUMsUUFBUSxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUMsQ0FBQTtJQUM3UixDQUFDO1NBQU0sQ0FBQztRQUNOLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLFlBQVksT0FBTyxDQUFDLElBQUksTUFBTSxPQUFPLENBQUMsS0FBSyxNQUFNLE9BQU8sQ0FBQyxNQUFNLE9BQU8sS0FBSyxDQUFDLE1BQU0sdUJBQXVCLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLElBQUksSUFBSSxHQUFHLENBQUMsU0FBUyxPQUFPLENBQUMsT0FBTyxFQUFFLEVBQUUsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFBO0lBQ3BOLENBQUM7SUFDRCxPQUFPLEdBQUcsTUFBTSxPQUFPLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxJQUFJLE9BQU8sQ0FBQTtJQUNqRCxNQUFNLEdBQUcsR0FBRyxNQUFNLGVBQWUsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sQ0FBQyxDQUFBO0lBQzFELE9BQU8sR0FBRyxDQUFBO0FBQ1osQ0FBQyxDQUFBO0FBRUQ7Ozs7Ozs7OztHQVNHO0FBQ0gsU0FBUyxZQUFZLENBQUMsS0FBYTtJQUNqQywrQkFBK0I7SUFDL0IsSUFBSSxRQUFRLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQztRQUNwQixPQUFPLENBQUMsS0FBSyxDQUFDLENBQUE7SUFDaEIsQ0FBQztJQUVELCtDQUErQztJQUMvQyxJQUFJLE1BQU0sQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztRQUN6QyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUE7SUFDaEIsQ0FBQztJQUVELDBDQUEwQztJQUMxQyxJQUFJLFFBQVEsQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDO1FBQ3BCLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQTtJQUNoQixDQUFDO0lBRUQsNkZBQTZGO0lBQzdGLE1BQU0sY0FBYyxHQUFHLGlDQUFpQyxDQUFDO0lBRXpELE1BQU0sWUFBWSxHQUFhLEVBQUUsQ0FBQztJQUNsQyxJQUFJLEtBQTZCLENBQUM7SUFFbEMsdUNBQXVDO0lBQ3ZDLE9BQU8sQ0FBQyxLQUFLLEdBQUcsY0FBYyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQyxLQUFLLElBQUksRUFBRSxDQUFDO1FBQ3JELFlBQVksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ2pDLENBQUM7SUFFRCw4QkFBOEI7SUFDOUIsSUFBSSxDQUFDLFlBQVksQ0FBQyxNQUFNLEVBQUUsQ0FBQztRQUN6QixPQUFPLEVBQUUsQ0FBQztJQUNaLENBQUM7SUFFRCxnREFBZ0Q7SUFDaEQsTUFBTSxLQUFLLEdBQWEsRUFBRSxDQUFDO0lBQzNCLEtBQUssSUFBSSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsR0FBRyxZQUFZLENBQUMsTUFBTSxFQUFFLENBQUMsRUFBRSxFQUFFLENBQUM7UUFDN0MsTUFBTSxLQUFLLEdBQUcsWUFBWSxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQzlCLE1BQU0sR0FBRyxHQUFHLENBQUMsR0FBRyxZQUFZLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsWUFBWSxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLE1BQU0sQ0FBQztRQUM3RSxNQUFNLElBQUksR0FBRyxLQUFLLENBQUMsU0FBUyxDQUFDLEtBQUssRUFBRSxHQUFHLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUNoRCxJQUFJLElBQUksRUFBRSxDQUFDO1lBQ1QsS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNuQixDQUFDO0lBQ0gsQ0FBQztJQUVELE9BQU8sS0FBSyxDQUFDO0FBQ2YsQ0FBQztBQUVELFNBQVMsYUFBYSxDQUFJLE1BQWE7SUFDckMsT0FBTyxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUMsV0FBVyxFQUFFLE9BQU8sRUFBRSxFQUFFO1FBQzVDLE9BQU8sV0FBVyxDQUFDLE1BQU0sQ0FBQyxPQUFPLENBQUMsQ0FBQztJQUNyQyxDQUFDLEVBQUUsRUFBUyxDQUFDLENBQUM7QUFDaEIsQ0FBQztBQUVELE1BQU0sQ0FBQyxNQUFNLEdBQUcsR0FBRyxLQUFLLEVBQUUsSUFBZSxFQUErQixFQUFFO0lBQ3hFLE1BQU0sR0FBRyxHQUF1QixFQUFFLENBQUE7SUFDbEMsTUFBTSxNQUFNLEdBQUcsSUFBSSxNQUFNLENBQVUsRUFBRSxRQUFRLEVBQUUsSUFBSSxDQUFDLFFBQVEsSUFBSSxDQUFDLEVBQUUsQ0FBQyxDQUFBO0lBQ3BFLElBQUksQ0FBQyxNQUFNLEdBQUcsTUFBTSxDQUFBO0lBRXBCLElBQUksSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ2pCLElBQUksUUFBUSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQzNCLElBQUksQ0FBQyxPQUFPLEdBQUcsQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUE7UUFDL0IsQ0FBQztRQUNELElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQzFCLDZDQUE2QztZQUM3QyxNQUFNLGVBQWUsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDLElBQVksRUFBRSxFQUFFLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxJQUFJLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFBO1lBQy9GLDRCQUE0QjtZQUM1QixNQUFNLFdBQVcsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDLElBQVksRUFBRSxFQUFFLENBQUMsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQTtZQUM3RixNQUFNLGNBQWMsR0FBRyxhQUFhLENBQUMsV0FBVyxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFBO1lBQ25FLCtDQUErQztZQUMvQyxJQUFJLENBQUMsT0FBTyxHQUFHLENBQUMsR0FBRyxlQUFlLEVBQUUsR0FBRyxjQUFjLENBQUMsQ0FBQTtRQUN4RCxDQUFDO0lBQ0gsQ0FBQztTQUFNLENBQUM7UUFDTixJQUFJLENBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQTtJQUNuQixDQUFDO0lBRUQsaURBQWlEO0lBQ2pELElBQUksSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ2pCLElBQUksUUFBUSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQzNCLElBQUksQ0FBQyxPQUFPLEdBQUcsQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUE7UUFDL0IsQ0FBQztRQUNELElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQzFCLE1BQU0sZUFBZSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUMsSUFBWSxFQUFFLEVBQUUsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7WUFDaEcsTUFBTSxXQUFXLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBQyxJQUFZLEVBQUUsRUFBRSxDQUFDLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7WUFDOUYsTUFBTSxjQUFjLEdBQUcsYUFBYSxDQUFDLFdBQVcsQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUMsQ0FBQztZQUNwRSxJQUFJLENBQUMsT0FBTyxHQUFHLENBQUMsR0FBRyxlQUFlLEVBQUUsR0FBRyxjQUFjLENBQUMsQ0FBQztRQUN6RCxDQUFDO0lBQ0gsQ0FBQztTQUFNLENBQUM7UUFDTixJQUFJLENBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQTtJQUNuQixDQUFDO0lBRUQsSUFBSSxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDZCxJQUFJLEtBQUssR0FBYSxFQUFFLENBQUE7UUFDeEIsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7WUFDdkIsS0FBSyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUE7UUFDbkIsQ0FBQzthQUFNLElBQUksTUFBTSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxNQUFNLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLElBQUksQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDLEdBQUcsS0FBSyxPQUFPLEVBQUUsQ0FBQztZQUMzRixLQUFLLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLEVBQUUsTUFBTSxDQUFPLElBQUksRUFBRSxDQUFBO1FBQzdDLENBQUM7YUFBTSxJQUFJLFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQztZQUMvQixNQUFNLElBQUksR0FBRyxVQUFVLENBQUMsYUFBYSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUMsS0FBSyxFQUFDO2dCQUM1RSxRQUFRLEVBQUUsSUFBSTthQUNmLENBQUMsQ0FBQTtZQUNGLEtBQUssR0FBRyxZQUFZLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFBO1FBQ2xDLENBQUM7YUFBTSxJQUFJLE1BQU0sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksTUFBTSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDO1lBQ2xELEtBQUssR0FBRyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQTtRQUNyQixDQUFDO2FBQU0sSUFBSSxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7WUFDL0IsS0FBSyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFBO1FBQzlCLENBQUM7UUFDRCxJQUFJLEtBQUssQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUFFLENBQUM7WUFDdkIsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsNkJBQTZCLElBQUksQ0FBQyxJQUFJLGFBQWEsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDLENBQUE7WUFDaEYsT0FBTyxHQUFHLENBQUE7UUFDWixDQUFDO1FBQ0QsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsY0FBYyxLQUFLLENBQUMsTUFBTSwyQkFBMkIsSUFBSSxDQUFDLElBQUksS0FBSyxDQUFDLENBQUE7UUFDckYsTUFBTSxPQUFPLEdBQUcsR0FBRyxFQUFFLENBQUE7UUFDckIsS0FBSyxNQUFNLElBQUksSUFBSSxLQUFLLEVBQUUsQ0FBQztZQUN6QixNQUFNLFFBQVEsR0FBRztnQkFDZixHQUFHLElBQUk7Z0JBQ1AsSUFBSSxFQUFFLElBQUk7Z0JBQ1YsU0FBUyxFQUFFLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRTthQUMxQixDQUFBO1lBQ0QsTUFBTSxLQUFLLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLEtBQUssSUFBSSxDQUFDLENBQUE7WUFDOUMsSUFBSSxLQUFLLEVBQUUsQ0FBQztnQkFDVixRQUFRLENBQUMsS0FBSyxHQUFHLElBQUksQ0FBQTtZQUN2QixDQUFDO1lBRUQsSUFBSSwyQkFBMkIsR0FBRyxDQUFDLGFBQWEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDO1lBRXhELFFBQVEsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxHQUFHLElBQUksQ0FBQyxPQUFPLEVBQUUsR0FBRywyQkFBMkIsQ0FBQyxDQUFDO1lBRXJFLE1BQU0sTUFBTSxHQUFHLE1BQU0sVUFBVSxDQUFDLFFBQVEsQ0FBQyxDQUFBO1lBQ3pDLElBQUksTUFBTSxLQUFLLFNBQVMsRUFBRSxDQUFDO2dCQUN6QixHQUFHLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFBO1lBQ2xCLENBQUM7UUFDSCxDQUFDO0lBQ0gsQ0FBQztTQUFNLENBQUM7UUFDTixNQUFNLE1BQU0sR0FBRyxNQUFNLFVBQVUsQ0FBQyxJQUFJLENBQUMsQ0FBQTtRQUNyQyxJQUFJLE1BQU0sS0FBSyxTQUFTLEVBQUUsQ0FBQztZQUN6QixHQUFHLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFBO1FBQ2xCLENBQUM7SUFDSCxDQUFDO0lBQ0QsT0FBTyxHQUFHLENBQUE7QUFDWixDQUFDLENBQUEifQ== \ No newline at end of file diff --git a/packages/kbot/logs/params.json b/packages/kbot/logs/params.json index 54f8eaf5..91b5fcf8 100644 --- a/packages/kbot/logs/params.json +++ b/packages/kbot/logs/params.json @@ -3,7 +3,7 @@ "messages": [ { "role": "user", - "content": "multiply 8 and 3. Return only the number, no explanation." + "content": "Provide a synonym for \"happy\". Return only the synonym, no explanation." }, { "role": "user", diff --git a/packages/kbot/package-lock.json b/packages/kbot/package-lock.json index f861d314..00653213 100644 --- a/packages/kbot/package-lock.json +++ b/packages/kbot/package-lock.json @@ -9,6 +9,7 @@ "version": "0.3.5", "license": "MIT", "dependencies": { + "@dmitryrechkin/json-schema-to-zod": "1.0.1", "@polymech/ai-tools": "file:../ai-tools", "@polymech/cache": "file:../cache", "@polymech/commons": "file:../commons", @@ -400,6 +401,14 @@ "node": ">=14.17.0" } }, + "node_modules/@dmitryrechkin/json-schema-to-zod": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@dmitryrechkin/json-schema-to-zod/-/json-schema-to-zod-1.0.1.tgz", + "integrity": "sha512-cG9gC4NMu/7JZqmRZy6uIb+l+kxek2GFQ0/qrhw7xeFK2l5B9yF9FVuujoqFPLRGDHNFYqtBWht7hY4KB0ngrA==", + "dependencies": { + "zod": "^3.23.8" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", diff --git a/packages/kbot/package.json b/packages/kbot/package.json index e8231ba7..37d40745 100644 --- a/packages/kbot/package.json +++ b/packages/kbot/package.json @@ -53,6 +53,7 @@ "examples:iterator-markdown:no-cache": "node dist-in/examples/core/iterator-markdown-example.js --no-cache" }, "dependencies": { + "@dmitryrechkin/json-schema-to-zod": "1.0.1", "@polymech/ai-tools": "file:../ai-tools", "@polymech/cache": "file:../cache", "@polymech/commons": "file:../commons", diff --git a/packages/kbot/src/commands/run.ts b/packages/kbot/src/commands/run.ts index 17fa82aa..dd3818b1 100644 --- a/packages/kbot/src/commands/run.ts +++ b/packages/kbot/src/commands/run.ts @@ -88,6 +88,7 @@ export const complete_options = async (opts: IKBotTask): Promise options) return options @@ -174,7 +175,6 @@ export const complete_params = async ( if (options.mode === E_Mode.TOOLS || options.mode === E_Mode.ASSISTANT) { params.tools = await loadTools(options) params.tool_choice = 'auto' - //params.parallel_tool_calls = false } return params diff --git a/packages/kbot/tests/unit/commons.ts b/packages/kbot/tests/unit/commons.ts index a1f751de..89289960 100644 --- a/packages/kbot/tests/unit/commons.ts +++ b/packages/kbot/tests/unit/commons.ts @@ -5,16 +5,46 @@ import { sync as write } from "@polymech/fs/write" import { sync as read } from "@polymech/fs/read" import { sync as exists } from "@polymech/fs/exists" import { sync as mkdirp } from "mkdirp" - +import { zodResponseFormat } from "openai/helpers/zod" +import { JSONSchemaToZod } from '@dmitryrechkin/json-schema-to-zod'; export enum ModelCategory { FAST = 'fast', LANGUAGE = 'language', TOOL = 'tool', ALL = 'all', CODING = 'coding', - FILES = 'file' + FILES = 'file', + TEST_EQUAL = '' } +export enum EqualityCheck { + DEFAULT = 'default', + JSON_EQUAL = 'json_equal', + LLM_EQUAL = 'llm_equal', + NONE = 'none' +} + +export type EqualityFn = (actual: string, expected: string) => Promise; + +export const EQUALITY_CHECKS: Record = { + [EqualityCheck.DEFAULT]: async (actual: string, expected: string): Promise => { + return (actual || '').trim().toLowerCase() === (expected || '').trim().toLowerCase(); + }, + [EqualityCheck.JSON_EQUAL]: async (actual: string, expected: string): Promise => { + try { + // we just stringify to normalize and compare + const actualJson = JSON.parse(actual.trim()); + const expectedJson = JSON.parse(expected.trim()); + return JSON.stringify(actualJson) === JSON.stringify(expectedJson); + } catch (e) { + return false; + } + }, + [EqualityCheck.NONE]: async (): Promise => { + return true; + }, +}; + export const getFastModels = (): string[] => { return [ E_OPENROUTER_MODEL.MODEL_ANTHROPIC_CLAUDE_SONNET_4, @@ -23,6 +53,12 @@ export const getFastModels = (): string[] => { ] } +export const getTestEqualModels = (): string[] => { + return [ + E_OPENROUTER_MODEL.MODEL_DEEPSEEK_DEEPSEEK_R1_FREE + ] +} + export const getCodingModels = (): string[] => { return [ E_OPENROUTER_MODEL.MODEL_NVIDIA_LLAMA_3_3_NEMOTRON_SUPER_49B_V1_FREE @@ -31,8 +67,7 @@ export const getCodingModels = (): string[] => { export const getFileModels = (): string[] => { return [ - E_OPENROUTER_MODEL.MODEL_OPENAI_GPT_4O_MINI, - E_OPENROUTER_MODEL.MODEL_GOOGLE_GEMINI_2_0_FLASH_EXP_FREE + E_OPENROUTER_MODEL.MODEL_OPENAI_GPT_4O ] } @@ -68,6 +103,8 @@ export const getDefaultModels = (category: ModelCategory = ModelCategory.FAST): return getCodingModels() case ModelCategory.FILES: return getFileModels() + case ModelCategory.TEST_EQUAL: + return getTestEqualModels() case ModelCategory.ALL: default: return [ @@ -184,6 +221,11 @@ export const runTest = async ( let defaultOptions = { filters: 'code' } + let format: any = null + if (options.format) { + const zodSchema = JSONSchemaToZod.convert(options.format); + format = zodResponseFormat(zodSchema, "format"); + } try { const result = await Promise.race([ run({ @@ -194,7 +236,7 @@ export const runTest = async ( logs: TEST_LOGS_PATH, preferences: TEST_PREFERENCES_PATH, logLevel: 2, - ...{ ...defaultOptions, ...options }, + ...{ ...defaultOptions, ...options, format }, onRun: async (options) => { model = options.model || 'unknown' router = options.model as string @@ -220,8 +262,10 @@ export const runTest = async ( reason: 'Model returned empty response' } } else { - const actual = result?.[0]?.trim()?.toLowerCase() || '' - const passed = actual === expected + const actual = result?.[0] || '' + const equalityCheck = options.equalityCheck || EqualityCheck.DEFAULT + const checkFn = EQUALITY_CHECKS[equalityCheck] || EQUALITY_CHECKS[EqualityCheck.DEFAULT] + const passed = await checkFn(actual, expected) testResult = { test: testName, @@ -233,7 +277,7 @@ export const runTest = async ( timestamp: new Date().toISOString(), passed, duration: Date.now() - startTime, - reason: passed ? undefined : `Expected ${expected}, but got ${actual}`, + reason: passed ? undefined : `Expected ${expected}, but got ${actual.trim()}`, } } } catch (e) { diff --git a/packages/kbot/tests/unit/files.test.ts b/packages/kbot/tests/unit/files.test.ts index 53cbb428..b881b3de 100644 --- a/packages/kbot/tests/unit/files.test.ts +++ b/packages/kbot/tests/unit/files.test.ts @@ -13,7 +13,8 @@ import { runTest, generateTestReport, getReportPaths, - ModelCategory + ModelCategory, + EqualityCheck } from './commons' import { IKBotOptions } from '@polymech/commons' @@ -25,6 +26,42 @@ describe('File Operations', () => { const TEST_LOG_PATH = getReportPaths('files', 'json') const TEST_REPORT_PATH = getReportPaths('files', 'md') + it.each(models)('should identify animals in image files with model %s', async (modelName) => { + const result = await runTest( + 'What animals are shown in these images?', + '["cat","fox"]', + 'file-inclusion', + modelName, + TEST_LOG_PATH, + 'completion', + { + include: ['tests/test-data/files/lazyfox.jpg', 'tests/test-data/files/cat.jpg'], + logLevel: 2, + equalityCheck: EqualityCheck.NONE, + format: { + type: "object", + properties: { + animals: { + type: "array", + items: { + type: "string" + }, + minItems: 2, + maxItems: 2 + } + }, + required: ["animals"] + } + } as IKBotOptions + ) + testResults.push(result) + const parsedResult = JSON.parse(result.result[0]?.trim()) + const animals = parsedResult.animals.map((s: string) => s.toLowerCase()) + expect(animals).toContain('cat') + expect(animals).toContain('fox') + expect(animals.length).toBe(2) + }, { timeout: TEST_TIMEOUT }) + it.each(models)('should process single file with model %s', async (modelName) => { const result = await runTest( 'What is the name of the algorithm implemented in these files? Return only the name.', @@ -53,6 +90,7 @@ describe('File Operations', () => { 'completion', { include: ['./tests/test-data/files/*.js'], + equalityCheck: EqualityCheck.NONE, logLevel: 2, format: { type: "object", @@ -70,11 +108,12 @@ describe('File Operations', () => { } } as IKBotOptions ) - testResults.push(result) + testResults.push(result) const parsedResult = JSON.parse(result.result[0]?.trim()) - expect(parsedResult.includes('bubble')) - expect(parsedResult.includes('factorial')) - expect(parsedResult).toHaveLength(2) + const algorithms = parsedResult.algorithms.map((s: string) => s.toLowerCase()) + expect(algorithms.some(a => a.includes('bubble'))).toBe(true) + expect(algorithms.some(a => a.includes('factorial'))).toBe(true) + expect(algorithms).toHaveLength(2) }, { timeout: TEST_TIMEOUT }) it.each(models)('should process files in glob subdirectory with model %s', async (modelName) => { @@ -84,46 +123,13 @@ describe('File Operations', () => { 'file-inclusion', modelName, TEST_LOG_PATH, - 'completion', + 'completion', { include: ['tests/test-data/files/glob/data.json'] } ) testResults.push(result) expect(result.result[0]?.trim()).toBe('Injection Barrel') }, { timeout: TEST_TIMEOUT }) - it.each(models)('should identify animals in image files with model %s', async (modelName) => { - const result = await runTest( - 'What animals are shown in these images? Return as JSON array.', - '["cat","fox"]', - 'file-inclusion', - modelName, - TEST_LOG_PATH, - 'completion', - { - include: ['tests/test-data/files/lazyfox.jpg'], - logLevel: 2, - format: { - type: "object", - properties: { - animals: { - type: "array", - items: { - type: "string" - }, - minItems: 2, - maxItems: 2 - } - }, - required: ["animals"] - } - } as IKBotOptions - ) - testResults.push(result) - const parsedResult = JSON.parse(result.result[0]?.trim()) - expect(parsedResult.includes('cat')) - expect(parsedResult.includes('fox')) - expect(parsedResult.length==2) - }, { timeout: TEST_TIMEOUT }) it('should generate markdown report', () => { generateTestReport(testResults, 'File Operations Test Results', TEST_REPORT_PATH) diff --git a/packages/kbot/tests/unit/reports/all.json b/packages/kbot/tests/unit/reports/all.json index 5da451f8..0bc3eec1 100644 --- a/packages/kbot/tests/unit/reports/all.json +++ b/packages/kbot/tests/unit/reports/all.json @@ -2613,6 +2613,2150 @@ "passed": true, "duration": 626, "category": "basic" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [ + "8" + ], + "expected": "8", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T21:19:15.716Z", + "passed": true, + "duration": 2024, + "category": "basic" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [ + "8" + ], + "expected": "8", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:19:16.361Z", + "passed": true, + "duration": 641, + "category": "basic" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [ + "8" + ], + "expected": "8", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T21:19:20.162Z", + "passed": true, + "duration": 3798, + "category": "basic" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [ + "24" + ], + "expected": "24", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T21:19:21.917Z", + "passed": true, + "duration": 1752, + "category": "basic" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [ + "24" + ], + "expected": "24", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:19:22.504Z", + "passed": true, + "duration": 585, + "category": "basic" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [ + "24" + ], + "expected": "24", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T21:19:25.779Z", + "passed": true, + "duration": 3272, + "category": "basic" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [ + "5" + ], + "expected": "5", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T21:19:27.557Z", + "passed": true, + "duration": 1775, + "category": "basic" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [ + "5" + ], + "expected": "5", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:19:28.041Z", + "passed": true, + "duration": 481, + "category": "basic" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [ + "5" + ], + "expected": "5", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T21:19:31.450Z", + "passed": true, + "duration": 3406, + "category": "basic" + }, + { + "test": "web_content", + "prompt": "Check if the content contains a section about Human prehistory. Reply with \"yes\" if it does, \"no\" if it does not.", + "result": [ + "yes" + ], + "expected": "yes", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T21:19:37.473Z", + "passed": true, + "duration": 6020, + "category": "basic" + }, + { + "test": "web_content", + "prompt": "Check if the content contains a section about Human prehistory. Reply with \"yes\" if it does, \"no\" if it does not.", + "result": [ + "yes" + ], + "expected": "yes", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:19:42.394Z", + "passed": true, + "duration": 4917, + "category": "basic" + }, + { + "test": "web_content", + "prompt": "Check if the content contains a section about Human prehistory. Reply with \"yes\" if it does, \"no\" if it does not.", + "result": [], + "expected": "yes", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T21:19:47.544Z", + "passed": false, + "duration": 5147, + "reason": "Model returned empty response", + "category": "basic" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:25:46.078Z", + "passed": true, + "duration": 824, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:25:53.366Z", + "passed": true, + "duration": 7284, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:25:54.218Z", + "passed": false, + "duration": 849, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:25:59.456Z", + "passed": false, + "duration": 5231, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:00.076Z", + "passed": true, + "duration": 616, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:02.225Z", + "passed": true, + "duration": 2146, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:05.871Z", + "passed": true, + "duration": 3643, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:09.058Z", + "passed": true, + "duration": 3183, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:42.944Z", + "passed": true, + "duration": 772, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:46.369Z", + "passed": true, + "duration": 3421, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:47.073Z", + "passed": false, + "duration": 700, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:48.594Z", + "passed": false, + "duration": 1514, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubbleSort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:49.375Z", + "passed": true, + "duration": 779, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:55.618Z", + "passed": true, + "duration": 6239, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"wildcat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:57.688Z", + "passed": false, + "duration": 2067, + "reason": "Expected [\"cat\",\"fox\"], but got [\n \"wildcat\",\n \"fox\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:00.508Z", + "passed": true, + "duration": 2815, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:27:38.292Z", + "passed": true, + "duration": 1023, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:42.531Z", + "passed": true, + "duration": 4235, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:27:43.285Z", + "passed": false, + "duration": 751, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:48.461Z", + "passed": false, + "duration": 5168, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:27:49.024Z", + "passed": true, + "duration": 559, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:54.942Z", + "passed": false, + "duration": 5915, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:27:56.766Z", + "passed": true, + "duration": 1819, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:59.278Z", + "passed": true, + "duration": 2508, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:28:17.298Z", + "passed": true, + "duration": 834, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:28:18.833Z", + "passed": true, + "duration": 1530, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:28:19.525Z", + "passed": false, + "duration": 688, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:28:23.761Z", + "passed": false, + "duration": 4229, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubbleSort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:28:24.280Z", + "passed": true, + "duration": 515, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:28:26.274Z", + "passed": true, + "duration": 1990, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:28:29.111Z", + "passed": true, + "duration": 2834, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:28:35.340Z", + "passed": false, + "duration": 6225, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:30:09.177Z", + "passed": true, + "duration": 1035, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:30:11.447Z", + "passed": true, + "duration": 2266, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:30:12.403Z", + "passed": false, + "duration": 952, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:30:18.660Z", + "passed": false, + "duration": 6250, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:30:19.412Z", + "passed": true, + "duration": 748, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:30:24.575Z", + "passed": false, + "duration": 5159, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:30:26.812Z", + "passed": true, + "duration": 2232, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:30:33.768Z", + "passed": false, + "duration": 6951, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:31:32.809Z", + "passed": true, + "duration": 941, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:31:35.174Z", + "passed": true, + "duration": 2360, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:31:49.546Z", + "passed": false, + "duration": 759, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:32:00.755Z", + "passed": false, + "duration": 7224, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubblesort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:32:01.351Z", + "passed": false, + "duration": 592, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:32:02.883Z", + "passed": false, + "duration": 1528, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:32:05.172Z", + "passed": false, + "duration": 2283, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:32:07.065Z", + "passed": false, + "duration": 1887, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:32:59.145Z", + "passed": true, + "duration": 883, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:33:01.231Z", + "passed": true, + "duration": 2081, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:33:01.922Z", + "passed": false, + "duration": 686, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:33:04.999Z", + "passed": false, + "duration": 3070, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubblesort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:33:07.789Z", + "passed": false, + "duration": 2785, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:33:09.512Z", + "passed": false, + "duration": 1718, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:33:14.818Z", + "passed": false, + "duration": 5303, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:33:17.979Z", + "passed": false, + "duration": 3156, + "reason": "Expected [\"cat\",\"fox\"], but got [\n \"cat\",\n \"fox\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:34:43.846Z", + "passed": true, + "duration": 1036, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:34:48.463Z", + "passed": true, + "duration": 4612, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:34:49.231Z", + "passed": false, + "duration": 763, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:34:55.136Z", + "passed": false, + "duration": 5897, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:34:55.630Z", + "passed": false, + "duration": 489, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:35:00.574Z", + "passed": false, + "duration": 4939, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:35:03.038Z", + "passed": false, + "duration": 2459, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:35:05.480Z", + "passed": false, + "duration": 2438, + "reason": "Expected [\"cat\",\"fox\"], but got [\n \"cat\",\n \"fox\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:36:08.730Z", + "passed": true, + "duration": 1322, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:36:12.605Z", + "passed": true, + "duration": 3870, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:36:13.472Z", + "passed": false, + "duration": 862, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubblesort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:36:58.567Z", + "passed": true, + "duration": 840, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:37:00.081Z", + "passed": true, + "duration": 1509, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:37:00.976Z", + "passed": false, + "duration": 891, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:37:02.031Z", + "passed": false, + "duration": 1048, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubblesort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:37:02.889Z", + "passed": false, + "duration": 854, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:37:04.623Z", + "passed": false, + "duration": 1730, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:37:07.559Z", + "passed": false, + "duration": 2933, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:37:10.561Z", + "passed": false, + "duration": 2998, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:38:15.039Z", + "passed": true, + "duration": 804, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:38:17.121Z", + "passed": true, + "duration": 2077, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:38:25.035Z", + "passed": false, + "duration": 797, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:39:30.971Z", + "passed": false, + "duration": 1721, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:53:07.100Z", + "passed": true, + "duration": 51668, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:53:13.381Z", + "passed": true, + "duration": 2388, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:53:14.200Z", + "passed": true, + "duration": 814, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:53:23.416Z", + "passed": true, + "duration": 2210, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:53:51.472Z", + "passed": true, + "duration": 8947, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:53:57.721Z", + "passed": false, + "duration": 6245, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:54:00.301Z", + "passed": true, + "duration": 2573, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:54:02.900Z", + "passed": true, + "duration": 2594, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:54:12.068Z", + "passed": true, + "duration": 792, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:54:13.596Z", + "passed": true, + "duration": 1522, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:54:14.332Z", + "passed": true, + "duration": 731, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:55:55.651Z", + "passed": false, + "duration": 7061, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:56:27.181Z", + "passed": true, + "duration": 3432, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:56:28.770Z", + "passed": true, + "duration": 1583, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:56:31.322Z", + "passed": true, + "duration": 2548, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:56:34.465Z", + "passed": true, + "duration": 3138, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:57:18.569Z", + "passed": true, + "duration": 867, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:57:20.692Z", + "passed": true, + "duration": 2117, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:57:21.814Z", + "passed": true, + "duration": 1117, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:57:23.423Z", + "passed": true, + "duration": 1603, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:57:24.145Z", + "passed": true, + "duration": 717, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:57:26.292Z", + "passed": true, + "duration": 2142, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:57:29.395Z", + "passed": true, + "duration": 3099, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:59:17.877Z", + "passed": true, + "duration": 4686, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:59:20.537Z", + "passed": true, + "duration": 2653, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:59:21.334Z", + "passed": true, + "duration": 793, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:59:25.325Z", + "passed": true, + "duration": 3986, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:59:25.873Z", + "passed": true, + "duration": 543, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:59:27.328Z", + "passed": true, + "duration": 1451, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "The image shows a wildcat on the left and a red fox on the right." + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:59:30.021Z", + "passed": true, + "duration": 2689, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:11:44.531Z", + "passed": false, + "duration": 26687, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "The image shows a cat and a fox.\n" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T22:12:19.670Z", + "passed": true, + "duration": 4801, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:18:06.005Z", + "passed": false, + "duration": 7536, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "{\"animals\":[\"cat\",\"fox\"]}" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:25:35.300Z", + "passed": true, + "duration": 20975, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "{\"animals\":[\"cat\",\"fox\"]}" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:26:40.769Z", + "passed": true, + "duration": 16606, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubbleSort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:26:41.541Z", + "passed": false, + "duration": 765, + "reason": "Expected bubble sort, but got bubbleSort", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "{\"algorithms\":[\"factorial\",\"bubbleSort\"]}" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:26:42.264Z", + "passed": true, + "duration": 718, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:26:43.192Z", + "passed": true, + "duration": 919, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "{\"animals\":[\"cat\",\"fox\"]}" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:27:59.823Z", + "passed": true, + "duration": 2365, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubbleSort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:28:00.651Z", + "passed": false, + "duration": 822, + "reason": "Expected bubble sort, but got bubbleSort", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "{\"algorithms\":[\"factorial\",\"bubbleSort\"]}" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:28:01.338Z", + "passed": true, + "duration": 682, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "{\"animals\":[\"cat\",\"fox\"]}" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:29:44.549Z", + "passed": true, + "duration": 3234, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubbleSort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:29:45.518Z", + "passed": false, + "duration": 961, + "reason": "Expected bubble sort, but got bubbleSort", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "{\"algorithms\":[\"factorial\",\"bubbleSort\"]}" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:29:46.232Z", + "passed": true, + "duration": 709, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:29:46.851Z", + "passed": true, + "duration": 614, + "category": "files" + }, + { + "test": "translation", + "prompt": "Translate \"Hello, world!\" to Spanish. Return only the translation, no explanation.", + "result": [ + "¡Hola, mundo!" + ], + "expected": "¡Hola, mundo!", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:21.412Z", + "passed": true, + "duration": 1560, + "category": "language" + }, + { + "test": "translation", + "prompt": "Translate \"Hello, world!\" to Spanish. Return only the translation, no explanation.", + "result": [ + "¡Hola, mundo!" + ], + "expected": "¡Hola, mundo!", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:22.869Z", + "passed": true, + "duration": 1451, + "category": "language" + }, + { + "test": "translation", + "prompt": "Translate \"Hello, world!\" to Spanish. Return only the translation, no explanation.", + "result": [ + "¡Hola, mundo!" + ], + "expected": "¡Hola, mundo!", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:30:28.307Z", + "passed": true, + "duration": 5434, + "category": "language" + }, + { + "test": "grammar", + "prompt": "Correct the grammar in: \"I goes to the store yesterday\". Return only the corrected sentence, no explanation.", + "result": [ + "I went to the store yesterday." + ], + "expected": "I went to the store yesterday", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:29.513Z", + "passed": false, + "duration": 1201, + "reason": "Expected I went to the store yesterday, but got I went to the store yesterday.", + "category": "language" + }, + { + "test": "grammar", + "prompt": "Correct the grammar in: \"I goes to the store yesterday\". Return only the corrected sentence, no explanation.", + "result": [ + "\"I went to the store yesterday.\"" + ], + "expected": "I went to the store yesterday", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:30.212Z", + "passed": false, + "duration": 695, + "reason": "Expected I went to the store yesterday, but got \"I went to the store yesterday.\"", + "category": "language" + }, + { + "test": "grammar", + "prompt": "Correct the grammar in: \"I goes to the store yesterday\". Return only the corrected sentence, no explanation.", + "result": [ + "I went to the store yesterday." + ], + "expected": "I went to the store yesterday", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:30:33.611Z", + "passed": false, + "duration": 3395, + "reason": "Expected I went to the store yesterday, but got I went to the store yesterday.", + "category": "language" + }, + { + "test": "summarization", + "prompt": "Summarize: \"The quick brown fox jumps over the dog\". Return only the summary, compact, no explanation.", + "result": [ + "A brown fox leaps over a dog." + ], + "expected": "A fox jumps over a dog", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:34.920Z", + "passed": false, + "duration": 1304, + "reason": "Expected A fox jumps over a dog, but got A brown fox leaps over a dog.", + "category": "language" + }, + { + "test": "summarization", + "prompt": "Summarize: \"The quick brown fox jumps over the dog\". Return only the summary, compact, no explanation.", + "result": [ + "A fox jumps over a dog." + ], + "expected": "A fox jumps over a dog", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:35.620Z", + "passed": false, + "duration": 692, + "reason": "Expected A fox jumps over a dog, but got A fox jumps over a dog.", + "category": "language" + }, + { + "test": "summarization", + "prompt": "Summarize: \"The quick brown fox jumps over the dog\". Return only the summary, compact, no explanation.", + "result": [ + "\"A quick brown fox leaps over a dog.\"" + ], + "expected": "A fox jumps over a dog", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:30:49.662Z", + "passed": false, + "duration": 14038, + "reason": "Expected A fox jumps over a dog, but got \"A quick brown fox leaps over a dog.\"", + "category": "language" + }, + { + "test": "language_detection", + "prompt": "Identify the language of: \"Bonjour, comment allez-vous?\". Return only the language name, no explanation.", + "result": [ + "French" + ], + "expected": "French", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:50.805Z", + "passed": true, + "duration": 1137, + "category": "language" + }, + { + "test": "language_detection", + "prompt": "Identify the language of: \"Bonjour, comment allez-vous?\". Return only the language name, no explanation.", + "result": [ + "French" + ], + "expected": "French", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:51.269Z", + "passed": true, + "duration": 459, + "category": "language" + }, + { + "test": "language_detection", + "prompt": "Identify the language of: \"Bonjour, comment allez-vous?\". Return only the language name, no explanation.", + "result": [ + "French" + ], + "expected": "French", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:30:55.198Z", + "passed": true, + "duration": 3924, + "category": "language" + }, + { + "test": "synonyms", + "prompt": "Provide a synonym for \"happy\". Return only the synonym, no explanation.", + "result": [ + "Joyful" + ], + "expected": "joyful", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:56.455Z", + "passed": true, + "duration": 1251, + "category": "language" + }, + { + "test": "synonyms", + "prompt": "Provide a synonym for \"happy\". Return only the synonym, no explanation.", + "result": [ + "Joyful" + ], + "expected": "joyful", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:57.083Z", + "passed": true, + "duration": 622, + "category": "language" + }, + { + "test": "synonyms", + "prompt": "Provide a synonym for \"happy\". Return only the synonym, no explanation.", + "result": [ + "joyful" + ], + "expected": "joyful", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:31:00.924Z", + "passed": true, + "duration": 3836, + "category": "language" } ], "highscores": [ @@ -2754,15 +4898,15 @@ { "test": "translation", "rankings": [ - { - "model": "openai/gpt-4o-mini", - "duration": 666, - "duration_secs": 0.666 - }, { "model": "openai/gpt-3.5-turbo", "duration": 818, "duration_secs": 0.818 + }, + { + "model": "openai/gpt-4o-mini", + "duration": 1451, + "duration_secs": 1.451 } ] }, @@ -2776,8 +4920,8 @@ }, { "model": "openai/gpt-4o-mini", - "duration": 1171, - "duration_secs": 1.171 + "duration": 695, + "duration_secs": 0.695 } ] }, @@ -2791,38 +4935,38 @@ }, { "model": "openai/gpt-4o-mini", - "duration": 699, - "duration_secs": 0.699 + "duration": 692, + "duration_secs": 0.692 } ] }, { "test": "language_detection", "rankings": [ + { + "model": "openai/gpt-4o-mini", + "duration": 459, + "duration_secs": 0.459 + }, { "model": "openai/gpt-3.5-turbo", "duration": 695, "duration_secs": 0.695 - }, - { - "model": "openai/gpt-4o-mini", - "duration": 776, - "duration_secs": 0.776 } ] }, { "test": "synonyms", "rankings": [ - { - "model": "openai/gpt-4o-mini", - "duration": 548, - "duration_secs": 0.548 - }, { "model": "openai/gpt-3.5-turbo", "duration": 570, "duration_secs": 0.57 + }, + { + "model": "openai/gpt-4o-mini", + "duration": 622, + "duration_secs": 0.622 } ] }, @@ -2830,14 +4974,14 @@ "test": "file-inclusion", "rankings": [ { - "model": "openai/gpt-4o-mini", - "duration": 2223, - "duration_secs": 2.223 + "model": "openai/gpt-4o", + "duration": 614, + "duration_secs": 0.614 }, { "model": "google/gemini-2.0-flash-exp:free", - "duration": 2404, - "duration_secs": 2.404 + "duration": 4801, + "duration_secs": 4.801 } ] }, @@ -2871,8 +5015,8 @@ "rankings": [ { "model": "openai/gpt-4o-mini", - "duration": 738, - "duration_secs": 0.738 + "duration": 641, + "duration_secs": 0.641 }, { "model": "openai/gpt-3.5-turbo", @@ -2894,15 +5038,15 @@ { "test": "multiplication", "rankings": [ + { + "model": "openai/gpt-4o-mini", + "duration": 585, + "duration_secs": 0.585 + }, { "model": "openai/gpt-3.5-turbo", "duration": 624, "duration_secs": 0.624 - }, - { - "model": "openai/gpt-4o-mini", - "duration": 626, - "duration_secs": 0.626 } ] }, @@ -2934,15 +5078,15 @@ { "test": "division", "rankings": [ + { + "model": "openai/gpt-4o-mini", + "duration": 481, + "duration_secs": 0.481 + }, { "model": "openai/gpt-3.5-turbo", "duration": 513, "duration_secs": 0.513 - }, - { - "model": "openai/gpt-4o-mini", - "duration": 895, - "duration_secs": 0.895 } ] }, @@ -2971,8 +5115,8 @@ }, { "model": "openai/gpt-4o-mini", - "duration": 4358, - "duration_secs": 4.358 + "duration": 4917, + "duration_secs": 4.917 } ] }, @@ -3027,5 +5171,5 @@ ] } ], - "lastUpdated": "2025-06-05T18:56:45.467Z" + "lastUpdated": "2025-06-05T22:31:00.927Z" } \ No newline at end of file diff --git a/packages/kbot/tests/unit/reports/basic.json b/packages/kbot/tests/unit/reports/basic.json index c62e014c..24c21875 100644 --- a/packages/kbot/tests/unit/reports/basic.json +++ b/packages/kbot/tests/unit/reports/basic.json @@ -180,6 +180,173 @@ "passed": true, "duration": 626, "category": "basic" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [ + "8" + ], + "expected": "8", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T21:19:15.716Z", + "passed": true, + "duration": 2024, + "category": "basic" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [ + "8" + ], + "expected": "8", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:19:16.361Z", + "passed": true, + "duration": 641, + "category": "basic" + }, + { + "test": "addition", + "prompt": "add 5 and 3. Return only the number, no explanation.", + "result": [ + "8" + ], + "expected": "8", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T21:19:20.162Z", + "passed": true, + "duration": 3798, + "category": "basic" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [ + "24" + ], + "expected": "24", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T21:19:21.917Z", + "passed": true, + "duration": 1752, + "category": "basic" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [ + "24" + ], + "expected": "24", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:19:22.504Z", + "passed": true, + "duration": 585, + "category": "basic" + }, + { + "test": "multiplication", + "prompt": "multiply 8 and 3. Return only the number, no explanation.", + "result": [ + "24" + ], + "expected": "24", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T21:19:25.779Z", + "passed": true, + "duration": 3272, + "category": "basic" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [ + "5" + ], + "expected": "5", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T21:19:27.557Z", + "passed": true, + "duration": 1775, + "category": "basic" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [ + "5" + ], + "expected": "5", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:19:28.041Z", + "passed": true, + "duration": 481, + "category": "basic" + }, + { + "test": "division", + "prompt": "divide 15 by 3. Return only the number, no explanation.", + "result": [ + "5" + ], + "expected": "5", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T21:19:31.450Z", + "passed": true, + "duration": 3406, + "category": "basic" + }, + { + "test": "web_content", + "prompt": "Check if the content contains a section about Human prehistory. Reply with \"yes\" if it does, \"no\" if it does not.", + "result": [ + "yes" + ], + "expected": "yes", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T21:19:37.473Z", + "passed": true, + "duration": 6020, + "category": "basic" + }, + { + "test": "web_content", + "prompt": "Check if the content contains a section about Human prehistory. Reply with \"yes\" if it does, \"no\" if it does not.", + "result": [ + "yes" + ], + "expected": "yes", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:19:42.394Z", + "passed": true, + "duration": 4917, + "category": "basic" + }, + { + "test": "web_content", + "prompt": "Check if the content contains a section about Human prehistory. Reply with \"yes\" if it does, \"no\" if it does not.", + "result": [], + "expected": "yes", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T21:19:47.544Z", + "passed": false, + "duration": 5147, + "reason": "Model returned empty response", + "category": "basic" } ], "highscores": [ @@ -188,8 +355,8 @@ "rankings": [ { "model": "openai/gpt-4o-mini", - "duration": 738, - "duration_secs": 0.738 + "duration": 641, + "duration_secs": 0.641 }, { "model": "openai/gpt-3.5-turbo", @@ -201,30 +368,30 @@ { "test": "multiplication", "rankings": [ + { + "model": "openai/gpt-4o-mini", + "duration": 585, + "duration_secs": 0.585 + }, { "model": "openai/gpt-3.5-turbo", "duration": 624, "duration_secs": 0.624 - }, - { - "model": "openai/gpt-4o-mini", - "duration": 626, - "duration_secs": 0.626 } ] }, { "test": "division", "rankings": [ + { + "model": "openai/gpt-4o-mini", + "duration": 481, + "duration_secs": 0.481 + }, { "model": "openai/gpt-3.5-turbo", "duration": 513, "duration_secs": 0.513 - }, - { - "model": "openai/gpt-4o-mini", - "duration": 895, - "duration_secs": 0.895 } ] }, @@ -238,11 +405,11 @@ }, { "model": "openai/gpt-4o-mini", - "duration": 4358, - "duration_secs": 4.358 + "duration": 4917, + "duration_secs": 4.917 } ] } ], - "lastUpdated": "2025-06-05T18:56:45.466Z" + "lastUpdated": "2025-06-05T21:19:47.545Z" } \ No newline at end of file diff --git a/packages/kbot/tests/unit/reports/basic.md b/packages/kbot/tests/unit/reports/basic.md index e1bceb54..25f4de44 100644 --- a/packages/kbot/tests/unit/reports/basic.md +++ b/packages/kbot/tests/unit/reports/basic.md @@ -6,89 +6,125 @@ | Test | Model | Duration (ms) | Duration (s) | |------|-------|--------------|--------------| -| addition | openai/gpt-4o-mini | 514 | 0.51 | -| addition | openai/gpt-3.5-turbo | 771 | 0.77 | -| multiplication | openai/gpt-3.5-turbo | 624 | 0.62 | -| multiplication | openai/gpt-4o-mini | 721 | 0.72 | -| division | openai/gpt-3.5-turbo | 513 | 0.51 | -| division | openai/gpt-4o-mini | 895 | 0.90 | -| web_content | openai/gpt-3.5-turbo | 220 | 0.22 | -| web_content | openai/gpt-4o-mini | 4358 | 4.36 | +| addition | openai/gpt-4o-mini | 641 | 0.64 | +| addition | anthropic/claude-sonnet-4 | 2024 | 2.02 | +| addition | deepseek/deepseek-r1:free | 3798 | 3.80 | +| multiplication | openai/gpt-4o-mini | 585 | 0.58 | +| multiplication | anthropic/claude-sonnet-4 | 1752 | 1.75 | +| multiplication | deepseek/deepseek-r1:free | 3272 | 3.27 | +| division | openai/gpt-4o-mini | 481 | 0.48 | +| division | anthropic/claude-sonnet-4 | 1775 | 1.77 | +| division | deepseek/deepseek-r1:free | 3406 | 3.41 | +| web_content | openai/gpt-4o-mini | 4917 | 4.92 | +| web_content | deepseek/deepseek-r1:free | 5147 | 5.15 | +| web_content | anthropic/claude-sonnet-4 | 6020 | 6.02 | ## Summary -- Total Tests: 8 -- Passed: 7 +- Total Tests: 12 +- Passed: 11 - Failed: 1 -- Success Rate: 87.50% -- Average Duration: 1077ms (1.08s) +- Success Rate: 91.67% +- Average Duration: 2818ms (2.82s) ## Failed Tests -### web_content - openai/gpt-3.5-turbo +### web_content - deepseek/deepseek-r1:free - Prompt: `Check if the content contains a section about Human prehistory. Reply with "yes" if it does, "no" if it does not.` - Expected: `yes` - Actual: `` -- Duration: 220ms (0.22s) +- Duration: 5147ms (5.15s) - Reason: Model returned empty response -- Timestamp: 6/5/2025, 8:46:11 PM +- Timestamp: 6/5/2025, 11:19:47 PM ## Passed Tests -### addition - openai/gpt-3.5-turbo +### addition - anthropic/claude-sonnet-4 - Prompt: `add 5 and 3. Return only the number, no explanation.` - Expected: `8` - Actual: `8` -- Duration: 771ms (0.77s) -- Timestamp: 6/5/2025, 8:46:08 PM +- Duration: 2024ms (2.02s) +- Timestamp: 6/5/2025, 11:19:15 PM ### addition - openai/gpt-4o-mini - Prompt: `add 5 and 3. Return only the number, no explanation.` - Expected: `8` - Actual: `8` -- Duration: 514ms (0.51s) -- Timestamp: 6/5/2025, 8:46:08 PM +- Duration: 641ms (0.64s) +- Timestamp: 6/5/2025, 11:19:16 PM -### multiplication - openai/gpt-3.5-turbo +### addition - deepseek/deepseek-r1:free + +- Prompt: `add 5 and 3. Return only the number, no explanation.` +- Expected: `8` +- Actual: `8` +- Duration: 3798ms (3.80s) +- Timestamp: 6/5/2025, 11:19:20 PM + +### multiplication - anthropic/claude-sonnet-4 - Prompt: `multiply 8 and 3. Return only the number, no explanation.` - Expected: `24` - Actual: `24` -- Duration: 624ms (0.62s) -- Timestamp: 6/5/2025, 8:46:09 PM +- Duration: 1752ms (1.75s) +- Timestamp: 6/5/2025, 11:19:21 PM ### multiplication - openai/gpt-4o-mini - Prompt: `multiply 8 and 3. Return only the number, no explanation.` - Expected: `24` - Actual: `24` -- Duration: 721ms (0.72s) -- Timestamp: 6/5/2025, 8:46:09 PM +- Duration: 585ms (0.58s) +- Timestamp: 6/5/2025, 11:19:22 PM -### division - openai/gpt-3.5-turbo +### multiplication - deepseek/deepseek-r1:free + +- Prompt: `multiply 8 and 3. Return only the number, no explanation.` +- Expected: `24` +- Actual: `24` +- Duration: 3272ms (3.27s) +- Timestamp: 6/5/2025, 11:19:25 PM + +### division - anthropic/claude-sonnet-4 - Prompt: `divide 15 by 3. Return only the number, no explanation.` - Expected: `5` - Actual: `5` -- Duration: 513ms (0.51s) -- Timestamp: 6/5/2025, 8:46:10 PM +- Duration: 1775ms (1.77s) +- Timestamp: 6/5/2025, 11:19:27 PM ### division - openai/gpt-4o-mini - Prompt: `divide 15 by 3. Return only the number, no explanation.` - Expected: `5` - Actual: `5` -- Duration: 895ms (0.90s) -- Timestamp: 6/5/2025, 8:46:11 PM +- Duration: 481ms (0.48s) +- Timestamp: 6/5/2025, 11:19:28 PM + +### division - deepseek/deepseek-r1:free + +- Prompt: `divide 15 by 3. Return only the number, no explanation.` +- Expected: `5` +- Actual: `5` +- Duration: 3406ms (3.41s) +- Timestamp: 6/5/2025, 11:19:31 PM + +### web_content - anthropic/claude-sonnet-4 + +- Prompt: `Check if the content contains a section about Human prehistory. Reply with "yes" if it does, "no" if it does not.` +- Expected: `yes` +- Actual: `yes` +- Duration: 6020ms (6.02s) +- Timestamp: 6/5/2025, 11:19:37 PM ### web_content - openai/gpt-4o-mini - Prompt: `Check if the content contains a section about Human prehistory. Reply with "yes" if it does, "no" if it does not.` - Expected: `yes` -- Actual: `Yes` -- Duration: 4358ms (4.36s) -- Timestamp: 6/5/2025, 8:46:15 PM +- Actual: `yes` +- Duration: 4917ms (4.92s) +- Timestamp: 6/5/2025, 11:19:42 PM diff --git a/packages/kbot/tests/unit/reports/files.json b/packages/kbot/tests/unit/reports/files.json index 9d20007c..8bbeeef2 100644 --- a/packages/kbot/tests/unit/reports/files.json +++ b/packages/kbot/tests/unit/reports/files.json @@ -235,6 +235,1767 @@ "duration": 2404, "reason": "Expected [\"cat\",\"fox\"], but got [\n \"cat\",\n \"fox\"\n]", "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:25:46.078Z", + "passed": true, + "duration": 824, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:25:53.366Z", + "passed": true, + "duration": 7284, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:25:54.218Z", + "passed": false, + "duration": 849, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:25:59.456Z", + "passed": false, + "duration": 5231, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:00.076Z", + "passed": true, + "duration": 616, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:02.225Z", + "passed": true, + "duration": 2146, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:05.871Z", + "passed": true, + "duration": 3643, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:09.058Z", + "passed": true, + "duration": 3183, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:42.944Z", + "passed": true, + "duration": 772, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:46.369Z", + "passed": true, + "duration": 3421, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:47.073Z", + "passed": false, + "duration": 700, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:48.594Z", + "passed": false, + "duration": 1514, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubbleSort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:49.375Z", + "passed": true, + "duration": 779, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:26:55.618Z", + "passed": true, + "duration": 6239, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"wildcat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:26:57.688Z", + "passed": false, + "duration": 2067, + "reason": "Expected [\"cat\",\"fox\"], but got [\n \"wildcat\",\n \"fox\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:00.508Z", + "passed": true, + "duration": 2815, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:27:38.292Z", + "passed": true, + "duration": 1023, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:42.531Z", + "passed": true, + "duration": 4235, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:27:43.285Z", + "passed": false, + "duration": 751, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:48.461Z", + "passed": false, + "duration": 5168, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:27:49.024Z", + "passed": true, + "duration": 559, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:54.942Z", + "passed": false, + "duration": 5915, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:27:56.766Z", + "passed": true, + "duration": 1819, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:27:59.278Z", + "passed": true, + "duration": 2508, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:28:17.298Z", + "passed": true, + "duration": 834, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:28:18.833Z", + "passed": true, + "duration": 1530, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:28:19.525Z", + "passed": false, + "duration": 688, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:28:23.761Z", + "passed": false, + "duration": 4229, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubbleSort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:28:24.280Z", + "passed": true, + "duration": 515, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:28:26.274Z", + "passed": true, + "duration": 1990, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:28:29.111Z", + "passed": true, + "duration": 2834, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:28:35.340Z", + "passed": false, + "duration": 6225, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:30:09.177Z", + "passed": true, + "duration": 1035, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:30:11.447Z", + "passed": true, + "duration": 2266, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:30:12.403Z", + "passed": false, + "duration": 952, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubbleSort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:30:18.660Z", + "passed": false, + "duration": 6250, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:30:19.412Z", + "passed": true, + "duration": 748, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:30:24.575Z", + "passed": false, + "duration": 5159, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:30:26.812Z", + "passed": true, + "duration": 2232, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:30:33.768Z", + "passed": false, + "duration": 6951, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:31:32.809Z", + "passed": true, + "duration": 941, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:31:35.174Z", + "passed": true, + "duration": 2360, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:31:49.546Z", + "passed": false, + "duration": 759, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:32:00.755Z", + "passed": false, + "duration": 7224, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubblesort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:32:01.351Z", + "passed": false, + "duration": 592, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:32:02.883Z", + "passed": false, + "duration": 1528, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:32:05.172Z", + "passed": false, + "duration": 2283, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:32:07.065Z", + "passed": false, + "duration": 1887, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:32:59.145Z", + "passed": true, + "duration": 883, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:33:01.231Z", + "passed": true, + "duration": 2081, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:33:01.922Z", + "passed": false, + "duration": 686, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:33:04.999Z", + "passed": false, + "duration": 3070, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubblesort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:33:07.789Z", + "passed": false, + "duration": 2785, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:33:09.512Z", + "passed": false, + "duration": 1718, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:33:14.818Z", + "passed": false, + "duration": 5303, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:33:17.979Z", + "passed": false, + "duration": 3156, + "reason": "Expected [\"cat\",\"fox\"], but got [\n \"cat\",\n \"fox\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:34:43.846Z", + "passed": true, + "duration": 1036, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:34:48.463Z", + "passed": true, + "duration": 4612, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:34:49.231Z", + "passed": false, + "duration": 763, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:34:55.136Z", + "passed": false, + "duration": 5897, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:34:55.630Z", + "passed": false, + "duration": 489, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:35:00.574Z", + "passed": false, + "duration": 4939, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:35:03.038Z", + "passed": false, + "duration": 2459, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:35:05.480Z", + "passed": false, + "duration": 2438, + "reason": "Expected [\"cat\",\"fox\"], but got [\n \"cat\",\n \"fox\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:36:08.730Z", + "passed": true, + "duration": 1322, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:36:12.605Z", + "passed": true, + "duration": 3870, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:36:13.472Z", + "passed": false, + "duration": 862, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubblesort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:36:58.567Z", + "passed": true, + "duration": 840, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:37:00.081Z", + "passed": true, + "duration": 1509, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:37:00.976Z", + "passed": false, + "duration": 891, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:37:02.031Z", + "passed": false, + "duration": 1048, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\n \"factorial\",\n \"bubblesort\"\n]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:37:02.889Z", + "passed": false, + "duration": 854, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:37:04.623Z", + "passed": false, + "duration": 1730, + "reason": "Expected Injection Barrel, but got injection barrel", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:37:07.559Z", + "passed": false, + "duration": 2933, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:37:10.561Z", + "passed": false, + "duration": 2998, + "reason": "Expected [\"cat\",\"fox\"], but got [\"cat\", \"fox\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:38:15.039Z", + "passed": true, + "duration": 804, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:38:17.121Z", + "passed": true, + "duration": 2077, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:38:25.035Z", + "passed": false, + "duration": 797, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:39:30.971Z", + "passed": false, + "duration": 1721, + "reason": "Expected [\"bubble sort\",\"factorial\"], but got [\"factorial\", \"bubblesort\"]", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:53:07.100Z", + "passed": true, + "duration": 51668, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:53:13.381Z", + "passed": true, + "duration": 2388, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:53:14.200Z", + "passed": true, + "duration": 814, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:53:23.416Z", + "passed": true, + "duration": 2210, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:53:51.472Z", + "passed": true, + "duration": 8947, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:53:57.721Z", + "passed": false, + "duration": 6245, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:54:00.301Z", + "passed": true, + "duration": 2573, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:54:02.900Z", + "passed": true, + "duration": 2594, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:54:12.068Z", + "passed": true, + "duration": 792, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:54:13.596Z", + "passed": true, + "duration": 1522, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:54:14.332Z", + "passed": true, + "duration": 731, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:55:55.651Z", + "passed": false, + "duration": 7061, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:56:27.181Z", + "passed": true, + "duration": 3432, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:56:28.770Z", + "passed": true, + "duration": 1583, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:56:31.322Z", + "passed": true, + "duration": 2548, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\"cat\", \"fox\"]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:56:34.465Z", + "passed": true, + "duration": 3138, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:57:18.569Z", + "passed": true, + "duration": 867, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:57:20.692Z", + "passed": true, + "duration": 2117, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\"factorial\", \"bubbleSort\"]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:57:21.814Z", + "passed": true, + "duration": 1117, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:57:23.423Z", + "passed": true, + "duration": 1603, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:57:24.145Z", + "passed": true, + "duration": 717, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:57:26.292Z", + "passed": true, + "duration": 2142, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images? Return as JSON array.", + "result": [ + "[\n \"cat\",\n \"fox\"\n]" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:57:29.395Z", + "passed": true, + "duration": 3099, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubble sort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:59:17.877Z", + "passed": true, + "duration": 4686, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "Bubble Sort\n" + ], + "expected": "bubble sort", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:59:20.537Z", + "passed": true, + "duration": 2653, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:59:21.334Z", + "passed": true, + "duration": 793, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "[\n \"factorial\",\n \"bubbleSort\"\n]" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:59:25.325Z", + "passed": true, + "duration": 3986, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:59:25.873Z", + "passed": true, + "duration": 543, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel\n" + ], + "expected": "Injection Barrel", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T21:59:27.328Z", + "passed": true, + "duration": 1451, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "The image shows a wildcat on the left and a red fox on the right." + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T21:59:30.021Z", + "passed": true, + "duration": 2689, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:11:44.531Z", + "passed": false, + "duration": 26687, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "The image shows a cat and a fox.\n" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "google/gemini-2.0-flash-exp:free", + "router": "google/gemini-2.0-flash-exp:free", + "timestamp": "2025-06-05T22:12:19.670Z", + "passed": true, + "duration": 4801, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:18:06.005Z", + "passed": false, + "duration": 7536, + "reason": "Model returned empty response", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "{\"animals\":[\"cat\",\"fox\"]}" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:25:35.300Z", + "passed": true, + "duration": 20975, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "{\"animals\":[\"cat\",\"fox\"]}" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:26:40.769Z", + "passed": true, + "duration": 16606, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubbleSort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:26:41.541Z", + "passed": false, + "duration": 765, + "reason": "Expected bubble sort, but got bubbleSort", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "{\"algorithms\":[\"factorial\",\"bubbleSort\"]}" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:26:42.264Z", + "passed": true, + "duration": 718, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:26:43.192Z", + "passed": true, + "duration": 919, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "{\"animals\":[\"cat\",\"fox\"]}" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:27:59.823Z", + "passed": true, + "duration": 2365, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubbleSort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:28:00.651Z", + "passed": false, + "duration": 822, + "reason": "Expected bubble sort, but got bubbleSort", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "{\"algorithms\":[\"factorial\",\"bubbleSort\"]}" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:28:01.338Z", + "passed": true, + "duration": 682, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What animals are shown in these images?", + "result": [ + "{\"animals\":[\"cat\",\"fox\"]}" + ], + "expected": "[\"cat\",\"fox\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:29:44.549Z", + "passed": true, + "duration": 3234, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the name of the algorithm implemented in these files? Return only the name.", + "result": [ + "bubbleSort" + ], + "expected": "bubble sort", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:29:45.518Z", + "passed": false, + "duration": 961, + "reason": "Expected bubble sort, but got bubbleSort", + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "List all algorithms implemented in these files, as JSON array.", + "result": [ + "{\"algorithms\":[\"factorial\",\"bubbleSort\"]}" + ], + "expected": "[\"bubble sort\",\"factorial\"]", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:29:46.232Z", + "passed": true, + "duration": 709, + "category": "files" + }, + { + "test": "file-inclusion", + "prompt": "What is the title of the product in data.json? Return only the title.", + "result": [ + "Injection Barrel" + ], + "expected": "Injection Barrel", + "model": "openai/gpt-4o", + "router": "openai/gpt-4o", + "timestamp": "2025-06-05T22:29:46.851Z", + "passed": true, + "duration": 614, + "category": "files" } ], "highscores": [ @@ -242,17 +2003,17 @@ "test": "file-inclusion", "rankings": [ { - "model": "openai/gpt-4o-mini", - "duration": 2223, - "duration_secs": 2.223 + "model": "openai/gpt-4o", + "duration": 614, + "duration_secs": 0.614 }, { "model": "google/gemini-2.0-flash-exp:free", - "duration": 2404, - "duration_secs": 2.404 + "duration": 4801, + "duration_secs": 4.801 } ] } ], - "lastUpdated": "2025-06-05T18:46:20.109Z" + "lastUpdated": "2025-06-05T22:29:46.852Z" } \ No newline at end of file diff --git a/packages/kbot/tests/unit/reports/files.md b/packages/kbot/tests/unit/reports/files.md index 648ab8ae..cee52d90 100644 --- a/packages/kbot/tests/unit/reports/files.md +++ b/packages/kbot/tests/unit/reports/files.md @@ -6,44 +6,27 @@ | Test | Model | Duration (ms) | Duration (s) | |------|-------|--------------|--------------| -| file-inclusion | openai/gpt-4o-mini | 2223 | 2.22 | -| file-inclusion | google/gemini-2.0-flash-exp:free | 2404 | 2.40 | +| file-inclusion | openai/gpt-4o | 614 | 0.61 | ## Summary -- Total Tests: 8 -- Passed: 2 -- Failed: 6 -- Success Rate: 25.00% -- Average Duration: 1671ms (1.67s) +- Total Tests: 4 +- Passed: 3 +- Failed: 1 +- Success Rate: 75.00% +- Average Duration: 1380ms (1.38s) ## Failed Tests -### file-inclusion - openai/gpt-4o-mini - -- Prompt: `What animals are shown in these images? Return as JSON array.` -- Expected: `["cat","fox"]` -- Actual: `["cat", "fox"]` -- Duration: 2223ms (2.22s) -- Reason: Expected ["cat","fox"], but got ["cat", "fox"] -- Timestamp: 6/5/2025, 8:46:17 PM - -### file-inclusion - google/gemini-2.0-flash-exp:free - -- Prompt: `What animals are shown in these images? Return as JSON array.` -- Expected: `["cat","fox"]` -- Actual: `[ - "cat", - "fox" -]` -- Duration: 2404ms (2.40s) -- Reason: Expected ["cat","fox"], but got [ - "cat", - "fox" -] -- Timestamp: 6/5/2025, 8:46:20 PM +*No failed tests* ## Passed Tests -*No passed tests* +### file-inclusion - openai/gpt-4o + +- Prompt: `What is the title of the product in data.json? Return only the title.` +- Expected: `Injection Barrel` +- Actual: `Injection Barrel` +- Duration: 614ms (0.61s) +- Timestamp: 6/6/2025, 12:29:46 AM diff --git a/packages/kbot/tests/unit/reports/language.json b/packages/kbot/tests/unit/reports/language.json index 90ed624e..fee85023 100644 --- a/packages/kbot/tests/unit/reports/language.json +++ b/packages/kbot/tests/unit/reports/language.json @@ -1451,21 +1451,237 @@ "passed": true, "duration": 2616, "category": "language" + }, + { + "test": "translation", + "prompt": "Translate \"Hello, world!\" to Spanish. Return only the translation, no explanation.", + "result": [ + "¡Hola, mundo!" + ], + "expected": "¡Hola, mundo!", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:21.412Z", + "passed": true, + "duration": 1560, + "category": "language" + }, + { + "test": "translation", + "prompt": "Translate \"Hello, world!\" to Spanish. Return only the translation, no explanation.", + "result": [ + "¡Hola, mundo!" + ], + "expected": "¡Hola, mundo!", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:22.869Z", + "passed": true, + "duration": 1451, + "category": "language" + }, + { + "test": "translation", + "prompt": "Translate \"Hello, world!\" to Spanish. Return only the translation, no explanation.", + "result": [ + "¡Hola, mundo!" + ], + "expected": "¡Hola, mundo!", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:30:28.307Z", + "passed": true, + "duration": 5434, + "category": "language" + }, + { + "test": "grammar", + "prompt": "Correct the grammar in: \"I goes to the store yesterday\". Return only the corrected sentence, no explanation.", + "result": [ + "I went to the store yesterday." + ], + "expected": "I went to the store yesterday", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:29.513Z", + "passed": false, + "duration": 1201, + "reason": "Expected I went to the store yesterday, but got I went to the store yesterday.", + "category": "language" + }, + { + "test": "grammar", + "prompt": "Correct the grammar in: \"I goes to the store yesterday\". Return only the corrected sentence, no explanation.", + "result": [ + "\"I went to the store yesterday.\"" + ], + "expected": "I went to the store yesterday", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:30.212Z", + "passed": false, + "duration": 695, + "reason": "Expected I went to the store yesterday, but got \"I went to the store yesterday.\"", + "category": "language" + }, + { + "test": "grammar", + "prompt": "Correct the grammar in: \"I goes to the store yesterday\". Return only the corrected sentence, no explanation.", + "result": [ + "I went to the store yesterday." + ], + "expected": "I went to the store yesterday", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:30:33.611Z", + "passed": false, + "duration": 3395, + "reason": "Expected I went to the store yesterday, but got I went to the store yesterday.", + "category": "language" + }, + { + "test": "summarization", + "prompt": "Summarize: \"The quick brown fox jumps over the dog\". Return only the summary, compact, no explanation.", + "result": [ + "A brown fox leaps over a dog." + ], + "expected": "A fox jumps over a dog", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:34.920Z", + "passed": false, + "duration": 1304, + "reason": "Expected A fox jumps over a dog, but got A brown fox leaps over a dog.", + "category": "language" + }, + { + "test": "summarization", + "prompt": "Summarize: \"The quick brown fox jumps over the dog\". Return only the summary, compact, no explanation.", + "result": [ + "A fox jumps over a dog." + ], + "expected": "A fox jumps over a dog", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:35.620Z", + "passed": false, + "duration": 692, + "reason": "Expected A fox jumps over a dog, but got A fox jumps over a dog.", + "category": "language" + }, + { + "test": "summarization", + "prompt": "Summarize: \"The quick brown fox jumps over the dog\". Return only the summary, compact, no explanation.", + "result": [ + "\"A quick brown fox leaps over a dog.\"" + ], + "expected": "A fox jumps over a dog", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:30:49.662Z", + "passed": false, + "duration": 14038, + "reason": "Expected A fox jumps over a dog, but got \"A quick brown fox leaps over a dog.\"", + "category": "language" + }, + { + "test": "language_detection", + "prompt": "Identify the language of: \"Bonjour, comment allez-vous?\". Return only the language name, no explanation.", + "result": [ + "French" + ], + "expected": "French", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:50.805Z", + "passed": true, + "duration": 1137, + "category": "language" + }, + { + "test": "language_detection", + "prompt": "Identify the language of: \"Bonjour, comment allez-vous?\". Return only the language name, no explanation.", + "result": [ + "French" + ], + "expected": "French", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:51.269Z", + "passed": true, + "duration": 459, + "category": "language" + }, + { + "test": "language_detection", + "prompt": "Identify the language of: \"Bonjour, comment allez-vous?\". Return only the language name, no explanation.", + "result": [ + "French" + ], + "expected": "French", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:30:55.198Z", + "passed": true, + "duration": 3924, + "category": "language" + }, + { + "test": "synonyms", + "prompt": "Provide a synonym for \"happy\". Return only the synonym, no explanation.", + "result": [ + "Joyful" + ], + "expected": "joyful", + "model": "anthropic/claude-sonnet-4", + "router": "anthropic/claude-sonnet-4", + "timestamp": "2025-06-05T22:30:56.455Z", + "passed": true, + "duration": 1251, + "category": "language" + }, + { + "test": "synonyms", + "prompt": "Provide a synonym for \"happy\". Return only the synonym, no explanation.", + "result": [ + "Joyful" + ], + "expected": "joyful", + "model": "openai/gpt-4o-mini", + "router": "openai/gpt-4o-mini", + "timestamp": "2025-06-05T22:30:57.083Z", + "passed": true, + "duration": 622, + "category": "language" + }, + { + "test": "synonyms", + "prompt": "Provide a synonym for \"happy\". Return only the synonym, no explanation.", + "result": [ + "joyful" + ], + "expected": "joyful", + "model": "deepseek/deepseek-r1:free", + "router": "deepseek/deepseek-r1:free", + "timestamp": "2025-06-05T22:31:00.924Z", + "passed": true, + "duration": 3836, + "category": "language" } ], "highscores": [ { "test": "translation", "rankings": [ - { - "model": "openai/gpt-4o-mini", - "duration": 666, - "duration_secs": 0.666 - }, { "model": "openai/gpt-3.5-turbo", "duration": 818, "duration_secs": 0.818 + }, + { + "model": "openai/gpt-4o-mini", + "duration": 1451, + "duration_secs": 1.451 } ] }, @@ -1479,8 +1695,8 @@ }, { "model": "openai/gpt-4o-mini", - "duration": 1171, - "duration_secs": 1.171 + "duration": 695, + "duration_secs": 0.695 } ] }, @@ -1494,41 +1710,41 @@ }, { "model": "openai/gpt-4o-mini", - "duration": 699, - "duration_secs": 0.699 + "duration": 692, + "duration_secs": 0.692 } ] }, { "test": "language_detection", "rankings": [ + { + "model": "openai/gpt-4o-mini", + "duration": 459, + "duration_secs": 0.459 + }, { "model": "openai/gpt-3.5-turbo", "duration": 695, "duration_secs": 0.695 - }, - { - "model": "openai/gpt-4o-mini", - "duration": 776, - "duration_secs": 0.776 } ] }, { "test": "synonyms", "rankings": [ - { - "model": "openai/gpt-4o-mini", - "duration": 548, - "duration_secs": 0.548 - }, { "model": "openai/gpt-3.5-turbo", "duration": 570, "duration_secs": 0.57 + }, + { + "model": "openai/gpt-4o-mini", + "duration": 622, + "duration_secs": 0.622 } ] } ], - "lastUpdated": "2025-06-05T18:56:08.627Z" + "lastUpdated": "2025-06-05T22:31:00.924Z" } \ No newline at end of file diff --git a/packages/kbot/tests/unit/reports/language.md b/packages/kbot/tests/unit/reports/language.md index 797d03d4..2102b9ea 100644 --- a/packages/kbot/tests/unit/reports/language.md +++ b/packages/kbot/tests/unit/reports/language.md @@ -6,164 +6,157 @@ | Test | Model | Duration (ms) | Duration (s) | |------|-------|--------------|--------------| -| translation | openai/gpt-4o-mini | 666 | 0.67 | -| translation | anthropic/claude-sonnet-4 | 1317 | 1.32 | -| translation | deepseek/deepseek-r1:free | 5397 | 5.40 | -| grammar | openai/gpt-4o-mini | 1171 | 1.17 | -| grammar | anthropic/claude-sonnet-4 | 1722 | 1.72 | -| grammar | deepseek/deepseek-r1:free | 5199 | 5.20 | -| summarization | openai/gpt-4o-mini | 699 | 0.70 | -| summarization | anthropic/claude-sonnet-4 | 1820 | 1.82 | -| summarization | deepseek/deepseek-r1:free | 7380 | 7.38 | -| language_detection | openai/gpt-4o-mini | 776 | 0.78 | -| language_detection | anthropic/claude-sonnet-4 | 1725 | 1.73 | -| language_detection | deepseek/deepseek-r1:free | 5247 | 5.25 | -| synonyms | openai/gpt-4o-mini | 548 | 0.55 | -| synonyms | anthropic/claude-sonnet-4 | 1967 | 1.97 | -| synonyms | deepseek/deepseek-r1:free | 2616 | 2.62 | +| translation | openai/gpt-4o-mini | 1451 | 1.45 | +| translation | anthropic/claude-sonnet-4 | 1560 | 1.56 | +| translation | deepseek/deepseek-r1:free | 5434 | 5.43 | +| grammar | openai/gpt-4o-mini | 695 | 0.69 | +| grammar | anthropic/claude-sonnet-4 | 1201 | 1.20 | +| grammar | deepseek/deepseek-r1:free | 3395 | 3.40 | +| summarization | openai/gpt-4o-mini | 692 | 0.69 | +| summarization | anthropic/claude-sonnet-4 | 1304 | 1.30 | +| summarization | deepseek/deepseek-r1:free | 14038 | 14.04 | +| language_detection | openai/gpt-4o-mini | 459 | 0.46 | +| language_detection | anthropic/claude-sonnet-4 | 1137 | 1.14 | +| language_detection | deepseek/deepseek-r1:free | 3924 | 3.92 | +| synonyms | openai/gpt-4o-mini | 622 | 0.62 | +| synonyms | anthropic/claude-sonnet-4 | 1251 | 1.25 | +| synonyms | deepseek/deepseek-r1:free | 3836 | 3.84 | ## Summary - Total Tests: 15 -- Passed: 2 -- Failed: 13 -- Success Rate: 13.33% -- Average Duration: 2550ms (2.55s) +- Passed: 9 +- Failed: 6 +- Success Rate: 60.00% +- Average Duration: 2733ms (2.73s) ## Failed Tests -### translation - anthropic/claude-sonnet-4 - -- Prompt: `Translate "Hello, world!" to Spanish. Return only the translation, no explanation.` -- Expected: `¡Hola, mundo!` -- Actual: `¡Hola, mundo!` -- Duration: 1317ms (1.32s) -- Reason: Expected ¡Hola, mundo!, but got ¡hola, mundo! -- Timestamp: 6/5/2025, 8:55:31 PM - -### translation - openai/gpt-4o-mini - -- Prompt: `Translate "Hello, world!" to Spanish. Return only the translation, no explanation.` -- Expected: `¡Hola, mundo!` -- Actual: `¡Hola, mundo!` -- Duration: 666ms (0.67s) -- Reason: Expected ¡Hola, mundo!, but got ¡hola, mundo! -- Timestamp: 6/5/2025, 8:55:32 PM - -### translation - deepseek/deepseek-r1:free - -- Prompt: `Translate "Hello, world!" to Spanish. Return only the translation, no explanation.` -- Expected: `¡Hola, mundo!` -- Actual: `¡Hola, mundo!` -- Duration: 5397ms (5.40s) -- Reason: Expected ¡Hola, mundo!, but got ¡hola, mundo! -- Timestamp: 6/5/2025, 8:55:37 PM - ### grammar - anthropic/claude-sonnet-4 - Prompt: `Correct the grammar in: "I goes to the store yesterday". Return only the corrected sentence, no explanation.` - Expected: `I went to the store yesterday` - Actual: `I went to the store yesterday.` -- Duration: 1722ms (1.72s) -- Reason: Expected I went to the store yesterday, but got i went to the store yesterday. -- Timestamp: 6/5/2025, 8:55:39 PM +- Duration: 1201ms (1.20s) +- Reason: Expected I went to the store yesterday, but got I went to the store yesterday. +- Timestamp: 6/6/2025, 12:30:29 AM ### grammar - openai/gpt-4o-mini - Prompt: `Correct the grammar in: "I goes to the store yesterday". Return only the corrected sentence, no explanation.` - Expected: `I went to the store yesterday` - Actual: `"I went to the store yesterday."` -- Duration: 1171ms (1.17s) -- Reason: Expected I went to the store yesterday, but got "i went to the store yesterday." -- Timestamp: 6/5/2025, 8:55:40 PM +- Duration: 695ms (0.69s) +- Reason: Expected I went to the store yesterday, but got "I went to the store yesterday." +- Timestamp: 6/6/2025, 12:30:30 AM ### grammar - deepseek/deepseek-r1:free - Prompt: `Correct the grammar in: "I goes to the store yesterday". Return only the corrected sentence, no explanation.` - Expected: `I went to the store yesterday` -- Actual: `"I went to the store yesterday."` -- Duration: 5199ms (5.20s) -- Reason: Expected I went to the store yesterday, but got "i went to the store yesterday." -- Timestamp: 6/5/2025, 8:55:45 PM +- Actual: `I went to the store yesterday.` +- Duration: 3395ms (3.40s) +- Reason: Expected I went to the store yesterday, but got I went to the store yesterday. +- Timestamp: 6/6/2025, 12:30:33 AM ### summarization - anthropic/claude-sonnet-4 - Prompt: `Summarize: "The quick brown fox jumps over the dog". Return only the summary, compact, no explanation.` - Expected: `A fox jumps over a dog` -- Actual: `A fox jumps over a dog.` -- Duration: 1820ms (1.82s) -- Reason: Expected A fox jumps over a dog, but got a fox jumps over a dog. -- Timestamp: 6/5/2025, 8:55:47 PM +- Actual: `A brown fox leaps over a dog.` +- Duration: 1304ms (1.30s) +- Reason: Expected A fox jumps over a dog, but got A brown fox leaps over a dog. +- Timestamp: 6/6/2025, 12:30:34 AM ### summarization - openai/gpt-4o-mini - Prompt: `Summarize: "The quick brown fox jumps over the dog". Return only the summary, compact, no explanation.` - Expected: `A fox jumps over a dog` - Actual: `A fox jumps over a dog.` -- Duration: 699ms (0.70s) -- Reason: Expected A fox jumps over a dog, but got a fox jumps over a dog. -- Timestamp: 6/5/2025, 8:55:48 PM +- Duration: 692ms (0.69s) +- Reason: Expected A fox jumps over a dog, but got A fox jumps over a dog. +- Timestamp: 6/6/2025, 12:30:35 AM ### summarization - deepseek/deepseek-r1:free - Prompt: `Summarize: "The quick brown fox jumps over the dog". Return only the summary, compact, no explanation.` - Expected: `A fox jumps over a dog` -- Actual: `A quick brown fox leaps over a dog.` -- Duration: 7380ms (7.38s) -- Reason: Expected A fox jumps over a dog, but got a quick brown fox leaps over a dog. -- Timestamp: 6/5/2025, 8:55:55 PM +- Actual: `"A quick brown fox leaps over a dog."` +- Duration: 14038ms (14.04s) +- Reason: Expected A fox jumps over a dog, but got "A quick brown fox leaps over a dog." +- Timestamp: 6/6/2025, 12:30:49 AM + +## Passed Tests + +### translation - anthropic/claude-sonnet-4 + +- Prompt: `Translate "Hello, world!" to Spanish. Return only the translation, no explanation.` +- Expected: `¡Hola, mundo!` +- Actual: `¡Hola, mundo!` +- Duration: 1560ms (1.56s) +- Timestamp: 6/6/2025, 12:30:21 AM + +### translation - openai/gpt-4o-mini + +- Prompt: `Translate "Hello, world!" to Spanish. Return only the translation, no explanation.` +- Expected: `¡Hola, mundo!` +- Actual: `¡Hola, mundo!` +- Duration: 1451ms (1.45s) +- Timestamp: 6/6/2025, 12:30:22 AM + +### translation - deepseek/deepseek-r1:free + +- Prompt: `Translate "Hello, world!" to Spanish. Return only the translation, no explanation.` +- Expected: `¡Hola, mundo!` +- Actual: `¡Hola, mundo!` +- Duration: 5434ms (5.43s) +- Timestamp: 6/6/2025, 12:30:28 AM ### language_detection - anthropic/claude-sonnet-4 - Prompt: `Identify the language of: "Bonjour, comment allez-vous?". Return only the language name, no explanation.` - Expected: `French` - Actual: `French` -- Duration: 1725ms (1.73s) -- Reason: Expected French, but got french -- Timestamp: 6/5/2025, 8:55:57 PM +- Duration: 1137ms (1.14s) +- Timestamp: 6/6/2025, 12:30:50 AM ### language_detection - openai/gpt-4o-mini - Prompt: `Identify the language of: "Bonjour, comment allez-vous?". Return only the language name, no explanation.` - Expected: `French` - Actual: `French` -- Duration: 776ms (0.78s) -- Reason: Expected French, but got french -- Timestamp: 6/5/2025, 8:55:58 PM +- Duration: 459ms (0.46s) +- Timestamp: 6/6/2025, 12:30:51 AM ### language_detection - deepseek/deepseek-r1:free - Prompt: `Identify the language of: "Bonjour, comment allez-vous?". Return only the language name, no explanation.` - Expected: `French` - Actual: `French` -- Duration: 5247ms (5.25s) -- Reason: Expected French, but got french -- Timestamp: 6/5/2025, 8:56:03 PM +- Duration: 3924ms (3.92s) +- Timestamp: 6/6/2025, 12:30:55 AM ### synonyms - anthropic/claude-sonnet-4 - Prompt: `Provide a synonym for "happy". Return only the synonym, no explanation.` - Expected: `joyful` -- Actual: `Content` -- Duration: 1967ms (1.97s) -- Reason: Expected joyful, but got content -- Timestamp: 6/5/2025, 8:56:05 PM - -## Passed Tests +- Actual: `Joyful` +- Duration: 1251ms (1.25s) +- Timestamp: 6/6/2025, 12:30:56 AM ### synonyms - openai/gpt-4o-mini - Prompt: `Provide a synonym for "happy". Return only the synonym, no explanation.` - Expected: `joyful` - Actual: `Joyful` -- Duration: 548ms (0.55s) -- Timestamp: 6/5/2025, 8:56:06 PM +- Duration: 622ms (0.62s) +- Timestamp: 6/6/2025, 12:30:57 AM ### synonyms - deepseek/deepseek-r1:free - Prompt: `Provide a synonym for "happy". Return only the synonym, no explanation.` - Expected: `joyful` -- Actual: `Joyful` -- Duration: 2616ms (2.62s) -- Timestamp: 6/5/2025, 8:56:08 PM +- Actual: `joyful` +- Duration: 3836ms (3.84s) +- Timestamp: 6/6/2025, 12:31:00 AM