import { describe, it, expect } from 'vitest' import * as path from 'node:path' import { sync as exists } from "@polymech/fs/exists" import { sync as read } from "@polymech/fs/read" import { sync as write } from "@polymech/fs/write" import { sync as mkdirp } from "mkdirp" import { VM } from 'vm2' import { getDefaultModels, TEST_BASE_PATH, TEST_LOGS_PATH, TEST_PREFERENCES_PATH, TEST_TIMEOUT, TestResult, runTest, generateTestReport, getReportPaths } from './commons' // Optionally override models for this specific test file const models = getDefaultModels() // Ensure test-data/code directory exists const TEST_CODE_DIR = path.resolve(__dirname, '../test-data/code') if (exists(TEST_CODE_DIR) !== 'directory') { mkdirp(TEST_CODE_DIR) } describe('Coding Capabilities', () => { let testResults: TestResult[] = [] const TEST_LOG_PATH = getReportPaths('coding', 'json') const TEST_REPORT_PATH = getReportPaths('coding', 'md') const executeCode = (code: string, context: any = {}): any => { const vm = new VM({ timeout: 1000, sandbox: context }) return vm.run(code) } it.each(models)('should generate and execute a simple function with model %s', async (modelName) => { const prompt = `Generate a JavaScript function that adds two numbers and returns the result. The function should be named 'add' and take two parameters 'a' and 'b'. Return only the function code, no explanation.` const result = await runTest( prompt, 'function add(a, b) { return a + b; }', 'simple_function', modelName, TEST_LOG_PATH ) testResults.push(result) // Save the code to a file const codePath = path.resolve(TEST_CODE_DIR, 'add.js') write(codePath, result.result[0]) // Execute the code const context = {} const addFunction = executeCode(result.result[0], context) expect(addFunction(5, 3)).toBe(8) }, { timeout: TEST_TIMEOUT }) it.each(models)('should generate and execute a factorial function with model %s', async (modelName) => { const prompt = `Generate a JavaScript function that calculates the factorial of a number. The function should be named 'factorial' and take one parameter 'n'. Return only the function code, no explanation.` const result = await runTest( prompt, 'function factorial(n) { return n <= 1 ? 1 : n * factorial(n - 1); }', 'factorial_function', modelName, TEST_LOG_PATH ) testResults.push(result) // Save the code to a file const codePath = path.resolve(TEST_CODE_DIR, 'factorial.js') write(codePath, result.result[0]) // Execute the code const context = {} const factorialFunction = executeCode(result.result[0], context) expect(factorialFunction(5)).toBe(120) }, { timeout: TEST_TIMEOUT }) it.each(models)('should generate and execute a fibonacci function with model %s', async (modelName) => { const prompt = `Generate a JavaScript function that calculates the nth Fibonacci number. The function should be named 'fibonacci' and take one parameter 'n'. Return only the function code, no explanation.` const result = await runTest( prompt, 'function fibonacci(n) { return n <= 1 ? n : fibonacci(n - 1) + fibonacci(n - 2); }', 'fibonacci_function', modelName, TEST_LOG_PATH ) testResults.push(result) // Save the code to a file const codePath = path.resolve(TEST_CODE_DIR, 'fibonacci.js') write(codePath, result.result[0]) // Execute the code const context = {} const fibonacciFunction = executeCode(result.result[0], context) expect(fibonacciFunction(6)).toBe(8) }, { timeout: TEST_TIMEOUT }) it('should generate markdown report', () => { generateTestReport(testResults, 'Coding Capabilities Test Results', TEST_REPORT_PATH) expect(exists(TEST_REPORT_PATH) === 'file').toBe(true) }) })