import { describe, it, expect } from 'vitest' import * as path from 'node:path' import { sync as exists } from "@polymech/fs/exists" import { sync as read } from "@polymech/fs/read" import { sync as write } from "@polymech/fs/write" import { sync as mkdirp } from "mkdirp" import { createContext, runInContext } from 'node:vm' import { getDefaultModels, TEST_BASE_PATH, TEST_LOGS_PATH, TEST_PREFERENCES_PATH, TEST_TIMEOUT, TestResult, runTest, generateTestReport, getReportPaths, ModelCategory } from './commons' // Simple slugify function to convert model names to file-safe strings const slugify = (str: string): string => { return str .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') } // Optionally override models for this specific test file const models = getDefaultModels(ModelCategory.FAST) // Ensure test-data/code directory exists const TEST_CODE_DIR = path.resolve(__dirname, '../test-data/code') if (exists(TEST_CODE_DIR) !== 'directory') { mkdirp(TEST_CODE_DIR) } describe('Coding Capabilities', () => { let testResults: TestResult[] = [] const TEST_LOG_PATH = getReportPaths('coding', 'json') const TEST_REPORT_PATH = getReportPaths('coding', 'md') const executeCode = (code: string, functionName: string): any => { const vmContext = createContext({}) // Wrap the code in a module pattern to avoid global scope pollution const wrappedCode = ` (function() { ${code} return ${functionName}; })() ` return runInContext(wrappedCode, vmContext, { timeout: 1000 }) } it.each(models)('should generate and execute a factorial function with model %s', async (modelName) => { const prompt = `Generate a JavaScript function that calculates the factorial of a number. The function should be named 'factorial' and take one parameter 'n'. Return only the function code, no explanation.` const result = await runTest( prompt, 'function factorial(n) { return n <= 1 ? 1 : n * factorial(n - 1); }', 'factorial_function', modelName, TEST_LOG_PATH ) testResults.push(result) // Save the code to a file with model name and prompt comment const modelSlug = slugify(modelName) const codePath = path.resolve(TEST_CODE_DIR, `factorial_${modelSlug}.js`) const codeWithComment = `/* Prompt: ${prompt} */\n\n${result.result[0]}` write(codePath, codeWithComment) // Execute the code const factorialFunction = executeCode(result.result[0], 'factorial') expect(factorialFunction(5)).toBe(120) }, { timeout: TEST_TIMEOUT }) it.each(models)('should generate and execute a bubble sort function with model %s', async (modelName) => { const prompt = `Generate a JavaScript function that implements bubble sort. The function should be named 'bubbleSort' and take one parameter 'arr' (an array of numbers). The function should return the sorted array. Return only the function code, no explanation.` const result = await runTest( prompt, 'function bubbleSort(arr) { const n = arr.length; for(let i = 0; i < n; i++) { for(let j = 0; j < n-i-1; j++) { if(arr[j] > arr[j+1]) { [arr[j], arr[j+1]] = [arr[j+1], arr[j]]; } } } return arr; }', 'bubble_sort_function', modelName, TEST_LOG_PATH ) testResults.push(result) // Save the code to a file with model name and prompt comment const modelSlug = slugify(modelName) const codePath = path.resolve(TEST_CODE_DIR, `bubble_sort_${modelSlug}.js`) const codeWithComment = `/* Prompt: ${prompt} */\n\n${result.result[0]}` write(codePath, codeWithComment) // Execute the code and test with various cases const bubbleSortFunction = executeCode(result.result[0], 'bubbleSort') expect(bubbleSortFunction([64, 34, 25, 12, 22, 11, 90])).toEqual([11, 12, 22, 25, 34, 64, 90]) expect(bubbleSortFunction([1])).toEqual([1]) expect(bubbleSortFunction([])).toEqual([]) expect(bubbleSortFunction([5, 2, 2, 1, 5])).toEqual([1, 2, 2, 5, 5]) }, { timeout: TEST_TIMEOUT }) it('should generate markdown report', () => { generateTestReport(testResults, 'Coding Capabilities Test Results', TEST_REPORT_PATH) expect(exists(TEST_REPORT_PATH) === 'file').toBe(true) }) })