122 lines
4.0 KiB
TypeScript
122 lines
4.0 KiB
TypeScript
import { describe, it, expect } from 'vitest'
|
|
import * as path from 'node:path'
|
|
import { sync as exists } from "@polymech/fs/exists"
|
|
import { sync as read } from "@polymech/fs/read"
|
|
import { sync as write } from "@polymech/fs/write"
|
|
import { sync as mkdirp } from "mkdirp"
|
|
import { VM } from 'vm2'
|
|
|
|
import {
|
|
getDefaultModels,
|
|
TEST_BASE_PATH,
|
|
TEST_LOGS_PATH,
|
|
TEST_PREFERENCES_PATH,
|
|
TEST_TIMEOUT,
|
|
TestResult,
|
|
runTest,
|
|
generateTestReport,
|
|
getReportPaths
|
|
} from './commons'
|
|
|
|
// Optionally override models for this specific test file
|
|
const models = getDefaultModels()
|
|
|
|
// Ensure test-data/code directory exists
|
|
const TEST_CODE_DIR = path.resolve(__dirname, '../test-data/code')
|
|
if (exists(TEST_CODE_DIR) !== 'directory') {
|
|
mkdirp(TEST_CODE_DIR)
|
|
}
|
|
|
|
describe('Coding Capabilities', () => {
|
|
let testResults: TestResult[] = []
|
|
const TEST_LOG_PATH = getReportPaths('coding', 'json')
|
|
const TEST_REPORT_PATH = getReportPaths('coding', 'md')
|
|
|
|
const executeCode = (code: string, context: any = {}): any => {
|
|
const vm = new VM({
|
|
timeout: 1000,
|
|
sandbox: context
|
|
})
|
|
return vm.run(code)
|
|
}
|
|
|
|
it.each(models)('should generate and execute a simple function with model %s', async (modelName) => {
|
|
const prompt = `Generate a JavaScript function that adds two numbers and returns the result.
|
|
The function should be named 'add' and take two parameters 'a' and 'b'.
|
|
Return only the function code, no explanation.`
|
|
|
|
const result = await runTest(
|
|
prompt,
|
|
'function add(a, b) { return a + b; }',
|
|
'simple_function',
|
|
modelName,
|
|
TEST_LOG_PATH
|
|
)
|
|
|
|
testResults.push(result)
|
|
|
|
// Save the code to a file
|
|
const codePath = path.resolve(TEST_CODE_DIR, 'add.js')
|
|
write(codePath, result.result[0])
|
|
|
|
// Execute the code
|
|
const context = {}
|
|
const addFunction = executeCode(result.result[0], context)
|
|
expect(addFunction(5, 3)).toBe(8)
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it.each(models)('should generate and execute a factorial function with model %s', async (modelName) => {
|
|
const prompt = `Generate a JavaScript function that calculates the factorial of a number.
|
|
The function should be named 'factorial' and take one parameter 'n'.
|
|
Return only the function code, no explanation.`
|
|
|
|
const result = await runTest(
|
|
prompt,
|
|
'function factorial(n) { return n <= 1 ? 1 : n * factorial(n - 1); }',
|
|
'factorial_function',
|
|
modelName,
|
|
TEST_LOG_PATH
|
|
)
|
|
|
|
testResults.push(result)
|
|
|
|
// Save the code to a file
|
|
const codePath = path.resolve(TEST_CODE_DIR, 'factorial.js')
|
|
write(codePath, result.result[0])
|
|
|
|
// Execute the code
|
|
const context = {}
|
|
const factorialFunction = executeCode(result.result[0], context)
|
|
expect(factorialFunction(5)).toBe(120)
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it.each(models)('should generate and execute a fibonacci function with model %s', async (modelName) => {
|
|
const prompt = `Generate a JavaScript function that calculates the nth Fibonacci number.
|
|
The function should be named 'fibonacci' and take one parameter 'n'.
|
|
Return only the function code, no explanation.`
|
|
|
|
const result = await runTest(
|
|
prompt,
|
|
'function fibonacci(n) { return n <= 1 ? n : fibonacci(n - 1) + fibonacci(n - 2); }',
|
|
'fibonacci_function',
|
|
modelName,
|
|
TEST_LOG_PATH
|
|
)
|
|
|
|
testResults.push(result)
|
|
|
|
// Save the code to a file
|
|
const codePath = path.resolve(TEST_CODE_DIR, 'fibonacci.js')
|
|
write(codePath, result.result[0])
|
|
|
|
// Execute the code
|
|
const context = {}
|
|
const fibonacciFunction = executeCode(result.result[0], context)
|
|
expect(fibonacciFunction(6)).toBe(8)
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it('should generate markdown report', () => {
|
|
generateTestReport(testResults, 'Coding Capabilities Test Results', TEST_REPORT_PATH)
|
|
expect(exists(TEST_REPORT_PATH) === 'file').toBe(true)
|
|
})
|
|
})
|