124 lines
4.2 KiB
TypeScript
124 lines
4.2 KiB
TypeScript
import { describe, it, expect } from 'vitest'
|
|
import * as path from 'node:path'
|
|
import { sync as exists } from "@polymech/fs/exists"
|
|
import { sync as read } from "@polymech/fs/read"
|
|
import { sync as write } from "@polymech/fs/write"
|
|
import { sync as mkdirp } from "mkdirp"
|
|
import { createContext, runInContext } from 'node:vm'
|
|
|
|
import {
|
|
getDefaultModels,
|
|
TEST_BASE_PATH,
|
|
TEST_LOGS_PATH,
|
|
TEST_PREFERENCES_PATH,
|
|
TEST_TIMEOUT,
|
|
TestResult,
|
|
runTest,
|
|
generateTestReport,
|
|
getReportPaths,
|
|
ModelCategory
|
|
} from './commons'
|
|
|
|
// Optionally override models for this specific test file
|
|
const models = getDefaultModels(ModelCategory.CODING)
|
|
|
|
// Ensure test-data/code directory exists
|
|
const TEST_CODE_DIR = path.resolve(__dirname, '../test-data/code')
|
|
if (exists(TEST_CODE_DIR) !== 'directory') {
|
|
mkdirp(TEST_CODE_DIR)
|
|
}
|
|
|
|
describe('Coding Capabilities', () => {
|
|
let testResults: TestResult[] = []
|
|
const TEST_LOG_PATH = getReportPaths('coding', 'json')
|
|
const TEST_REPORT_PATH = getReportPaths('coding', 'md')
|
|
|
|
const executeCode = (code: string, functionName: string): any => {
|
|
const vmContext = createContext({})
|
|
// Wrap the code in a module pattern to avoid global scope pollution
|
|
const wrappedCode = `
|
|
(function() {
|
|
${code}
|
|
return ${functionName};
|
|
})()
|
|
`
|
|
return runInContext(wrappedCode, vmContext, { timeout: 1000 })
|
|
}
|
|
|
|
it.each(models)('should generate and execute a simple function with model %s', async (modelName) => {
|
|
const prompt = `Generate a JavaScript function that adds two numbers and returns the result.
|
|
The function should be named 'add' and take two parameters 'a' and 'b'.
|
|
Return only the function code, no explanation.`
|
|
|
|
const result = await runTest(
|
|
prompt,
|
|
'function add(a, b) { return a + b; }',
|
|
'simple_function',
|
|
modelName,
|
|
TEST_LOG_PATH
|
|
)
|
|
|
|
testResults.push(result)
|
|
|
|
// Save the code to a file
|
|
const codePath = path.resolve(TEST_CODE_DIR, 'add.js')
|
|
write(codePath, result.result[0])
|
|
|
|
// Execute the code
|
|
const addFunction = executeCode(result.result[0], 'add')
|
|
expect(addFunction(5, 3)).toBe(8)
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it.each(models)('should generate and execute a factorial function with model %s', async (modelName) => {
|
|
const prompt = `Generate a JavaScript function that calculates the factorial of a number.
|
|
The function should be named 'factorial' and take one parameter 'n'.
|
|
Return only the function code, no explanation.`
|
|
|
|
const result = await runTest(
|
|
prompt,
|
|
'function factorial(n) { return n <= 1 ? 1 : n * factorial(n - 1); }',
|
|
'factorial_function',
|
|
modelName,
|
|
TEST_LOG_PATH
|
|
)
|
|
|
|
testResults.push(result)
|
|
|
|
// Save the code to a file
|
|
const codePath = path.resolve(TEST_CODE_DIR, 'factorial.js')
|
|
write(codePath, result.result[0])
|
|
|
|
// Execute the code
|
|
const factorialFunction = executeCode(result.result[0], 'factorial')
|
|
expect(factorialFunction(5)).toBe(120)
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it.each(models)('should generate and execute a fibonacci function with model %s', async (modelName) => {
|
|
const prompt = `Generate a JavaScript function that calculates the nth Fibonacci number.
|
|
The function should be named 'fibonacci' and take one parameter 'n'.
|
|
Return only the function code, no explanation.`
|
|
|
|
const result = await runTest(
|
|
prompt,
|
|
'function fibonacci(n) { return n <= 1 ? n : fibonacci(n - 1) + fibonacci(n - 2); }',
|
|
'fibonacci_function',
|
|
modelName,
|
|
TEST_LOG_PATH
|
|
)
|
|
|
|
testResults.push(result)
|
|
|
|
// Save the code to a file
|
|
const codePath = path.resolve(TEST_CODE_DIR, 'fibonacci.js')
|
|
write(codePath, result.result[0])
|
|
|
|
// Execute the code
|
|
const fibonacciFunction = executeCode(result.result[0], 'fibonacci')
|
|
expect(fibonacciFunction(6)).toBe(8)
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it('should generate markdown report', () => {
|
|
generateTestReport(testResults, 'Coding Capabilities Test Results', TEST_REPORT_PATH)
|
|
expect(exists(TEST_REPORT_PATH) === 'file').toBe(true)
|
|
})
|
|
})
|