120 lines
4.4 KiB
TypeScript
120 lines
4.4 KiB
TypeScript
import { describe, it, expect } from 'vitest'
|
|
import * as path from 'node:path'
|
|
import { sync as exists } from "@polymech/fs/exists"
|
|
import { sync as read } from "@polymech/fs/read"
|
|
import { sync as write } from "@polymech/fs/write"
|
|
import { sync as mkdirp } from "mkdirp"
|
|
import { createContext, runInContext } from 'node:vm'
|
|
|
|
import {
|
|
getDefaultModels,
|
|
TEST_BASE_PATH,
|
|
TEST_LOGS_PATH,
|
|
TEST_PREFERENCES_PATH,
|
|
TEST_TIMEOUT,
|
|
TestResult,
|
|
runTest,
|
|
generateTestReport,
|
|
getReportPaths,
|
|
ModelCategory
|
|
} from './commons'
|
|
|
|
// Simple slugify function to convert model names to file-safe strings
|
|
const slugify = (str: string): string => {
|
|
return str
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9]+/g, '-')
|
|
.replace(/^-+|-+$/g, '')
|
|
}
|
|
|
|
// Optionally override models for this specific test file
|
|
const models = getDefaultModels(ModelCategory.FAST)
|
|
|
|
// Ensure test-data/code directory exists
|
|
const TEST_CODE_DIR = path.resolve(__dirname, '../test-data/code')
|
|
if (exists(TEST_CODE_DIR) !== 'directory') {
|
|
mkdirp(TEST_CODE_DIR)
|
|
}
|
|
|
|
describe('Coding Capabilities', () => {
|
|
let testResults: TestResult[] = []
|
|
const TEST_LOG_PATH = getReportPaths('coding', 'json')
|
|
const TEST_REPORT_PATH = getReportPaths('coding', 'md')
|
|
|
|
const executeCode = (code: string, functionName: string): any => {
|
|
const vmContext = createContext({})
|
|
// Wrap the code in a module pattern to avoid global scope pollution
|
|
const wrappedCode = `
|
|
(function() {
|
|
${code}
|
|
return ${functionName};
|
|
})()
|
|
`
|
|
return runInContext(wrappedCode, vmContext, { timeout: 1000 })
|
|
}
|
|
|
|
it.each(models)('should generate and execute a factorial function with model %s', async (modelName) => {
|
|
const prompt = `Generate a JavaScript function that calculates the factorial of a number.
|
|
The function should be named 'factorial' and take one parameter 'n'.
|
|
Return only the function code, no explanation.`
|
|
|
|
const result = await runTest(
|
|
prompt,
|
|
'function factorial(n) { return n <= 1 ? 1 : n * factorial(n - 1); }',
|
|
'factorial_function',
|
|
modelName,
|
|
TEST_LOG_PATH
|
|
)
|
|
|
|
testResults.push(result)
|
|
|
|
// Save the code to a file with model name and prompt comment
|
|
const modelSlug = slugify(modelName)
|
|
const codePath = path.resolve(TEST_CODE_DIR, `factorial_${modelSlug}.js`)
|
|
const codeWithComment = `/* Prompt:
|
|
${prompt}
|
|
*/\n\n${result.result[0]}`
|
|
write(codePath, codeWithComment)
|
|
|
|
// Execute the code
|
|
const factorialFunction = executeCode(result.result[0], 'factorial')
|
|
expect(factorialFunction(5)).toBe(120)
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it.each(models)('should generate and execute a bubble sort function with model %s', async (modelName) => {
|
|
const prompt = `Generate a JavaScript function that implements bubble sort.
|
|
The function should be named 'bubbleSort' and take one parameter 'arr' (an array of numbers).
|
|
The function should return the sorted array.
|
|
Return only the function code, no explanation.`
|
|
|
|
const result = await runTest(
|
|
prompt,
|
|
'function bubbleSort(arr) { const n = arr.length; for(let i = 0; i < n; i++) { for(let j = 0; j < n-i-1; j++) { if(arr[j] > arr[j+1]) { [arr[j], arr[j+1]] = [arr[j+1], arr[j]]; } } } return arr; }',
|
|
'bubble_sort_function',
|
|
modelName,
|
|
TEST_LOG_PATH
|
|
)
|
|
|
|
testResults.push(result)
|
|
|
|
// Save the code to a file with model name and prompt comment
|
|
const modelSlug = slugify(modelName)
|
|
const codePath = path.resolve(TEST_CODE_DIR, `bubble_sort_${modelSlug}.js`)
|
|
const codeWithComment = `/* Prompt:
|
|
${prompt}
|
|
*/\n\n${result.result[0]}`
|
|
write(codePath, codeWithComment)
|
|
|
|
// Execute the code and test with various cases
|
|
const bubbleSortFunction = executeCode(result.result[0], 'bubbleSort')
|
|
expect(bubbleSortFunction([64, 34, 25, 12, 22, 11, 90])).toEqual([11, 12, 22, 25, 34, 64, 90])
|
|
expect(bubbleSortFunction([1])).toEqual([1])
|
|
expect(bubbleSortFunction([])).toEqual([])
|
|
expect(bubbleSortFunction([5, 2, 2, 1, 5])).toEqual([1, 2, 2, 5, 5])
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it('should generate markdown report', () => {
|
|
generateTestReport(testResults, 'Coding Capabilities Test Results', TEST_REPORT_PATH)
|
|
expect(exists(TEST_REPORT_PATH) === 'file').toBe(true)
|
|
})
|
|
})
|