95 lines
3.5 KiB
TypeScript
95 lines
3.5 KiB
TypeScript
import { describe, it, expect } from 'vitest'
|
|
import * as path from 'node:path'
|
|
import { sync as exists } from "@polymech/fs/exists"
|
|
import { sync as read } from "@polymech/fs/read"
|
|
import { z } from 'zod'
|
|
|
|
import {
|
|
getDefaultModels,
|
|
TEST_BASE_PATH,
|
|
TEST_LOGS_PATH,
|
|
TEST_PREFERENCES_PATH,
|
|
TEST_TIMEOUT,
|
|
TestResult,
|
|
runTest,
|
|
generateTestReport,
|
|
getReportPaths,
|
|
ModelCategory
|
|
} from './commons'
|
|
|
|
// Use tool models for this test file
|
|
const models = getDefaultModels(ModelCategory.TOOL)
|
|
|
|
// Define the Zod schema for equation results
|
|
const equationResultSchema = z.object({
|
|
equation: z.string(),
|
|
result: z.string()
|
|
})
|
|
|
|
const equationResultsSchema = z.array(equationResultSchema)
|
|
|
|
describe('LLM Tools Operations', () => {
|
|
let testResults: TestResult[] = []
|
|
const TEST_LOG_PATH = getReportPaths('tools', 'json')
|
|
const TEST_REPORT_PATH = getReportPaths('tools', 'md')
|
|
const TEST_DATA_PATH = path.resolve('./tests/units/tools.test.md')
|
|
const TEST_DATA_DIR = path.resolve(__dirname, 'test-data')
|
|
|
|
it.each(models)('should read and solve equations with model %s', async (modelName) => {
|
|
const result = await runTest(
|
|
`Read the file at ${TEST_DATA_PATH} and solve all equations. Return the results in the specified JSON format.`,
|
|
JSON.stringify([
|
|
{ equation: "2x + 5 = 13", result: "4" },
|
|
{ equation: "3y - 7 = 20", result: "9" },
|
|
{ equation: "4z + 8 = 32", result: "6" }
|
|
]),
|
|
'equation_solving',
|
|
modelName,
|
|
TEST_LOG_PATH,
|
|
'tools',
|
|
{ format: equationResultsSchema }
|
|
)
|
|
testResults.push(result)
|
|
expect(result.result[0]?.trim()).toBeTruthy()
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it.each(models)('should write and read back data with model %s', async (modelName) => {
|
|
const testData = { test: "data", timestamp: new Date().toISOString() }
|
|
const testFilePath = path.resolve(TEST_DATA_DIR, 'test-data.json')
|
|
|
|
const result = await runTest(
|
|
`Write the following data to ${testFilePath} and then read it back: ${JSON.stringify(testData)}. Return the read data in JSON format.`,
|
|
JSON.stringify(testData),
|
|
'file_operations',
|
|
modelName,
|
|
TEST_LOG_PATH,
|
|
'tools',
|
|
{
|
|
disable: ['npm,git,interact,terminal,search,web,email,user']
|
|
}
|
|
)
|
|
testResults.push(result)
|
|
expect(result.result[0]?.trim()).toBeTruthy()
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it.each(models)('should list directory contents with model %s', async (modelName) => {
|
|
const result = await runTest(
|
|
`List all files in the directory ${TEST_DATA_DIR}. Return the list as a JSON array of filenames.`,
|
|
'[]', // We don't know exact contents, just verify we get a valid JSON array
|
|
'directory_listing',
|
|
modelName,
|
|
TEST_LOG_PATH,
|
|
'tools',
|
|
{
|
|
disable: ['npm,git,interact,terminal,search,web,email,user']
|
|
}
|
|
)
|
|
testResults.push(result)
|
|
expect(result.result[0]?.trim()).toBeTruthy()
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it('should generate markdown report', () => {
|
|
generateTestReport(testResults, 'LLM Tools Test Results', TEST_REPORT_PATH)
|
|
expect(exists(TEST_REPORT_PATH) === 'file').toBe(true)
|
|
})
|
|
})
|