138 lines
5.1 KiB
TypeScript
138 lines
5.1 KiB
TypeScript
import { describe, it, expect } from 'vitest'
|
|
import * as path from 'node:path'
|
|
import { sync as exists } from "@polymech/fs/exists"
|
|
import { sync as read } from "@polymech/fs/read"
|
|
|
|
import {
|
|
getDefaultModels,
|
|
TEST_BASE_PATH,
|
|
TEST_LOGS_PATH,
|
|
TEST_PREFERENCES_PATH,
|
|
TEST_TIMEOUT,
|
|
TestResult,
|
|
runTest,
|
|
generateTestReport,
|
|
getReportPaths,
|
|
ModelCategory,
|
|
EqualityCheck
|
|
} from './commons'
|
|
import { IKBotOptions } from '@polymech/commons'
|
|
|
|
// Optionally override models for this specific test file
|
|
const models = getDefaultModels(ModelCategory.FILES)
|
|
|
|
describe('File Operations', () => {
|
|
let testResults: TestResult[] = []
|
|
const TEST_LOG_PATH = getReportPaths('files', 'json')
|
|
const TEST_REPORT_PATH = getReportPaths('files', 'md')
|
|
|
|
it.each(models)('should identify animals in image files with model %s', async (modelName) => {
|
|
const result = await runTest(
|
|
'What animals are shown in these images?',
|
|
'["cat","fox"]',
|
|
'file-inclusion',
|
|
modelName,
|
|
TEST_LOG_PATH,
|
|
'completion',
|
|
{
|
|
include: ['tests/test-data/files/lazyfox.jpg', 'tests/test-data/files/cat.jpg'],
|
|
logLevel: 2,
|
|
equalityCheck: EqualityCheck.NONE,
|
|
format: {
|
|
type: "object",
|
|
properties: {
|
|
animals: {
|
|
type: "array",
|
|
items: {
|
|
type: "string"
|
|
},
|
|
minItems: 2,
|
|
maxItems: 2
|
|
}
|
|
},
|
|
required: ["animals"]
|
|
}
|
|
} as IKBotOptions
|
|
)
|
|
testResults.push(result)
|
|
const parsedResult = JSON.parse(result.result[0]?.trim())
|
|
const animals = parsedResult.animals.map((s: string) => s.toLowerCase())
|
|
expect(animals).toContain('cat')
|
|
expect(animals).toContain('fox')
|
|
expect(animals.length).toBe(2)
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it.each(models)('should process single file with model %s', async (modelName) => {
|
|
const result = await runTest(
|
|
'What is the name of the algorithm implemented in these files? Return only the name.',
|
|
'bubble sort',
|
|
'file-inclusion',
|
|
modelName,
|
|
TEST_LOG_PATH,
|
|
'completion',
|
|
{ include: ['tests/test-data/files/bubblesort.js'] }
|
|
)
|
|
testResults.push(result)
|
|
expect(result.result[0]?.trim()?.toLowerCase()).toContain('bubble')
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
|
|
it.each(models)('should process multiple files with model %s', async (modelName) => {
|
|
console.log('Starting test for model:', modelName)
|
|
console.log('About to run test with debugger')
|
|
|
|
const result = await runTest(
|
|
'List all algorithms implemented in these files, as JSON array.',
|
|
'["bubble sort","factorial"]',
|
|
'file-inclusion',
|
|
modelName,
|
|
TEST_LOG_PATH,
|
|
'completion',
|
|
{
|
|
include: ['./tests/test-data/files/*.js'],
|
|
equalityCheck: EqualityCheck.NONE,
|
|
logLevel: 2,
|
|
format: {
|
|
type: "object",
|
|
properties: {
|
|
algorithms: {
|
|
type: "array",
|
|
items: {
|
|
type: "string"
|
|
},
|
|
minItems: 2,
|
|
maxItems: 2
|
|
}
|
|
},
|
|
required: ["algorithms"]
|
|
}
|
|
} as IKBotOptions
|
|
)
|
|
testResults.push(result)
|
|
const parsedResult = JSON.parse(result.result[0]?.trim())
|
|
const algorithms = parsedResult.algorithms.map((s: string) => s.toLowerCase())
|
|
expect(algorithms.some(a => a.includes('bubble'))).toBe(true)
|
|
expect(algorithms.some(a => a.includes('factorial'))).toBe(true)
|
|
expect(algorithms).toHaveLength(2)
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
it.each(models)('should process files in glob subdirectory with model %s', async (modelName) => {
|
|
const result = await runTest(
|
|
'What is the title of the product in data.json? Return only the title.',
|
|
'Injection Barrel',
|
|
'file-inclusion',
|
|
modelName,
|
|
TEST_LOG_PATH,
|
|
'completion',
|
|
{ include: ['tests/test-data/files/glob/data.json'] }
|
|
)
|
|
testResults.push(result)
|
|
expect(result.result[0]?.trim()).toBe('Injection Barrel')
|
|
}, { timeout: TEST_TIMEOUT })
|
|
|
|
|
|
it('should generate markdown report', () => {
|
|
generateTestReport(testResults, 'File Operations Test Results', TEST_REPORT_PATH)
|
|
expect(exists(TEST_REPORT_PATH) === 'file').toBe(true)
|
|
})
|
|
})
|