mono/packages/kbot/tests/unit/basic.test.ts

import { describe, it, expect } from 'vitest'
import { run } from '../../src/index'
import * as path from 'node:path'
import { sync as write } from "@polymech/fs/write"
import { sync as read } from "@polymech/fs/read"
import { sync as exists } from "@polymech/fs/exists"
import { models, TEST_BASE_PATH, TEST_LOGS_PATH, TEST_PREFERENCES_PATH, TestResult } from './commons'

const TEST_LOG_PATH = path.resolve(__dirname, './basic.json')

describe('Basic Operations', () => {
  let testResults: TestResult[] = []
  
  // Load existing results if any
  if (exists(TEST_LOG_PATH)) {
    const data = read(TEST_LOG_PATH, 'json')
    testResults = Array.isArray(data) ? data : []
  }

  it.each(models)('should add two numbers with model %s', async (modelName) => {
    const prompt = 'add 5 and 3. Return only the number, no explanation.'
    const expected = '8'
    let model = 'unknown'
    let router = 'unknown'
    
    const result = await run({
      prompt,
      mode: 'completion',
      model: modelName,
      path: TEST_BASE_PATH,
      logs: TEST_LOGS_PATH,
      preferences: TEST_PREFERENCES_PATH,
      onRun: async (options) => {
        model = options.model || 'unknown'
        router = options.router || 'unknown'
        return options
      }
    }) as string[]    

    const actual = result?.[0]?.trim() || ''
    const passed = actual === expected
    expect(actual).toEqual(expected)

    // Add test result to array
    testResults.push({
      test: 'addition',
      prompt,
      result: result || [],
      expected,
      model,
      router,
      timestamp: new Date().toISOString(),
      passed,
      reason: !actual ? 'No result returned from model' : passed ? undefined : `Expected ${expected}, but got ${actual}`
    })

    // Write all results to the same file
    write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2))
  })

  it.each(models)('should multiply two numbers with model %s', async (modelName) => {
    const prompt = 'multiply 8 and 3. Return only the number, no explanation.'
    const expected = '24'
    let model = 'unknown'
    let router = 'unknown'
    
    const result = await run({
      prompt,
      mode: 'completion',
      model: modelName,
      path: TEST_BASE_PATH,
      logs: TEST_LOGS_PATH,
      preferences: TEST_PREFERENCES_PATH,
      onRun: async (options) => {
        model = options.model || 'unknown'
        router = options.router || 'unknown'
        return options
      }
    }) as string[]

    const actual = result?.[0]?.trim() || ''
    const passed = actual === expected
    expect(actual).toEqual(expected)

    // Add test result to array
    testResults.push({
      test: 'multiplication',
      prompt,
      result: result || [],
      expected,
      model,
      router,
      timestamp: new Date().toISOString(),
      passed,
      reason: !actual ? 'No result returned from model' : passed ? undefined : `Expected ${expected}, but got ${actual}`
    })

    // Write all results to the same file
    write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2))
  })

  it.each(models)('should divide two numbers with model %s', async (modelName) => {
    const prompt = 'divide 15 by 3. Return only the number, no explanation.'
    const expected = '5'
    let model = 'unknown'
    let router = 'unknown'
    
    const result = await run({
      prompt,
      mode: 'completion',
      model: modelName,
      path: TEST_BASE_PATH,
      logs: TEST_LOGS_PATH,
      preferences: TEST_PREFERENCES_PATH,
      onRun: async (options) => {
        model = options.model || 'unknown'
        router = options.router || 'unknown'
        return options
      }
    }) as string[]

    const actual = result?.[0]?.trim() || ''
    const passed = actual === expected
    expect(actual).toEqual(expected)

    // Add test result to array
    testResults.push({
      test: 'division',
      prompt,
      result: result || [],
      expected,
      model,
      router,
      timestamp: new Date().toISOString(),
      passed,
      reason: !actual ? 'No result returned from model' : passed ? undefined : `Expected ${expected}, but got ${actual}`
    })

    // Write all results to the same file
    write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2))
  })

  it('should generate markdown report', () => {
    // Group results by test and model
    const latestResults = new Map<string, Map<string, TestResult>>()
    
    // Get only the latest result for each test+model combination
    testResults.forEach(result => {
      if (!latestResults.has(result.test)) {
        latestResults.set(result.test, new Map())
      }
      const testMap = latestResults.get(result.test)!
      const existingResult = testMap.get(result.model)
      if (!existingResult || new Date(result.timestamp) > new Date(existingResult.timestamp)) {
        testMap.set(result.model, result)
      }
    })

    // Generate markdown report
    let report = '# Basic Operations Test Results\n\n'
    
    // First list failed tests
    report += '## Failed Tests\n\n'
    for (const [testName, modelResults] of latestResults) {
      for (const [model, result] of modelResults) {
        if (!result.passed) {
          report += `### ${testName} - ${model}\n`
          report += `- Prompt: \`${result.prompt}\`\n`
          report += `- Expected: \`${result.expected}\`\n`
          report += `- Actual: \`${result.result[0] || ''}\`\n`
          report += `- Reason: ${result.reason}\n`
          report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n`
        }
      }
    }

    // Then list passed tests
    report += '## Passed Tests\n\n'
    for (const [testName, modelResults] of latestResults) {
      for (const [model, result] of modelResults) {
        if (result.passed) {
          report += `### ${testName} - ${model}\n`
          report += `- Prompt: \`${result.prompt}\`\n`
          report += `- Expected: \`${result.expected}\`\n`
          report += `- Actual: \`${result.result[0] || ''}\`\n`
          report += `- Timestamp: ${new Date(result.timestamp).toLocaleString()}\n\n`
        }
      }
    }

    // Write report to file
    const reportPath = path.resolve(__dirname, './basic-report.md')
    write(reportPath, report)

    // Verify report was written
    expect(exists(reportPath) === 'file').toBe(true)
  })
})