mono/packages/kbot/tests/unit/basic.test.ts

import { describe, it, expect } from 'vitest'
import { run } from '../../src/index'
import * as path from 'node:path'
import { sync as write } from "@polymech/fs/write"
import { sync as read } from "@polymech/fs/read"
import { sync as exists } from "@polymech/fs/exists"

import { E_OPENROUTER_MODEL_FREE, E_OPENAI_MODEL  } from '../../src/index'

const models = [
    E_OPENROUTER_MODEL_FREE.MODEL_FREE_DEEPSEEK_DEEPSEEK_CHAT_FREE, 
    E_OPENROUTER_MODEL_FREE.MODEL_FREE_GOOGLE_GEMINI_2_0_FLASH_EXP_FREE,
    E_OPENAI_MODEL.MODEL_GPT_4
]

const TEST_BASE_PATH = path.resolve(__dirname, '../../')
const TEST_LOGS_PATH = path.resolve(__dirname, '../../logs')
const TEST_PREFERENCES_PATH = path.resolve(__dirname, '../../preferences.md')
const TEST_LOG_PATH = path.resolve(__dirname, './basic.json')

interface TestResult {
  test: string;
  prompt: string;
  result: string[];
  expected: string;
  model: string;
  router: string;
  timestamp: string;
  passed: boolean;
  reason?: string;
}

describe('Basic Operations', () => {
  let testResults: TestResult[] = []
  
  // Load existing results if any
  if (exists(TEST_LOG_PATH)) {
    const data = read(TEST_LOG_PATH, 'json')
    testResults = Array.isArray(data) ? data : []
  }

  it.each(models)('should add two numbers with model %s', async (modelName) => {
    const prompt = 'add 5 and 3. Return only the number, no explanation.'
    const expected = '8'
    let model = 'unknown'
    let router = 'unknown'
    
    const result = await run({
      prompt,
      mode: 'completion',
      model: modelName,
      path: TEST_BASE_PATH,
      logs: TEST_LOGS_PATH,
      preferences: TEST_PREFERENCES_PATH,
      onRun: async (options) => {
        model = options.model || 'unknown'
        router = options.router || 'unknown'
        return options
      }
    }) as string[]    
    
    const actual = result.map(r => r.trim())[0]
    const passed = actual === expected
    expect(actual).toEqual(expected)

    // Add test result to array
    testResults.push({
      test: 'addition',
      prompt,
      result,
      expected,
      model,
      router,
      timestamp: new Date().toISOString(),
      passed,
      reason: passed ? undefined : `Expected ${expected}, but got ${actual}`
    })

    // Write all results to the same file
    write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2))
  })

  it.each(models)('should multiply two numbers with model %s', async (modelName) => {
    const prompt = 'multiply 4 and 6. Return only the number, no explanation.'
    const expected = '24'
    let model = 'unknown'
    let router = 'unknown'
    
    const result = await run({
      prompt,
      mode: 'completion',
      model: modelName,
      path: TEST_BASE_PATH,
      logs: TEST_LOGS_PATH,
      preferences: TEST_PREFERENCES_PATH,
      onRun: async (options) => {
        model = options.model || 'unknown'
        router = options.router || 'unknown'
        return options
      }
    }) as string[]    

    const actual = result.map(r => r.trim())[0]
    const passed = actual === expected
    expect(actual).toEqual(expected)

    // Add test result to array
    testResults.push({
      test: 'multiplication',
      prompt,
      result,
      expected,
      model,
      router,
      timestamp: new Date().toISOString(),
      passed,
      reason: passed ? undefined : `Expected ${expected}, but got ${actual}`
    })

    // Write all results to the same file
    write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2))
  })

  it.each(models)('should divide two numbers with model %s', async (modelName) => {
    const prompt = 'divide 15 by 3. Return only the number, no explanation.'
    const expected = '5'
    let model = 'unknown'
    let router = 'unknown'
    
    const result = await run({
      prompt,
      mode: 'completion',
      model: modelName,
      path: TEST_BASE_PATH,
      logs: TEST_LOGS_PATH,
      preferences: TEST_PREFERENCES_PATH,
      onRun: async (options) => {
        model = options.model || 'unknown'
        router = options.router || 'unknown'
        return options
      }
    }) as string[]    

    const actual = result.map(r => r.trim())[0]
    const passed = actual === expected
    expect(actual).toEqual(expected)

    // Add test result to array
    testResults.push({
      test: 'division',
      prompt,
      result,
      expected,
      model,
      router,
      timestamp: new Date().toISOString(),
      passed,
      reason: passed ? undefined : `Expected ${expected}, but got ${actual}`
    })

    // Write all results to the same file
    write(TEST_LOG_PATH, JSON.stringify(testResults, null, 2))
  })
})