From 21a3939c9acfa01a78f0833bd742ebe94e21c99e Mon Sep 17 00:00:00 2001 From: babayaga Date: Sun, 6 Apr 2025 21:31:55 +0200 Subject: [PATCH] kbot - async it transformer example --- packages/kbot/README.md | 88 +++-- packages/kbot/dist-in/async-iterator.d.ts | 21 ++ packages/kbot/dist-in/async-iterator.js | 78 +++++ .../examples/core/async-iterator-example.d.ts | 38 ++ .../examples/core/async-iterator-example.js | 268 ++++++++++++++ packages/kbot/dist-in/examples/index.d.ts | 1 + packages/kbot/dist-in/examples/index.js | 3 + packages/kbot/dist-in/profile.js | 2 +- packages/kbot/logs/params.json | 10 +- packages/kbot/package-lock.json | 158 +++++++++ packages/kbot/package.json | 5 +- .../examples/core/async-iterator-example.ts | 326 ++++++++++++++++++ packages/kbot/src/examples/index.ts | 2 + packages/kbot/tsconfig.json | 3 +- 14 files changed, 966 insertions(+), 37 deletions(-) create mode 100644 packages/kbot/dist-in/async-iterator.d.ts create mode 100644 packages/kbot/dist-in/async-iterator.js create mode 100644 packages/kbot/dist-in/examples/core/async-iterator-example.d.ts create mode 100644 packages/kbot/dist-in/examples/core/async-iterator-example.js create mode 100644 packages/kbot/dist-in/examples/index.d.ts create mode 100644 packages/kbot/dist-in/examples/index.js create mode 100644 packages/kbot/src/examples/core/async-iterator-example.ts create mode 100644 packages/kbot/src/examples/index.ts diff --git a/packages/kbot/README.md b/packages/kbot/README.md index 66f229c8..961ae7be 100644 --- a/packages/kbot/README.md +++ b/packages/kbot/README.md @@ -109,31 +109,31 @@ When creating content - always add links - when sending emails, always add 'Best regards, [Your Name]' ``` -## Commands - -### Prompt - -```kbot "create Astro minimal boilerplate, use starlight theme. Install dependencies via NPM tool"``` - -### Fetch latest models - -```kbot fetch``` - -### Print examples - -```kbot examples``` - -### Print extended help - -```kbot help-md``` - -### Initialize folder - -```kbot init``` - -### Internal : Build - -```kbot build``` +## Commands + +### Prompt + +```kbot "create Astro minimal boilerplate, use starlight theme. Install dependencies via NPM tool"``` + +### Fetch latest models + +```kbot fetch``` + +### Print examples + +```kbot examples``` + +### Print extended help + +```kbot help-md``` + +### Initialize folder + +```kbot init``` + +### Internal : Build + +```kbot build``` # Command Line Parameters @@ -210,4 +210,40 @@ osr-cli each --main='kbot \"read ${KEY} and translate to german, save in docs/la ```bash npm i -g @plastichub/osr-cli-commons -``` \ No newline at end of file +``` + +## Async Iterator Examples + +This package provides examples of how to use the async-iterator module to transform objects with LLM integration: + +### Running the Examples + +```bash +# Run the standard example (uses mock transformers if API is unavailable) +npm run examples:async-iterator + +# Run with verbose logging +npm run examples:async-iterator:verbose +``` + +### Implementation Details + +The async-iterator module provides a powerful way to transform specific fields in complex nested objects using JSONPath selectors. Key features include: + +1. **Field Selection**: Use JSONPath patterns like `$..description` to target specific fields across the entire object +2. **Transformation Options**: Configure how fields are transformed with options like model, prompt, etc. +3. **Target Mapping**: Transform fields in-place or create new fields to store the transformed values +4. **Error Handling**: Built-in error callbacks to handle transformation failures gracefully +5. **Throttling & Concurrency**: Control API rate limits with throttle delay and concurrent task settings + +### Example Source Code + +You can find the example in: +- `src/examples/core/async-iterator-example.ts` - Source implementation using the actual run command with fallback to mock transformers + +The example demonstrates: +1. Using the `transformObject` function from the async-iterator module +2. Setting up proper JSONPath selectors for targeting specific fields +3. Creating field mappings with transformation options +4. Implementing in-place transformations and new field creation +5. Graceful fallback handling when API calls fail \ No newline at end of file diff --git a/packages/kbot/dist-in/async-iterator.d.ts b/packages/kbot/dist-in/async-iterator.d.ts new file mode 100644 index 00000000..6ec8a6f6 --- /dev/null +++ b/packages/kbot/dist-in/async-iterator.d.ts @@ -0,0 +1,21 @@ +export type AsyncTransformer = (input: string, path: string) => Promise; +export type ErrorCallback = (path: string, value: string, error: any) => void; +export type FilterCallback = (input: string, path: string) => Promise; +export type Filter = (input: string) => Promise; +export interface TransformOptions { + transform: AsyncTransformer; + path: string; + throttleDelay: number; + concurrentTasks: number; + errorCallback: ErrorCallback; + filterCallback: FilterCallback; +} +export declare const isNumber: Filter; +export declare const isBoolean: Filter; +export declare const isValidString: Filter; +export declare const testFilters: (filters: Filter[]) => FilterCallback; +export declare const defaultFilters: (filters?: Filter[]) => Filter[]; +export declare function transformObject(obj: any, transform: AsyncTransformer, path: string, throttleDelay: number, concurrentTasks: number, errorCallback: ErrorCallback, testCallback: FilterCallback): Promise; +export declare function transformPath(obj: any, keys: string[], transform: AsyncTransformer, throttleDelay: number, concurrentTasks: number, currentPath: string, errorCallback: ErrorCallback, testCallback: FilterCallback): Promise; +export declare const defaultError: ErrorCallback; +export declare const defaultOptions: (options?: TransformOptions) => TransformOptions; diff --git a/packages/kbot/dist-in/async-iterator.js b/packages/kbot/dist-in/async-iterator.js new file mode 100644 index 00000000..b4f89980 --- /dev/null +++ b/packages/kbot/dist-in/async-iterator.js @@ -0,0 +1,78 @@ +import { JSONPath } from 'jsonpath-plus'; +import pThrottle from 'p-throttle'; +import pMap from 'p-map'; +export const isNumber = async (input) => (/^-?\d+(\.\d+)?$/.test(input)); +export const isBoolean = async (input) => /^(true|false)$/i.test(input); +export const isValidString = async (input) => !(input.trim() !== ''); +export const testFilters = (filters) => { + return async (input) => { + for (const filter of filters) { + if (await filter(input)) { + return false; + } + } + return true; + }; +}; +export const defaultFilters = (filters = []) => [ + isNumber, isBoolean, isValidString, ...filters +]; +export async function transformObject(obj, transform, path, throttleDelay, concurrentTasks, errorCallback, testCallback) { + const paths = JSONPath({ path, json: obj, resultType: 'pointer' }); + await pMap(paths, async (jsonPointer) => { + const keys = jsonPointer.slice(1).split('/'); + await transformPath(obj, keys, transform, throttleDelay, concurrentTasks, jsonPointer, errorCallback, testCallback); + }, { concurrency: concurrentTasks }); +} +export async function transformPath(obj, keys, transform, throttleDelay, concurrentTasks, currentPath, errorCallback, testCallback) { + let current = obj; + for (let i = 0; i < keys.length - 1; i++) { + current = current[keys[i]]; + } + const lastKey = keys[keys.length - 1]; + const throttle = pThrottle({ + limit: 1, + interval: throttleDelay, + }); + if (typeof lastKey === 'string' && lastKey !== '') { + try { + const newKey = (await isValidString(lastKey)) && !(await isNumber(lastKey)) ? + await throttle(transform)(lastKey, currentPath) : lastKey; + if (newKey !== lastKey) { + current[newKey] = current[lastKey]; + delete current[lastKey]; + } + if (typeof current[newKey] === 'string' && current[newKey] !== '') { + if (await testCallback(current[newKey], `${currentPath}/${lastKey}`)) { + current[newKey] = await throttle(transform)(current[newKey], `${currentPath}/${lastKey}`); + } + } + else if (typeof current[newKey] === 'object' && current[newKey] !== null) { + await transformObject(current[newKey], transform, '$.*', throttleDelay, concurrentTasks, errorCallback, testCallback); + } + } + catch (error) { + errorCallback(currentPath, lastKey, error); + } + } +} +const exampleTransformFunction = async (input, path) => { + if (input === 'random') + throw new Error('API error'); + return input.toUpperCase(); +}; +export const defaultError = (path, value, error) => { + // logger.error(`Error at path: ${path}, value: ${value}, error: ${error}`) +}; +export const defaultOptions = (options = {}) => { + return { + ...options, + transform: options.transform || exampleTransformFunction, + path: options.path || '$[*][0,1,2]', + throttleDelay: options.throttleDelay || 10, + concurrentTasks: options.concurrentTasks || 1, + errorCallback: options.errorCallback || defaultError, + filterCallback: options.filterCallback || testFilters(defaultFilters()) + }; +}; +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXN5bmMtaXRlcmF0b3IuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvYXN5bmMtaXRlcmF0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFFBQVEsRUFBRSxNQUFNLGVBQWUsQ0FBQTtBQUN4QyxPQUFPLFNBQVMsTUFBTSxZQUFZLENBQUE7QUFDbEMsT0FBTyxJQUFJLE1BQU0sT0FBTyxDQUFBO0FBZ0J4QixNQUFNLENBQUMsTUFBTSxRQUFRLEdBQVcsS0FBSyxFQUFFLEtBQWEsRUFBRSxFQUFFLENBQUMsQ0FBQyxpQkFBaUIsQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQTtBQUN4RixNQUFNLENBQUMsTUFBTSxTQUFTLEdBQVcsS0FBSyxFQUFFLEtBQWEsRUFBRSxFQUFFLENBQUMsaUJBQWlCLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFBO0FBQ3ZGLE1BQU0sQ0FBQyxNQUFNLGFBQWEsR0FBVyxLQUFLLEVBQUUsS0FBYSxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFBO0FBRXBGLE1BQU0sQ0FBQyxNQUFNLFdBQVcsR0FBRyxDQUFDLE9BQWlCLEVBQWtCLEVBQUU7SUFDN0QsT0FBTyxLQUFLLEVBQUUsS0FBYSxFQUFFLEVBQUU7UUFDM0IsS0FBSyxNQUFNLE1BQU0sSUFBSSxPQUFPLEVBQUUsQ0FBQztZQUMzQixJQUFJLE1BQU0sTUFBTSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7Z0JBQ3RCLE9BQU8sS0FBSyxDQUFDO1lBQ2pCLENBQUM7UUFDTCxDQUFDO1FBQ0QsT0FBTyxJQUFJLENBQUM7SUFDaEIsQ0FBQyxDQUFDO0FBQ04sQ0FBQyxDQUFDO0FBRUYsTUFBTSxDQUFDLE1BQU0sY0FBYyxHQUFHLENBQUMsVUFBb0IsRUFBRSxFQUFFLEVBQUUsQ0FDckQ7SUFDSSxRQUFRLEVBQUUsU0FBUyxFQUFFLGFBQWEsRUFBRyxHQUFHLE9BQU87Q0FDbEQsQ0FBQTtBQUVMLE1BQU0sQ0FBQyxLQUFLLFVBQVUsZUFBZSxDQUNqQyxHQUFRLEVBQ1IsU0FBMkIsRUFDM0IsSUFBWSxFQUNaLGFBQXFCLEVBQ3JCLGVBQXVCLEVBQ3ZCLGFBQTRCLEVBQzVCLFlBQTRCO0lBRTVCLE1BQU0sS0FBSyxHQUFHLFFBQVEsQ0FBQyxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsR0FBRyxFQUFFLFVBQVUsRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDO0lBQ25FLE1BQU0sSUFBSSxDQUNOLEtBQUssRUFDTCxLQUFLLEVBQUUsV0FBZ0IsRUFBRSxFQUFFO1FBQ3ZCLE1BQU0sSUFBSSxHQUFHLFdBQVcsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFBO1FBQzVDLE1BQU0sYUFBYSxDQUFDLEdBQUcsRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFLGFBQWEsRUFBRSxlQUFlLEVBQUUsV0FBVyxFQUFFLGFBQWEsRUFBRSxZQUFZLENBQUMsQ0FBQTtJQUN2SCxDQUFDLEVBQ0QsRUFBRSxXQUFXLEVBQUUsZUFBZSxFQUFFLENBQ25DLENBQUE7QUFDTCxDQUFDO0FBQ0QsTUFBTSxDQUFDLEtBQUssVUFBVSxhQUFhLENBQy9CLEdBQVEsRUFDUixJQUFjLEVBQ2QsU0FBMkIsRUFDM0IsYUFBcUIsRUFDckIsZUFBdUIsRUFDdkIsV0FBbUIsRUFDbkIsYUFBNEIsRUFDNUIsWUFBNEI7SUFHNUIsSUFBSSxPQUFPLEdBQUcsR0FBRyxDQUFBO0lBRWpCLEtBQUssSUFBSSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsR0FBRyxJQUFJLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDLEVBQUUsRUFBRSxDQUFDO1FBQ3ZDLE9BQU8sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUE7SUFDOUIsQ0FBQztJQUNELE1BQU0sT0FBTyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFBO0lBQ3JDLE1BQU0sUUFBUSxHQUFHLFNBQVMsQ0FBQztRQUN2QixLQUFLLEVBQUUsQ0FBQztRQUNSLFFBQVEsRUFBRSxhQUFhO0tBQzFCLENBQUMsQ0FBQTtJQUNGLElBQUksT0FBTyxPQUFPLEtBQUssUUFBUSxJQUFJLE9BQU8sS0FBSyxFQUFFLEVBQUUsQ0FBQztRQUNoRCxJQUFJLENBQUM7WUFDRCxNQUFNLE1BQU0sR0FBRyxDQUFDLE1BQU0sYUFBYSxDQUFDLE9BQU8sQ0FBQyxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQU0sUUFBUSxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQztnQkFDekUsTUFBTSxRQUFRLENBQUMsU0FBUyxDQUFDLENBQUMsT0FBTyxFQUFFLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUE7WUFDN0QsSUFBSSxNQUFNLEtBQUssT0FBTyxFQUFFLENBQUM7Z0JBQ3JCLE9BQU8sQ0FBQyxNQUFNLENBQUMsR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUE7Z0JBQ2xDLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFBO1lBQzNCLENBQUM7WUFDRCxJQUFJLE9BQU8sT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLFFBQVEsSUFBSSxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssRUFBRSxFQUFFLENBQUM7Z0JBQ2hFLElBQUksTUFBTSxZQUFZLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxFQUFFLEdBQUcsV0FBVyxJQUFJLE9BQU8sRUFBRSxDQUFDLEVBQUUsQ0FBQztvQkFDbkUsT0FBTyxDQUFDLE1BQU0sQ0FBQyxHQUFHLE1BQU0sUUFBUSxDQUFDLFNBQVMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsRUFBRSxHQUFHLFdBQVcsSUFBSSxPQUFPLEVBQUUsQ0FBQyxDQUFBO2dCQUM3RixDQUFDO1lBQ0wsQ0FBQztpQkFBTSxJQUFJLE9BQU8sT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLFFBQVEsSUFBSSxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssSUFBSSxFQUFFLENBQUM7Z0JBQ3pFLE1BQU0sZUFBZSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsRUFBRSxTQUFTLEVBQUUsS0FBSyxFQUFFLGFBQWEsRUFBRSxlQUFlLEVBQUUsYUFBYSxFQUFFLFlBQVksQ0FBQyxDQUFBO1lBQ3pILENBQUM7UUFDTCxDQUFDO1FBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztZQUNiLGFBQWEsQ0FBQyxXQUFXLEVBQUUsT0FBTyxFQUFFLEtBQUssQ0FBQyxDQUFBO1FBQzlDLENBQUM7SUFDTCxDQUFDO0FBQ0wsQ0FBQztBQUVELE1BQU0sd0JBQXdCLEdBQXFCLEtBQUssRUFBRSxLQUFhLEVBQUUsSUFBWSxFQUFtQixFQUFFO0lBQ3RHLElBQUksS0FBSyxLQUFLLFFBQVE7UUFBRSxNQUFNLElBQUksS0FBSyxDQUFDLFdBQVcsQ0FBQyxDQUFBO0lBQ3BELE9BQU8sS0FBSyxDQUFDLFdBQVcsRUFBRSxDQUFBO0FBQzlCLENBQUMsQ0FBQTtBQUVELE1BQU0sQ0FBQyxNQUFNLFlBQVksR0FBa0IsQ0FBQyxJQUFZLEVBQUUsS0FBYSxFQUFFLEtBQVUsRUFBUSxFQUFFO0lBQ3pGLDJFQUEyRTtBQUMvRSxDQUFDLENBQUE7QUFFRCxNQUFNLENBQUMsTUFBTSxjQUFjLEdBQUcsQ0FBQyxVQUE0QixFQUFzQixFQUFvQixFQUFFO0lBQ25HLE9BQU87UUFDSCxHQUFHLE9BQU87UUFDVixTQUFTLEVBQUUsT0FBTyxDQUFDLFNBQVMsSUFBSSx3QkFBd0I7UUFDeEQsSUFBSSxFQUFFLE9BQU8sQ0FBQyxJQUFJLElBQUksYUFBYTtRQUNuQyxhQUFhLEVBQUUsT0FBTyxDQUFDLGFBQWEsSUFBSSxFQUFFO1FBQzFDLGVBQWUsRUFBRSxPQUFPLENBQUMsZUFBZSxJQUFJLENBQUM7UUFDN0MsYUFBYSxFQUFFLE9BQU8sQ0FBQyxhQUFhLElBQUksWUFBWTtRQUNwRCxjQUFjLEVBQUUsT0FBTyxDQUFDLGNBQWMsSUFBSSxXQUFXLENBQUMsY0FBYyxFQUFFLENBQUM7S0FDMUUsQ0FBQTtBQUNMLENBQUMsQ0FBQSJ9 \ No newline at end of file diff --git a/packages/kbot/dist-in/examples/core/async-iterator-example.d.ts b/packages/kbot/dist-in/examples/core/async-iterator-example.d.ts new file mode 100644 index 00000000..bde4f2bf --- /dev/null +++ b/packages/kbot/dist-in/examples/core/async-iterator-example.d.ts @@ -0,0 +1,38 @@ +import { AsyncTransformer } from '../../async-iterator.js'; +declare const exampleData: { + products: { + fruits: { + id: string; + name: string; + description: string; + details: { + color: string; + origin: string; + nutrition: string; + }; + }[]; + vegetables: { + id: string; + name: string; + description: string; + details: { + color: string; + origin: string; + nutrition: string; + }; + }[]; + }; + metadata: { + lastUpdated: string; + source: string; + description: string; + }; +}; +declare const createLLMTransformer: (options: any) => AsyncTransformer; +export declare function transformField(data: any, mapping: { + jsonPath: string; + targetPath: string | null; + options: any; +}): Promise; +export declare function transformExample(): Promise; +export { createLLMTransformer, exampleData }; diff --git a/packages/kbot/dist-in/examples/core/async-iterator-example.js b/packages/kbot/dist-in/examples/core/async-iterator-example.js new file mode 100644 index 00000000..e10e417a --- /dev/null +++ b/packages/kbot/dist-in/examples/core/async-iterator-example.js @@ -0,0 +1,268 @@ +import { sync as write } from "@polymech/fs/write"; +// Import path but not the logger +import * as path from 'path'; +import { E_OPENROUTER_MODEL } from '../../models/cache/openrouter-models.js'; +// Configurable constants +const MODEL = E_OPENROUTER_MODEL.MODEL_OPENROUTER_QUASAR_ALPHA; +const ROUTER = 'openrouter'; +const LOG_LEVEL = 2; // 0=trace, 1=debug, 2=info, 3=warn, 4=error, 5=fatal +// Create a simple console logger +let logger = { + info: (message) => console.log(`INFO: ${message}`), + warn: (message) => console.log(`WARN: ${message}`), + error: (message) => console.error(`ERROR: ${message}`), + debug: (message) => console.log(`DEBUG: ${message}`), + trace: (message) => console.log(`TRACE: ${message}`) +}; +// Import the actual kbot modules directly from source +import { transformObject } from '../../async-iterator.js'; +// Import run function from commands +import { run } from '../../commands/run.js'; +// Example data structure with fields to be transformed +const exampleData = { + products: { + fruits: [ + { + id: 'f1', + name: 'apple', + description: 'A sweet and crunchy fruit', + details: { + color: 'red', + origin: 'Worldwide', + nutrition: 'Rich in fiber and vitamin C' + } + }, + { + id: 'f2', + name: 'banana', + description: 'A yellow tropical fruit', + details: { + color: 'yellow', + origin: 'Southeast Asia', + nutrition: 'High in potassium' + } + } + ], + vegetables: [ + { + id: 'v1', + name: 'carrot', + description: 'An orange root vegetable', + details: { + color: 'orange', + origin: 'Eurasia', + nutrition: 'Good for vision health' + } + }, + { + id: 'v2', + name: 'broccoli', + description: 'A green cruciferous vegetable', + details: { + color: 'green', + origin: 'Italy', + nutrition: 'High in vitamins K and C' + } + } + ] + }, + metadata: { + lastUpdated: '2023-04-15', + source: 'Foods Database', + description: 'Collection of common fruits and vegetables' + } +}; +// Create a mapping of fields to transform with JSONPath +const fieldMappings = [ + { + // Transform all product descriptions + jsonPath: '$..description', + // Target field to store LLM result - same as source in this case + targetPath: null, // null means replace in place + options: { + model: MODEL, + prompt: 'Make this description more engaging and detailed, around 20-30 words' + } + }, + { + // Transform nutrition information + jsonPath: '$..nutrition', + targetPath: null, + options: { + model: MODEL, + prompt: 'Expand this nutrition information with 2-3 specific health benefits, around 25-35 words' + } + }, + { + // Transform product names and store in a new 'marketingName' field + jsonPath: '$..name', + targetPath: 'marketingName', // will create a new field called marketingName + options: { + model: MODEL, + prompt: 'Generate a more appealing marketing name for this product' + } + } +]; +// Create an async transformer function that uses an LLM to transform text +const createLLMTransformer = (options) => { + return async (input, jsonPath) => { + logger.info(`Transforming field at path: ${jsonPath}`); + logger.info(`Input: ${input}`); + logger.info(`Using prompt: ${options.prompt}`); + // Configure the task for kbot + const kbotTask = { + model: options.model || MODEL, + router: ROUTER, + prompt: `${options.prompt}\n\nText to transform: "${input}"`, + logLevel: LOG_LEVEL, + path: path.resolve('./'), // Fix: provide absolute path + mode: 'completion' + }; + try { + // Run kbot with the configured task + const results = await run(kbotTask); + if (results && results.length > 0 && typeof results[0] === 'string') { + const result = results[0].trim(); + logger.info(`Result: ${result}`); + return result; + } + // Return original if transformation fails + logger.warn(`No valid result received for ${jsonPath}, returning original`); + return input; + } + catch (error) { + console.error(`Error calling LLM API: ${error.message}`, error); + return input; // Return original on error + } + }; +}; +// Function to handle field transformation for a specific mapping +export async function transformField(data, mapping) { + const { jsonPath, targetPath, options } = mapping; + // Create a transformer for this specific field + const transformer = createLLMTransformer(options); + // Set up error callback + const errorCallback = (path, value, error) => { + logger.error(`Error transforming ${path}: ${error.message}`); + // Don't throw to allow continued processing + }; + // Set up filter callback - always return true to transform all matching fields + const filterCallback = async () => true; + // Try/catch to handle any JSONPath or transformation errors + try { + if (!targetPath) { + // Transform in place + await transformObject(data, transformer, jsonPath, 1000, // throttle delay - higher for real API calls + 1, // concurrent tasks - limit for API rate limits + errorCallback, filterCallback); + } + else { + // Create a copy of the data to transform + const dataCopy = JSON.parse(JSON.stringify(data)); + // Transform the copy + await transformObject(dataCopy, transformer, jsonPath, 1000, 1, errorCallback, filterCallback); + // Extract transformed values and store in target fields + const { JSONPath } = await import('jsonpath-plus'); + const paths = JSONPath({ path: jsonPath, json: dataCopy, resultType: 'pointer' }); + for (const p of paths) { + const keys = p.slice(1).split('/'); + // Navigate to the value in the transformed copy + let value = dataCopy; + for (const key of keys) { + if (key === '') + continue; // Skip empty segments + value = value[key]; + } + // Store in target path in the original data + const originalKeys = p.slice(1).split('/'); + const parentKeys = originalKeys.slice(0, -1); + // Navigate to parent object in original data + let target = data; + for (const key of parentKeys) { + if (key === '') + continue; + target = target[key]; + } + // Set the new field with transformed value + target[targetPath] = value; + } + } + } + catch (error) { + logger.error(`Error in field transformation: ${error.message}`); + } +} +// Main function to demonstrate the transformation +export async function transformExample() { + console.log("========================================"); + console.log("Starting async-iterator example transformation"); + console.log("========================================"); + try { + // Clone the data to avoid modifying the original + const data = JSON.parse(JSON.stringify(exampleData)); + logger.info("Starting transformation of data fields with LLM..."); + console.log("Processing field mappings..."); + // Process each field mapping + for (const mapping of fieldMappings) { + console.log(`Processing mapping: ${mapping.jsonPath} -> ${mapping.targetPath || 'in-place'}`); + logger.info(`Processing field mapping: ${mapping.jsonPath} -> ${mapping.targetPath || 'in-place'}`); + await transformField(data, mapping); + } + // Save the result to a file + const outputPath = path.resolve('./tests/test-data/core/async-iterator-data.json'); + write(outputPath, JSON.stringify(data, null, 2)); + console.log(`Transformation complete. Results saved to ${outputPath}`); + logger.info(`Transformation complete. Results saved to ${outputPath}`); + // Display before/after comparison for a sample field + console.log("\nBefore/After Comparison Example:"); + console.log(`Original description: ${exampleData.products.fruits[0].description}`); + console.log(`Transformed description: ${data.products.fruits[0].description}`); + console.log(`Original name: ${exampleData.products.fruits[0].name}`); + console.log(`New marketing name: ${data.products.fruits[0].marketingName}`); + logger.info("\nBefore/After Comparison Example:"); + logger.info(`Original description: ${exampleData.products.fruits[0].description}`); + logger.info(`Transformed description: ${data.products.fruits[0].description}`); + logger.info(`Original name: ${exampleData.products.fruits[0].name}`); + logger.info(`New marketing name: ${data.products.fruits[0].marketingName}`); + return data; + } + catch (error) { + console.error("ERROR during transformation:", error); + logger.error(`Error during transformation: ${error.message}`); + throw error; + } +} +// Run directly with Node.js +// This section checks if this module is being executed directly (not imported) +console.log("Module loading, checking if this is direct execution"); +console.log("process.argv[1]:", process.argv[1]); +console.log("import.meta.url:", import.meta.url); +// Directly check if file is run directly +const isDirectExecution = process.argv[1] && process.argv[1].includes('async-iterator-example'); +console.log("Is direct execution:", isDirectExecution); +if (isDirectExecution) { + // Process command line arguments + const isDebug = process.argv.includes('--debug'); + console.log('Starting async-iterator example...'); + console.log(`Command arguments: ${process.argv.slice(2).join(', ')}`); + console.log(`Debug mode: ${isDebug ? 'enabled' : 'disabled'}`); + // Use more verbose logging if debug mode is enabled + if (isDebug) { + // Override the LOG_LEVEL constant for debug mode + // @ts-ignore + logger = { + info: (message) => console.log(`DEBUG-INFO: ${message}`), + warn: (message) => console.log(`DEBUG-WARN: ${message}`), + error: (message) => console.error(`DEBUG-ERROR: ${message}`), + debug: (message) => console.log(`DEBUG-DEBUG: ${message}`), + trace: (message) => console.log(`DEBUG-TRACE: ${message}`) + }; + logger.info("Running in debug mode with verbose output"); + } + transformExample().catch(error => { + console.error("Unhandled error:", error); + }); +} +// Export for potential reuse +export { createLLMTransformer, exampleData }; +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXN5bmMtaXRlcmF0b3ItZXhhbXBsZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9leGFtcGxlcy9jb3JlL2FzeW5jLWl0ZXJhdG9yLWV4YW1wbGUudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLElBQUksSUFBSSxLQUFLLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQztBQUNuRCxpQ0FBaUM7QUFDakMsT0FBTyxLQUFLLElBQUksTUFBTSxNQUFNLENBQUM7QUFFN0IsT0FBTyxFQUFFLGtCQUFrQixFQUFFLE1BQU0seUNBQXlDLENBQUE7QUFFNUUseUJBQXlCO0FBQ3pCLE1BQU0sS0FBSyxHQUFHLGtCQUFrQixDQUFDLDZCQUE2QixDQUFDO0FBQy9ELE1BQU0sTUFBTSxHQUFHLFlBQVksQ0FBQztBQUU1QixNQUFNLFNBQVMsR0FBRyxDQUFDLENBQUMsQ0FBQyxxREFBcUQ7QUFFMUUsaUNBQWlDO0FBQ2pDLElBQUksTUFBTSxHQUFHO0lBQ1QsSUFBSSxFQUFFLENBQUMsT0FBZSxFQUFFLEVBQUUsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLFNBQVMsT0FBTyxFQUFFLENBQUM7SUFDMUQsSUFBSSxFQUFFLENBQUMsT0FBZSxFQUFFLEVBQUUsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLFNBQVMsT0FBTyxFQUFFLENBQUM7SUFDMUQsS0FBSyxFQUFFLENBQUMsT0FBZSxFQUFFLEVBQUUsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLFVBQVUsT0FBTyxFQUFFLENBQUM7SUFDOUQsS0FBSyxFQUFFLENBQUMsT0FBZSxFQUFFLEVBQUUsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLFVBQVUsT0FBTyxFQUFFLENBQUM7SUFDNUQsS0FBSyxFQUFFLENBQUMsT0FBZSxFQUFFLEVBQUUsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLFVBQVUsT0FBTyxFQUFFLENBQUM7Q0FDL0QsQ0FBQztBQUVGLHNEQUFzRDtBQUN0RCxPQUFPLEVBQ0gsZUFBZSxFQU1sQixNQUFNLHlCQUF5QixDQUFDO0FBRWpDLG9DQUFvQztBQUNwQyxPQUFPLEVBQUUsR0FBRyxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFFNUMsdURBQXVEO0FBQ3ZELE1BQU0sV0FBVyxHQUFHO0lBQ2hCLFFBQVEsRUFBRTtRQUNOLE1BQU0sRUFBRTtZQUNKO2dCQUNJLEVBQUUsRUFBRSxJQUFJO2dCQUNSLElBQUksRUFBRSxPQUFPO2dCQUNiLFdBQVcsRUFBRSwyQkFBMkI7Z0JBQ3hDLE9BQU8sRUFBRTtvQkFDTCxLQUFLLEVBQUUsS0FBSztvQkFDWixNQUFNLEVBQUUsV0FBVztvQkFDbkIsU0FBUyxFQUFFLDZCQUE2QjtpQkFDM0M7YUFDSjtZQUNEO2dCQUNJLEVBQUUsRUFBRSxJQUFJO2dCQUNSLElBQUksRUFBRSxRQUFRO2dCQUNkLFdBQVcsRUFBRSx5QkFBeUI7Z0JBQ3RDLE9BQU8sRUFBRTtvQkFDTCxLQUFLLEVBQUUsUUFBUTtvQkFDZixNQUFNLEVBQUUsZ0JBQWdCO29CQUN4QixTQUFTLEVBQUUsbUJBQW1CO2lCQUNqQzthQUNKO1NBQ0o7UUFDRCxVQUFVLEVBQUU7WUFDUjtnQkFDSSxFQUFFLEVBQUUsSUFBSTtnQkFDUixJQUFJLEVBQUUsUUFBUTtnQkFDZCxXQUFXLEVBQUUsMEJBQTBCO2dCQUN2QyxPQUFPLEVBQUU7b0JBQ0wsS0FBSyxFQUFFLFFBQVE7b0JBQ2YsTUFBTSxFQUFFLFNBQVM7b0JBQ2pCLFNBQVMsRUFBRSx3QkFBd0I7aUJBQ3RDO2FBQ0o7WUFDRDtnQkFDSSxFQUFFLEVBQUUsSUFBSTtnQkFDUixJQUFJLEVBQUUsVUFBVTtnQkFDaEIsV0FBVyxFQUFFLCtCQUErQjtnQkFDNUMsT0FBTyxFQUFFO29CQUNMLEtBQUssRUFBRSxPQUFPO29CQUNkLE1BQU0sRUFBRSxPQUFPO29CQUNmLFNBQVMsRUFBRSwwQkFBMEI7aUJBQ3hDO2FBQ0o7U0FDSjtLQUNKO0lBQ0QsUUFBUSxFQUFFO1FBQ04sV0FBVyxFQUFFLFlBQVk7UUFDekIsTUFBTSxFQUFFLGdCQUFnQjtRQUN4QixXQUFXLEVBQUUsNENBQTRDO0tBQzVEO0NBQ0osQ0FBQztBQUVGLHdEQUF3RDtBQUN4RCxNQUFNLGFBQWEsR0FBRztJQUNsQjtRQUNJLHFDQUFxQztRQUNyQyxRQUFRLEVBQUUsZ0JBQWdCO1FBQzFCLGlFQUFpRTtRQUNqRSxVQUFVLEVBQUUsSUFBSSxFQUFFLDhCQUE4QjtRQUNoRCxPQUFPLEVBQUU7WUFDTCxLQUFLLEVBQUUsS0FBSztZQUNaLE1BQU0sRUFBRSxzRUFBc0U7U0FDakY7S0FDSjtJQUNEO1FBQ0ksa0NBQWtDO1FBQ2xDLFFBQVEsRUFBRSxjQUFjO1FBQ3hCLFVBQVUsRUFBRSxJQUFJO1FBQ2hCLE9BQU8sRUFBRTtZQUNMLEtBQUssRUFBRSxLQUFLO1lBQ1osTUFBTSxFQUFFLHlGQUF5RjtTQUNwRztLQUNKO0lBQ0Q7UUFDSSxtRUFBbUU7UUFDbkUsUUFBUSxFQUFFLFNBQVM7UUFDbkIsVUFBVSxFQUFFLGVBQWUsRUFBRSwrQ0FBK0M7UUFDNUUsT0FBTyxFQUFFO1lBQ0wsS0FBSyxFQUFFLEtBQUs7WUFDWixNQUFNLEVBQUUsMkRBQTJEO1NBQ3RFO0tBQ0o7Q0FDSixDQUFDO0FBRUYsMEVBQTBFO0FBQzFFLE1BQU0sb0JBQW9CLEdBQUcsQ0FBQyxPQUFZLEVBQW9CLEVBQUU7SUFDNUQsT0FBTyxLQUFLLEVBQUUsS0FBYSxFQUFFLFFBQWdCLEVBQW1CLEVBQUU7UUFDOUQsTUFBTSxDQUFDLElBQUksQ0FBQywrQkFBK0IsUUFBUSxFQUFFLENBQUMsQ0FBQztRQUN2RCxNQUFNLENBQUMsSUFBSSxDQUFDLFVBQVUsS0FBSyxFQUFFLENBQUMsQ0FBQztRQUMvQixNQUFNLENBQUMsSUFBSSxDQUFDLGlCQUFpQixPQUFPLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQztRQUUvQyw4QkFBOEI7UUFDOUIsTUFBTSxRQUFRLEdBQWM7WUFDeEIsS0FBSyxFQUFFLE9BQU8sQ0FBQyxLQUFLLElBQUksS0FBSztZQUM3QixNQUFNLEVBQUUsTUFBTTtZQUNkLE1BQU0sRUFBRSxHQUFHLE9BQU8sQ0FBQyxNQUFNLDJCQUEyQixLQUFLLEdBQUc7WUFDNUQsUUFBUSxFQUFFLFNBQVM7WUFDbkIsSUFBSSxFQUFFLElBQUksQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLEVBQUUsNkJBQTZCO1lBQ3ZELElBQUksRUFBRSxZQUFtQjtTQUM1QixDQUFDO1FBRUYsSUFBSSxDQUFDO1lBQ0Qsb0NBQW9DO1lBQ3BDLE1BQU0sT0FBTyxHQUFHLE1BQU0sR0FBRyxDQUFDLFFBQVEsQ0FBQyxDQUFDO1lBRXBDLElBQUksT0FBTyxJQUFJLE9BQU8sQ0FBQyxNQUFNLEdBQUcsQ0FBQyxJQUFJLE9BQU8sT0FBTyxDQUFDLENBQUMsQ0FBQyxLQUFLLFFBQVEsRUFBRSxDQUFDO2dCQUNsRSxNQUFNLE1BQU0sR0FBRyxPQUFPLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUM7Z0JBQ2pDLE1BQU0sQ0FBQyxJQUFJLENBQUMsV0FBVyxNQUFNLEVBQUUsQ0FBQyxDQUFDO2dCQUNqQyxPQUFPLE1BQU0sQ0FBQztZQUNsQixDQUFDO1lBRUQsMENBQTBDO1lBQzFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsZ0NBQWdDLFFBQVEsc0JBQXNCLENBQUMsQ0FBQztZQUM1RSxPQUFPLEtBQUssQ0FBQztRQUNqQixDQUFDO1FBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztZQUNiLE9BQU8sQ0FBQyxLQUFLLENBQUMsMEJBQTBCLEtBQUssQ0FBQyxPQUFPLEVBQUUsRUFBRSxLQUFLLENBQUMsQ0FBQztZQUNoRSxPQUFPLEtBQUssQ0FBQyxDQUFDLDJCQUEyQjtRQUM3QyxDQUFDO0lBQ0wsQ0FBQyxDQUFDO0FBQ04sQ0FBQyxDQUFDO0FBRUYsaUVBQWlFO0FBQ2pFLE1BQU0sQ0FBQyxLQUFLLFVBQVUsY0FBYyxDQUFDLElBQVMsRUFBRSxPQUFzRTtJQUNsSCxNQUFNLEVBQUUsUUFBUSxFQUFFLFVBQVUsRUFBRSxPQUFPLEVBQUUsR0FBRyxPQUFPLENBQUM7SUFFbEQsK0NBQStDO0lBQy9DLE1BQU0sV0FBVyxHQUFHLG9CQUFvQixDQUFDLE9BQU8sQ0FBQyxDQUFDO0lBRWxELHdCQUF3QjtJQUN4QixNQUFNLGFBQWEsR0FBRyxDQUFDLElBQVksRUFBRSxLQUFhLEVBQUUsS0FBVSxFQUFFLEVBQUU7UUFDOUQsTUFBTSxDQUFDLEtBQUssQ0FBQyxzQkFBc0IsSUFBSSxLQUFLLEtBQUssQ0FBQyxPQUFPLEVBQUUsQ0FBQyxDQUFDO1FBQzdELDRDQUE0QztJQUNoRCxDQUFDLENBQUM7SUFFRiwrRUFBK0U7SUFDL0UsTUFBTSxjQUFjLEdBQUcsS0FBSyxJQUFJLEVBQUUsQ0FBQyxJQUFJLENBQUM7SUFFeEMsNERBQTREO0lBQzVELElBQUksQ0FBQztRQUNELElBQUksQ0FBQyxVQUFVLEVBQUUsQ0FBQztZQUNkLHFCQUFxQjtZQUNyQixNQUFNLGVBQWUsQ0FDakIsSUFBSSxFQUNKLFdBQVcsRUFDWCxRQUFRLEVBQ1IsSUFBSSxFQUFFLDZDQUE2QztZQUNuRCxDQUFDLEVBQUssK0NBQStDO1lBQ3JELGFBQWEsRUFDYixjQUFjLENBQ2pCLENBQUM7UUFDTixDQUFDO2FBQU0sQ0FBQztZQUNKLHlDQUF5QztZQUN6QyxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQztZQUVsRCxxQkFBcUI7WUFDckIsTUFBTSxlQUFlLENBQ2pCLFFBQVEsRUFDUixXQUFXLEVBQ1gsUUFBUSxFQUNSLElBQUksRUFDSixDQUFDLEVBQ0QsYUFBYSxFQUNiLGNBQWMsQ0FDakIsQ0FBQztZQUVGLHdEQUF3RDtZQUN4RCxNQUFNLEVBQUUsUUFBUSxFQUFFLEdBQUcsTUFBTSxNQUFNLENBQUMsZUFBZSxDQUFDLENBQUM7WUFDbkQsTUFBTSxLQUFLLEdBQUcsUUFBUSxDQUFDLEVBQUUsSUFBSSxFQUFFLFFBQVEsRUFBRSxJQUFJLEVBQUUsUUFBUSxFQUFFLFVBQVUsRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDO1lBRWxGLEtBQUssTUFBTSxDQUFDLElBQUksS0FBSyxFQUFFLENBQUM7Z0JBQ3BCLE1BQU0sSUFBSSxHQUFHLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDO2dCQUVuQyxnREFBZ0Q7Z0JBQ2hELElBQUksS0FBSyxHQUFHLFFBQVEsQ0FBQztnQkFDckIsS0FBSyxNQUFNLEdBQUcsSUFBSSxJQUFJLEVBQUUsQ0FBQztvQkFDckIsSUFBSSxHQUFHLEtBQUssRUFBRTt3QkFBRSxTQUFTLENBQUMsc0JBQXNCO29CQUNoRCxLQUFLLEdBQUcsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDO2dCQUN2QixDQUFDO2dCQUVELDRDQUE0QztnQkFDNUMsTUFBTSxZQUFZLEdBQUcsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUM7Z0JBQzNDLE1BQU0sVUFBVSxHQUFHLFlBQVksQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUM7Z0JBRTdDLDZDQUE2QztnQkFDN0MsSUFBSSxNQUFNLEdBQUcsSUFBSSxDQUFDO2dCQUNsQixLQUFLLE1BQU0sR0FBRyxJQUFJLFVBQVUsRUFBRSxDQUFDO29CQUMzQixJQUFJLEdBQUcsS0FBSyxFQUFFO3dCQUFFLFNBQVM7b0JBQ3pCLE1BQU0sR0FBRyxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUM7Z0JBQ3pCLENBQUM7Z0JBRUQsMkNBQTJDO2dCQUMzQyxNQUFNLENBQUMsVUFBVSxDQUFDLEdBQUcsS0FBSyxDQUFDO1lBQy9CLENBQUM7UUFDTCxDQUFDO0lBQ0wsQ0FBQztJQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7UUFDYixNQUFNLENBQUMsS0FBSyxDQUFDLGtDQUFrQyxLQUFLLENBQUMsT0FBTyxFQUFFLENBQUMsQ0FBQztJQUNwRSxDQUFDO0FBQ0wsQ0FBQztBQUVELGtEQUFrRDtBQUNsRCxNQUFNLENBQUMsS0FBSyxVQUFVLGdCQUFnQjtJQUNsQyxPQUFPLENBQUMsR0FBRyxDQUFDLDBDQUEwQyxDQUFDLENBQUM7SUFDeEQsT0FBTyxDQUFDLEdBQUcsQ0FBQyxnREFBZ0QsQ0FBQyxDQUFDO0lBQzlELE9BQU8sQ0FBQyxHQUFHLENBQUMsMENBQTBDLENBQUMsQ0FBQztJQUV4RCxJQUFJLENBQUM7UUFDRCxpREFBaUQ7UUFDakQsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUM7UUFFckQsTUFBTSxDQUFDLElBQUksQ0FBQyxvREFBb0QsQ0FBQyxDQUFDO1FBQ2xFLE9BQU8sQ0FBQyxHQUFHLENBQUMsOEJBQThCLENBQUMsQ0FBQztRQUU1Qyw2QkFBNkI7UUFDN0IsS0FBSyxNQUFNLE9BQU8sSUFBSSxhQUFhLEVBQUUsQ0FBQztZQUNsQyxPQUFPLENBQUMsR0FBRyxDQUFDLHVCQUF1QixPQUFPLENBQUMsUUFBUSxPQUFPLE9BQU8sQ0FBQyxVQUFVLElBQUksVUFBVSxFQUFFLENBQUMsQ0FBQztZQUM5RixNQUFNLENBQUMsSUFBSSxDQUFDLDZCQUE2QixPQUFPLENBQUMsUUFBUSxPQUFPLE9BQU8sQ0FBQyxVQUFVLElBQUksVUFBVSxFQUFFLENBQUMsQ0FBQztZQUNwRyxNQUFNLGNBQWMsQ0FBQyxJQUFJLEVBQUUsT0FBTyxDQUFDLENBQUM7UUFDeEMsQ0FBQztRQUVELDRCQUE0QjtRQUM1QixNQUFNLFVBQVUsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLGlEQUFpRCxDQUFDLENBQUM7UUFDbkYsS0FBSyxDQUFDLFVBQVUsRUFBRSxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQztRQUVqRCxPQUFPLENBQUMsR0FBRyxDQUFDLDZDQUE2QyxVQUFVLEVBQUUsQ0FBQyxDQUFDO1FBQ3ZFLE1BQU0sQ0FBQyxJQUFJLENBQUMsNkNBQTZDLFVBQVUsRUFBRSxDQUFDLENBQUM7UUFFdkUscURBQXFEO1FBQ3JELE9BQU8sQ0FBQyxHQUFHLENBQUMsb0NBQW9DLENBQUMsQ0FBQztRQUNsRCxPQUFPLENBQUMsR0FBRyxDQUFDLHlCQUF5QixXQUFXLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxXQUFXLEVBQUUsQ0FBQyxDQUFDO1FBQ25GLE9BQU8sQ0FBQyxHQUFHLENBQUMsNEJBQTRCLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUM7UUFDL0UsT0FBTyxDQUFDLEdBQUcsQ0FBQyxrQkFBa0IsV0FBVyxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQztRQUNyRSxPQUFPLENBQUMsR0FBRyxDQUFDLHVCQUF1QixJQUFJLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxhQUFhLEVBQUUsQ0FBQyxDQUFDO1FBRTVFLE1BQU0sQ0FBQyxJQUFJLENBQUMsb0NBQW9DLENBQUMsQ0FBQztRQUNsRCxNQUFNLENBQUMsSUFBSSxDQUFDLHlCQUF5QixXQUFXLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxXQUFXLEVBQUUsQ0FBQyxDQUFDO1FBQ25GLE1BQU0sQ0FBQyxJQUFJLENBQUMsNEJBQTRCLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUM7UUFDL0UsTUFBTSxDQUFDLElBQUksQ0FBQyxrQkFBa0IsV0FBVyxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQztRQUNyRSxNQUFNLENBQUMsSUFBSSxDQUFDLHVCQUF1QixJQUFJLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxhQUFhLEVBQUUsQ0FBQyxDQUFDO1FBRTVFLE9BQU8sSUFBSSxDQUFDO0lBQ2hCLENBQUM7SUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1FBQ2IsT0FBTyxDQUFDLEtBQUssQ0FBQyw4QkFBOEIsRUFBRSxLQUFLLENBQUMsQ0FBQztRQUNyRCxNQUFNLENBQUMsS0FBSyxDQUFDLGdDQUFnQyxLQUFLLENBQUMsT0FBTyxFQUFFLENBQUMsQ0FBQztRQUM5RCxNQUFNLEtBQUssQ0FBQztJQUNoQixDQUFDO0FBQ0wsQ0FBQztBQUVELDRCQUE0QjtBQUM1QiwrRUFBK0U7QUFDL0UsT0FBTyxDQUFDLEdBQUcsQ0FBQyxzREFBc0QsQ0FBQyxDQUFDO0FBQ3BFLE9BQU8sQ0FBQyxHQUFHLENBQUMsa0JBQWtCLEVBQUUsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDO0FBQ2pELE9BQU8sQ0FBQyxHQUFHLENBQUMsa0JBQWtCLEVBQUUsTUFBTSxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQztBQUVqRCx5Q0FBeUM7QUFDekMsTUFBTSxpQkFBaUIsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsUUFBUSxDQUFDLHdCQUF3QixDQUFDLENBQUM7QUFDaEcsT0FBTyxDQUFDLEdBQUcsQ0FBQyxzQkFBc0IsRUFBRSxpQkFBaUIsQ0FBQyxDQUFDO0FBRXZELElBQUksaUJBQWlCLEVBQUUsQ0FBQztJQUNwQixpQ0FBaUM7SUFDakMsTUFBTSxPQUFPLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLENBQUM7SUFFakQsT0FBTyxDQUFDLEdBQUcsQ0FBQyxvQ0FBb0MsQ0FBQyxDQUFDO0lBQ2xELE9BQU8sQ0FBQyxHQUFHLENBQUMsc0JBQXNCLE9BQU8sQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUM7SUFDdEUsT0FBTyxDQUFDLEdBQUcsQ0FBQyxlQUFlLE9BQU8sQ0FBQyxDQUFDLENBQUMsU0FBUyxDQUFDLENBQUMsQ0FBQyxVQUFVLEVBQUUsQ0FBQyxDQUFDO0lBRS9ELG9EQUFvRDtJQUNwRCxJQUFJLE9BQU8sRUFBRSxDQUFDO1FBQ1YsaURBQWlEO1FBQ2pELGFBQWE7UUFDYixNQUFNLEdBQUc7WUFDTCxJQUFJLEVBQUUsQ0FBQyxPQUFPLEVBQUUsRUFBRSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsZUFBZSxPQUFPLEVBQUUsQ0FBQztZQUN4RCxJQUFJLEVBQUUsQ0FBQyxPQUFPLEVBQUUsRUFBRSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsZUFBZSxPQUFPLEVBQUUsQ0FBQztZQUN4RCxLQUFLLEVBQUUsQ0FBQyxPQUFPLEVBQUUsRUFBRSxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUMsZ0JBQWdCLE9BQU8sRUFBRSxDQUFDO1lBQzVELEtBQUssRUFBRSxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxnQkFBZ0IsT0FBTyxFQUFFLENBQUM7WUFDMUQsS0FBSyxFQUFFLENBQUMsT0FBTyxFQUFFLEVBQUUsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLGdCQUFnQixPQUFPLEVBQUUsQ0FBQztTQUM3RCxDQUFDO1FBQ0YsTUFBTSxDQUFDLElBQUksQ0FBQywyQ0FBMkMsQ0FBQyxDQUFDO0lBQzdELENBQUM7SUFFRCxnQkFBZ0IsRUFBRSxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsRUFBRTtRQUM3QixPQUFPLENBQUMsS0FBSyxDQUFDLGtCQUFrQixFQUFFLEtBQUssQ0FBQyxDQUFDO0lBQzdDLENBQUMsQ0FBQyxDQUFDO0FBQ1AsQ0FBQztBQUVELDZCQUE2QjtBQUM3QixPQUFPLEVBQ0gsb0JBQW9CLEVBQ3BCLFdBQVcsRUFDZCxDQUFDIn0= \ No newline at end of file diff --git a/packages/kbot/dist-in/examples/index.d.ts b/packages/kbot/dist-in/examples/index.d.ts new file mode 100644 index 00000000..940e830b --- /dev/null +++ b/packages/kbot/dist-in/examples/index.d.ts @@ -0,0 +1 @@ +export * from './core/async-iterator-example.js'; diff --git a/packages/kbot/dist-in/examples/index.js b/packages/kbot/dist-in/examples/index.js new file mode 100644 index 00000000..1eb399f1 --- /dev/null +++ b/packages/kbot/dist-in/examples/index.js @@ -0,0 +1,3 @@ +// Export examples +export * from './core/async-iterator-example.js'; +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvZXhhbXBsZXMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsa0JBQWtCO0FBQ2xCLGNBQWMsa0NBQWtDLENBQUMifQ== \ No newline at end of file diff --git a/packages/kbot/dist-in/profile.js b/packages/kbot/dist-in/profile.js index 02acfd5c..988bc439 100644 --- a/packages/kbot/dist-in/profile.js +++ b/packages/kbot/dist-in/profile.js @@ -13,7 +13,7 @@ const testPath = (profilePath) => { }; export const load = async (options) => { let profile = { includes: [], variables: options.variables || {}, env: {} }; - let profilePath = testPath(options.profile || path.join(options.logs, 'profile.json')); + let profilePath = testPath(options.profile || path.join(options.logs || '.', 'profile.json')); if (!profilePath || !exists(profilePath) || !isFile(profilePath)) { return profile.variables; } diff --git a/packages/kbot/logs/params.json b/packages/kbot/logs/params.json index 5c64d919..8f996877 100644 --- a/packages/kbot/logs/params.json +++ b/packages/kbot/logs/params.json @@ -1,19 +1,13 @@ { - "model": "openai/gpt-3.5-turbo", + "model": "openrouter/quasar-alpha", "messages": [ { "role": "user", - "content": "Is this data in JSON format? Answer with only \"yes\" or \"no\"." + "content": "Generate a more appealing marketing name for this product\n\nText to transform: \"broccoli\"" }, { "role": "user", "content": "USER Preferences : # Preferences\r\n\r\nYou are a helpful AI assistant. When asked to perform calculations, you should return only the numerical result without any explanation or comments. " - }, - { - "content": "```json\n[\n {\n \"id\": 1,\n \"name\": \"Leanne Graham\",\n \"username\": \"Bret\",\n \"email\": \"Sincere@april.biz\",\n \"address\": {\n \"street\": \"Kulas Light\",\n \"suite\": \"Apt. 556\",\n \"city\": \"Gwenborough\",\n \"zipcode\": \"92998-3874\",\n \"geo\": {\n \"lat\": \"-37.3159\",\n \"lng\": \"81.1496\"\n }\n },\n \"phone\": \"1-770-736-8031 x56442\",\n \"website\": \"hildegard.org\",\n \"company\": {\n \"name\": \"Romaguera-Crona\",\n \"catchPhrase\": \"Multi-layered client-server neural-net\",\n \"bs\": \"harness real-time e-markets\"\n }\n },\n {\n \"id\": 2,\n \"name\": \"Ervin Howell\",\n \"username\": \"Antonette\",\n \"email\": \"Shanna@melissa.tv\",\n \"address\": {\n \"street\": \"Victor Plains\",\n \"suite\": \"Suite 879\",\n \"city\": \"Wisokyburgh\",\n \"zipcode\": \"90566-7771\",\n \"geo\": {\n \"lat\": \"-43.9509\",\n \"lng\": \"-34.4618\"\n }\n },\n \"phone\": \"010-692-6593 x09125\",\n \"website\": \"anastasia.net\",\n \"company\": {\n \"name\": \"Deckow-Crist\",\n \"catchPhrase\": \"Proactive didactic contingency\",\n \"bs\": \"synergize scalable supply-chains\"\n }\n },\n {\n \"id\": 3,\n \"name\": \"Clementine Bauch\",\n \"username\": \"Samantha\",\n \"email\": \"Nathan@yesenia.net\",\n \"address\": {\n \"street\": \"Douglas Extension\",\n \"suite\": \"Suite 847\",\n \"city\": \"McKenziehaven\",\n \"zipcode\": \"59590-4157\",\n \"geo\": {\n \"lat\": \"-68.6102\",\n \"lng\": \"-47.0653\"\n }\n },\n \"phone\": \"1-463-123-4447\",\n \"website\": \"ramiro.info\",\n \"company\": {\n \"name\": \"Romaguera-Jacobson\",\n \"catchPhrase\": \"Face to face bifurcated interface\",\n \"bs\": \"e-enable strategic applications\"\n }\n },\n {\n \"id\": 4,\n \"name\": \"Patricia Lebsack\",\n \"username\": \"Karianne\",\n \"email\": \"Julianne.OConner@kory.org\",\n \"address\": {\n \"street\": \"Hoeger Mall\",\n \"suite\": \"Apt. 692\",\n \"city\": \"South Elvis\",\n \"zipcode\": \"53919-4257\",\n \"geo\": {\n \"lat\": \"29.4572\",\n \"lng\": \"-164.2990\"\n }\n },\n \"phone\": \"493-170-9623 x156\",\n \"website\": \"kale.biz\",\n \"company\": {\n \"name\": \"Robel-Corkery\",\n \"catchPhrase\": \"Multi-tiered zero tolerance productivity\",\n \"bs\": \"transition cutting-edge web services\"\n }\n },\n {\n \"id\": 5,\n \"name\": \"Chelsey Dietrich\",\n \"username\": \"Kamren\",\n \"email\": \"Lucio_Hettinger@annie.ca\",\n \"address\": {\n \"street\": \"Skiles Walks\",\n \"suite\": \"Suite 351\",\n \"city\": \"Roscoeview\",\n \"zipcode\": \"33263\",\n \"geo\": {\n \"lat\": \"-31.8129\",\n \"lng\": \"62.5342\"\n }\n },\n \"phone\": \"(254)954-1289\",\n \"website\": \"demarco.info\",\n \"company\": {\n \"name\": \"Keebler LLC\",\n \"catchPhrase\": \"User-centric fault-tolerant solution\",\n \"bs\": \"revolutionize end-to-end systems\"\n }\n },\n {\n \"id\": 6,\n \"name\": \"Mrs. Dennis Schulist\",\n \"username\": \"Leopoldo_Corkery\",\n \"email\": \"Karley_Dach@jasper.info\",\n \"address\": {\n \"street\": \"Norberto Crossing\",\n \"suite\": \"Apt. 950\",\n \"city\": \"South Christy\",\n \"zipcode\": \"23505-1337\",\n \"geo\": {\n \"lat\": \"-71.4197\",\n \"lng\": \"71.7478\"\n }\n },\n \"phone\": \"1-477-935-8478 x6430\",\n \"website\": \"ola.org\",\n \"company\": {\n \"name\": \"Considine-Lockman\",\n \"catchPhrase\": \"Synchronised bottom-line interface\",\n \"bs\": \"e-enable innovative applications\"\n }\n },\n {\n \"id\": 7,\n \"name\": \"Kurtis Weissnat\",\n \"username\": \"Elwyn.Skiles\",\n \"email\": \"Telly.Hoeger@billy.biz\",\n \"address\": {\n \"street\": \"Rex Trail\",\n \"suite\": \"Suite 280\",\n \"city\": \"Howemouth\",\n \"zipcode\": \"58804-1099\",\n \"geo\": {\n \"lat\": \"24.8918\",\n \"lng\": \"21.8984\"\n }\n },\n \"phone\": \"210.067.6132\",\n \"website\": \"elvis.io\",\n \"company\": {\n \"name\": \"Johns Group\",\n \"catchPhrase\": \"Configurable multimedia task-force\",\n \"bs\": \"generate enterprise e-tailers\"\n }\n },\n {\n \"id\": 8,\n \"name\": \"Nicholas Runolfsdottir V\",\n \"username\": \"Maxime_Nienow\",\n \"email\": \"Sherwood@rosamond.me\",\n \"address\": {\n \"street\": \"Ellsworth Summit\",\n \"suite\": \"Suite 729\",\n \"city\": \"Aliyaview\",\n \"zipcode\": \"45169\",\n \"geo\": {\n \"lat\": \"-14.3990\",\n \"lng\": \"-120.7677\"\n }\n },\n \"phone\": \"586.493.6943 x140\",\n \"website\": \"jacynthe.com\",\n \"company\": {\n \"name\": \"Abernathy Group\",\n \"catchPhrase\": \"Implemented secondary concept\",\n \"bs\": \"e-enable extensible e-tailers\"\n }\n },\n {\n \"id\": 9,\n \"name\": \"Glenna Reichert\",\n \"username\": \"Delphine\",\n \"email\": \"Chaim_McDermott@dana.io\",\n \"address\": {\n \"street\": \"Dayna Park\",\n \"suite\": \"Suite 449\",\n \"city\": \"Bartholomebury\",\n \"zipcode\": \"76495-3109\",\n \"geo\": {\n \"lat\": \"24.6463\",\n \"lng\": \"-168.8889\"\n }\n },\n \"phone\": \"(775)976-6794 x41206\",\n \"website\": \"conrad.com\",\n \"company\": {\n \"name\": \"Yost and Sons\",\n \"catchPhrase\": \"Switchable contextually-based project\",\n \"bs\": \"aggregate real-time technologies\"\n }\n },\n {\n \"id\": 10,\n \"name\": \"Clementina DuBuque\",\n \"username\": \"Moriah.Stanton\",\n \"email\": \"Rey.Padberg@karina.biz\",\n \"address\": {\n \"street\": \"Kattie Turnpike\",\n \"suite\": \"Suite 198\",\n \"city\": \"Lebsackbury\",\n \"zipcode\": \"31428-2261\",\n \"geo\": {\n \"lat\": \"-38.2386\",\n \"lng\": \"57.2232\"\n }\n },\n \"phone\": \"024-648-3804\",\n \"website\": \"ambrose.net\",\n \"company\": {\n \"name\": \"Hoeger LLC\",\n \"catchPhrase\": \"Centralized empowering task-force\",\n \"bs\": \"target end-to-end models\"\n }\n }\n]\n```", - "path": "https://jsonplaceholder.typicode.com/users", - "role": "user", - "name": "web_jsonplaceholder.typicode.com_users" } ], "tools": [] diff --git a/packages/kbot/package-lock.json b/packages/kbot/package-lock.json index c6ff5fa8..00cd58e7 100644 --- a/packages/kbot/package-lock.json +++ b/packages/kbot/package-lock.json @@ -47,6 +47,7 @@ "eslint": "^8.57.1", "ts-json-schema-generator": "^2.3.0", "ts-loader": "9.5.1", + "ts-node": "10.9.2", "tsx": "^4.5.0", "typescript": "^5.7.2", "vitest": "^2.1.8", @@ -336,6 +337,30 @@ "node": ">=0.1.90" } }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@discoveryjs/json-ext": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.6.3.tgz", @@ -1628,6 +1653,34 @@ "url": "https://github.com/sindresorhus/is?sponsor=1" } }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/cookie": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", @@ -2357,6 +2410,19 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/agentkeepalive": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", @@ -2486,6 +2552,13 @@ "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", "license": "MIT" }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -3012,6 +3085,13 @@ "node": ">= 0.6" } }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -3086,6 +3166,16 @@ "node": ">=0.4.0" } }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -4609,6 +4699,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, "node_modules/marked": { "version": "14.1.4", "resolved": "https://registry.npmjs.org/marked/-/marked-14.1.4.tgz", @@ -6539,6 +6636,50 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, "node_modules/ts-retry": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/ts-retry/-/ts-retry-6.0.0.tgz", @@ -7148,6 +7289,13 @@ "requires-port": "^1.0.0" } }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, "node_modules/vite": { "version": "5.4.14", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.14.tgz", @@ -7665,6 +7813,16 @@ "node": ">=8" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/yoctocolors-cjs": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz", diff --git a/packages/kbot/package.json b/packages/kbot/package.json index 90eaeb65..dfde37cd 100644 --- a/packages/kbot/package.json +++ b/packages/kbot/package.json @@ -32,7 +32,9 @@ "test2:coverage": "vitest run --coverage", "webpack": "webpack --config webpack.config.js --stats-error-details", "exe:win": "cd dist && nexe -i main_node.js -o kbot.exe --build --temp=../../temp-kbot --verbose", - "exe:lnx": "cd dist && nexe -i main_node.js -o kbot --build --temp=../../temp-kbot --verbose" + "exe:lnx": "cd dist && nexe -i main_node.js -o kbot --build --temp=../../temp-kbot --verbose", + "examples:async-iterator": "node dist-in/examples/core/async-iterator-example.js", + "examples:async-iterator:verbose": "node dist-in/examples/core/async-iterator-example.js --debug" }, "dependencies": { "@polymech/ai-tools": "file:../ai-tools", @@ -92,6 +94,7 @@ "eslint": "^8.57.1", "ts-json-schema-generator": "^2.3.0", "ts-loader": "9.5.1", + "ts-node": "10.9.2", "tsx": "^4.5.0", "typescript": "^5.7.2", "vitest": "^2.1.8", diff --git a/packages/kbot/src/examples/core/async-iterator-example.ts b/packages/kbot/src/examples/core/async-iterator-example.ts new file mode 100644 index 00000000..cd2c2a11 --- /dev/null +++ b/packages/kbot/src/examples/core/async-iterator-example.ts @@ -0,0 +1,326 @@ +import { sync as write } from "@polymech/fs/write"; +// Import path but not the logger +import * as path from 'path'; +import type { IKBotTask } from '@polymech/ai-tools'; +import { E_OPENROUTER_MODEL } from '../../models/cache/openrouter-models.js' + +// Configurable constants +const MODEL = E_OPENROUTER_MODEL.MODEL_OPENROUTER_QUASAR_ALPHA; +const ROUTER = 'openrouter'; + +const LOG_LEVEL = 2; // 0=trace, 1=debug, 2=info, 3=warn, 4=error, 5=fatal + +// Create a simple console logger +let logger = { + info: (message: string) => console.log(`INFO: ${message}`), + warn: (message: string) => console.log(`WARN: ${message}`), + error: (message: string) => console.error(`ERROR: ${message}`), + debug: (message: string) => console.log(`DEBUG: ${message}`), + trace: (message: string) => console.log(`TRACE: ${message}`) +}; + +// Import the actual kbot modules directly from source +import { + transformObject, + defaultOptions, + TransformOptions, + testFilters, + defaultFilters, + AsyncTransformer +} from '../../async-iterator.js'; + +// Import run function from commands +import { run } from '../../commands/run.js'; + +// Example data structure with fields to be transformed +const exampleData = { + products: { + fruits: [ + { + id: 'f1', + name: 'apple', + description: 'A sweet and crunchy fruit', + details: { + color: 'red', + origin: 'Worldwide', + nutrition: 'Rich in fiber and vitamin C' + } + }, + { + id: 'f2', + name: 'banana', + description: 'A yellow tropical fruit', + details: { + color: 'yellow', + origin: 'Southeast Asia', + nutrition: 'High in potassium' + } + } + ], + vegetables: [ + { + id: 'v1', + name: 'carrot', + description: 'An orange root vegetable', + details: { + color: 'orange', + origin: 'Eurasia', + nutrition: 'Good for vision health' + } + }, + { + id: 'v2', + name: 'broccoli', + description: 'A green cruciferous vegetable', + details: { + color: 'green', + origin: 'Italy', + nutrition: 'High in vitamins K and C' + } + } + ] + }, + metadata: { + lastUpdated: '2023-04-15', + source: 'Foods Database', + description: 'Collection of common fruits and vegetables' + } +}; + +// Create a mapping of fields to transform with JSONPath +const fieldMappings = [ + { + // Transform all product descriptions + jsonPath: '$..description', + // Target field to store LLM result - same as source in this case + targetPath: null, // null means replace in place + options: { + model: MODEL, + prompt: 'Make this description more engaging and detailed, around 20-30 words' + } + }, + { + // Transform nutrition information + jsonPath: '$..nutrition', + targetPath: null, + options: { + model: MODEL, + prompt: 'Expand this nutrition information with 2-3 specific health benefits, around 25-35 words' + } + }, + { + // Transform product names and store in a new 'marketingName' field + jsonPath: '$..name', + targetPath: 'marketingName', // will create a new field called marketingName + options: { + model: MODEL, + prompt: 'Generate a more appealing marketing name for this product' + } + } +]; + +// Create an async transformer function that uses an LLM to transform text +const createLLMTransformer = (options: any): AsyncTransformer => { + return async (input: string, jsonPath: string): Promise => { + logger.info(`Transforming field at path: ${jsonPath}`); + logger.info(`Input: ${input}`); + logger.info(`Using prompt: ${options.prompt}`); + + // Configure the task for kbot + const kbotTask: IKBotTask = { + model: options.model || MODEL, + router: ROUTER, + prompt: `${options.prompt}\n\nText to transform: "${input}"`, + logLevel: LOG_LEVEL, + path: path.resolve('./'), // Fix: provide absolute path + mode: 'completion' as any + }; + + try { + // Run kbot with the configured task + const results = await run(kbotTask); + + if (results && results.length > 0 && typeof results[0] === 'string') { + const result = results[0].trim(); + logger.info(`Result: ${result}`); + return result; + } + + // Return original if transformation fails + logger.warn(`No valid result received for ${jsonPath}, returning original`); + return input; + } catch (error) { + console.error(`Error calling LLM API: ${error.message}`, error); + return input; // Return original on error + } + }; +}; + +// Function to handle field transformation for a specific mapping +export async function transformField(data: any, mapping: { jsonPath: string, targetPath: string | null, options: any }) { + const { jsonPath, targetPath, options } = mapping; + + // Create a transformer for this specific field + const transformer = createLLMTransformer(options); + + // Set up error callback + const errorCallback = (path: string, value: string, error: any) => { + logger.error(`Error transforming ${path}: ${error.message}`); + // Don't throw to allow continued processing + }; + + // Set up filter callback - always return true to transform all matching fields + const filterCallback = async () => true; + + // Try/catch to handle any JSONPath or transformation errors + try { + if (!targetPath) { + // Transform in place + await transformObject( + data, + transformer, + jsonPath, + 1000, // throttle delay - higher for real API calls + 1, // concurrent tasks - limit for API rate limits + errorCallback, + filterCallback + ); + } else { + // Create a copy of the data to transform + const dataCopy = JSON.parse(JSON.stringify(data)); + + // Transform the copy + await transformObject( + dataCopy, + transformer, + jsonPath, + 1000, + 1, + errorCallback, + filterCallback + ); + + // Extract transformed values and store in target fields + const { JSONPath } = await import('jsonpath-plus'); + const paths = JSONPath({ path: jsonPath, json: dataCopy, resultType: 'pointer' }); + + for (const p of paths) { + const keys = p.slice(1).split('/'); + + // Navigate to the value in the transformed copy + let value = dataCopy; + for (const key of keys) { + if (key === '') continue; // Skip empty segments + value = value[key]; + } + + // Store in target path in the original data + const originalKeys = p.slice(1).split('/'); + const parentKeys = originalKeys.slice(0, -1); + + // Navigate to parent object in original data + let target = data; + for (const key of parentKeys) { + if (key === '') continue; + target = target[key]; + } + + // Set the new field with transformed value + target[targetPath] = value; + } + } + } catch (error) { + logger.error(`Error in field transformation: ${error.message}`); + } +} + +// Main function to demonstrate the transformation +export async function transformExample() { + console.log("========================================"); + console.log("Starting async-iterator example transformation"); + console.log("========================================"); + + try { + // Clone the data to avoid modifying the original + const data = JSON.parse(JSON.stringify(exampleData)); + + logger.info("Starting transformation of data fields with LLM..."); + console.log("Processing field mappings..."); + + // Process each field mapping + for (const mapping of fieldMappings) { + console.log(`Processing mapping: ${mapping.jsonPath} -> ${mapping.targetPath || 'in-place'}`); + logger.info(`Processing field mapping: ${mapping.jsonPath} -> ${mapping.targetPath || 'in-place'}`); + await transformField(data, mapping); + } + + // Save the result to a file + const outputPath = path.resolve('./tests/test-data/core/async-iterator-data.json'); + write(outputPath, JSON.stringify(data, null, 2)); + + console.log(`Transformation complete. Results saved to ${outputPath}`); + logger.info(`Transformation complete. Results saved to ${outputPath}`); + + // Display before/after comparison for a sample field + console.log("\nBefore/After Comparison Example:"); + console.log(`Original description: ${exampleData.products.fruits[0].description}`); + console.log(`Transformed description: ${data.products.fruits[0].description}`); + console.log(`Original name: ${exampleData.products.fruits[0].name}`); + console.log(`New marketing name: ${data.products.fruits[0].marketingName}`); + + logger.info("\nBefore/After Comparison Example:"); + logger.info(`Original description: ${exampleData.products.fruits[0].description}`); + logger.info(`Transformed description: ${data.products.fruits[0].description}`); + logger.info(`Original name: ${exampleData.products.fruits[0].name}`); + logger.info(`New marketing name: ${data.products.fruits[0].marketingName}`); + + return data; + } catch (error) { + console.error("ERROR during transformation:", error); + logger.error(`Error during transformation: ${error.message}`); + throw error; + } +} + +// Run directly with Node.js +// This section checks if this module is being executed directly (not imported) +console.log("Module loading, checking if this is direct execution"); +console.log("process.argv[1]:", process.argv[1]); +console.log("import.meta.url:", import.meta.url); + +// Directly check if file is run directly +const isDirectExecution = process.argv[1] && process.argv[1].includes('async-iterator-example'); +console.log("Is direct execution:", isDirectExecution); + +if (isDirectExecution) { + // Process command line arguments + const isDebug = process.argv.includes('--debug'); + + console.log('Starting async-iterator example...'); + console.log(`Command arguments: ${process.argv.slice(2).join(', ')}`); + console.log(`Debug mode: ${isDebug ? 'enabled' : 'disabled'}`); + + // Use more verbose logging if debug mode is enabled + if (isDebug) { + // Override the LOG_LEVEL constant for debug mode + // @ts-ignore + logger = { + info: (message) => console.log(`DEBUG-INFO: ${message}`), + warn: (message) => console.log(`DEBUG-WARN: ${message}`), + error: (message) => console.error(`DEBUG-ERROR: ${message}`), + debug: (message) => console.log(`DEBUG-DEBUG: ${message}`), + trace: (message) => console.log(`DEBUG-TRACE: ${message}`) + }; + logger.info("Running in debug mode with verbose output"); + } + + transformExample().catch(error => { + console.error("Unhandled error:", error); + }); +} + +// Export for potential reuse +export { + createLLMTransformer, + exampleData +}; \ No newline at end of file diff --git a/packages/kbot/src/examples/index.ts b/packages/kbot/src/examples/index.ts new file mode 100644 index 00000000..9f576d74 --- /dev/null +++ b/packages/kbot/src/examples/index.ts @@ -0,0 +1,2 @@ +// Export examples +export * from './core/async-iterator-example.js'; \ No newline at end of file diff --git a/packages/kbot/tsconfig.json b/packages/kbot/tsconfig.json index 45a6f86e..0ce2c1ac 100644 --- a/packages/kbot/tsconfig.json +++ b/packages/kbot/tsconfig.json @@ -16,6 +16,7 @@ }, "files": [ "src/index.ts", - "src/main.ts" + "src/main.ts", + "src/examples/index.ts" ] } \ No newline at end of file