kbot - async it transformer example
This commit is contained in:
parent
902ef97467
commit
2be8a59103
File diff suppressed because one or more lines are too long
@ -3,7 +3,7 @@
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Expand this nutrition information with 2-3 specific health benefits, around 25-35 words\n\nText to transform: \"Rich in fiber and vitamin C\""
|
||||
"content": "Generate a more appealing marketing name for this product\n\nText to transform: \"broccoli\""
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
|
||||
@ -1,16 +1,12 @@
|
||||
import { sync as write } from "@polymech/fs/write";
|
||||
// Import path but not the logger
|
||||
import * as path from 'path';
|
||||
import type { IKBotTask } from '@polymech/ai-tools';
|
||||
import { E_OPENROUTER_MODEL } from '../../models/cache/openrouter-models.js'
|
||||
|
||||
// Configurable constants
|
||||
const MODEL = E_OPENROUTER_MODEL.MODEL_OPENROUTER_QUASAR_ALPHA;
|
||||
const ROUTER = 'openrouter';
|
||||
const LOG_LEVEL = 2;
|
||||
|
||||
const LOG_LEVEL = 2; // 0=trace, 1=debug, 2=info, 3=warn, 4=error, 5=fatal
|
||||
|
||||
// Create a simple console logger
|
||||
let logger = {
|
||||
info: (message: string) => console.log(`INFO: ${message}`),
|
||||
warn: (message: string) => console.log(`WARN: ${message}`),
|
||||
@ -19,7 +15,6 @@ let logger = {
|
||||
trace: (message: string) => console.log(`TRACE: ${message}`)
|
||||
};
|
||||
|
||||
// Import the actual kbot modules directly from source
|
||||
import {
|
||||
transformObject,
|
||||
defaultOptions,
|
||||
@ -29,10 +24,8 @@ import {
|
||||
AsyncTransformer
|
||||
} from '../../async-iterator.js';
|
||||
|
||||
// Import run function from commands
|
||||
import { run } from '../../commands/run.js';
|
||||
|
||||
// Example data structure with fields to be transformed
|
||||
const exampleData = {
|
||||
products: {
|
||||
fruits: [
|
||||
@ -87,20 +80,16 @@ const exampleData = {
|
||||
}
|
||||
};
|
||||
|
||||
// Create a mapping of fields to transform with JSONPath
|
||||
const fieldMappings = [
|
||||
{
|
||||
// Transform all product descriptions
|
||||
jsonPath: '$..description',
|
||||
// Target field to store LLM result - same as source in this case
|
||||
targetPath: null, // null means replace in place
|
||||
targetPath: null,
|
||||
options: {
|
||||
model: MODEL,
|
||||
prompt: 'Make this description more engaging and detailed, around 20-30 words'
|
||||
}
|
||||
},
|
||||
{
|
||||
// Transform nutrition information
|
||||
jsonPath: '$..nutrition',
|
||||
targetPath: null,
|
||||
options: {
|
||||
@ -109,9 +98,8 @@ const fieldMappings = [
|
||||
}
|
||||
},
|
||||
{
|
||||
// Transform product names and store in a new 'marketingName' field
|
||||
jsonPath: '$..name',
|
||||
targetPath: 'marketingName', // will create a new field called marketingName
|
||||
targetPath: 'marketingName',
|
||||
options: {
|
||||
model: MODEL,
|
||||
prompt: 'Generate a more appealing marketing name for this product'
|
||||
@ -119,25 +107,22 @@ const fieldMappings = [
|
||||
}
|
||||
];
|
||||
|
||||
// Create an async transformer function that uses an LLM to transform text
|
||||
const createLLMTransformer = (options: any): AsyncTransformer => {
|
||||
return async (input: string, jsonPath: string): Promise<string> => {
|
||||
logger.info(`Transforming field at path: ${jsonPath}`);
|
||||
logger.info(`Input: ${input}`);
|
||||
logger.info(`Using prompt: ${options.prompt}`);
|
||||
|
||||
// Configure the task for kbot
|
||||
const kbotTask: IKBotTask = {
|
||||
model: options.model || MODEL,
|
||||
router: ROUTER,
|
||||
prompt: `${options.prompt}\n\nText to transform: "${input}"`,
|
||||
logLevel: LOG_LEVEL,
|
||||
path: path.resolve('./'), // Fix: provide absolute path
|
||||
path: path.resolve('./'),
|
||||
mode: 'completion' as any
|
||||
};
|
||||
|
||||
try {
|
||||
// Run kbot with the configured task
|
||||
const results = await run(kbotTask);
|
||||
|
||||
if (results && results.length > 0 && typeof results[0] === 'string') {
|
||||
@ -146,50 +131,37 @@ const createLLMTransformer = (options: any): AsyncTransformer => {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Return original if transformation fails
|
||||
logger.warn(`No valid result received for ${jsonPath}, returning original`);
|
||||
return input;
|
||||
} catch (error) {
|
||||
console.error(`Error calling LLM API: ${error.message}`, error);
|
||||
return input; // Return original on error
|
||||
return input;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
// Function to handle field transformation for a specific mapping
|
||||
export async function transformField(data: any, mapping: { jsonPath: string, targetPath: string | null, options: any }) {
|
||||
const { jsonPath, targetPath, options } = mapping;
|
||||
|
||||
// Create a transformer for this specific field
|
||||
const transformer = createLLMTransformer(options);
|
||||
|
||||
// Set up error callback
|
||||
const errorCallback = (path: string, value: string, error: any) => {
|
||||
logger.error(`Error transforming ${path}: ${error.message}`);
|
||||
// Don't throw to allow continued processing
|
||||
};
|
||||
|
||||
// Set up filter callback - always return true to transform all matching fields
|
||||
const filterCallback = async () => true;
|
||||
|
||||
// Try/catch to handle any JSONPath or transformation errors
|
||||
try {
|
||||
if (!targetPath) {
|
||||
// Transform in place
|
||||
await transformObject(
|
||||
data,
|
||||
transformer,
|
||||
jsonPath,
|
||||
1000, // throttle delay - higher for real API calls
|
||||
1, // concurrent tasks - limit for API rate limits
|
||||
1000,
|
||||
1,
|
||||
errorCallback,
|
||||
filterCallback
|
||||
);
|
||||
} else {
|
||||
// Create a copy of the data to transform
|
||||
const dataCopy = JSON.parse(JSON.stringify(data));
|
||||
|
||||
// Transform the copy
|
||||
await transformObject(
|
||||
dataCopy,
|
||||
transformer,
|
||||
@ -200,32 +172,25 @@ export async function transformField(data: any, mapping: { jsonPath: string, tar
|
||||
filterCallback
|
||||
);
|
||||
|
||||
// Extract transformed values and store in target fields
|
||||
const { JSONPath } = await import('jsonpath-plus');
|
||||
const paths = JSONPath({ path: jsonPath, json: dataCopy, resultType: 'pointer' });
|
||||
|
||||
for (const p of paths) {
|
||||
const keys = p.slice(1).split('/');
|
||||
|
||||
// Navigate to the value in the transformed copy
|
||||
let value = dataCopy;
|
||||
for (const key of keys) {
|
||||
if (key === '') continue; // Skip empty segments
|
||||
if (key === '') continue;
|
||||
value = value[key];
|
||||
}
|
||||
|
||||
// Store in target path in the original data
|
||||
const originalKeys = p.slice(1).split('/');
|
||||
const parentKeys = originalKeys.slice(0, -1);
|
||||
|
||||
// Navigate to parent object in original data
|
||||
let target = data;
|
||||
for (const key of parentKeys) {
|
||||
if (key === '') continue;
|
||||
target = target[key];
|
||||
}
|
||||
|
||||
// Set the new field with transformed value
|
||||
target[targetPath] = value;
|
||||
}
|
||||
}
|
||||
@ -234,34 +199,28 @@ export async function transformField(data: any, mapping: { jsonPath: string, tar
|
||||
}
|
||||
}
|
||||
|
||||
// Main function to demonstrate the transformation
|
||||
export async function transformExample() {
|
||||
console.log("========================================");
|
||||
console.log("Starting async-iterator example transformation");
|
||||
console.log("========================================");
|
||||
|
||||
try {
|
||||
// Clone the data to avoid modifying the original
|
||||
const data = JSON.parse(JSON.stringify(exampleData));
|
||||
|
||||
logger.info("Starting transformation of data fields with LLM...");
|
||||
console.log("Processing field mappings...");
|
||||
|
||||
// Process each field mapping
|
||||
for (const mapping of fieldMappings) {
|
||||
console.log(`Processing mapping: ${mapping.jsonPath} -> ${mapping.targetPath || 'in-place'}`);
|
||||
logger.info(`Processing field mapping: ${mapping.jsonPath} -> ${mapping.targetPath || 'in-place'}`);
|
||||
await transformField(data, mapping);
|
||||
}
|
||||
|
||||
// Save the result to a file
|
||||
const outputPath = path.resolve('./tests/test-data/core/async-iterator-data.json');
|
||||
write(outputPath, JSON.stringify(data, null, 2));
|
||||
|
||||
console.log(`Transformation complete. Results saved to ${outputPath}`);
|
||||
logger.info(`Transformation complete. Results saved to ${outputPath}`);
|
||||
|
||||
// Display before/after comparison for a sample field
|
||||
console.log("\nBefore/After Comparison Example:");
|
||||
console.log(`Original description: ${exampleData.products.fruits[0].description}`);
|
||||
console.log(`Transformed description: ${data.products.fruits[0].description}`);
|
||||
@ -282,28 +241,20 @@ export async function transformExample() {
|
||||
}
|
||||
}
|
||||
|
||||
// Run directly with Node.js
|
||||
// This section checks if this module is being executed directly (not imported)
|
||||
console.log("Module loading, checking if this is direct execution");
|
||||
console.log("process.argv[1]:", process.argv[1]);
|
||||
console.log("import.meta.url:", import.meta.url);
|
||||
|
||||
// Directly check if file is run directly
|
||||
const isDirectExecution = process.argv[1] && process.argv[1].includes('async-iterator-example');
|
||||
console.log("Is direct execution:", isDirectExecution);
|
||||
|
||||
if (isDirectExecution) {
|
||||
// Process command line arguments
|
||||
const isDebug = process.argv.includes('--debug');
|
||||
|
||||
console.log('Starting async-iterator example...');
|
||||
console.log(`Command arguments: ${process.argv.slice(2).join(', ')}`);
|
||||
console.log(`Debug mode: ${isDebug ? 'enabled' : 'disabled'}`);
|
||||
|
||||
// Use more verbose logging if debug mode is enabled
|
||||
if (isDebug) {
|
||||
// Override the LOG_LEVEL constant for debug mode
|
||||
// @ts-ignore
|
||||
logger = {
|
||||
info: (message) => console.log(`DEBUG-INFO: ${message}`),
|
||||
warn: (message) => console.log(`DEBUG-WARN: ${message}`),
|
||||
@ -319,8 +270,7 @@ if (isDirectExecution) {
|
||||
});
|
||||
}
|
||||
|
||||
// Export for potential reuse
|
||||
export {
|
||||
createLLMTransformer,
|
||||
exampleData
|
||||
};
|
||||
};
|
||||
Loading…
Reference in New Issue
Block a user