polymech/mono
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kbot - async it transformer example

Browse Source
This commit is contained in:
lovebird 2025-04-06 21:31:55 +02:00
parent 4129021b67
commit 21a3939c9a
14 changed files with 966 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
packages/kbot/README.md
View File

@ -109,31 +109,31 @@ When creating content
- always add links
- when sending emails, always add 'Best regards, [Your Name]'
```
## Commands
### Prompt
```kbot "create Astro minimal boilerplate, use starlight theme. Install dependencies via NPM tool"```
### Fetch latest models
```kbot fetch```
### Print examples
```kbot examples```
### Print extended help
```kbot help-md```
### Initialize folder
```kbot init```
### Internal : Build
```kbot build```
## Commands
### Prompt
```kbot "create Astro minimal boilerplate, use starlight theme. Install dependencies via NPM tool"```
### Fetch latest models
```kbot fetch```
### Print examples
```kbot examples```
### Print extended help
```kbot help-md```
### Initialize folder
```kbot init```
### Internal : Build
```kbot build```
# Command Line Parameters
@ -210,4 +210,40 @@ osr-cli each --main='kbot \"read ${KEY} and translate to german, save in docs/la
```bash
npm i -g @plastichub/osr-cli-commons
```
```
## Async Iterator Examples
This package provides examples of how to use the async-iterator module to transform objects with LLM integration:
### Running the Examples
```bash
# Run the standard example (uses mock transformers if API is unavailable)
npm run examples:async-iterator
# Run with verbose logging
npm run examples:async-iterator:verbose
```
### Implementation Details
The async-iterator module provides a powerful way to transform specific fields in complex nested objects using JSONPath selectors. Key features include:
1. **Field Selection**: Use JSONPath patterns like `$..description` to target specific fields across the entire object
2. **Transformation Options**: Configure how fields are transformed with options like model, prompt, etc.
3. **Target Mapping**: Transform fields in-place or create new fields to store the transformed values
4. **Error Handling**: Built-in error callbacks to handle transformation failures gracefully
5. **Throttling & Concurrency**: Control API rate limits with throttle delay and concurrent task settings
### Example Source Code
You can find the example in:
- `src/examples/core/async-iterator-example.ts` - Source implementation using the actual run command with fallback to mock transformers
The example demonstrates:
1. Using the `transformObject` function from the async-iterator module
2. Setting up proper JSONPath selectors for targeting specific fields
3. Creating field mappings with transformation options
4. Implementing in-place transformations and new field creation
5. Graceful fallback handling when API calls fail

21
packages/kbot/dist-in/async-iterator.d.ts vendored Normal file
View File

@ -0,0 +1,21 @@
export type AsyncTransformer = (input: string, path: string) => Promise<string>;
export type ErrorCallback = (path: string, value: string, error: any) => void;
export type FilterCallback = (input: string, path: string) => Promise<boolean>;
export type Filter = (input: string) => Promise<boolean>;
export interface TransformOptions {
transform: AsyncTransformer;
path: string;
throttleDelay: number;
concurrentTasks: number;
errorCallback: ErrorCallback;
filterCallback: FilterCallback;
}
export declare const isNumber: Filter;
export declare const isBoolean: Filter;
export declare const isValidString: Filter;
export declare const testFilters: (filters: Filter[]) => FilterCallback;
export declare const defaultFilters: (filters?: Filter[]) => Filter[];
export declare function transformObject(obj: any, transform: AsyncTransformer, path: string, throttleDelay: number, concurrentTasks: number, errorCallback: ErrorCallback, testCallback: FilterCallback): Promise<void>;
export declare function transformPath(obj: any, keys: string[], transform: AsyncTransformer, throttleDelay: number, concurrentTasks: number, currentPath: string, errorCallback: ErrorCallback, testCallback: FilterCallback): Promise<void>;
export declare const defaultError: ErrorCallback;
export declare const defaultOptions: (options?: TransformOptions) => TransformOptions;

78
packages/kbot/dist-in/async-iterator.js Normal file
View File

@ -0,0 +1,78 @@
import { JSONPath } from 'jsonpath-plus';
import pThrottle from 'p-throttle';
import pMap from 'p-map';
export const isNumber = async (input) => (/^-?\d+(\.\d+)?$/.test(input));
export const isBoolean = async (input) => /^(true|false)$/i.test(input);
export const isValidString = async (input) => !(input.trim() !== '');
export const testFilters = (filters) => {
return async (input) => {
for (const filter of filters) {
if (await filter(input)) {
return false;
}
}
return true;
};
};
export const defaultFilters = (filters = []) => [
isNumber, isBoolean, isValidString, ...filters
];
export async function transformObject(obj, transform, path, throttleDelay, concurrentTasks, errorCallback, testCallback) {
const paths = JSONPath({ path, json: obj, resultType: 'pointer' });
await pMap(paths, async (jsonPointer) => {
const keys = jsonPointer.slice(1).split('/');
await transformPath(obj, keys, transform, throttleDelay, concurrentTasks, jsonPointer, errorCallback, testCallback);
}, { concurrency: concurrentTasks });
}
export async function transformPath(obj, keys, transform, throttleDelay, concurrentTasks, currentPath, errorCallback, testCallback) {
let current = obj;
for (let i = 0; i < keys.length - 1; i++) {
current = current[keys[i]];
}
const lastKey = keys[keys.length - 1];
const throttle = pThrottle({
limit: 1,
interval: throttleDelay,
});
if (typeof lastKey === 'string' && lastKey !== '') {
try {
const newKey = (await isValidString(lastKey)) && !(await isNumber(lastKey)) ?
await throttle(transform)(lastKey, currentPath) : lastKey;
if (newKey !== lastKey) {
current[newKey] = current[lastKey];
delete current[lastKey];
}
if (typeof current[newKey] === 'string' && current[newKey] !== '') {
if (await testCallback(current[newKey], `${currentPath}/${lastKey}`)) {
current[newKey] = await throttle(transform)(current[newKey], `${currentPath}/${lastKey}`);
}
}
else if (typeof current[newKey] === 'object' && current[newKey] !== null) {
await transformObject(current[newKey], transform, '$.*', throttleDelay, concurrentTasks, errorCallback, testCallback);
}
}
catch (error) {
errorCallback(currentPath, lastKey, error);
}
}
}
const exampleTransformFunction = async (input, path) => {
if (input === 'random')
throw new Error('API error');
return input.toUpperCase();
};
export const defaultError = (path, value, error) => {
// logger.error(`Error at path: ${path}, value: ${value}, error: ${error}`)
};
export const defaultOptions = (options = {}) => {
return {
...options,
transform: options.transform || exampleTransformFunction,
path: options.path || '$[*][0,1,2]',
throttleDelay: options.throttleDelay || 10,
concurrentTasks: options.concurrentTasks || 1,
errorCallback: options.errorCallback || defaultError,
filterCallback: options.filterCallback || testFilters(defaultFilters())
};
};
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXN5bmMtaXRlcmF0b3IuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvYXN5bmMtaXRlcmF0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFFBQVEsRUFBRSxNQUFNLGVBQWUsQ0FBQTtBQUN4QyxPQUFPLFNBQVMsTUFBTSxZQUFZLENBQUE7QUFDbEMsT0FBTyxJQUFJLE1BQU0sT0FBTyxDQUFBO0FBZ0J4QixNQUFNLENBQUMsTUFBTSxRQUFRLEdBQVcsS0FBSyxFQUFFLEtBQWEsRUFBRSxFQUFFLENBQUMsQ0FBQyxpQkFBaUIsQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQTtBQUN4RixNQUFNLENBQUMsTUFBTSxTQUFTLEdBQVcsS0FBSyxFQUFFLEtBQWEsRUFBRSxFQUFFLENBQUMsaUJBQWlCLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFBO0FBQ3ZGLE1BQU0sQ0FBQyxNQUFNLGFBQWEsR0FBVyxLQUFLLEVBQUUsS0FBYSxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFBO0FBRXBGLE1BQU0sQ0FBQyxNQUFNLFdBQVcsR0FBRyxDQUFDLE9BQWlCLEVBQWtCLEVBQUU7SUFDN0QsT0FBTyxLQUFLLEVBQUUsS0FBYSxFQUFFLEVBQUU7UUFDM0IsS0FBSyxNQUFNLE1BQU0sSUFBSSxPQUFPLEVBQUUsQ0FBQztZQUMzQixJQUFJLE1BQU0sTUFBTSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7Z0JBQ3RCLE9BQU8sS0FBSyxDQUFDO1lBQ2pCLENBQUM7UUFDTCxDQUFDO1FBQ0QsT0FBTyxJQUFJLENBQUM7SUFDaEIsQ0FBQyxDQUFDO0FBQ04sQ0FBQyxDQUFDO0FBRUYsTUFBTSxDQUFDLE1BQU0sY0FBYyxHQUFHLENBQUMsVUFBb0IsRUFBRSxFQUFFLEVBQUUsQ0FDckQ7SUFDSSxRQUFRLEVBQUUsU0FBUyxFQUFFLGFBQWEsRUFBRyxHQUFHLE9BQU87Q0FDbEQsQ0FBQTtBQUVMLE1BQU0sQ0FBQyxLQUFLLFVBQVUsZUFBZSxDQUNqQyxHQUFRLEVBQ1IsU0FBMkIsRUFDM0IsSUFBWSxFQUNaLGFBQXFCLEVBQ3JCLGVBQXVCLEVBQ3ZCLGFBQTRCLEVBQzVCLFlBQTRCO0lBRTVCLE1BQU0sS0FBSyxHQUFHLFFBQVEsQ0FBQyxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsR0FBRyxFQUFFLFVBQVUsRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDO0lBQ25FLE1BQU0sSUFBSSxDQUNOLEtBQUssRUFDTCxLQUFLLEVBQUUsV0FBZ0IsRUFBRSxFQUFFO1FBQ3ZCLE1BQU0sSUFBSSxHQUFHLFdBQVcsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFBO1FBQzVDLE1BQU0sYUFBYSxDQUFDLEdBQUcsRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFLGFBQWEsRUFBRSxlQUFlLEVBQUUsV0FBVyxFQUFFLGFBQWEsRUFBRSxZQUFZLENBQUMsQ0FBQTtJQUN2SCxDQUFDLEVBQ0QsRUFBRSxXQUFXLEVBQUUsZUFBZSxFQUFFLENBQ25DLENBQUE7QUFDTCxDQUFDO0FBQ0QsTUFBTSxDQUFDLEtBQUssVUFBVSxhQUFhLENBQy9CLEdBQVEsRUFDUixJQUFjLEVBQ2QsU0FBMkIsRUFDM0IsYUFBcUIsRUFDckIsZUFBdUIsRUFDdkIsV0FBbUIsRUFDbkIsYUFBNEIsRUFDNUIsWUFBNEI7SUFHNUIsSUFBSSxPQUFPLEdBQUcsR0FBRyxDQUFBO0lBRWpCLEtBQUssSUFBSSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsR0FBRyxJQUFJLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDLEVBQUUsRUFBRSxDQUFDO1FBQ3ZDLE9BQU8sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUE7SUFDOUIsQ0FBQztJQUNELE1BQU0sT0FBTyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFBO0lBQ3JDLE1BQU0sUUFBUSxHQUFHLFNBQVMsQ0FBQztRQUN2QixLQUFLLEVBQUUsQ0FBQztRQUNSLFFBQVEsRUFBRSxhQUFhO0tBQzFCLENBQUMsQ0FBQTtJQUNGLElBQUksT0FBTyxPQUFPLEtBQUssUUFBUSxJQUFJLE9BQU8sS0FBSyxFQUFFLEVBQUUsQ0FBQztRQUNoRCxJQUFJLENBQUM7WUFDRCxNQUFNLE1BQU0sR0FBRyxDQUFDLE1BQU0sYUFBYSxDQUFDLE9BQU8sQ0FBQyxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQU0sUUFBUSxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQztnQkFDekUsTUFBTSxRQUFRLENBQUMsU0FBUyxDQUFDLENBQUMsT0FBTyxFQUFFLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUE7WUFDN0QsSUFBSSxNQUFNLEtBQUssT0FBTyxFQUFFLENBQUM7Z0JBQ3JCLE9BQU8sQ0FBQyxNQUFNLENBQUMsR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUE7Z0JBQ2xDLE9BQU8sT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFBO1lBQzNCLENBQUM7WUFDRCxJQUFJLE9BQU8sT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLFFBQVEsSUFBSSxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssRUFBRSxFQUFFLENBQUM7Z0JBQ2hFLElBQUksTUFBTSxZQUFZLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxFQUFFLEdBQUcsV0FBVyxJQUFJLE9BQU8sRUFBRSxDQUFDLEVBQUUsQ0FBQztvQkFDbkUsT0FBTyxDQUFDLE1BQU0sQ0FBQyxHQUFHLE1BQU0sUUFBUSxDQUFDLFNBQVMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsRUFBRSxHQUFHLFdBQVcsSUFBSSxPQUFPLEVBQUUsQ0FBQyxDQUFBO2dCQUM3RixDQUFDO1lBQ0wsQ0FBQztpQkFBTSxJQUFJLE9BQU8sT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLFFBQVEsSUFBSSxPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssSUFBSSxFQUFFLENBQUM7Z0JBQ3pFLE1BQU0sZUFBZSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsRUFBRSxTQUFTLEVBQUUsS0FBSyxFQUFFLGFBQWEsRUFBRSxlQUFlLEVBQUUsYUFBYSxFQUFFLFlBQVksQ0FBQyxDQUFBO1lBQ3pILENBQUM7UUFDTCxDQUFDO1FBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztZQUNiLGFBQWEsQ0FBQyxXQUFXLEVBQUUsT0FBTyxFQUFFLEtBQUssQ0FBQyxDQUFBO1FBQzlDLENBQUM7SUFDTCxDQUFDO0FBQ0wsQ0FBQztBQUVELE1BQU0sd0JBQXdCLEdBQXFCLEtBQUssRUFBRSxLQUFhLEVBQUUsSUFBWSxFQUFtQixFQUFFO0lBQ3RHLElBQUksS0FBSyxLQUFLLFFBQVE7UUFBRSxNQUFNLElBQUksS0FBSyxDQUFDLFdBQVcsQ0FBQyxDQUFBO0lBQ3BELE9BQU8sS0FBSyxDQUFDLFdBQVcsRUFBRSxDQUFBO0FBQzlCLENBQUMsQ0FBQTtBQUVELE1BQU0sQ0FBQyxNQUFNLFlBQVksR0FBa0IsQ0FBQyxJQUFZLEVBQUUsS0FBYSxFQUFFLEtBQVUsRUFBUSxFQUFFO0lBQ3pGLDJFQUEyRTtBQUMvRSxDQUFDLENBQUE7QUFFRCxNQUFNLENBQUMsTUFBTSxjQUFjLEdBQUcsQ0FBQyxVQUE0QixFQUFzQixFQUFvQixFQUFFO0lBQ25HLE9BQU87UUFDSCxHQUFHLE9BQU87UUFDVixTQUFTLEVBQUUsT0FBTyxDQUFDLFNBQVMsSUFBSSx3QkFBd0I7UUFDeEQsSUFBSSxFQUFFLE9BQU8sQ0FBQyxJQUFJLElBQUksYUFBYTtRQUNuQyxhQUFhLEVBQUUsT0FBTyxDQUFDLGFBQWEsSUFBSSxFQUFFO1FBQzFDLGVBQWUsRUFBRSxPQUFPLENBQUMsZUFBZSxJQUFJLENBQUM7UUFDN0MsYUFBYSxFQUFFLE9BQU8sQ0FBQyxhQUFhLElBQUksWUFBWTtRQUNwRCxjQUFjLEVBQUUsT0FBTyxDQUFDLGNBQWMsSUFBSSxXQUFXLENBQUMsY0FBYyxFQUFFLENBQUM7S0FDMUUsQ0FBQTtBQUNMLENBQUMsQ0FBQSJ9

38
packages/kbot/dist-in/examples/core/async-iterator-example.d.ts vendored Normal file
View File

@ -0,0 +1,38 @@
import { AsyncTransformer } from '../../async-iterator.js';
declare const exampleData: {
products: {
fruits: {
id: string;
name: string;
description: string;
details: {
color: string;
origin: string;
nutrition: string;
};
}[];
vegetables: {
id: string;
name: string;
description: string;
details: {
color: string;
origin: string;
nutrition: string;
};
}[];
};
metadata: {
lastUpdated: string;
source: string;
description: string;
};
};
declare const createLLMTransformer: (options: any) => AsyncTransformer;
export declare function transformField(data: any, mapping: {
jsonPath: string;
targetPath: string | null;
options: any;
}): Promise<void>;
export declare function transformExample(): Promise<any>;
export { createLLMTransformer, exampleData };

268
packages/kbot/dist-in/examples/core/async-iterator-example.js Normal file
View File

File diff suppressed because one or more lines are too long

1
packages/kbot/dist-in/examples/index.d.ts vendored Normal file
View File

@ -0,0 +1 @@
export * from './core/async-iterator-example.js';

3
packages/kbot/dist-in/examples/index.js Normal file
View File

@ -0,0 +1,3 @@
// Export examples
export * from './core/async-iterator-example.js';
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvZXhhbXBsZXMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsa0JBQWtCO0FBQ2xCLGNBQWMsa0NBQWtDLENBQUMifQ==

2
packages/kbot/dist-in/profile.js
View File

@ -13,7 +13,7 @@ const testPath = (profilePath) => {
};
export const load = async (options) => {
let profile = { includes: [], variables: options.variables || {}, env: {} };
let profilePath = testPath(options.profile || path.join(options.logs, 'profile.json'));
let profilePath = testPath(options.profile || path.join(options.logs || '.', 'profile.json'));
if (!profilePath || !exists(profilePath) || !isFile(profilePath)) {
return profile.variables;
}

10
packages/kbot/logs/params.json
View File

File diff suppressed because one or more lines are too long

158
packages/kbot/package-lock.json generated
View File

@ -47,6 +47,7 @@
"eslint": "^8.57.1",
"ts-json-schema-generator": "^2.3.0",
"ts-loader": "9.5.1",
"ts-node": "10.9.2",
"tsx": "^4.5.0",
"typescript": "^5.7.2",
"vitest": "^2.1.8",
@ -336,6 +337,30 @@
"node": ">=0.1.90"
}
},
"node_modules/@cspotcode/source-map-support": {
"version": "0.8.1",
"resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
"integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jridgewell/trace-mapping": "0.3.9"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz",
"integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jridgewell/resolve-uri": "^3.0.3",
"@jridgewell/sourcemap-codec": "^1.4.10"
}
},
"node_modules/@discoveryjs/json-ext": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.6.3.tgz",
@ -1628,6 +1653,34 @@
"url": "https://github.com/sindresorhus/is?sponsor=1"
}
},
"node_modules/@tsconfig/node10": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz",
"integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==",
"dev": true,
"license": "MIT"
},
"node_modules/@tsconfig/node12": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz",
"integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==",
"dev": true,
"license": "MIT"
},
"node_modules/@tsconfig/node14": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz",
"integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==",
"dev": true,
"license": "MIT"
},
"node_modules/@tsconfig/node16": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
"integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/cookie": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz",
@ -2357,6 +2410,19 @@
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
}
},
"node_modules/acorn-walk": {
"version": "8.3.4",
"resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz",
"integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==",
"dev": true,
"license": "MIT",
"dependencies": {
"acorn": "^8.11.0"
},
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/agentkeepalive": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz",
@ -2486,6 +2552,13 @@
"integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==",
"license": "MIT"
},
"node_modules/arg": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
"integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
"dev": true,
"license": "MIT"
},
"node_modules/argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
@ -3012,6 +3085,13 @@
"node": ">= 0.6"
}
},
"node_modules/create-require": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"dev": true,
"license": "MIT"
},
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@ -3086,6 +3166,16 @@
"node": ">=0.4.0"
}
},
"node_modules/diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
"integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
"dev": true,
"license": "BSD-3-Clause",
"engines": {
"node": ">=0.3.1"
}
},
"node_modules/dir-glob": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
@ -4609,6 +4699,13 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/make-error": {
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
"integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
"dev": true,
"license": "ISC"
},
"node_modules/marked": {
"version": "14.1.4",
"resolved": "https://registry.npmjs.org/marked/-/marked-14.1.4.tgz",
@ -6539,6 +6636,50 @@
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/ts-node": {
"version": "10.9.2",
"resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
"integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@cspotcode/source-map-support": "^0.8.0",
"@tsconfig/node10": "^1.0.7",
"@tsconfig/node12": "^1.0.7",
"@tsconfig/node14": "^1.0.0",
"@tsconfig/node16": "^1.0.2",
"acorn": "^8.4.1",
"acorn-walk": "^8.1.1",
"arg": "^4.1.0",
"create-require": "^1.1.0",
"diff": "^4.0.1",
"make-error": "^1.1.1",
"v8-compile-cache-lib": "^3.0.1",
"yn": "3.1.1"
},
"bin": {
"ts-node": "dist/bin.js",
"ts-node-cwd": "dist/bin-cwd.js",
"ts-node-esm": "dist/bin-esm.js",
"ts-node-script": "dist/bin-script.js",
"ts-node-transpile-only": "dist/bin-transpile.js",
"ts-script": "dist/bin-script-deprecated.js"
},
"peerDependencies": {
"@swc/core": ">=1.2.50",
"@swc/wasm": ">=1.2.50",
"@types/node": "*",
"typescript": ">=2.7"
},
"peerDependenciesMeta": {
"@swc/core": {
"optional": true
},
"@swc/wasm": {
"optional": true
}
}
},
"node_modules/ts-retry": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/ts-retry/-/ts-retry-6.0.0.tgz",
@ -7148,6 +7289,13 @@
"requires-port": "^1.0.0"
}
},
"node_modules/v8-compile-cache-lib": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
"integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
"dev": true,
"license": "MIT"
},
"node_modules/vite": {
"version": "5.4.14",
"resolved": "https://registry.npmjs.org/vite/-/vite-5.4.14.tgz",
@ -7665,6 +7813,16 @@
"node": ">=8"
}
},
"node_modules/yn": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
"integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/yoctocolors-cjs": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz",

5
packages/kbot/package.json
View File

@ -32,7 +32,9 @@
"test2:coverage": "vitest run --coverage",
"webpack": "webpack --config webpack.config.js --stats-error-details",
"exe:win": "cd dist && nexe -i main_node.js -o kbot.exe --build --temp=../../temp-kbot --verbose",
"exe:lnx": "cd dist && nexe -i main_node.js -o kbot --build --temp=../../temp-kbot --verbose"
"exe:lnx": "cd dist && nexe -i main_node.js -o kbot --build --temp=../../temp-kbot --verbose",
"examples:async-iterator": "node dist-in/examples/core/async-iterator-example.js",
"examples:async-iterator:verbose": "node dist-in/examples/core/async-iterator-example.js --debug"
},
"dependencies": {
"@polymech/ai-tools": "file:../ai-tools",
@ -92,6 +94,7 @@
"eslint": "^8.57.1",
"ts-json-schema-generator": "^2.3.0",
"ts-loader": "9.5.1",
"ts-node": "10.9.2",
"tsx": "^4.5.0",
"typescript": "^5.7.2",
"vitest": "^2.1.8",

326
packages/kbot/src/examples/core/async-iterator-example.ts Normal file
View File

@ -0,0 +1,326 @@
import { sync as write } from "@polymech/fs/write";
// Import path but not the logger
import * as path from 'path';
import type { IKBotTask } from '@polymech/ai-tools';
import { E_OPENROUTER_MODEL } from '../../models/cache/openrouter-models.js'
// Configurable constants
const MODEL = E_OPENROUTER_MODEL.MODEL_OPENROUTER_QUASAR_ALPHA;
const ROUTER = 'openrouter';
const LOG_LEVEL = 2; // 0=trace, 1=debug, 2=info, 3=warn, 4=error, 5=fatal
// Create a simple console logger
let logger = {
info: (message: string) => console.log(`INFO: ${message}`),
warn: (message: string) => console.log(`WARN: ${message}`),
error: (message: string) => console.error(`ERROR: ${message}`),
debug: (message: string) => console.log(`DEBUG: ${message}`),
trace: (message: string) => console.log(`TRACE: ${message}`)
};
// Import the actual kbot modules directly from source
import {
transformObject,
defaultOptions,
TransformOptions,
testFilters,
defaultFilters,
AsyncTransformer
} from '../../async-iterator.js';
// Import run function from commands
import { run } from '../../commands/run.js';
// Example data structure with fields to be transformed
const exampleData = {
products: {
fruits: [
{
id: 'f1',
name: 'apple',
description: 'A sweet and crunchy fruit',
details: {
color: 'red',
origin: 'Worldwide',
nutrition: 'Rich in fiber and vitamin C'
}
},
{
id: 'f2',
name: 'banana',
description: 'A yellow tropical fruit',
details: {
color: 'yellow',
origin: 'Southeast Asia',
nutrition: 'High in potassium'
}
}
],
vegetables: [
{
id: 'v1',
name: 'carrot',
description: 'An orange root vegetable',
details: {
color: 'orange',
origin: 'Eurasia',
nutrition: 'Good for vision health'
}
},
{
id: 'v2',
name: 'broccoli',
description: 'A green cruciferous vegetable',
details: {
color: 'green',
origin: 'Italy',
nutrition: 'High in vitamins K and C'
}
}
]
},
metadata: {
lastUpdated: '2023-04-15',
source: 'Foods Database',
description: 'Collection of common fruits and vegetables'
}
};
// Create a mapping of fields to transform with JSONPath
const fieldMappings = [
{
// Transform all product descriptions
jsonPath: '$..description',
// Target field to store LLM result - same as source in this case
targetPath: null, // null means replace in place
options: {
model: MODEL,
prompt: 'Make this description more engaging and detailed, around 20-30 words'
}
},
{
// Transform nutrition information
jsonPath: '$..nutrition',
targetPath: null,
options: {
model: MODEL,
prompt: 'Expand this nutrition information with 2-3 specific health benefits, around 25-35 words'
}
},
{
// Transform product names and store in a new 'marketingName' field
jsonPath: '$..name',
targetPath: 'marketingName', // will create a new field called marketingName
options: {
model: MODEL,
prompt: 'Generate a more appealing marketing name for this product'
}
}
];
// Create an async transformer function that uses an LLM to transform text
const createLLMTransformer = (options: any): AsyncTransformer => {
return async (input: string, jsonPath: string): Promise<string> => {
logger.info(`Transforming field at path: ${jsonPath}`);
logger.info(`Input: ${input}`);
logger.info(`Using prompt: ${options.prompt}`);
// Configure the task for kbot
const kbotTask: IKBotTask = {
model: options.model || MODEL,
router: ROUTER,
prompt: `${options.prompt}\n\nText to transform: "${input}"`,
logLevel: LOG_LEVEL,
path: path.resolve('./'), // Fix: provide absolute path
mode: 'completion' as any
};
try {
// Run kbot with the configured task
const results = await run(kbotTask);
if (results && results.length > 0 && typeof results[0] === 'string') {
const result = results[0].trim();
logger.info(`Result: ${result}`);
return result;
}
// Return original if transformation fails
logger.warn(`No valid result received for ${jsonPath}, returning original`);
return input;
} catch (error) {
console.error(`Error calling LLM API: ${error.message}`, error);
return input; // Return original on error
}
};
};
// Function to handle field transformation for a specific mapping
export async function transformField(data: any, mapping: { jsonPath: string, targetPath: string | null, options: any }) {
const { jsonPath, targetPath, options } = mapping;
// Create a transformer for this specific field
const transformer = createLLMTransformer(options);
// Set up error callback
const errorCallback = (path: string, value: string, error: any) => {
logger.error(`Error transforming ${path}: ${error.message}`);
// Don't throw to allow continued processing
};
// Set up filter callback - always return true to transform all matching fields
const filterCallback = async () => true;
// Try/catch to handle any JSONPath or transformation errors
try {
if (!targetPath) {
// Transform in place
await transformObject(
data,
transformer,
jsonPath,
1000, // throttle delay - higher for real API calls
1, // concurrent tasks - limit for API rate limits
errorCallback,
filterCallback
);
} else {
// Create a copy of the data to transform
const dataCopy = JSON.parse(JSON.stringify(data));
// Transform the copy
await transformObject(
dataCopy,
transformer,
jsonPath,
1000,
1,
errorCallback,
filterCallback
);
// Extract transformed values and store in target fields
const { JSONPath } = await import('jsonpath-plus');
const paths = JSONPath({ path: jsonPath, json: dataCopy, resultType: 'pointer' });
for (const p of paths) {
const keys = p.slice(1).split('/');
// Navigate to the value in the transformed copy
let value = dataCopy;
for (const key of keys) {
if (key === '') continue; // Skip empty segments
value = value[key];
}
// Store in target path in the original data
const originalKeys = p.slice(1).split('/');
const parentKeys = originalKeys.slice(0, -1);
// Navigate to parent object in original data
let target = data;
for (const key of parentKeys) {
if (key === '') continue;
target = target[key];
}
// Set the new field with transformed value
target[targetPath] = value;
}
}
} catch (error) {
logger.error(`Error in field transformation: ${error.message}`);
}
}
// Main function to demonstrate the transformation
export async function transformExample() {
console.log("========================================");
console.log("Starting async-iterator example transformation");
console.log("========================================");
try {
// Clone the data to avoid modifying the original
const data = JSON.parse(JSON.stringify(exampleData));
logger.info("Starting transformation of data fields with LLM...");
console.log("Processing field mappings...");
// Process each field mapping
for (const mapping of fieldMappings) {
console.log(`Processing mapping: ${mapping.jsonPath} -> ${mapping.targetPath || 'in-place'}`);
logger.info(`Processing field mapping: ${mapping.jsonPath} -> ${mapping.targetPath || 'in-place'}`);
await transformField(data, mapping);
}
// Save the result to a file
const outputPath = path.resolve('./tests/test-data/core/async-iterator-data.json');
write(outputPath, JSON.stringify(data, null, 2));
console.log(`Transformation complete. Results saved to ${outputPath}`);
logger.info(`Transformation complete. Results saved to ${outputPath}`);
// Display before/after comparison for a sample field
console.log("\nBefore/After Comparison Example:");
console.log(`Original description: ${exampleData.products.fruits[0].description}`);
console.log(`Transformed description: ${data.products.fruits[0].description}`);
console.log(`Original name: ${exampleData.products.fruits[0].name}`);
console.log(`New marketing name: ${data.products.fruits[0].marketingName}`);
logger.info("\nBefore/After Comparison Example:");
logger.info(`Original description: ${exampleData.products.fruits[0].description}`);
logger.info(`Transformed description: ${data.products.fruits[0].description}`);
logger.info(`Original name: ${exampleData.products.fruits[0].name}`);
logger.info(`New marketing name: ${data.products.fruits[0].marketingName}`);
return data;
} catch (error) {
console.error("ERROR during transformation:", error);
logger.error(`Error during transformation: ${error.message}`);
throw error;
}
}
// Run directly with Node.js
// This section checks if this module is being executed directly (not imported)
console.log("Module loading, checking if this is direct execution");
console.log("process.argv[1]:", process.argv[1]);
console.log("import.meta.url:", import.meta.url);
// Directly check if file is run directly
const isDirectExecution = process.argv[1] && process.argv[1].includes('async-iterator-example');
console.log("Is direct execution:", isDirectExecution);
if (isDirectExecution) {
// Process command line arguments
const isDebug = process.argv.includes('--debug');
console.log('Starting async-iterator example...');
console.log(`Command arguments: ${process.argv.slice(2).join(', ')}`);
console.log(`Debug mode: ${isDebug ? 'enabled' : 'disabled'}`);
// Use more verbose logging if debug mode is enabled
if (isDebug) {
// Override the LOG_LEVEL constant for debug mode
// @ts-ignore
logger = {
info: (message) => console.log(`DEBUG-INFO: ${message}`),
warn: (message) => console.log(`DEBUG-WARN: ${message}`),
error: (message) => console.error(`DEBUG-ERROR: ${message}`),
debug: (message) => console.log(`DEBUG-DEBUG: ${message}`),
trace: (message) => console.log(`DEBUG-TRACE: ${message}`)
};
logger.info("Running in debug mode with verbose output");
}
transformExample().catch(error => {
console.error("Unhandled error:", error);
});
}
// Export for potential reuse
export {
createLLMTransformer,
exampleData
};

2
packages/kbot/src/examples/index.ts Normal file
View File

@ -0,0 +1,2 @@
// Export examples
export * from './core/async-iterator-example.js';

3
packages/kbot/tsconfig.json
View File

@ -16,6 +16,7 @@
},
"files": [
"src/index.ts",
"src/main.ts"
"src/main.ts",
"src/examples/index.ts"
]
}