latest

2025-04-06 16:31:25 +02:00 · 2025-04-06 16:31:25 +02:00 · 29f2e4d848
commit 29f2e4d848
parent a4c91ef82a
4 changed files with 592 additions and 0 deletions
--- a/packages/kbot/README.md
+++ b/packages/kbot/README.md
@ -0,0 +1,213 @@
+# @plastichub/kbot
+
+AI-powered command-line tool for code modifications and project management that supports multiple AI models and routers.
+
+## Overview
+
+Code-bot is a powerful CLI tool that helps developers automate code modifications, handle project management tasks, and integrate with various AI models for intelligent code and content assistance.
+
+## Quick Start
+
+### Installation Steps
+
+KBot requires Node.js to run. It's recommended to use Node.js version 18 or higher.
+
+1. Visit the official [Node.js website](https://nodejs.org/)
+2. Download the LTS (Long Term Support) version for your operating system
+3. Follow the installation wizard
+4. Verify installation by opening a terminal and running:
+   ```bash
+   node --version
+   npm --version
+   ```
+
+### API Keys
+
+KBot supports both OpenRouter and OpenAI APIs. You'll need at least one of these set up.
+
+#### OpenRouter API (Recommended)
+
+1. Visit [OpenRouter](https://openrouter.ai/)
+2. Sign up for an account
+3. Navigate to the API Keys section
+4. Create a new API key
+
+#### OpenAI API (Optional)
+
+1. Go to [OpenAI's platform](https://platform.openai.com/)
+2. Create an account or sign in
+3. Navigate to API keys section
+4. Create a new secret key
+
+### Installation using Node NPM package manager
+
+```bash
+npm install -g @plastichub/kbot
+```
+
+## Configuration
+
+### API Keys Setup
+
+Create configuration at `$HOME/.osr/.config.json` (or export OSR_CONFIG with path to config.json):
+
+```json
+{
+  "openrouter": {
+    "key": "your-openrouter-key"
+  },
+  "openai": {
+    "key": "your-openai-key"
+  },
+   "email": {
+      "newsletter": {
+          "host": "host.org",
+          "port": 465,
+          "debug": true,
+          "transactionLog": true,
+          "auth": {
+              "user": "foo@bar.com",
+              "pass": "pass"
+          }
+      }
+  },
+  "google": {
+      "cse": "custom search engine id",
+      "api_key": "google custom search api key"
+  },
+  "serpapi": {
+      "key": "your SerpAPI key (optional, used for web searches(places, google maps))"
+  },
+  "deepseek": {
+      "key": "your SerpAPI key (optional, used for web searches(places, google maps))"
+  },
+}
+```
+
+### Preferences Setup
+
+Optionally, create `.kbot/preferences.md` in your project directory to customize AI interactions:
+
+```markdown
+## My Preferences
+
+Gender : male
+Location : New York, USA (eg: `send me all saunas next to me`)
+Language : English
+Occupation : software developer, Typescript
+Age : 30+
+
+## Contacts
+
+My email address : example@email.com (eg: `send me latest hacker news`)
+My wife's email address ("Anne") : example@email.com (eg: `send email to my wife, with latest local news')
+
+## Content
+
+When creating content
+- always Markdown
+- always add links
+- when sending emails, always add 'Best regards, [Your Name]'
+```
+## Commands
+
+### Prompt
+
+```kbot "create Astro minimal boilerplate, use starlight theme. Install dependencies via NPM tool"```
+
+### Fetch latest models
+
+```kbot fetch```
+
+### Print examples
+
+```kbot examples```
+
+### Print extended help
+
+```kbot help-md```
+
+### Initialize folder
+
+```kbot init```
+
+### Internal : Build
+
+```kbot build```
+
+# Command Line Parameters
+
+This document describes all available command line parameters.
+
+## Core Parameters
+
+| Parameter | Description | Default | Required |
+|-----------|-------------|---------|----------|
+| `path` | Target directory | `.` | No |
+| `prompt` | The prompt. Supports file paths and environment variables | `./prompt.md` | No |
+| `output` | Optional output path for modified files (Tool mode only) | - | No |
+| `dst` | Optional destination path for the result, will substitute ${MODEL} and ${ROUTER} in the path. | - | No |
+| `model` | AI model to use for processing | `anthropic/claude-3.5-sonnet` | No |
+| `router` | Router to use: openai or openrouter | `openrouter` | No |
+| `mode` | Chat completion mode: "completion" (without tools) or "tools" | `tools` | No |
+
+## Advanced Parameters
+
+| Parameter | Description | Default | Required |
+|-----------|-------------|---------|----------|
+| `each` | Target directory | `.` | No |
+| `dry` | Dry run - only write out parameters without making API calls | `false` | No |
+
+## File Selection & Tools
+
+| Parameter | Description | Default | Required |
+|-----------|-------------|---------|----------|
+| `include` | Glob patterns to match files for processing. Supports multiple patterns, e.g. `--include=src/*.tsx,src/*.ts --include=package.json` | - | No |
+| `disable` | Disable tools categories | `[]` | No |
+| `disableTools` | List of specific tools to disable | `[]` | No |
+
+## Configuration & Profiles
+
+| Parameter | Description | Default | Required |
+|-----------|-------------|---------|----------|
+| `profile` | Path to profile for variables. Supports environment variables | `${POLYMECH-ROOT}/profile.json` | No |
+| `env` | Environment (in profile) | `default` | No |
+| `config` | Path to JSON configuration file (API keys). Supports environment variables | - | No |
+| `preferences` | Path to preferences file (location, email, gender, etc). Supports environment variables | `./.kbot/preferences.md` | No |
+
+## Debugging & Logging
+
+| Parameter | Description | Default | Required |
+|-----------|-------------|---------|----------|
+| `logLevel` | Logging level for the application (0-4) | `2` | No |
+| `logs` | Logging directory | `./.kbot` | No |
+| `dump` | Create a script | - | No |
+
+
+# Working on Larger Directories
+
+Since LLMs (Large Language Models) and providers are limited to very small 'context windows', it's necessary to feed them with smaller chunks instead. This document explains how to process larger directories efficiently.
+
+## Directory Processing Example
+
+Here's an example of how to walk through files and process them:
+
+```bash
+osr-cli each --main='kbot \"read ${KEY} and translate to german, save in docs/language code/filename.md\" --include=\"${REL}\" --include=\".kbot/preferences.md\"' --list="./docs/*.md" --cwd=.
+```
+
+### Parameter Explanation
+
+- `each`: Command to process multiple files iteratively
+- `--main`: The main command (`kbot`) to execute for each file
+  - `--include=\"${REL}\"` instructs kbot to include the current selected path
+  - `--include=\".kbot/preferences.md\"` instructs kbot to include additional preferences about the task (eg: translation specifics)
+- `--list`: Specifies the file pattern to match
+  - Supports include patterns (e.g., `"./docs/*.md"`)
+- `--cwd`: Sets the current working directory for the command execution. Default is the current directory (`.`)
+
+**Note** requires `@plastichub/osr-cli-commons` to be installed globally:
+
+```bash
+npm i -g @plastichub/osr-cli-commons
+```
--- a/packages/kbot/src/async-iterator.ts
+++ b/packages/kbot/src/async-iterator.ts
@ -0,0 +1,119 @@
+import { JSONPath } from 'jsonpath-plus'
+import pThrottle from 'p-throttle'
+import pMap from 'p-map'
+
+export type AsyncTransformer = (input: string, path: string) => Promise<string>
+export type ErrorCallback = (path: string, value: string, error: any) => void
+export type FilterCallback = (input: string, path: string) => Promise<boolean>
+export type Filter = (input: string) => Promise<boolean>
+
+export interface TransformOptions {
+    transform: AsyncTransformer
+    path: string
+    throttleDelay: number
+    concurrentTasks: number
+    errorCallback: ErrorCallback
+    filterCallback: FilterCallback
+}
+
+export const isNumber: Filter = async (input: string) => (/^-?\d+(\.\d+)?$/.test(input))
+export const isBoolean: Filter = async (input: string) => /^(true|false)$/i.test(input)
+export const isValidString: Filter = async (input: string) => !(input.trim() !== '')
+
+export const testFilters = (filters: Filter[]): FilterCallback => {
+    return async (input: string) => {
+        for (const filter of filters) {
+            if (await filter(input)) {
+                return false;
+            }
+        }
+        return true;
+    };
+};
+
+export const defaultFilters = (filters: Filter[] = []) =>
+    [
+        isNumber, isBoolean, isValidString,  ...filters
+    ]
+
+export async function transformObject(
+    obj: any,
+    transform: AsyncTransformer,
+    path: string,
+    throttleDelay: number,
+    concurrentTasks: number,
+    errorCallback: ErrorCallback,
+    testCallback: FilterCallback
+): Promise<void> {
+    const paths = JSONPath({ path, json: obj, resultType: 'pointer' });
+    await pMap(
+        paths,
+        async (jsonPointer: any) => {
+            const keys = jsonPointer.slice(1).split('/')
+            await transformPath(obj, keys, transform, throttleDelay, concurrentTasks, jsonPointer, errorCallback, testCallback)
+        },
+        { concurrency: concurrentTasks }
+    )
+}
+export async function transformPath(
+    obj: any,
+    keys: string[],
+    transform: AsyncTransformer,
+    throttleDelay: number,
+    concurrentTasks: number,
+    currentPath: string,
+    errorCallback: ErrorCallback,
+    testCallback: FilterCallback
+): Promise<void> {
+    
+    let current = obj
+    
+    for (let i = 0; i < keys.length - 1; i++) {
+        current = current[keys[i]]
+    }
+    const lastKey = keys[keys.length - 1]
+    const throttle = pThrottle({
+        limit: 1,
+        interval: throttleDelay,
+    })
+    if (typeof lastKey === 'string' && lastKey !== '') {
+        try {
+            const newKey = (await isValidString(lastKey)) && !(await isNumber(lastKey)) ? 
+                await throttle(transform)(lastKey, currentPath) : lastKey
+            if (newKey !== lastKey) {
+                current[newKey] = current[lastKey]
+                delete current[lastKey]
+            }
+            if (typeof current[newKey] === 'string' && current[newKey] !== '') {
+                if (await testCallback(current[newKey], `${currentPath}/${lastKey}`)) {
+                    current[newKey] = await throttle(transform)(current[newKey], `${currentPath}/${lastKey}`)
+                }
+            } else if (typeof current[newKey] === 'object' && current[newKey] !== null) {
+                await transformObject(current[newKey], transform, '$.*', throttleDelay, concurrentTasks, errorCallback, testCallback)
+            }
+        } catch (error) {
+            errorCallback(currentPath, lastKey, error)
+        }
+    }
+}
+
+const exampleTransformFunction: AsyncTransformer = async (input: string, path: string): Promise<string> => {
+    if (input === 'random') throw new Error('API error')
+    return input.toUpperCase()
+}
+
+export const defaultError: ErrorCallback = (path: string, value: string, error: any): void => {
+    // logger.error(`Error at path: ${path}, value: ${value}, error: ${error}`)
+}
+
+export const defaultOptions = (options: TransformOptions = {} as TransformOptions): TransformOptions => {
+    return {
+        ...options,
+        transform: options.transform || exampleTransformFunction,
+        path: options.path || '$[*][0,1,2]',
+        throttleDelay: options.throttleDelay || 10,
+        concurrentTasks: options.concurrentTasks || 1,
+        errorCallback: options.errorCallback || defaultError,
+        filterCallback: options.filterCallback || testFilters(defaultFilters())
+    }
+}
--- a/packages/kbot/src/reference/translate_commons.ts
+++ b/packages/kbot/src/reference/translate_commons.ts
@ -0,0 +1,204 @@
+import * as path from 'path'
+import { isString, isArray, isObject, isNumber } from '@polymech/core/primitives'
+import { sync as read } from "@polymech/fs/read"
+import { sync as write } from "@polymech/fs/write"
+import { minify as minify_html } from 'html-minifier-terser'
+
+import { IOptions, TranslateFilter } from '../types.js'
+import { store, get } from './store.js'
+import * as deepl from './deepl.js'
+
+const minify = false
+
+import {
+    defaultFilters,
+    defaultOptions,
+    transformObject,
+    TransformOptions,
+    testFilters
+} from '../async-iterator.js'
+
+import { update } from './glossary.js'
+
+import { createLogger } from '@polymech/log'
+export let logger = createLogger('i18n')
+
+export const clean = (text: string = "") => text.trim()
+export const extension = (file: string) => path.parse(file).ext
+
+export const getTranslation = (translations: any, all: boolean = false) => {
+    if (!all) {
+        if (translations && translations[0] && translations[0].text) {
+            return translations[0].text
+        }
+    } else {
+        return translations
+    }
+    return false
+}
+export const storeSet = (storePath: string, text: string, translation: string, file: string = '') => {
+    const store = read(storePath, 'json') || {}
+    store[text] = clean(translation)
+    write(storePath, store)
+}
+export const storeGet = (storePath: string, text: string, file: string = '') => {
+    const db = read(storePath, 'json') || {}
+    if (db[text]) {
+        return db[text]
+    }
+}
+export const translateObjectAIT = async (obj: any, src: string, options: IOptions) => {
+    const opts: TransformOptions = defaultOptions({
+        throttleDelay: 100,
+        concurrentTasks: 1,
+        path: options.query,
+        filterCallback: testFilters(
+            defaultFilters([
+                async (input) => !options.keys.includes(input)
+            ])
+        ),
+        transform: async (input: string, path: string) => {
+            if ((isNumber(input) || parseInt(input))) {
+                return input
+            }
+            const stored = get(options.store, input as string, options)
+            if (stored) {
+                return stored
+            }
+            const translated = await _translate(input, src, options)
+            if (translated) {
+                if (options.store) {
+                    store(options.store, input, translated, options)
+                }
+                return translated
+            }
+            return input
+        },
+        errorCallback: (path: string, value: string, error: any) => {
+            logger.error(`Error at path: ${path}, value: ${value}, error: ${error}`)
+            return value
+        }
+    } as TransformOptions)
+    try {
+        await transformObject(obj, opts.transform, opts.path, opts.throttleDelay, opts.concurrentTasks, opts.errorCallback, opts.filterCallback)
+        return obj
+    } catch (error) {
+        logger.error('Translation failed:', error)
+    }
+}
+export const translateDeepL = async (
+    text: string,
+    srcLang: string = 'EN',
+    dstLang: string = 'DE',
+    dOptions: deepl.IDeepLOptions,
+    options: IOptions = {},
+    file: string = '') => {
+
+    if (minify) {
+        text = await minify_html(text, {
+            collapseWhitespace: true
+        })
+    }
+    let glossary
+    try {
+        glossary = await update(srcLang.toLowerCase(), dstLang.toLowerCase(), options)
+    } catch (e) {
+        logger.warn('Error updating glossary', e.message)
+    }
+    const deeplOptions = {
+        preserve_formatting: '1',
+        tag_handling: ["xml"],
+        ...dOptions,
+        text: text,
+        target_lang: dstLang as deepl.DeepLLanguages,
+        source_lang: srcLang as deepl.DeepLLanguages,
+        formality: options.formality || 'default',
+    } as deepl.IDeepLOptions
+
+    if (glossary && glossary.glossaryId) {
+        deeplOptions.glossary_id = glossary.glossaryId
+    }
+    // logger.info('Translate:', deeplOptions)
+    let ret: any = await deepl.translate_deepl(deeplOptions) as deepl.IDeepLResponse
+    if (!ret) {
+        logger.error('Translate failed : ' + text, file)
+        return false
+    }
+    ret = ret?.data
+    if (options.filters) {
+        (ret.translations).forEach((t, i) => {
+            (options.filters as TranslateFilter[]).forEach((f) => {
+                ret.translations[i].text = f(text, t.text, file)
+            })
+        })
+    }
+    return ret.translations
+}
+const _translate = async (value: string, src: string, options: IOptions) => {
+
+    const translations = await translateDeepL(value as string, options.srcLang, options.dstLang,
+        {
+            auth_key: options.api_key,
+            formality: options.formality || 'default',
+            free_api: false
+        } as any, options, src)
+
+    return getTranslation(translations)
+}
+export const translateObject = async (obj: any, src: string, options: IOptions) => {
+    if (isNumber(obj)) {
+        return obj
+    }
+    if (isString(obj) && !obj.trim().length) {
+        return obj
+    }
+
+    if (isString(obj) && options.store) {
+        const stored = get(options.store, obj as string, options)
+        if (stored && options.cache) {
+            return stored
+        }
+        const ret = await _translate(obj as string, src, options)
+        if (ret && options.store) {
+            store(options.store, obj, ret, options)
+            return ret
+        } else {
+            console.error('Error translating : ', obj)
+        }
+        return obj
+    }
+    if (isObject(obj) || isArray(obj)) {
+        for await (const [key, value] of Object.entries(obj)) {
+            if (!obj[key]) {
+                continue
+            }
+            if (!isString(key)) {
+                continue
+            }
+            if (isString(value) && options.keys && !options.keys.includes(key)) {
+                continue
+            }
+            if (isString(value)) {
+                const stored = get(options.store, value as string, options)
+                if (stored && options.cache) {
+                    obj[key] = stored
+                } else {
+                    obj[key] = await _translate(value as string, src, options)
+                    if (options.store) {
+                        store(options.store, value, obj[key], options)
+                    }
+                }
+            } else if (isObject(value)) {
+                obj[key] = await translateObject(value, src, options)
+            } else if (isArray(value)) {
+                let i = 0
+                for await (const v of value) {
+                    if (!v) continue
+                    value[i] = await translateObject(v, src, options)
+                    i++
+                }
+            }
+        }
+    }
+    return obj
+}
--- a/packages/kbot/src/reference/translate_xls.ts
+++ b/packages/kbot/src/reference/translate_xls.ts
@ -0,0 +1,56 @@
+import * as path from 'path'
+import { JSONPath } from 'jsonpath-plus'
+import { get_cached, set_cached } from '@polymech/cache'
+import XLSX from 'xlsx'
+import { OSR_CACHE } from '@polymech/commons'
+import { sync as read } from "@polymech/fs/read"
+import { sync as exists } from "@polymech/fs/exists"
+import { sync as mkdir } from "@polymech/fs/dir"
+import { MODULE_NAME } from '../constants.js'
+import { IOptions } from '../types.js'
+import { logger, translateObjectAIT } from './translate_commons.js'
+
+export const translateXLS = async (src: string, dst: string, options: IOptions) => {
+    logger.debug(`Translating ${src} to ${dst}`)
+    if (!exists(src)) {
+        logger.error(`Translating ${src} : not found`)
+        return
+    }
+    const dstDir = path.parse(dst).dir
+    mkdir(dstDir)
+    const osr_cache = OSR_CACHE()
+    const cached = await get_cached(src, { keys: options.keys }, MODULE_NAME)
+    if (osr_cache && cached && options.cache && exists(dst)) {
+        return cached
+    }
+    const data = read(src, 'buffer') as Buffer
+    const workbook = XLSX.read(data)
+    const worksheet = workbook.Sheets[workbook.SheetNames[0]];
+    const raw_data: any[] = XLSX.utils.sheet_to_json(worksheet, { header: 1, blankrows: false, raw: false, skipHidden: true });
+    const queryResult = JSONPath(
+        {
+            path: options.query,
+            json: raw_data,
+
+        })
+    let translated
+    try {
+        translated = await translateObjectAIT(raw_data, src, {
+            ...options,
+            keys: queryResult
+        })
+    } catch (error) {
+        logger.error(`Error translating XLSX ${src}`, error)
+        return
+    }
+    if (!translated) {
+        logger.error(`Error translating XLSX ${src}`)
+        return
+    }
+    const sheetOut = XLSX.utils.json_to_sheet(translated, { skipHeader: true })
+    const workbookOut = XLSX.utils.book_new()
+    XLSX.utils.book_append_sheet(workbookOut, sheetOut, workbook.SheetNames[0])
+    XLSX.writeFileXLSX(workbookOut, dst)
+    osr_cache && options.cache && await set_cached(src, { keys: options.keys }, MODULE_NAME, translated)
+    return dst
+}