i18n | async iterator

.gitignore

@@ -7,3 +7,4 @@ node_modules
 .env
 .cache
 storybook-static/
+.kbot

eslint.config.js

@@ -40,7 +40,6 @@ export default [
 			'@typescript-eslint/no-empty-object-type': 'off',
 			// Todo: do we want these?
 			'no-var': 'off',
-      
 			'regexp/prefer-regexp-exec': 'off',
 			'@typescript-eslint/no-duplicate-enum-values': 'off',
 			'@typescript-eslint/no-unsafe-function-type': 'off',

packages/commons/dist/async-iterator.d.ts

@@ -0,0 +1,21 @@
+export type AsyncTransformer = (input: string, path: string) => Promise<string>;
+export type ErrorCallback = (path: string, value: string, error: any) => void;
+export type FilterCallback = (input: string, path: string) => Promise<boolean>;
+export type Filter = (input: string) => Promise<boolean>;
+export interface TransformOptions {
+    transform: AsyncTransformer;
+    path: string;
+    throttleDelay: number;
+    concurrentTasks: number;
+    errorCallback: ErrorCallback;
+    filterCallback: FilterCallback;
+}
+export declare const isNumber: Filter;
+export declare const isBoolean: Filter;
+export declare const isValidString: Filter;
+export declare const testFilters: (filters: Filter[]) => FilterCallback;
+export declare const defaultFilters: (filters?: Filter[]) => Filter[];
+export declare function transformObject(obj: any, transform: AsyncTransformer, path: string, throttleDelay: number, concurrentTasks: number, errorCallback: ErrorCallback, testCallback: FilterCallback): Promise<void>;
+export declare function transformPath(obj: any, keys: string[], transform: AsyncTransformer, throttleDelay: number, concurrentTasks: number, currentPath: string, errorCallback: ErrorCallback, testCallback: FilterCallback): Promise<void>;
+export declare const defaultError: ErrorCallback;
+export declare const defaultOptions: (options?: TransformOptions) => TransformOptions;

packages/commons/dist/async-iterator.js

@@ -0,0 +1,77 @@
+import { JSONPath } from 'jsonpath-plus';
+import pThrottle from 'p-throttle';
+import pMap from 'p-map';
+export const isNumber = async (input) => (/^-?\d+(\.\d+)?$/.test(input));
+export const isBoolean = async (input) => /^(true|false)$/i.test(input);
+export const isValidString = async (input) => !(input.trim() !== '');
+export const testFilters = (filters) => {
+    return async (input) => {
+        for (const filter of filters) {
+            if (await filter(input)) {
+                return false;
+            }
+        }
+        return true;
+    };
+};
+export const defaultFilters = (filters = []) => [
+    isNumber, isBoolean, isValidString, ...filters
+];
+export async function transformObject(obj, transform, path, throttleDelay, concurrentTasks, errorCallback, testCallback) {
+    const paths = JSONPath({ path, json: obj, resultType: 'pointer' });
+    await pMap(paths, async (jsonPointer) => {
+        const keys = jsonPointer.slice(1).split('/');
+        await transformPath(obj, keys, transform, throttleDelay, concurrentTasks, jsonPointer, errorCallback, testCallback);
+    }, { concurrency: concurrentTasks });
+}
+export async function transformPath(obj, keys, transform, throttleDelay, concurrentTasks, currentPath, errorCallback, testCallback) {
+    let current = obj;
+    for (let i = 0; i < keys.length - 1; i++) {
+        current = current[keys[i]];
+    }
+    const lastKey = keys[keys.length - 1];
+    const throttle = pThrottle({
+        limit: 1,
+        interval: throttleDelay,
+    });
+    if (typeof lastKey === 'string' && lastKey !== '') {
+        try {
+            const newKey = isValidString(lastKey) && !isNumber(lastKey) ? await throttle(transform)(lastKey, currentPath) : lastKey;
+            if (newKey !== lastKey) {
+                current[newKey] = current[lastKey];
+                delete current[lastKey];
+            }
+            if (typeof current[newKey] === 'string' && current[newKey] !== '') {
+                if (await testCallback(current[newKey], `${currentPath}/${lastKey}`)) {
+                    current[newKey] = await throttle(transform)(current[newKey], `${currentPath}/${lastKey}`);
+                }
+            }
+            else if (typeof current[newKey] === 'object' && current[newKey] !== null) {
+                await transformObject(current[newKey], transform, '$.*', throttleDelay, concurrentTasks, errorCallback, testCallback);
+            }
+        }
+        catch (error) {
+            errorCallback(currentPath, lastKey, error);
+        }
+    }
+}
+const exampleTransformFunction = async (input, path) => {
+    if (input === 'random')
+        throw new Error('API error');
+    return input.toUpperCase();
+};
+export const defaultError = (path, value, error) => {
+    // logger.error(`Error at path: ${path}, value: ${value}, error: ${error}`)
+};
+export const defaultOptions = (options = {}) => {
+    return {
+        transform: exampleTransformFunction,
+        path: options.path || '$[*][0,1,2]',
+        throttleDelay: 10,
+        concurrentTasks: 1,
+        errorCallback: defaultError,
+        filterCallback: testFilters(defaultFilters()),
+        ...options
+    };
+};
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiYXN5bmMtaXRlcmF0b3IuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvYXN5bmMtaXRlcmF0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFFBQVEsRUFBRSxNQUFNLGVBQWUsQ0FBQTtBQUN4QyxPQUFPLFNBQVMsTUFBTSxZQUFZLENBQUE7QUFDbEMsT0FBTyxJQUFJLE1BQU0sT0FBTyxDQUFBO0FBZ0J4QixNQUFNLENBQUMsTUFBTSxRQUFRLEdBQVcsS0FBSyxFQUFFLEtBQWEsRUFBRSxFQUFFLENBQUMsQ0FBQyxpQkFBaUIsQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQTtBQUN4RixNQUFNLENBQUMsTUFBTSxTQUFTLEdBQVcsS0FBSyxFQUFFLEtBQWEsRUFBRSxFQUFFLENBQUMsaUJBQWlCLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFBO0FBQ3ZGLE1BQU0sQ0FBQyxNQUFNLGFBQWEsR0FBVyxLQUFLLEVBQUUsS0FBYSxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFBO0FBRXBGLE1BQU0sQ0FBQyxNQUFNLFdBQVcsR0FBRyxDQUFDLE9BQWlCLEVBQWtCLEVBQUU7SUFDN0QsT0FBTyxLQUFLLEVBQUUsS0FBYSxFQUFFLEVBQUU7UUFDM0IsS0FBSyxNQUFNLE1BQU0sSUFBSSxPQUFPLEVBQUUsQ0FBQztZQUMzQixJQUFJLE1BQU0sTUFBTSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUM7Z0JBQ3RCLE9BQU8sS0FBSyxDQUFDO1lBQ2pCLENBQUM7UUFDTCxDQUFDO1FBQ0QsT0FBTyxJQUFJLENBQUM7SUFDaEIsQ0FBQyxDQUFDO0FBQ04sQ0FBQyxDQUFDO0FBRUYsTUFBTSxDQUFDLE1BQU0sY0FBYyxHQUFHLENBQUMsVUFBb0IsRUFBRSxFQUFFLEVBQUUsQ0FDckQ7SUFDSSxRQUFRLEVBQUUsU0FBUyxFQUFFLGFBQWEsRUFBRyxHQUFHLE9BQU87Q0FDbEQsQ0FBQTtBQUVMLE1BQU0sQ0FBQyxLQUFLLFVBQVUsZUFBZSxDQUNqQyxHQUFRLEVBQ1IsU0FBMkIsRUFDM0IsSUFBWSxFQUNaLGFBQXFCLEVBQ3JCLGVBQXVCLEVBQ3ZCLGFBQTRCLEVBQzVCLFlBQTRCO0lBRTVCLE1BQU0sS0FBSyxHQUFHLFFBQVEsQ0FBQyxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsR0FBRyxFQUFFLFVBQVUsRUFBRSxTQUFTLEVBQUUsQ0FBQyxDQUFDO0lBQ25FLE1BQU0sSUFBSSxDQUNOLEtBQUssRUFDTCxLQUFLLEVBQUUsV0FBZ0IsRUFBRSxFQUFFO1FBQ3ZCLE1BQU0sSUFBSSxHQUFHLFdBQVcsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFBO1FBQzVDLE1BQU0sYUFBYSxDQUFDLEdBQUcsRUFBRSxJQUFJLEVBQUUsU0FBUyxFQUFFLGFBQWEsRUFBRSxlQUFlLEVBQUUsV0FBVyxFQUFFLGFBQWEsRUFBRSxZQUFZLENBQUMsQ0FBQTtJQUN2SCxDQUFDLEVBQ0QsRUFBRSxXQUFXLEVBQUUsZUFBZSxFQUFFLENBQ25DLENBQUE7QUFDTCxDQUFDO0FBQ0QsTUFBTSxDQUFDLEtBQUssVUFBVSxhQUFhLENBQy9CLEdBQVEsRUFDUixJQUFjLEVBQ2QsU0FBMkIsRUFDM0IsYUFBcUIsRUFDckIsZUFBdUIsRUFDdkIsV0FBbUIsRUFDbkIsYUFBNEIsRUFDNUIsWUFBNEI7SUFHNUIsSUFBSSxPQUFPLEdBQUcsR0FBRyxDQUFBO0lBRWpCLEtBQUssSUFBSSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsR0FBRyxJQUFJLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRSxDQUFDLEVBQUUsRUFBRSxDQUFDO1FBQ3ZDLE9BQU8sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUE7SUFDOUIsQ0FBQztJQUNELE1BQU0sT0FBTyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFBO0lBQ3JDLE1BQU0sUUFBUSxHQUFHLFNBQVMsQ0FBQztRQUN2QixLQUFLLEVBQUUsQ0FBQztRQUNSLFFBQVEsRUFBRSxhQUFhO0tBQzFCLENBQUMsQ0FBQTtJQUVGLElBQUksT0FBTyxPQUFPLEtBQUssUUFBUSxJQUFJLE9BQU8sS0FBSyxFQUFFLEVBQUUsQ0FBQztRQUNoRCxJQUFJLENBQUM7WUFDRCxNQUFNLE1BQU0sR0FBRyxhQUFhLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFDLE1BQU0sUUFBUSxDQUFDLFNBQVMsQ0FBQyxDQUFDLE9BQU8sRUFBRSxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUMsT0FBTyxDQUFBO1lBQ3ZILElBQUksTUFBTSxLQUFLLE9BQU8sRUFBRSxDQUFDO2dCQUNyQixPQUFPLENBQUMsTUFBTSxDQUFDLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFBO2dCQUNsQyxPQUFPLE9BQU8sQ0FBQyxPQUFPLENBQUMsQ0FBQTtZQUMzQixDQUFDO1lBQ0QsSUFBSSxPQUFPLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxRQUFRLElBQUksT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDO2dCQUNoRSxJQUFJLE1BQU0sWUFBWSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsRUFBRSxHQUFHLFdBQVcsSUFBSSxPQUFPLEVBQUUsQ0FBQyxFQUFFLENBQUM7b0JBQ25FLE9BQU8sQ0FBQyxNQUFNLENBQUMsR0FBRyxNQUFNLFFBQVEsQ0FBQyxTQUFTLENBQUMsQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLEVBQUUsR0FBRyxXQUFXLElBQUksT0FBTyxFQUFFLENBQUMsQ0FBQTtnQkFDN0YsQ0FBQztZQUNMLENBQUM7aUJBQU0sSUFBSSxPQUFPLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxRQUFRLElBQUksT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLElBQUksRUFBRSxDQUFDO2dCQUN6RSxNQUFNLGVBQWUsQ0FBQyxPQUFPLENBQUMsTUFBTSxDQUFDLEVBQUUsU0FBUyxFQUFFLEtBQUssRUFBRSxhQUFhLEVBQUUsZUFBZSxFQUFFLGFBQWEsRUFBRSxZQUFZLENBQUMsQ0FBQTtZQUN6SCxDQUFDO1FBQ0wsQ0FBQztRQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7WUFDYixhQUFhLENBQUMsV0FBVyxFQUFFLE9BQU8sRUFBRSxLQUFLLENBQUMsQ0FBQTtRQUM5QyxDQUFDO0lBQ0wsQ0FBQztBQUNMLENBQUM7QUFFRCxNQUFNLHdCQUF3QixHQUFxQixLQUFLLEVBQUUsS0FBYSxFQUFFLElBQVksRUFBbUIsRUFBRTtJQUN0RyxJQUFJLEtBQUssS0FBSyxRQUFRO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyxXQUFXLENBQUMsQ0FBQTtJQUNwRCxPQUFPLEtBQUssQ0FBQyxXQUFXLEVBQUUsQ0FBQTtBQUM5QixDQUFDLENBQUE7QUFFRCxNQUFNLENBQUMsTUFBTSxZQUFZLEdBQWtCLENBQUMsSUFBWSxFQUFFLEtBQWEsRUFBRSxLQUFVLEVBQVEsRUFBRTtJQUN6RiwyRUFBMkU7QUFDL0UsQ0FBQyxDQUFBO0FBRUQsTUFBTSxDQUFDLE1BQU0sY0FBYyxHQUFHLENBQUMsVUFBNEIsRUFBc0IsRUFBb0IsRUFBRTtJQUNuRyxPQUFPO1FBQ0gsU0FBUyxFQUFFLHdCQUF3QjtRQUNuQyxJQUFJLEVBQUcsT0FBTyxDQUFDLElBQUksSUFBSSxhQUFhO1FBQ3BDLGFBQWEsRUFBRSxFQUFFO1FBQ2pCLGVBQWUsRUFBRSxDQUFDO1FBQ2xCLGFBQWEsRUFBRSxZQUFZO1FBQzNCLGNBQWMsRUFBRSxXQUFXLENBQUMsY0FBYyxFQUFFLENBQUM7UUFDN0MsR0FBRyxPQUFPO0tBQ2IsQ0FBQTtBQUNMLENBQUMsQ0FBQSJ9

packages/commons/package.json

@@ -67,6 +67,9 @@
     "env-var": "^7.5.0",
     "glob": "^10.4.5",
     "js-yaml": "^4.1.0",
+    "jsonpath-plus": "^8.1.0",
+    "p-map": "^7.0.3",
+    "p-throttle": "^4.1.1",
     "tslog": "^3.3.3",
     "tsup": "^8.3.5",
     "yargs": "^17.7.2",

packages/commons/src/async-iterator.ts

@@ -0,0 +1,119 @@
+import { JSONPath } from 'jsonpath-plus'
+import pThrottle from 'p-throttle'
+import pMap from 'p-map'
+
+export type AsyncTransformer = (input: string, path: string) => Promise<string>
+export type ErrorCallback = (path: string, value: string, error: any) => void
+export type FilterCallback = (input: string, path: string) => Promise<boolean>
+export type Filter = (input: string) => Promise<boolean>
+
+export interface TransformOptions {
+    transform: AsyncTransformer
+    path: string
+    throttleDelay: number
+    concurrentTasks: number
+    errorCallback: ErrorCallback
+    filterCallback: FilterCallback
+}
+
+export const isNumber: Filter = async (input: string) => (/^-?\d+(\.\d+)?$/.test(input))
+export const isBoolean: Filter = async (input: string) => /^(true|false)$/i.test(input)
+export const isValidString: Filter = async (input: string) => !(input.trim() !== '')
+
+export const testFilters = (filters: Filter[]): FilterCallback => {
+    return async (input: string) => {
+        for (const filter of filters) {
+            if (await filter(input)) {
+                return false;
+            }
+        }
+        return true;
+    };
+};
+
+export const defaultFilters = (filters: Filter[] = []) =>
+    [
+        isNumber, isBoolean, isValidString,  ...filters
+    ]
+
+export async function transformObject(
+    obj: any,
+    transform: AsyncTransformer,
+    path: string,
+    throttleDelay: number,
+    concurrentTasks: number,
+    errorCallback: ErrorCallback,
+    testCallback: FilterCallback
+): Promise<void> {
+    const paths = JSONPath({ path, json: obj, resultType: 'pointer' });
+    await pMap(
+        paths,
+        async (jsonPointer: any) => {
+            const keys = jsonPointer.slice(1).split('/')
+            await transformPath(obj, keys, transform, throttleDelay, concurrentTasks, jsonPointer, errorCallback, testCallback)
+        },
+        { concurrency: concurrentTasks }
+    )
+}
+export async function transformPath(
+    obj: any,
+    keys: string[],
+    transform: AsyncTransformer,
+    throttleDelay: number,
+    concurrentTasks: number,
+    currentPath: string,
+    errorCallback: ErrorCallback,
+    testCallback: FilterCallback
+): Promise<void> {
+    
+    let current = obj
+    
+    for (let i = 0; i < keys.length - 1; i++) {
+        current = current[keys[i]]
+    }
+    const lastKey = keys[keys.length - 1]
+    const throttle = pThrottle({
+        limit: 1,
+        interval: throttleDelay,
+    })
+
+    if (typeof lastKey === 'string' && lastKey !== '') {
+        try {
+            const newKey = isValidString(lastKey) && !isNumber(lastKey) ? await throttle(transform)(lastKey, currentPath) : lastKey
+            if (newKey !== lastKey) {
+                current[newKey] = current[lastKey]
+                delete current[lastKey]
+            }
+            if (typeof current[newKey] === 'string' && current[newKey] !== '') {
+                if (await testCallback(current[newKey], `${currentPath}/${lastKey}`)) {
+                    current[newKey] = await throttle(transform)(current[newKey], `${currentPath}/${lastKey}`)
+                }
+            } else if (typeof current[newKey] === 'object' && current[newKey] !== null) {
+                await transformObject(current[newKey], transform, '$.*', throttleDelay, concurrentTasks, errorCallback, testCallback)
+            }
+        } catch (error) {
+            errorCallback(currentPath, lastKey, error)
+        }
+    }
+}
+
+const exampleTransformFunction: AsyncTransformer = async (input: string, path: string): Promise<string> => {
+    if (input === 'random') throw new Error('API error')
+    return input.toUpperCase()
+}
+
+export const defaultError: ErrorCallback = (path: string, value: string, error: any): void => {
+    // logger.error(`Error at path: ${path}, value: ${value}, error: ${error}`)
+}
+
+export const defaultOptions = (options: TransformOptions = {} as TransformOptions): TransformOptions => {
+    return {
+        transform: exampleTransformFunction,
+        path:  options.path || '$[*][0,1,2]',
+        throttleDelay: 10,
+        concurrentTasks: 1,
+        errorCallback: defaultError,
+        filterCallback: testFilters(defaultFilters()),
+        ...options
+    }
+}

packages/i18n/src/lib/translate_commons.ts

@@ -64,7 +64,7 @@ export const translateObjectAIT = async (obj: any, src: string, options: IOption
             const stored = get(options.store, input as string, options)
             if (stored) {
                 return stored
-            } else {
+            }
             const translated = await _translate(input, src, options)
             if (translated) {
                 if (options.store) {
@@ -73,7 +73,6 @@ export const translateObjectAIT = async (obj: any, src: string, options: IOption
                 return translated
             }
             return input
-            }
         },
         errorCallback: (path: string, value: string, error: any) => {
             logger.error(`Error at path: ${path}, value: ${value}, error: ${error}`)