polymech/mono
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kbot iterator network options

Browse Source
This commit is contained in:
lovebird 2025-04-07 13:01:13 +02:00
parent 07fe877eee
commit 3c2ee7fd07
14 changed files with 233 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
packages/kbot/dist-in/async-iterator.d.ts vendored
View File

@ -2,42 +2,40 @@ export type AsyncTransformer = (input: string, path: string) => Promise<string>;
export type ErrorCallback = (path: string, value: string, error: unknown) => void;
export type FilterCallback = (input: string, path: string) => Promise<boolean>;
export type Filter = (input: string) => Promise<boolean>;
export interface TransformOptions {
transform: AsyncTransformer;
path: string;
throttleDelay: number;
concurrentTasks: number;
errorCallback: ErrorCallback;
filterCallback: FilterCallback;
targetPath?: string | null;
export interface INetworkOptions {
throttleDelay?: number;
concurrentTasks?: number;
maxRetries?: number;
retryDelay?: number;
}
export declare const DEFAULT_NETWORK_OPTIONS: Required<INetworkOptions>;
export interface TransformOptions {
transform: AsyncTransformer;
path: string;
network?: INetworkOptions;
errorCallback: ErrorCallback;
filterCallback: FilterCallback;
targetPath?: string | null;
}
export interface GlobalOptions {
throttleDelay?: number;
concurrentTasks?: number;
network?: INetworkOptions;
errorCallback?: ErrorCallback;
filterCallback?: FilterCallback;
maxRetries?: number;
retryDelay?: number;
}
export declare const isNumber: Filter;
export declare const isBoolean: Filter;
export declare const isValidString: Filter;
export declare const testFilters: (filters: Filter[]) => FilterCallback;
export declare const defaultFilters: (filters?: Filter[]) => Filter[];
export declare function transformObject(obj: Record<string, any>, transform: AsyncTransformer, path: string, throttleDelay: number, concurrentTasks: number, errorCallback: ErrorCallback, testCallback: FilterCallback, maxRetries?: number, retryDelay?: number): Promise<void>;
export declare function transformPath(obj: Record<string, any>, keys: string[], transform: AsyncTransformer, throttleDelay: number, concurrentTasks: number, currentPath: string, errorCallback: ErrorCallback, testCallback: FilterCallback, maxRetries?: number, retryDelay?: number): Promise<void>;
export declare function transformObject(obj: Record<string, any>, transform: AsyncTransformer, path: string, networkOptions: Required<INetworkOptions>, errorCallback: ErrorCallback, testCallback: FilterCallback): Promise<void>;
export declare function transformPath(obj: Record<string, any>, keys: string[], transform: AsyncTransformer, networkOptions: Required<INetworkOptions>, currentPath: string, errorCallback: ErrorCallback, testCallback: FilterCallback): Promise<void>;
export declare const defaultError: ErrorCallback;
export interface TransformWithOptionsInput {
jsonPath: string;
targetPath?: string | null;
throttleDelay?: number;
concurrentTasks?: number;
network?: INetworkOptions;
errorCallback?: ErrorCallback;
filterCallback?: FilterCallback;
maxRetries?: number;
retryDelay?: number;
}
export declare function transformObjectWithOptions(obj: Record<string, any>, transform: AsyncTransformer, options: TransformWithOptionsInput): Promise<void>;
export declare const defaultOptions: (options?: Partial<TransformOptions>) => TransformOptions;

43
packages/kbot/dist-in/async-iterator.js
View File

File diff suppressed because one or more lines are too long

12
packages/kbot/dist-in/examples/core/async-iterator-example.js
View File

File diff suppressed because one or more lines are too long

17
packages/kbot/dist-in/examples/core/iterator-factory-example.js
View File

File diff suppressed because one or more lines are too long

21
packages/kbot/dist-in/iterator.d.ts vendored
View File

@ -1,5 +1,5 @@
import { IKBotTask } from '@polymech/ai-tools';
import { AsyncTransformer, ErrorCallback, FilterCallback } from './async-iterator.js';
import { AsyncTransformer, ErrorCallback, FilterCallback, INetworkOptions } from './async-iterator.js';
/**
* Notes for LLM modifications
*
@ -16,8 +16,6 @@ export interface FieldMapping {
jsonPath: string;
targetPath?: string | null;
options?: IKBotTask;
maxRetries?: number;
retryDelay?: number;
}
export interface IteratorFactory {
transform: (mappings: FieldMapping[]) => Promise<void>;
@ -29,25 +27,14 @@ export interface CacheConfig {
expiration?: number;
}
export interface IOptions {
throttleDelay?: number;
concurrentTasks?: number;
network?: INetworkOptions;
errorCallback?: ErrorCallback;
filterCallback?: FilterCallback;
transformerFactory?: (options: IKBotTask) => AsyncTransformer;
maxRetries?: number;
retryDelay?: number;
logger?: ILogger;
cacheConfig?: CacheConfig;
}
export { INetworkOptions };
export declare function createLLMTransformer(options: IKBotTask, logger?: ILogger, cacheConfig?: CacheConfig): AsyncTransformer;
export declare function createIterator(obj: Record<string, any>, optionsMixin: Partial<IKBotTask>, globalOptions?: IOptions): IteratorFactory;
export declare function transformWithMappings(obj: Record<string, any>, createTransformer: (options: IKBotTask) => AsyncTransformer, mappings: FieldMapping[], globalOptions?: {
throttleDelay?: number;
concurrentTasks?: number;
errorCallback?: ErrorCallback;
filterCallback?: FilterCallback;
maxRetries?: number;
retryDelay?: number;
logger?: ILogger;
cacheConfig?: CacheConfig;
}): Promise<void>;
export declare function transformWithMappings(obj: Record<string, any>, createTransformer: (options: IKBotTask) => AsyncTransformer, mappings: FieldMapping[], globalOptions?: IOptions): Promise<void>;

19
packages/kbot/dist-in/iterator.js
View File

File diff suppressed because one or more lines are too long

2
packages/kbot/logs/params.json
View File

@ -3,7 +3,7 @@
"messages": [
{
"role": "user",
"content": "Generate a more appealing marketing name for this product\n\nText to transform: \"apple2\""
"content": "Generate a more appealing marketing name for this product\n\nText to transform: \"broccoli\""
},
{
"role": "user",

101
packages/kbot/src/async-iterator.ts
View File

@ -8,25 +8,33 @@ export type ErrorCallback = (path: string, value: string, error: unknown) => voi
export type FilterCallback = (input: string, path: string) => Promise<boolean>
export type Filter = (input: string) => Promise<boolean>
export interface INetworkOptions {
throttleDelay?: number;
concurrentTasks?: number;
maxRetries?: number;
retryDelay?: number;
}
export const DEFAULT_NETWORK_OPTIONS: Required<INetworkOptions> = {
throttleDelay: 1000,
concurrentTasks: 1,
maxRetries: 3,
retryDelay: 2000
};
export interface TransformOptions {
transform: AsyncTransformer
path: string
throttleDelay: number
concurrentTasks: number
network?: INetworkOptions
errorCallback: ErrorCallback
filterCallback: FilterCallback
targetPath?: string | null
maxRetries?: number
retryDelay?: number
}
export interface GlobalOptions {
throttleDelay?: number
concurrentTasks?: number
network?: INetworkOptions
errorCallback?: ErrorCallback
filterCallback?: FilterCallback
maxRetries?: number
retryDelay?: number
}
// Sleep utility for retry mechanism
@ -56,12 +64,9 @@ export async function transformObject(
obj: Record<string, any>,
transform: AsyncTransformer,
path: string,
throttleDelay: number,
concurrentTasks: number,
networkOptions: Required<INetworkOptions>,
errorCallback: ErrorCallback,
testCallback: FilterCallback,
maxRetries: number = 3,
retryDelay: number = 2000
testCallback: FilterCallback
): Promise<void> {
const paths = JSONPath({ path, json: obj, resultType: 'pointer' });
await pMap(
@ -72,16 +77,13 @@ export async function transformObject(
obj,
keys,
transform,
throttleDelay,
concurrentTasks,
networkOptions,
jsonPointer,
errorCallback,
testCallback,
maxRetries,
retryDelay
testCallback
)
},
{ concurrency: concurrentTasks }
{ concurrency: networkOptions.concurrentTasks }
)
}
@ -89,13 +91,10 @@ export async function transformPath(
obj: Record<string, any>,
keys: string[],
transform: AsyncTransformer,
throttleDelay: number,
concurrentTasks: number,
networkOptions: Required<INetworkOptions>,
currentPath: string,
errorCallback: ErrorCallback,
testCallback: FilterCallback,
maxRetries: number = 3,
retryDelay: number = 2000
testCallback: FilterCallback
): Promise<void> {
let current: Record<string, any> = obj
@ -109,7 +108,7 @@ export async function transformPath(
const lastKey = keys[keys.length - 1]
const throttle = pThrottle({
limit: 1,
interval: throttleDelay,
interval: networkOptions.throttleDelay,
})
if (typeof lastKey === 'string' && lastKey !== '') {
if (typeof current[lastKey] === 'string' && current[lastKey] !== '') {
@ -119,7 +118,7 @@ export async function transformPath(
let success = false;
let lastError: unknown;
while (attempts < maxRetries && !success) {
while (attempts < networkOptions.maxRetries && !success) {
try {
current[lastKey] = await throttle(transform)(current[lastKey], `${currentPath}/${lastKey}`);
success = true;
@ -127,9 +126,9 @@ export async function transformPath(
lastError = error;
attempts++;
if (attempts < maxRetries) {
if (attempts < networkOptions.maxRetries) {
// Exponential backoff: retry delay increases with each attempt
const backoffDelay = retryDelay * Math.pow(2, attempts - 1);
const backoffDelay = networkOptions.retryDelay * Math.pow(2, attempts - 1);
await sleep(backoffDelay);
}
}
@ -144,12 +143,9 @@ export async function transformPath(
current[lastKey] as Record<string, any>,
transform,
'$.*',
throttleDelay,
concurrentTasks,
networkOptions,
errorCallback,
testCallback,
maxRetries,
retryDelay
testCallback
)
}
}
@ -162,12 +158,9 @@ export const defaultError: ErrorCallback = (path: string, value: string, error:
export interface TransformWithOptionsInput {
jsonPath: string
targetPath?: string | null
throttleDelay?: number
concurrentTasks?: number
network?: INetworkOptions
errorCallback?: ErrorCallback
filterCallback?: FilterCallback
maxRetries?: number
retryDelay?: number
}
export async function transformObjectWithOptions(
@ -178,26 +171,25 @@ export async function transformObjectWithOptions(
const {
jsonPath,
targetPath = null,
throttleDelay = 1000,
concurrentTasks = 1,
network = {},
errorCallback = defaultError,
filterCallback = testFilters(defaultFilters()),
maxRetries = 3,
retryDelay = 2000
filterCallback = testFilters(defaultFilters())
} = options;
const networkOptions: Required<INetworkOptions> = {
...DEFAULT_NETWORK_OPTIONS,
...network
};
// If targetPath is null, directly transform the object at jsonPath
if (!targetPath) {
return transformObject(
obj,
transform,
jsonPath,
throttleDelay,
concurrentTasks,
networkOptions,
errorCallback,
filterCallback,
maxRetries,
retryDelay
filterCallback
);
}
@ -209,12 +201,9 @@ export async function transformObjectWithOptions(
dataCopy,
transform,
jsonPath,
throttleDelay,
concurrentTasks,
networkOptions,
errorCallback,
filterCallback,
maxRetries,
retryDelay
filterCallback
);
// Get paths from original object
@ -251,14 +240,14 @@ export async function transformObjectWithOptions(
}
export const defaultOptions = (options: Partial<TransformOptions> = {}): TransformOptions => {
const network = { ...DEFAULT_NETWORK_OPTIONS, ...options.network };
return {
transform: options.transform,
path: options.path || '$[*][0,1,2]',
throttleDelay: options.throttleDelay || 10,
concurrentTasks: options.concurrentTasks || 1,
network,
errorCallback: options.errorCallback || defaultError,
filterCallback: options.filterCallback || testFilters(defaultFilters()),
maxRetries: options.maxRetries || 3,
retryDelay: options.retryDelay || 2000
targetPath: options.targetPath
}
}

14
packages/kbot/src/examples/core/async-iterator-example.ts
View File

@ -22,7 +22,8 @@ import {
import {
FieldMapping,
createIterator
createIterator,
INetworkOptions
} from '../../iterator.js';
import { run } from '../../commands/run.js';
@ -136,13 +137,20 @@ export async function transformExample() {
mode: E_Mode.COMPLETION
};
// Network configuration
const networkOptions: INetworkOptions = {
throttleDelay: 1000,
concurrentTasks: 1,
maxRetries: 3,
retryDelay: 2000
};
// Create an iterator factory instance
const iterator = createIterator(
data,
globalOptionsMixin,
{
throttleDelay: 1000,
concurrentTasks: 1,
network: networkOptions,
errorCallback,
filterCallback: async () => true,
transformerFactory: createLLMTransformer

18
packages/kbot/src/examples/core/iterator-factory-example.ts
View File

@ -4,7 +4,7 @@ import * as fs from 'fs';
import type { IKBotTask } from '@polymech/ai-tools';
import { E_OPENROUTER_MODEL } from '../../models/cache/openrouter-models.js';
import { E_Mode } from '../../zod_schema.js';
import { FieldMapping, createIterator, createLLMTransformer, CacheConfig } from '../../iterator.js';
import { FieldMapping, createIterator, createLLMTransformer, CacheConfig, INetworkOptions } from '../../iterator.js';
/**
* Notes for LLM modifications
@ -31,7 +31,7 @@ const exampleData = {
fruits: [
{
id: 'f1',
name: 'apple2',
name: 'apple',
description: 'A sweet and crunchy fruit',
details: {
color: 'red',
@ -83,6 +83,14 @@ const errorCallback = (path: string, value: string, error: any) => {
logger.error(`Error transforming ${path}: ${error.message}`);
};
// Network configuration
const networkOptions: INetworkOptions = {
throttleDelay: 1000,
concurrentTasks: 1,
maxRetries: 3,
retryDelay: 2000
};
// Cache configuration
const cacheConfig: CacheConfig = {
enabled: true,
@ -111,8 +119,7 @@ export async function factoryExample() {
data,
globalOptionsMixin,
{
throttleDelay: 1000,
concurrentTasks: 1,
network: networkOptions,
errorCallback,
filterCallback: async () => true,
transformerFactory: (options) => createLLMTransformer(options, logger, cacheConfig),
@ -153,8 +160,7 @@ export async function factoryExample() {
data2,
globalOptionsMixin,
{
throttleDelay: 1000,
concurrentTasks: 1,
network: networkOptions,
errorCallback,
filterCallback: async () => true,
transformerFactory: (options) => createLLMTransformer(options, logger, cacheConfig),

52
packages/kbot/src/iterator.ts
View File

@ -1,5 +1,15 @@
import { IKBotTask } from '@polymech/ai-tools'
import { AsyncTransformer, ErrorCallback, FilterCallback, defaultError, defaultFilters, testFilters, transformObjectWithOptions } from './async-iterator.js'
import {
AsyncTransformer,
ErrorCallback,
FilterCallback,
defaultError,
defaultFilters,
testFilters,
transformObjectWithOptions,
INetworkOptions,
DEFAULT_NETWORK_OPTIONS
} from './async-iterator.js'
import { run } from './commands/run.js'
import { get_cached_object, set_cached_object, rm_cached_object } from "@polymech/cache"
import { deepClone } from "@polymech/core/objects"
@ -44,8 +54,6 @@ export interface FieldMapping {
jsonPath: string
targetPath?: string | null
options?: IKBotTask
maxRetries?: number
retryDelay?: number
}
export interface IteratorFactory {
@ -60,13 +68,10 @@ export interface CacheConfig {
}
export interface IOptions {
throttleDelay?: number;
concurrentTasks?: number;
network?: INetworkOptions;
errorCallback?: ErrorCallback;
filterCallback?: FilterCallback;
transformerFactory?: (options: IKBotTask) => AsyncTransformer;
maxRetries?: number;
retryDelay?: number;
logger?: ILogger;
cacheConfig?: CacheConfig;
}
@ -77,6 +82,9 @@ const DEFAULT_CACHE_CONFIG: Required<CacheConfig> = {
expiration: 7 * 24 * 60 * 60 // 7 days in seconds
};
// Re-export INetworkOptions for other modules to use
export { INetworkOptions };
export function createLLMTransformer(
options: IKBotTask,
logger: ILogger = dummyLogger,
@ -145,17 +153,19 @@ export function createIterator(
globalOptions: IOptions = {}
): IteratorFactory {
const {
throttleDelay = 1000,
concurrentTasks = 1,
network = {},
errorCallback = defaultError,
filterCallback = testFilters(defaultFilters()),
transformerFactory,
maxRetries = 3,
retryDelay = 2000,
logger = dummyLogger,
cacheConfig
} = globalOptions;
const networkOptions: Required<INetworkOptions> = {
...DEFAULT_NETWORK_OPTIONS,
...network
};
const config: Required<CacheConfig> = { ...DEFAULT_CACHE_CONFIG, ...cacheConfig };
const defaultTransformerFactory = (options: IKBotTask): AsyncTransformer => {
@ -212,7 +222,7 @@ export function createIterator(
const transformedObj = JSON.parse(JSON.stringify(obj));
for (const mapping of mappings) {
const mergedOptions = { ...optionsMixin, ...mapping.options } as IKBotTask;
const { jsonPath, targetPath = null, maxRetries: mappingRetries, retryDelay: mappingRetryDelay } = mapping;
const { jsonPath, targetPath = null } = mapping;
const transformer = createTransformer(mergedOptions);
await transformObjectWithOptions(
@ -221,12 +231,9 @@ export function createIterator(
{
jsonPath,
targetPath,
throttleDelay,
concurrentTasks,
network: networkOptions,
errorCallback,
filterCallback,
maxRetries: mappingRetries || maxRetries,
retryDelay: mappingRetryDelay || retryDelay
filterCallback
}
);
}
@ -253,16 +260,7 @@ export async function transformWithMappings(
obj: Record<string, any>,
createTransformer: (options: IKBotTask) => AsyncTransformer,
mappings: FieldMapping[],
globalOptions: {
throttleDelay?: number;
concurrentTasks?: number;
errorCallback?: ErrorCallback;
filterCallback?: FilterCallback;
maxRetries?: number;
retryDelay?: number;
logger?: ILogger;
cacheConfig?: CacheConfig;
} = {}
globalOptions: IOptions = {}
): Promise<void> {
const iterator = createIterator(obj, {}, globalOptions);
await iterator.transform(mappings);

22
packages/kbot/tests/test-data/core/async-iterator-data.json
View File

@ -4,22 +4,22 @@
{
"id": "f1",
"name": "apple",
"description": "A deliciously sweet fruit bursting with juicy flavor and a satisfying crunch, offering a refreshing taste experience in every bite, perfect for a healthy, invigorating snack.",
"description": "A deliciously juicy fruit that bursts with natural sweetness and offers a satisfying crunch, making every bite a refreshing and energizing treat for your senses.",
"details": {
"color": "red",
"origin": "Worldwide",
"nutrition": "Rich in fiber and vitamin C, it supports healthy digestion, helps regulate blood sugar levels, and boosts immune function by protecting against infections and promoting collagen production for skin health."
"nutrition": "Rich in fiber and vitamin C, which supports healthy digestion and promotes regular bowel movements. Vitamin C boosts the immune system, aids in collagen production, and helps protect cells from oxidative damage."
},
"marketingName": "Crimson Orchard Delight"
"marketingName": "Orchard Bliss"
},
{
"id": "f2",
"name": "banana",
"description": "A vibrant, sun-kissed tropical fruit with a bright yellow skin, offering a sweet, juicy flavor bursting with tropical essence and a refreshing, exotic aroma.",
"description": "A vibrant, sun-kissed tropical fruit with a smooth golden skin, juicy flesh bursting with sweet, tangy flavor, and an irresistibly refreshing aroma that captures the essence of summer.",
"details": {
"color": "yellow",
"origin": "Southeast Asia",
"nutrition": "High in potassium, which helps regulate blood pressure, supports proper nerve and muscle function, and reduces the risk of kidney stones by maintaining healthy electrolyte balance in the body."
"nutrition": "High in potassium, which helps regulate blood pressure, supports proper muscle and nerve function, and aids in maintaining fluid balance in the body, contributing to overall cardiovascular and muscular health."
},
"marketingName": "Golden Tropic Delight"
}
@ -28,22 +28,22 @@
{
"id": "v1",
"name": "carrot",
"description": "A vibrant, earthy-hued root vegetable bursting with natural sweetness and rich in vitamins, perfect for roasting, juicing, or enjoying raw for a crunchy snack.",
"description": "A vibrant, nutrient-packed orange root vegetable with a crisp texture and sweet, earthy flavor, often enjoyed fresh, roasted, or blended into nourishing soups and juices.",
"details": {
"color": "orange",
"origin": "Eurasia",
"nutrition": "Supports eye health by protecting against age-related macular degeneration, maintains healthy night vision, and provides antioxidants that reduce the risk of cataracts, contributing to overall visual clarity and longevity."
"nutrition": "Supports sharp vision by protecting eye cells from oxidative stress, reduces risk of age-related macular degeneration, and helps maintain retinal health for overall improved eye function and long-term visual clarity."
},
"marketingName": "Golden Harvest Ruby"
"marketingName": "Golden Crunch"
},
{
"id": "v2",
"name": "broccoli",
"description": "A vibrant, nutrient-rich green vegetable from the cruciferous family, packed with antioxidants and vitamins, known for its crisp texture and subtly peppery, earthy flavor.",
"description": "A vibrant green, nutrient-packed cruciferous vegetable known for its crisp texture and subtle peppery flavor, rich in vitamins, minerals, and powerful antioxidants promoting overall health.",
"details": {
"color": "green",
"origin": "Italy",
"nutrition": "Rich in vitamins K and C, this supports healthy blood clotting, strengthens the immune system, and promotes collagen production, which benefits skin health and helps maintain strong bones."
"nutrition": "Rich in vitamins K and C, this supports strong immune function, promotes healthy blood clotting, and enhances collagen production for skin health. Together, they also help reduce inflammation and support bone strength."
},
"marketingName": "Emerald Crown Veggie"
}
@ -52,6 +52,6 @@
"metadata": {
"lastUpdated": "2023-04-15",
"source": "Foods Database",
"description": "A vibrant assortment of fresh, colorful fruits and vegetables, bursting with natural flavors and essential nutrients—perfect for creating healthy, delicious meals every day."
"description": "A vibrant assortment of fresh, colorful fruits and crisp vegetables, featuring a diverse selection of popular, nutrient-rich produce perfect for healthy snacks, delicious recipes, and vibrant meals."
}
}

4
packages/kbot/tests/test-data/core/iterator-factory-data.json
View File

@ -3,14 +3,14 @@
"fruits": [
{
"id": "f1",
"name": "apple2",
"name": "apple",
"description": "A deliciously sweet fruit with a satisfying crunch, bursting with juicy flavor and vibrant freshness in every bite, perfect for a refreshing and healthy snack.",
"details": {
"color": "red",
"origin": "Worldwide",
"nutrition": "Rich in fiber and vitamin D, this promotes digestive health, supports gut microbiome balance, and helps maintain strong bones and immune function through enhanced calcium absorption and regulation of immune responses."
},
"marketingName": "Apple Fusion"
"marketingName": "Crimson Orchard Delight"
},
{
"id": "f2",

57
packages/kbot/tests/unit/core/async-iterator.test.ts
View File

@ -7,7 +7,9 @@ import {
defaultFilters,
AsyncTransformer,
ErrorCallback,
FilterCallback
FilterCallback,
INetworkOptions,
DEFAULT_NETWORK_OPTIONS
} from '../../../src/async-iterator'
// Helper function to manually transform paths in the test data based on mockFilterCallback and mockTransform
@ -300,12 +302,13 @@ describe('async-iterator', () => {
// Empty object - this test passes without changes
const emptyObj = {}
const networkOptions = { ...DEFAULT_NETWORK_OPTIONS, throttleDelay: 10, concurrentTasks: 1 };
await transformObject(
emptyObj,
mockTransform,
'$.items[*].name',
10,
1,
networkOptions,
mockErrorCallback,
mockFilterCallback
)
@ -321,8 +324,7 @@ describe('async-iterator', () => {
testObj,
mockTransform,
'',
10,
1,
networkOptions,
mockErrorCallback,
mockFilterCallback
)
@ -476,12 +478,13 @@ describe('async-iterator', () => {
// Now test with the actual transformObject function to ensure it works with arrays
const actualArray = JSON.parse(JSON.stringify(arrayData))
const networkOptions = { ...DEFAULT_NETWORK_OPTIONS, throttleDelay: 10, concurrentTasks: 1 };
await transformObject(
actualArray,
transformer,
'$[*].[name,description]', // JSONPath expression targeting name and description in array items
10,
1,
networkOptions,
mockErrorCallback,
filter
)
@ -498,5 +501,45 @@ describe('async-iterator', () => {
expect(actualArray[0].id).toBe('1')
expect(actualArray[0].type).toBe('fresh')
})
it('should transform objects with the default filter callback', async () => {
const path = '$..description'
const testObj = JSON.parse(JSON.stringify(testData))
const networkOptions = {...DEFAULT_NETWORK_OPTIONS, throttleDelay: 10, concurrentTasks: 2}
await transformObject(
testObj,
mockTransform,
path,
networkOptions,
mockErrorCallback,
mockFilterCallback
)
expect(testObj.items[0].description).toBe('A FRUIT')
expect(testObj.items[1].description).toBe('YELLOW FRUIT')
expect(testObj.items[2].description).toBe('ORANGE FRUIT')
expect(testObj.metadata.description).toBe('COLLECTION OF FRUITS')
})
it('should use provided options over defaults', async () => {
const path = '$..description'
const testObj = JSON.parse(JSON.stringify(testData))
const networkOptions = {...DEFAULT_NETWORK_OPTIONS, throttleDelay: 50, concurrentTasks: 3}
await transformObject(
testObj,
mockTransform,
path,
networkOptions,
mockErrorCallback,
mockFilterCallback
)
expect(testObj.items[0].description).toBe('A FRUIT')
expect(testObj.items[1].description).toBe('YELLOW FRUIT')
expect(testObj.items[2].description).toBe('ORANGE FRUIT')
expect(testObj.metadata.description).toBe('COLLECTION OF FRUITS')
})
})
})