transcribe glob

packages/kbot/dist-in/commands/transcribe.d.ts

@@ -1,3 +1,4 @@
 import { IKBotTask } from '@polymech/ai-tools';
+export declare const default_sort: (files: string[]) => string[];
 export declare const TranscribeOptionsSchema: () => any;
 export declare const transcribeCommand: (opts: IKBotTask) => Promise<void>;

packages/kbot/dist-in/lib/transcribe.d.ts

@@ -1,2 +1,2 @@
 import { IKBotTask } from '@polymech/ai-tools';
-export declare const transcribe: (options: IKBotTask) => Promise<any>;
+export declare const transcribe: (options: IKBotTask) => Promise<string>;

packages/kbot/dist-in/lib/transcribe.js

@@ -1,7 +1,6 @@
 import * as fs from 'fs';
 import { toFile } from "openai";
 import { sync as exists } from '@polymech/fs/exists';
-import { sync as write } from '@polymech/fs/write';
 import { createClient } from '../client.js';
 const createBuffer = (path) => {
     try {
@@ -17,21 +16,21 @@ export const transcribe = async (options) => {
     const client = createClient(options);
     if (!client) {
         options.logger.error('Failed to create client');
-        return;
+        return '';
     }
     if (!options.include || options.include.length === 0) {
         options.logger.error('No source file provided via --include');
-        return;
+        return '';
     }
     const sourceFile = options.include[0];
     if (!exists(sourceFile)) {
         options.logger.error('Source file does not exist', sourceFile);
-        return;
+        return '';
     }
     const file = await toFile(createBuffer(sourceFile), 'audio.mp3', { type: 'audio/mpeg' });
     if (!file) {
         options.logger.error('Error converting source to file');
-        return;
+        return '';
     }
     const completion = await client.audio.transcriptions.create({
         model: 'whisper-1',
@@ -40,16 +39,9 @@ export const transcribe = async (options) => {
     });
     if (!completion) {
         options.logger.error('OpenAI response is empty');
-        return;
+        return '';
     }
     const text_content = completion.text || '';
-    if (options.dst) {
-        write(options.dst, text_content);
-    }
-    else {
-        process.stdout.write(text_content);
-    }
-    // options.logger.debug('OpenAI Transcribe response:', completion)
-    return completion;
+    return text_content;
 };
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHJhbnNjcmliZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9saWIvdHJhbnNjcmliZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLElBQUksQ0FBQTtBQUN4QixPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sUUFBUSxDQUFBO0FBQy9CLE9BQU8sRUFBRSxJQUFJLElBQUksTUFBTSxFQUFFLE1BQU0scUJBQXFCLENBQUE7QUFDcEQsT0FBTyxFQUFFLElBQUksSUFBSSxLQUFLLEVBQUUsTUFBTSxvQkFBb0IsQ0FBQTtBQUVsRCxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sY0FBYyxDQUFBO0FBRTNDLE1BQU0sWUFBWSxHQUFHLENBQUMsSUFBWSxFQUFpQixFQUFFO0lBQ2pELElBQUksQ0FBQztRQUNELE1BQU0sTUFBTSxHQUFHLEVBQUUsQ0FBQyxZQUFZLENBQUMsSUFBSSxDQUFDLENBQUE7UUFDcEMsT0FBTyxNQUFNLENBQUM7SUFDbEIsQ0FBQztJQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7UUFDYixPQUFPLENBQUMsS0FBSyxDQUFDLHdCQUF3QixFQUFFLEtBQUssQ0FBQyxDQUFDO1FBQy9DLE9BQU8sSUFBSSxDQUFDO0lBQ2hCLENBQUM7QUFDTCxDQUFDLENBQUE7QUFFRCxNQUFNLENBQUMsTUFBTSxVQUFVLEdBQUcsS0FBSyxFQUFFLE9BQWtCLEVBQUUsRUFBRTtJQUNuRCxNQUFNLE1BQU0sR0FBRyxZQUFZLENBQUMsT0FBTyxDQUFDLENBQUE7SUFDcEMsSUFBSSxDQUFDLE1BQU0sRUFBRSxDQUFDO1FBQ1YsT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMseUJBQXlCLENBQUMsQ0FBQTtRQUMvQyxPQUFNO0lBQ1YsQ0FBQztJQUVELElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsTUFBTSxLQUFLLENBQUMsRUFBRSxDQUFDO1FBQ25ELE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLHVDQUF1QyxDQUFDLENBQUE7UUFDN0QsT0FBTztJQUNYLENBQUM7SUFFRCxNQUFNLFVBQVUsR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFDO0lBRXRDLElBQUksQ0FBQyxNQUFNLENBQUMsVUFBVSxDQUFDLEVBQUUsQ0FBQztRQUN0QixPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyw0QkFBNEIsRUFBRSxVQUFVLENBQUMsQ0FBQTtRQUM5RCxPQUFPO0lBQ1gsQ0FBQztJQUVELE1BQU0sSUFBSSxHQUFHLE1BQU0sTUFBTSxDQUFDLFlBQVksQ0FBQyxVQUFVLENBQUMsRUFBRSxXQUFXLEVBQUUsRUFBRSxJQUFJLEVBQUUsWUFBWSxFQUFFLENBQUMsQ0FBQztJQUN6RixJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDUixPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxpQ0FBaUMsQ0FBQyxDQUFBO1FBQ3ZELE9BQU87SUFDWCxDQUFDO0lBRUQsTUFBTSxVQUFVLEdBQVEsTUFBTSxNQUFNLENBQUMsS0FBSyxDQUFDLGNBQWMsQ0FBQyxNQUFNLENBQUM7UUFDN0QsS0FBSyxFQUFFLFdBQVc7UUFDbEIsSUFBSSxFQUFFLElBQUk7UUFDVixlQUFlLEVBQUcsT0FBZSxDQUFDLGVBQWUsSUFBSSxjQUFjO0tBQ3RFLENBQUMsQ0FBQTtJQUVGLElBQUksQ0FBQyxVQUFVLEVBQUUsQ0FBQztRQUNkLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLDBCQUEwQixDQUFDLENBQUE7UUFDaEQsT0FBTztJQUNYLENBQUM7SUFFRCxNQUFNLFlBQVksR0FBRyxVQUFVLENBQUMsSUFBSSxJQUFJLEVBQUUsQ0FBQztJQUUzQyxJQUFJLE9BQU8sQ0FBQyxHQUFHLEVBQUUsQ0FBQztRQUNkLEtBQUssQ0FBQyxPQUFPLENBQUMsR0FBRyxFQUFFLFlBQVksQ0FBQyxDQUFBO0lBQ3BDLENBQUM7U0FBTSxDQUFDO1FBQ0osT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsWUFBWSxDQUFDLENBQUE7SUFDdEMsQ0FBQztJQUVELGtFQUFrRTtJQUNsRSxPQUFPLFVBQVUsQ0FBQTtBQUNyQixDQUFDLENBQUEifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHJhbnNjcmliZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9saWIvdHJhbnNjcmliZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxNQUFNLElBQUksQ0FBQTtBQUN4QixPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sUUFBUSxDQUFBO0FBQy9CLE9BQU8sRUFBRSxJQUFJLElBQUksTUFBTSxFQUFFLE1BQU0scUJBQXFCLENBQUE7QUFHcEQsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLGNBQWMsQ0FBQTtBQUUzQyxNQUFNLFlBQVksR0FBRyxDQUFDLElBQVksRUFBaUIsRUFBRTtJQUNqRCxJQUFJLENBQUM7UUFDRCxNQUFNLE1BQU0sR0FBRyxFQUFFLENBQUMsWUFBWSxDQUFDLElBQUksQ0FBQyxDQUFBO1FBQ3BDLE9BQU8sTUFBTSxDQUFDO0lBQ2xCLENBQUM7SUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1FBQ2IsT0FBTyxDQUFDLEtBQUssQ0FBQyx3QkFBd0IsRUFBRSxLQUFLLENBQUMsQ0FBQztRQUMvQyxPQUFPLElBQUksQ0FBQztJQUNoQixDQUFDO0FBQ0wsQ0FBQyxDQUFBO0FBRUQsTUFBTSxDQUFDLE1BQU0sVUFBVSxHQUFHLEtBQUssRUFBRSxPQUFrQixFQUFtQixFQUFFO0lBQ3BFLE1BQU0sTUFBTSxHQUFHLFlBQVksQ0FBQyxPQUFPLENBQUMsQ0FBQTtJQUNwQyxJQUFJLENBQUMsTUFBTSxFQUFFLENBQUM7UUFDVixPQUFPLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyx5QkFBeUIsQ0FBQyxDQUFBO1FBQy9DLE9BQU8sRUFBRSxDQUFBO0lBQ2IsQ0FBQztJQUVELElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsTUFBTSxLQUFLLENBQUMsRUFBRSxDQUFDO1FBQ25ELE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLHVDQUF1QyxDQUFDLENBQUE7UUFDN0QsT0FBTyxFQUFFLENBQUM7SUFDZCxDQUFDO0lBRUQsTUFBTSxVQUFVLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUV0QyxJQUFJLENBQUMsTUFBTSxDQUFDLFVBQVUsQ0FBQyxFQUFFLENBQUM7UUFDdEIsT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsNEJBQTRCLEVBQUUsVUFBVSxDQUFDLENBQUE7UUFDOUQsT0FBTyxFQUFFLENBQUM7SUFDZCxDQUFDO0lBRUQsTUFBTSxJQUFJLEdBQUcsTUFBTSxNQUFNLENBQUMsWUFBWSxDQUFDLFVBQVUsQ0FBQyxFQUFFLFdBQVcsRUFBRSxFQUFFLElBQUksRUFBRSxZQUFZLEVBQUUsQ0FBQyxDQUFDO0lBQ3pGLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUNSLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLGlDQUFpQyxDQUFDLENBQUE7UUFDdkQsT0FBTyxFQUFFLENBQUM7SUFDZCxDQUFDO0lBRUQsTUFBTSxVQUFVLEdBQVEsTUFBTSxNQUFNLENBQUMsS0FBSyxDQUFDLGNBQWMsQ0FBQyxNQUFNLENBQUM7UUFDN0QsS0FBSyxFQUFFLFdBQVc7UUFDbEIsSUFBSSxFQUFFLElBQUk7UUFDVixlQUFlLEVBQUcsT0FBZSxDQUFDLGVBQWUsSUFBSSxjQUFjO0tBQ3RFLENBQUMsQ0FBQTtJQUVGLElBQUksQ0FBQyxVQUFVLEVBQUUsQ0FBQztRQUNkLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLDBCQUEwQixDQUFDLENBQUE7UUFDaEQsT0FBTyxFQUFFLENBQUM7SUFDZCxDQUFDO0lBRUQsTUFBTSxZQUFZLEdBQUcsVUFBVSxDQUFDLElBQUksSUFBSSxFQUFFLENBQUE7SUFDMUMsT0FBTyxZQUFZLENBQUE7QUFDdkIsQ0FBQyxDQUFBIn0=

packages/kbot/src/commands/transcribe.ts

@@ -1,5 +1,7 @@
 import * as path from 'node:path'
+import * as fs from 'node:fs'
 import { isString, isArray } from '@polymech/core/primitives'
+import pMap from 'p-map'
 import { hasMagic } from 'glob'
 import { sync as exists } from '@polymech/fs/exists'
 import { forward_slash, resolve, pathInfoEx } from '@polymech/commons'
@@ -8,10 +10,29 @@ import { IKBotTask } from '@polymech/ai-tools'
 import { OptionsSchema } from '../zod_schema.js'
 import { transcribe } from '../lib/transcribe.js'
 import { isWebUrl } from '../glob.js'
-import { default_sort } from './run.js'
+
 import { getLogger } from '../index.js'
 import { variables } from '../variables.js'
 
+export const default_sort = (files: string[]): string[] => {
+    const getSortableParts = (filename: string) => {
+        const baseName = path.parse(filename).name;
+        const match = baseName.match(/^(\d+)_?(.*)$/); // Match leading numbers
+        const numPart = match ? parseInt(match[1], 10) : NaN;
+        const textPart = match ? match[2] : baseName; // Extract text part
+  
+        return { numPart, textPart };
+    }
+    return files.sort((a, b) => {
+        const { numPart: aNum, textPart: aText } = getSortableParts(a)
+        const { numPart: bNum, textPart: bText } = getSortableParts(b)
+        if (!isNaN(aNum) && !isNaN(bNum)) {
+            return aNum - bNum || aText.localeCompare(bText, undefined, { numeric: true, sensitivity: 'base' })
+        }
+        return aText.localeCompare(bText, undefined, { numeric: true, sensitivity: 'base' })
+    })
+  }
+
 export const TranscribeOptionsSchema = () => {
     return OptionsSchema().pick({
         include: true,
@@ -89,7 +110,7 @@ export const transcribeCommand = async (opts: IKBotTask) => {
             const info = pathInfoEx(forward_slash(path.resolve(resolve(includePath))), false, {
                 absolute: true,
             })
-            files.push(...default_sort(info.FILES))
+            files.push(...info.FILES)
         } else if (exists(includePath)) {
             files.push(includePath)
         }
@@ -100,9 +121,11 @@ export const transcribeCommand = async (opts: IKBotTask) => {
         return
     }
 
+    files = default_sort(files)
+
     opts.logger.info(`Found ${files.length} files to transcribe.`)
 
-    for (const file of files) {
+    const mapper = async (file: string) => {
         const fileInfo = path.parse(file)
         const CWD = process.cwd()
         const current_variables = {
@@ -120,18 +143,32 @@ export const transcribeCommand = async (opts: IKBotTask) => {
             include: [file],
             variables: current_variables
         };
-
-        if (!itemOpts.dst) {
-            itemOpts.dst = '${SRC_DIR}/${SRC_NAME}.md';
-        }
-        itemOpts.dst = path.resolve(resolve(itemOpts.dst, itemOpts.alt, itemOpts.variables))
-
-
         opts.logger.info(`Transcribing ${file}...`)
-        if(itemOpts.dst) {
-            opts.logger.info(`Output will be saved to ${itemOpts.dst}`)
-        }
+        const transcribedText = await transcribe(itemOpts)
+        return { transcribedText, itemOpts }
+    };
 
-        await transcribe(itemOpts)
+    const results = await pMap(files, mapper, { concurrency: 1 });
+
+    let resolvedDstPath: string | undefined;
+    if (opts.dst) {
+        resolvedDstPath = path.resolve(resolve(opts.dst, opts.alt, opts.variables));
+        if (fs.existsSync(resolvedDstPath)) {
+            fs.unlinkSync(resolvedDstPath);
+        }
+        const allText = results.map(r => r.transcribedText).filter(Boolean).join('\n\n')
+        if (allText) {
+            fs.writeFileSync(resolvedDstPath, allText + '\n');
+            opts.logger.info(`Wrote all transcriptions to ${resolvedDstPath}`);
+        }
+    } else {
+        for (const { transcribedText, itemOpts } of results) {
+            if (transcribedText) {
+                const defaultDstTemplate = '${SRC_DIR}/${SRC_NAME}.md';
+                const defaultDstPath = path.resolve(resolve(defaultDstTemplate, itemOpts.alt, itemOpts.variables));
+                fs.writeFileSync(defaultDstPath, transcribedText);
+                opts.logger.info(`Output will be saved to ${defaultDstPath}`);
+            }
+        }
     }
 }

packages/kbot/src/lib/transcribe.ts

@@ -15,29 +15,29 @@ const createBuffer = (path: string): Buffer | null => {
     }
 }
 
-export const transcribe = async (options: IKBotTask) => {
+export const transcribe = async (options: IKBotTask): Promise<string> => {
     const client = createClient(options)
     if (!client) {
         options.logger.error('Failed to create client')
-        return
+        return ''
     }
 
     if (!options.include || options.include.length === 0) {
         options.logger.error('No source file provided via --include')
-        return;
+        return '';
     }
 
     const sourceFile = options.include[0];
 
     if (!exists(sourceFile)) {
         options.logger.error('Source file does not exist', sourceFile)
-        return;
+        return '';
     }
 
     const file = await toFile(createBuffer(sourceFile), 'audio.mp3', { type: 'audio/mpeg' });
     if (!file) {
         options.logger.error('Error converting source to file')
-        return;
+        return '';
     }
 
     const completion: any = await client.audio.transcriptions.create({
@@ -48,17 +48,9 @@ export const transcribe = async (options: IKBotTask) => {
 
     if (!completion) {
         options.logger.error('OpenAI response is empty')
-        return;
+        return '';
     }
 
-    const text_content = completion.text || '';
-
-    if (options.dst) {
-        write(options.dst, text_content)
-    } else {
-        process.stdout.write(text_content)
-    }
-
-    // options.logger.debug('OpenAI Transcribe response:', completion)
-    return completion
+    const text_content = completion.text || ''
+    return text_content
 }

packages/kbot/tests/unit/transcribe/sequence.md

@@ -0,0 +1,5 @@
+Sequence one started now.
+
+Sequence 2 started now.
+
+The result is 100.

packages/kbot/tests/unit/transcribe/test.md

@@ -1 +0,0 @@
-The lazy fox jumps over the cat.

packages/kbot/tests/unit/transcribe/transcribe.test.ts

@@ -9,19 +9,26 @@ import { IKBotTask } from '@polymech/ai-tools'
 
 const TEST_DATA_DIR = './tests/unit/transcribe'
 const TEST_MP3 = path.join(TEST_DATA_DIR, 'test.mp3')
-const TEST_TIMEOUT = 30000 // 30 seconds
+const TEST_TIMEOUT = 60000 // Increased timeout for multiple files
 
 describe('Transcribe Command', () => {
 
     const defaultOutputFile = path.resolve(path.join(TEST_DATA_DIR, 'test.md'))
+    const sequenceOutputFile = path.resolve(path.join(TEST_DATA_DIR, 'sequence.md'))
 
-    beforeAll(() => {
+    const cleanupFiles = () => {
         if (fs.existsSync(defaultOutputFile)) {
             fs.unlinkSync(defaultOutputFile)
         }
-    })
+        if (fs.existsSync(sequenceOutputFile)) {
+         //   fs.unlinkSync(sequenceOutputFile)
+        }
+    }
 
-    it('should transcribe an audio file and save the output to a default markdown file', async () => {
+    beforeAll(cleanupFiles)
+    afterAll(cleanupFiles)
+
+    it('should transcribe a single audio file and save the output to a default markdown file', async () => {
         const options: IKBotTask = {
             include: [TEST_MP3],
             router: 'openai',
@@ -40,4 +47,26 @@ describe('Transcribe Command', () => {
         expect(lowerCaseResult).toContain("cat")
 
     }, TEST_TIMEOUT)
+
+    it('should transcribe multiple audio files from a glob pattern and append to a single destination file', async () => {
+        const options: IKBotTask = {
+            include: [`${TEST_DATA_DIR}/Saturday*.mp3`],
+            dst: sequenceOutputFile,
+            router: 'openai',
+            logLevel: 2,
+        }
+
+        await transcribeCommand(options);
+
+        expect(exists(sequenceOutputFile)).toBe('file');
+
+        const result = read(sequenceOutputFile, 'text') as string;
+        expect(result).toBeDefined();
+
+        const lowerCaseResult = result.toLowerCase();
+        expect(lowerCaseResult).toContain('one');
+        expect(lowerCaseResult).toMatch(/two|2/);
+        expect(lowerCaseResult).toMatch(/hundred|100/);
+
+    }, TEST_TIMEOUT);
 })