polymech/mono
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kbot image chat :)

Browse Source
This commit is contained in:
babayaga 2025-09-17 19:50:22 +02:00
parent 2c5bacfae0
commit b34d17191e
5 changed files with 203 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
packages/kbot/dist-in/commands/images.js
View File

File diff suppressed because one or more lines are too long

BIN
packages/kbot/dist/win-64/tauri-app.exe vendored
View File

Binary file not shown.

19
packages/kbot/gui/tauri-app/src-tauri/src/lib.rs
View File

@ -282,7 +282,8 @@ pub fn run() {
send_ipc_message,
request_config_from_images,
forward_config_to_frontend,
forward_image_to_frontend
forward_image_to_frontend,
generate_image_via_backend
])
.setup(|app| {
let app_handle = app.handle().clone();
@ -355,6 +356,22 @@ pub fn run() {
}
}
}
"generation_result" => {
eprintln!("[RUST LOG]: Forwarding generation result to frontend");
if let Err(e) = app_handle.emit("generation-result", &command) {
eprintln!("[RUST LOG]: Failed to emit generation-result: {}", e);
} else {
eprintln!("[RUST LOG]: Generation result emitted successfully");
}
}
"generation_error" => {
eprintln!("[RUST LOG]: Forwarding generation error to frontend");
if let Err(e) = app_handle.emit("generation-error", &command) {
eprintln!("[RUST LOG]: Failed to emit generation-error: {}", e);
} else {
eprintln!("[RUST LOG]: Generation error emitted successfully");
}
}
_ => {
eprintln!("[RUST LOG]: Unknown command: {}", cmd);
}

133
packages/kbot/gui/tauri-app/src/App.tsx
View File

@ -197,99 +197,42 @@ function App() {
const generateImage = async (promptText: string, includeImages: ImageFile[] = []) => {
if (!apiKey) {
console.error('No API key available for image generation');
addDebugMessage('error', 'No API key available for image generation');
return;
}
setIsGenerating(true);
addDebugMessage('info', `🎨 Starting image generation via backend: "${promptText}"`);
try {
console.log('Starting image generation...');
console.log('API key available:', !!apiKey);
console.log('Include images count:', includeImages.length);
// Use the images.ts backend instead of direct API calls
const filePaths = includeImages.map(img => img.path);
const genDst = dst || `generated_${Date.now()}.png`;
// Use Tauri's HTTP client directly instead of Google SDK (which has fetch issues in Tauri)
console.log('Using Tauri HTTP client for API calls...');
// Prepare the request payload for Google Gemini API
const parts: any[] = [];
if (includeImages.length > 0) {
// Add image parts for editing
for (const imageFile of includeImages) {
const base64Match = imageFile.src.match(/^data:([^;]+);base64,(.+)$/);
if (base64Match) {
const mimeType = base64Match[1];
const base64Data = base64Match[2];
parts.push({
inlineData: {
mimeType,
data: base64Data
}
});
}
}
}
// Add text prompt
parts.push({ text: promptText });
const requestBody = {
contents: [{
parts: parts
}]
};
console.log('Making API call with parts:', parts.length);
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-image-preview:generateContent?key=${apiKey}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(requestBody)
});
if (!response.ok) {
throw new Error(`API request failed: ${response.status} ${response.statusText}`);
}
const data = await response.json();
console.log('API call completed successfully');
// Extract generated image from response
const candidates = data.candidates;
if (candidates && candidates[0]?.content?.parts) {
for (const part of candidates[0].content.parts) {
if (part.inlineData) {
const generatedImage: GeneratedImage = {
id: Date.now().toString(),
src: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`,
addDebugMessage('info', 'Sending generation request to images.ts backend', {
prompt: promptText,
timestamp: Date.now(),
saved: false
};
files: filePaths,
dst: genDst
});
setGeneratedImages(prev => [...prev, generatedImage]);
console.log('Generated new image:', generatedImage.id);
return;
}
}
}
// Send generation request via Tauri command
await safeInvoke('generate_image_via_backend', {
prompt: promptText,
files: filePaths,
dst: genDst
});
addDebugMessage('info', '📤 Generation request sent to backend');
throw new Error('No image found in API response');
} catch (error) {
console.error('Image generation failed:', error);
const err: any = error;
const errorDetails = {
message: err?.message || 'Unknown error',
name: err?.name || 'Unknown',
stack: err?.stack || 'No stack trace',
toString: error?.toString() || 'No string representation'
};
console.error('Error details:', errorDetails);
safeInvoke('log_error_to_console', { error: `[Frontend Error] Image generation failed: ${JSON.stringify(errorDetails)}` });
} finally {
addDebugMessage('error', 'Failed to send generation request', {
error: error instanceof Error ? error.message : JSON.stringify(error)
});
setIsGenerating(false);
}
// Note: setIsGenerating(false) will be called when we receive the generated image
// or error response from the backend
};
// Theme management
@ -461,6 +404,34 @@ function App() {
}
});
// Listen for generation results
await listen('generation-result', (event: any) => {
const resultData = event.payload;
addDebugMessage('info', '🎨 Generation completed', resultData);
if (resultData.success && resultData.base64) {
const generatedImage: GeneratedImage = {
id: Date.now().toString(),
src: `data:image/png;base64,${resultData.base64}`,
prompt: resultData.prompt || 'Generated image',
timestamp: Date.now(),
saved: false
};
setGeneratedImages(prev => [...prev, generatedImage]);
addDebugMessage('info', '✅ Generated image added to gallery');
}
setIsGenerating(false);
});
// Listen for generation errors
await listen('generation-error', (event: any) => {
const errorData = event.payload;
addDebugMessage('error', '❌ Generation failed', errorData);
setIsGenerating(false);
});
addDebugMessage('info', 'Tauri event listeners set up');
// NOW request config from images.ts (after listeners are ready)

70
packages/kbot/src/commands/images.ts
View File

@ -14,7 +14,6 @@ import { variables } from '../variables.js';
import { resolve } from '@polymech/commons';
import { spawn } from 'node:child_process';
import { loadConfig } from '../config.js';
import { createIPCClient, IPCClient } from '../lib/ipc.js';
function getGuiAppPath(): string {
@ -113,7 +112,7 @@ async function launchGuiAndGetPrompt(argv: any): Promise<string | null> {
let output = '';
let errorOutput = '';
tauriProcess.stdout.on('data', (data) => {
tauriProcess.stdout.on('data', async (data) => {
const chunk = data.toString();
// Check for config requests from the GUI
@ -173,6 +172,73 @@ async function launchGuiAndGetPrompt(argv: any): Promise<string | null> {
logger.error(`Failed to send image: ${imagePath}`, error.message);
}
}
} else if (message.type === 'generate_request') {
logger.info('📨 Received generation request from GUI');
// Process the generation request using our existing image generation logic
try {
const genPrompt = message.prompt;
const genFiles = message.files || [];
const genDst = message.dst;
logger.info(`🎨 Starting image generation: "${genPrompt}"`);
let imageBuffer: Buffer | null = null;
if (genFiles.length > 0) {
// Image editing
logger.info(`Editing image(s) "${genFiles.join(', ')}" with prompt: "${genPrompt}"`);
const parsedOptions = ImageOptionsSchema().parse({
...argv,
prompt: genPrompt,
include: genFiles,
dst: genDst
});
imageBuffer = await editImage(genPrompt, genFiles, parsedOptions);
} else {
// Image creation
logger.info(`Creating image with prompt: "${genPrompt}"`);
const parsedOptions = ImageOptionsSchema().parse({
...argv,
prompt: genPrompt,
dst: genDst
});
imageBuffer = await createImage(genPrompt, parsedOptions);
}
if (imageBuffer) {
const base64Result = imageBuffer.toString('base64');
// Send the generated image back to the GUI
const resultResponse = {
cmd: 'forward_image_to_frontend',
base64: base64Result,
mimeType: 'image/png',
filename: path.basename(genDst)
};
tauriProcess.stdin?.write(JSON.stringify(resultResponse) + '\n');
logger.info(`✅ Generated image sent to GUI: ${genDst}`);
} else {
logger.error('❌ Failed to generate image');
// Send error back to GUI
const errorResponse = {
cmd: 'generation_error',
error: 'Failed to generate image'
};
tauriProcess.stdin?.write(JSON.stringify(errorResponse) + '\n');
}
} catch (error) {
logger.error('❌ Generation error:', error.message);
// Send error back to GUI
const errorResponse = {
cmd: 'generation_error',
error: error.message
};
tauriProcess.stdin?.write(JSON.stringify(errorResponse) + '\n');
}
}
} catch (e) {
// Not a JSON message, add to regular output