import type { Base64ImageSource, ImageBlockParam, } from '@anthropic-ai/sdk/resources/messages.mjs' import { API_IMAGE_MAX_BASE64_SIZE, IMAGE_MAX_HEIGHT, IMAGE_MAX_WIDTH, IMAGE_TARGET_RAW_SIZE, } from '../constants/apiLimits.js' import { logEvent } from '../services/analytics/index.js' import { getImageProcessor, type SharpFunction, type SharpInstance, } from '../tools/FileReadTool/imageProcessor.js' import { logForDebugging } from './debug.js' import { errorMessage } from './errors.js' import { formatFileSize } from './format.js' import { logError } from './log.js' type ImageMediaType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp' // Error type constants for analytics (numeric to comply with logEvent restrictions) const ERROR_TYPE_MODULE_LOAD = 1 const ERROR_TYPE_PROCESSING = 2 const ERROR_TYPE_UNKNOWN = 3 const ERROR_TYPE_PIXEL_LIMIT = 4 const ERROR_TYPE_MEMORY = 5 const ERROR_TYPE_TIMEOUT = 6 const ERROR_TYPE_VIPS = 7 const ERROR_TYPE_PERMISSION = 8 /** * Error thrown when image resizing fails and the image exceeds the API limit. */ export class ImageResizeError extends Error { constructor(message: string) { super(message) this.name = 'ImageResizeError' } } /** * Classifies image processing errors for analytics. * * Uses error codes when available (Node.js module errors), falls back to * message matching for libraries like sharp that don't expose error codes. */ function classifyImageError(error: unknown): number { // Check for Node.js error codes first (more reliable than string matching) if (error instanceof Error) { const errorWithCode = error as Error & { code?: string } if ( errorWithCode.code === 'MODULE_NOT_FOUND' || errorWithCode.code === 'ERR_MODULE_NOT_FOUND' || errorWithCode.code === 'ERR_DLOPEN_FAILED' ) { return ERROR_TYPE_MODULE_LOAD } if (errorWithCode.code === 'EACCES' || errorWithCode.code === 'EPERM') { return ERROR_TYPE_PERMISSION } if (errorWithCode.code === 'ENOMEM') { return ERROR_TYPE_MEMORY } } // Fall back to message matching for errors without codes // Note: sharp doesn't expose error codes, so we must match on messages const message = errorMessage(error) // Module loading errors from our native wrapper if (message.includes('Native image processor module not available')) { return ERROR_TYPE_MODULE_LOAD } // Sharp/vips processing errors (format detection, corrupt data, etc.) if ( message.includes('unsupported image format') || message.includes('Input buffer') || message.includes('Input file is missing') || message.includes('Input file has corrupt header') || message.includes('corrupt header') || message.includes('corrupt image') || message.includes('premature end') || message.includes('zlib: data error') || message.includes('zero width') || message.includes('zero height') ) { return ERROR_TYPE_PROCESSING } // Pixel/dimension limit errors from sharp/vips if ( message.includes('pixel limit') || message.includes('too many pixels') || message.includes('exceeds pixel') || message.includes('image dimensions') ) { return ERROR_TYPE_PIXEL_LIMIT } // Memory allocation failures if ( message.includes('out of memory') || message.includes('Cannot allocate') || message.includes('memory allocation') ) { return ERROR_TYPE_MEMORY } // Timeout errors if (message.includes('timeout') || message.includes('timed out')) { return ERROR_TYPE_TIMEOUT } // Vips-specific errors (VipsJpeg, VipsPng, VipsWebp, etc.) if (message.includes('Vips')) { return ERROR_TYPE_VIPS } return ERROR_TYPE_UNKNOWN } /** * Computes a simple numeric hash of a string for analytics grouping. * Uses djb2 algorithm, returning a 32-bit unsigned integer. */ function hashString(str: string): number { let hash = 5381 for (let i = 0; i < str.length; i++) { hash = ((hash << 5) + hash + str.charCodeAt(i)) | 0 } return hash >>> 0 } export type ImageDimensions = { originalWidth?: number originalHeight?: number displayWidth?: number displayHeight?: number } export interface ResizeResult { buffer: Buffer mediaType: string dimensions?: ImageDimensions } interface ImageCompressionContext { imageBuffer: Buffer metadata: { width?: number; height?: number; format?: string } format: string maxBytes: number originalSize: number } interface CompressedImageResult { base64: string mediaType: Base64ImageSource['media_type'] originalSize: number } /** * Extracted from FileReadTool's readImage function * Resizes image buffer to meet size and dimension constraints */ export async function maybeResizeAndDownsampleImageBuffer( imageBuffer: Buffer, originalSize: number, ext: string, ): Promise { if (imageBuffer.length === 0) { // Empty buffer would fall through the catch block below (sharp throws // "Unable to determine image format"), and the fallback's size check // `0 ≤ 5MB` would pass it through, yielding an empty base64 string // that the API rejects with `image cannot be empty`. throw new ImageResizeError('Image file is empty (0 bytes)') } try { const sharp = await getImageProcessor() const image = sharp(imageBuffer) const metadata = await image.metadata() const mediaType = metadata.format ?? ext // Normalize "jpg" to "jpeg" for media type compatibility const normalizedMediaType = mediaType === 'jpg' ? 'jpeg' : mediaType // If dimensions aren't available from metadata if (!metadata.width || !metadata.height) { if (originalSize > IMAGE_TARGET_RAW_SIZE) { // Create fresh sharp instance for compression const compressedBuffer = await sharp(imageBuffer) .jpeg({ quality: 80 }) .toBuffer() return { buffer: compressedBuffer, mediaType: 'jpeg' } } // Return without dimensions if we can't determine them return { buffer: imageBuffer, mediaType: normalizedMediaType } } // Store original dimensions (guaranteed to be defined here) const originalWidth = metadata.width const originalHeight = metadata.height // Calculate dimensions while maintaining aspect ratio let width = originalWidth let height = originalHeight // Check if the original file just works if ( originalSize <= IMAGE_TARGET_RAW_SIZE && width <= IMAGE_MAX_WIDTH && height <= IMAGE_MAX_HEIGHT ) { return { buffer: imageBuffer, mediaType: normalizedMediaType, dimensions: { originalWidth, originalHeight, displayWidth: width, displayHeight: height, }, } } const needsDimensionResize = width > IMAGE_MAX_WIDTH || height > IMAGE_MAX_HEIGHT const isPng = normalizedMediaType === 'png' // If dimensions are within limits but file is too large, try compression first // This preserves full resolution when possible if (!needsDimensionResize && originalSize > IMAGE_TARGET_RAW_SIZE) { // For PNGs, try PNG compression first to preserve transparency if (isPng) { // Create fresh sharp instance for each compression attempt const pngCompressed = await sharp(imageBuffer) .png({ compressionLevel: 9, palette: true }) .toBuffer() if (pngCompressed.length <= IMAGE_TARGET_RAW_SIZE) { return { buffer: pngCompressed, mediaType: 'png', dimensions: { originalWidth, originalHeight, displayWidth: width, displayHeight: height, }, } } } // Try JPEG compression (lossy but much smaller) for (const quality of [80, 60, 40, 20]) { // Create fresh sharp instance for each attempt const compressedBuffer = await sharp(imageBuffer) .jpeg({ quality }) .toBuffer() if (compressedBuffer.length <= IMAGE_TARGET_RAW_SIZE) { return { buffer: compressedBuffer, mediaType: 'jpeg', dimensions: { originalWidth, originalHeight, displayWidth: width, displayHeight: height, }, } } } // Quality reduction alone wasn't enough, fall through to resize } // Constrain dimensions if needed if (width > IMAGE_MAX_WIDTH) { height = Math.round((height * IMAGE_MAX_WIDTH) / width) width = IMAGE_MAX_WIDTH } if (height > IMAGE_MAX_HEIGHT) { width = Math.round((width * IMAGE_MAX_HEIGHT) / height) height = IMAGE_MAX_HEIGHT } // IMPORTANT: Always create fresh sharp(imageBuffer) instances for each operation. // The native image-processor-napi module doesn't properly apply format conversions // when reusing a sharp instance after calling toBuffer(). This caused a bug where // all compression attempts (PNG, JPEG at various qualities) returned identical sizes. logForDebugging(`Resizing to ${width}x${height}`) const resizedImageBuffer = await sharp(imageBuffer) .resize(width, height, { fit: 'inside', withoutEnlargement: true, }) .toBuffer() // If still too large after resize, try compression if (resizedImageBuffer.length > IMAGE_TARGET_RAW_SIZE) { // For PNGs, try PNG compression first to preserve transparency if (isPng) { const pngCompressed = await sharp(imageBuffer) .resize(width, height, { fit: 'inside', withoutEnlargement: true, }) .png({ compressionLevel: 9, palette: true }) .toBuffer() if (pngCompressed.length <= IMAGE_TARGET_RAW_SIZE) { return { buffer: pngCompressed, mediaType: 'png', dimensions: { originalWidth, originalHeight, displayWidth: width, displayHeight: height, }, } } } // Try JPEG with progressively lower quality for (const quality of [80, 60, 40, 20]) { const compressedBuffer = await sharp(imageBuffer) .resize(width, height, { fit: 'inside', withoutEnlargement: true, }) .jpeg({ quality }) .toBuffer() if (compressedBuffer.length <= IMAGE_TARGET_RAW_SIZE) { return { buffer: compressedBuffer, mediaType: 'jpeg', dimensions: { originalWidth, originalHeight, displayWidth: width, displayHeight: height, }, } } } // If still too large, resize smaller and compress aggressively const smallerWidth = Math.min(width, 1000) const smallerHeight = Math.round( (height * smallerWidth) / Math.max(width, 1), ) logForDebugging('Still too large, compressing with JPEG') const compressedBuffer = await sharp(imageBuffer) .resize(smallerWidth, smallerHeight, { fit: 'inside', withoutEnlargement: true, }) .jpeg({ quality: 20 }) .toBuffer() logForDebugging(`JPEG compressed buffer size: ${compressedBuffer.length}`) return { buffer: compressedBuffer, mediaType: 'jpeg', dimensions: { originalWidth, originalHeight, displayWidth: smallerWidth, displayHeight: smallerHeight, }, } } return { buffer: resizedImageBuffer, mediaType: normalizedMediaType, dimensions: { originalWidth, originalHeight, displayWidth: width, displayHeight: height, }, } } catch (error) { // Log the error and emit analytics event logError(error as Error) const errorType = classifyImageError(error) const errorMsg = errorMessage(error) logEvent('tengu_image_resize_failed', { original_size_bytes: originalSize, error_type: errorType, error_message_hash: hashString(errorMsg), }) // Detect actual format from magic bytes instead of trusting extension const detected = detectImageFormatFromBuffer(imageBuffer) const normalizedExt = detected.slice(6) // Remove 'image/' prefix // Calculate the base64 size (API limit is on base64-encoded length) const base64Size = Math.ceil((originalSize * 4) / 3) // Size-under-5MB does not imply dimensions-under-cap. Don't return the // raw buffer if the PNG header says it's oversized — fall through to // ImageResizeError instead. PNG sig is 8 bytes, IHDR dims at 16-24. const overDim = imageBuffer.length >= 24 && imageBuffer[0] === 0x89 && imageBuffer[1] === 0x50 && imageBuffer[2] === 0x4e && imageBuffer[3] === 0x47 && (imageBuffer.readUInt32BE(16) > IMAGE_MAX_WIDTH || imageBuffer.readUInt32BE(20) > IMAGE_MAX_HEIGHT) // If original image's base64 encoding is within API limit, allow it through uncompressed if (base64Size <= API_IMAGE_MAX_BASE64_SIZE && !overDim) { logEvent('tengu_image_resize_fallback', { original_size_bytes: originalSize, base64_size_bytes: base64Size, error_type: errorType, }) return { buffer: imageBuffer, mediaType: normalizedExt } } // Image is too large and we failed to compress it - fail with user-friendly error throw new ImageResizeError( overDim ? `Unable to resize image — dimensions exceed the ${IMAGE_MAX_WIDTH}x${IMAGE_MAX_HEIGHT}px limit and image processing failed. ` + `Please resize the image to reduce its pixel dimensions.` : `Unable to resize image (${formatFileSize(originalSize)} raw, ${formatFileSize(base64Size)} base64). ` + `The image exceeds the 5MB API limit and compression failed. ` + `Please resize the image manually or use a smaller image.`, ) } } export interface ImageBlockWithDimensions { block: ImageBlockParam dimensions?: ImageDimensions } /** * Resizes an image content block if needed * Takes an image ImageBlockParam and returns a resized version if necessary * Also returns dimension information for coordinate mapping */ export async function maybeResizeAndDownsampleImageBlock( imageBlock: ImageBlockParam, ): Promise { // Only process base64 images if (imageBlock.source.type !== 'base64') { return { block: imageBlock } } // Decode base64 to buffer const imageBuffer = Buffer.from(imageBlock.source.data, 'base64') const originalSize = imageBuffer.length // Extract extension from media type const mediaType = imageBlock.source.media_type const ext = mediaType?.split('/')[1] || 'png' // Resize if needed const resized = await maybeResizeAndDownsampleImageBuffer( imageBuffer, originalSize, ext, ) // Return resized image block with dimension info return { block: { type: 'image', source: { type: 'base64', media_type: `image/${resized.mediaType}` as Base64ImageSource['media_type'], data: resized.buffer.toString('base64'), }, }, dimensions: resized.dimensions, } } /** * Compresses an image buffer to fit within a maximum byte size. * * Uses a multi-strategy fallback approach because simple compression often fails for * large screenshots, high-resolution photos, or images with complex gradients. Each * strategy is progressively more aggressive to handle edge cases where earlier * strategies produce files still exceeding the size limit. * * Strategy (from FileReadTool): * 1. Try to preserve original format (PNG, JPEG, WebP) with progressive resizing * 2. For PNG: Use palette optimization and color reduction if needed * 3. Last resort: Convert to JPEG with aggressive compression * * This ensures images fit within context windows while maintaining format when possible. */ export async function compressImageBuffer( imageBuffer: Buffer, maxBytes: number = IMAGE_TARGET_RAW_SIZE, originalMediaType?: string, ): Promise { // Extract format from originalMediaType if provided (e.g., "image/png" -> "png") const fallbackFormat = originalMediaType?.split('/')[1] || 'jpeg' const normalizedFallback = fallbackFormat === 'jpg' ? 'jpeg' : fallbackFormat try { const sharp = await getImageProcessor() const metadata = await sharp(imageBuffer).metadata() const format = metadata.format || normalizedFallback const originalSize = imageBuffer.length const context: ImageCompressionContext = { imageBuffer, metadata, format, maxBytes, originalSize, } // If image is already within size limit, return as-is without processing if (originalSize <= maxBytes) { return createCompressedImageResult(imageBuffer, format, originalSize) } // Try progressive resizing with format preservation const resizedResult = await tryProgressiveResizing(context, sharp) if (resizedResult) { return resizedResult } // For PNG, try palette optimization if (format === 'png') { const palettizedResult = await tryPalettePNG(context, sharp) if (palettizedResult) { return palettizedResult } } // Try JPEG conversion with moderate compression const jpegResult = await tryJPEGConversion(context, 50, sharp) if (jpegResult) { return jpegResult } // Last resort: ultra-compressed JPEG return await createUltraCompressedJPEG(context, sharp) } catch (error) { // Log the error and emit analytics event logError(error as Error) const errorType = classifyImageError(error) const errorMsg = errorMessage(error) logEvent('tengu_image_compress_failed', { original_size_bytes: imageBuffer.length, max_bytes: maxBytes, error_type: errorType, error_message_hash: hashString(errorMsg), }) // If original image is within the requested limit, allow it through if (imageBuffer.length <= maxBytes) { // Detect actual format from magic bytes instead of trusting the provided media type const detected = detectImageFormatFromBuffer(imageBuffer) return { base64: imageBuffer.toString('base64'), mediaType: detected, originalSize: imageBuffer.length, } } // Image is too large and compression failed - throw error throw new ImageResizeError( `Unable to compress image (${formatFileSize(imageBuffer.length)}) to fit within ${formatFileSize(maxBytes)}. ` + `Please use a smaller image.`, ) } } /** * Compresses an image buffer to fit within a token limit. * Converts tokens to bytes using the formula: maxBytes = (maxTokens / 0.125) * 0.75 */ export async function compressImageBufferWithTokenLimit( imageBuffer: Buffer, maxTokens: number, originalMediaType?: string, ): Promise { // Convert token limit to byte limit // base64 uses about 4/3 the original size, so we reverse this const maxBase64Chars = Math.floor(maxTokens / 0.125) const maxBytes = Math.floor(maxBase64Chars * 0.75) return compressImageBuffer(imageBuffer, maxBytes, originalMediaType) } /** * Compresses an image block to fit within a maximum byte size. * Wrapper around compressImageBuffer for ImageBlockParam. */ export async function compressImageBlock( imageBlock: ImageBlockParam, maxBytes: number = IMAGE_TARGET_RAW_SIZE, ): Promise { // Only process base64 images if (imageBlock.source.type !== 'base64') { return imageBlock } // Decode base64 to buffer const imageBuffer = Buffer.from(imageBlock.source.data, 'base64') // Check if already within size limit if (imageBuffer.length <= maxBytes) { return imageBlock } // Compress the image const compressed = await compressImageBuffer(imageBuffer, maxBytes) return { type: 'image', source: { type: 'base64', media_type: compressed.mediaType, data: compressed.base64, }, } } // Helper functions for compression pipeline function createCompressedImageResult( buffer: Buffer, mediaType: string, originalSize: number, ): CompressedImageResult { const normalizedMediaType = mediaType === 'jpg' ? 'jpeg' : mediaType return { base64: buffer.toString('base64'), mediaType: `image/${normalizedMediaType}` as Base64ImageSource['media_type'], originalSize, } } async function tryProgressiveResizing( context: ImageCompressionContext, sharp: SharpFunction, ): Promise { const scalingFactors = [1.0, 0.75, 0.5, 0.25] for (const scalingFactor of scalingFactors) { const newWidth = Math.round( (context.metadata.width || 2000) * scalingFactor, ) const newHeight = Math.round( (context.metadata.height || 2000) * scalingFactor, ) let resizedImage = sharp(context.imageBuffer).resize(newWidth, newHeight, { fit: 'inside', withoutEnlargement: true, }) // Apply format-specific optimizations resizedImage = applyFormatOptimizations(resizedImage, context.format) const resizedBuffer = await resizedImage.toBuffer() if (resizedBuffer.length <= context.maxBytes) { return createCompressedImageResult( resizedBuffer, context.format, context.originalSize, ) } } return null } function applyFormatOptimizations( image: SharpInstance, format: string, ): SharpInstance { switch (format) { case 'png': return image.png({ compressionLevel: 9, palette: true, }) case 'jpeg': case 'jpg': return image.jpeg({ quality: 80 }) case 'webp': return image.webp({ quality: 80 }) default: return image } } async function tryPalettePNG( context: ImageCompressionContext, sharp: SharpFunction, ): Promise { const palettePng = await sharp(context.imageBuffer) .resize(800, 800, { fit: 'inside', withoutEnlargement: true, }) .png({ compressionLevel: 9, palette: true, colors: 64, // Reduce colors to 64 for better compression }) .toBuffer() if (palettePng.length <= context.maxBytes) { return createCompressedImageResult(palettePng, 'png', context.originalSize) } return null } async function tryJPEGConversion( context: ImageCompressionContext, quality: number, sharp: SharpFunction, ): Promise { const jpegBuffer = await sharp(context.imageBuffer) .resize(600, 600, { fit: 'inside', withoutEnlargement: true, }) .jpeg({ quality }) .toBuffer() if (jpegBuffer.length <= context.maxBytes) { return createCompressedImageResult(jpegBuffer, 'jpeg', context.originalSize) } return null } async function createUltraCompressedJPEG( context: ImageCompressionContext, sharp: SharpFunction, ): Promise { const ultraCompressedBuffer = await sharp(context.imageBuffer) .resize(400, 400, { fit: 'inside', withoutEnlargement: true, }) .jpeg({ quality: 20 }) .toBuffer() return createCompressedImageResult( ultraCompressedBuffer, 'jpeg', context.originalSize, ) } /** * Detect image format from a buffer using magic bytes * @param buffer Buffer containing image data * @returns Media type string (e.g., 'image/png', 'image/jpeg') or 'image/png' as default */ export function detectImageFormatFromBuffer(buffer: Buffer): ImageMediaType { if (buffer.length < 4) return 'image/png' // default // Check PNG signature if ( buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47 ) { return 'image/png' } // Check JPEG signature (FFD8FF) if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { return 'image/jpeg' } // Check GIF signature (GIF87a or GIF89a) if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) { return 'image/gif' } // Check WebP signature (RIFF....WEBP) if ( buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46 ) { if ( buffer.length >= 12 && buffer[8] === 0x57 && buffer[9] === 0x45 && buffer[10] === 0x42 && buffer[11] === 0x50 ) { return 'image/webp' } } // Default to PNG if unknown return 'image/png' } /** * Detect image format from base64 data using magic bytes * @param base64Data Base64 encoded image data * @returns Media type string (e.g., 'image/png', 'image/jpeg') or 'image/png' as default */ export function detectImageFormatFromBase64( base64Data: string, ): ImageMediaType { try { const buffer = Buffer.from(base64Data, 'base64') return detectImageFormatFromBuffer(buffer) } catch { // Default to PNG on any error return 'image/png' } } /** * Creates a text description of image metadata including dimensions and source path. * Returns null if no useful metadata is available. */ export function createImageMetadataText( dims: ImageDimensions, sourcePath?: string, ): string | null { const { originalWidth, originalHeight, displayWidth, displayHeight } = dims // Skip if dimensions are not available or invalid // Note: checks for undefined/null and zero to prevent division by zero if ( !originalWidth || !originalHeight || !displayWidth || !displayHeight || displayWidth <= 0 || displayHeight <= 0 ) { // If we have a source path but no valid dimensions, still return source info if (sourcePath) { return `[Image source: ${sourcePath}]` } return null } // Check if image was resized const wasResized = originalWidth !== displayWidth || originalHeight !== displayHeight // Only include metadata if there's useful info (resized or has source path) if (!wasResized && !sourcePath) { return null } // Build metadata parts const parts: string[] = [] if (sourcePath) { parts.push(`source: ${sourcePath}`) } if (wasResized) { const scaleFactor = originalWidth / displayWidth parts.push( `original ${originalWidth}x${originalHeight}, displayed at ${displayWidth}x${displayHeight}. Multiply coordinates by ${scaleFactor.toFixed(2)} to map to original image.`, ) } return `[Image: ${parts.join(', ')}]` }