diff --git a/src/services/mcp/client.ts b/src/services/mcp/client.ts index cec272642..571527b57 100644 --- a/src/services/mcp/client.ts +++ b/src/services/mcp/client.ts @@ -74,7 +74,10 @@ import { } from '../../utils/errors.js' import { getMCPUserAgent } from '../../utils/http.js' import { maybeNotifyIDEConnected } from '../../utils/ide.js' -import { maybeResizeAndDownsampleImageBuffer } from '../../utils/imageResizer.js' +import { + type ImageLimits, + maybeResizeAndDownsampleImageBuffer, +} from '../../utils/imageResizer.js' import { logMCPDebug, logMCPError } from '../../utils/log.js' import { getBinaryBlobSavedMessage, @@ -102,6 +105,7 @@ import { isPersistError, persistToolResult, } from '../../utils/toolResultStorage.js' +import { checkStatsigFeatureGate_CACHED_MAY_BE_STALE } from '../analytics/growthbook.js' import { type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, logEvent, @@ -2486,15 +2490,23 @@ export function prefetchAllMcpResources( export async function transformResultContent( resultContent: PromptMessage['content'], serverName: string, + limits?: ImageLimits, + includeMeta = false, ): Promise> { switch (resultContent.type) { - case 'text': - return [ - { - type: 'text', - text: resultContent.text, - }, - ] + case 'text': { + const block: ContentBlockParam = { + type: 'text', + text: resultContent.text, + } + if (includeMeta) { + const meta = resultContent._meta + if (meta) { + ;(block as { _meta?: unknown })._meta = meta + } + } + return [block] + } case 'audio': { const audioData = resultContent as { type: 'audio' @@ -2516,6 +2528,7 @@ export async function transformResultContent( imageBuffer, imageBuffer.length, ext, + limits, ) return [ { @@ -2551,6 +2564,7 @@ export async function transformResultContent( imageBuffer, imageBuffer.length, ext, + limits, ) const content: MessageParam['content'] = [] if (prefix) { @@ -2671,6 +2685,7 @@ export async function transformMCPResult( result: unknown, tool: string, // Tool name for validation (e.g., "search") name: string, // Server name for transformation (e.g., "slack") + limits?: ImageLimits, // Image processing limits, plumbed to transformResultContent ): Promise { if (result && typeof result === 'object') { if ('toolResult' in result) { @@ -2694,7 +2709,9 @@ export async function transformMCPResult( if ('content' in result && Array.isArray(result.content)) { const transformedContent = ( await Promise.all( - result.content.map(item => transformResultContent(item, name)), + result.content.map(item => + transformResultContent(item, name, limits, true), + ), ) ).flat() return { @@ -2729,8 +2746,15 @@ export async function processMCPResult( result: unknown, tool: string, // Tool name for validation (e.g., "search") name: string, // Server name for IDE check and transformation (e.g., "slack") + limits?: ImageLimits, // Image processing limits, plumbed to transformMCPResult + skipLargeOutput = false, // If true, skip large-output handling for non-image content ): Promise { - const { content, type, schema } = await transformMCPResult(result, tool, name) + const { content, type, schema } = await transformMCPResult( + result, + tool, + name, + limits, + ) // IDE tools are not going to the model directly, so we don't need to // handle large output. @@ -2738,6 +2762,12 @@ export async function processMCPResult( return content } + // Caller opted out of large-output handling (e.g., result already truncated + // upstream); only continue if the content has images that may need handling. + if (skipLargeOutput && !contentContainsImages(content)) { + return content + } + // Check if content needs truncation (i.e., is too large) if (!(await mcpContentNeedsTruncation(content))) { return content @@ -2775,9 +2805,15 @@ export async function processMCPResult( // Generate a unique ID for the persisted file (server__tool-timestamp) const timestamp = Date.now() const persistId = `mcp-${normalizeNameForMCP(name)}-${normalizeNameForMCP(tool)}-${timestamp}` - // Convert to string for persistence (persistToolResult expects string or specific block types) + // When the large-string format gate is on, unwrap a single bare text block + // (no annotations, no _meta) into raw text so the model gets plain text in + // the persisted file instead of a JSON-wrapped block. The `_meta` check is + // why transformResultContent preserves _meta on text blocks. + const unwrappedText = unwrapSingleTextBlock(content) const contentStr = - typeof content === 'string' ? content : jsonStringify(content, null, 2) + typeof content === 'string' + ? content + : (unwrappedText ?? jsonStringify(content, null, 2)) const persistResult = await persistToolResult(contentStr, persistId) if (isPersistError(persistResult)) { @@ -2798,7 +2834,10 @@ export async function processMCPResult( persistedSizeChars: persistResult.originalSize, } as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS) - const formatDescription = getFormatDescription(type, schema) + const formatDescription = + unwrappedText !== undefined + ? getFormatDescription('toolResult') + : getFormatDescription(type, schema) return getLargeOutputInstructions( persistResult.filepath, persistResult.originalSize, @@ -2806,6 +2845,39 @@ export async function processMCPResult( ) } +/** + * Returns true when the large-string output format is enabled (matches + * binary's `sf8()`). When enabled, processMCPResult unwraps a single bare + * text block to raw text for persistence instead of JSON-wrapping it. + * + * Gating sources, in order: + * 1. MCP_TRUNCATION_PROMPT_OVERRIDE env var (anything except "legacy" enables) + * 2. Statsig gate `tengu_mcp_subagent_prompt` + */ +function isLargeStringFormatEnabled(): boolean { + const override = process.env.MCP_TRUNCATION_PROMPT_OVERRIDE + if (override) return override !== 'legacy' + return checkStatsigFeatureGate_CACHED_MAY_BE_STALE( + 'tengu_mcp_subagent_prompt', + ) +} + +/** + * Unwraps a single bare text content block to its raw text when the + * large-string format gate is on. Returns undefined when the gate is off, + * the content is not an array, the array doesn't contain exactly one text + * block, or the block carries annotations or _meta. Matches binary mg5's + * `M=...` computation. + */ +function unwrapSingleTextBlock(content: MCPToolResult): string | undefined { + if (!isLargeStringFormatEnabled()) return undefined + if (!Array.isArray(content) || content.length !== 1) return undefined + const block = content[0] + if (!block || block.type !== 'text') return undefined + if ('annotations' in block || '_meta' in block) return undefined + return block.text +} + /** * Call an MCP tool, handling UrlElicitationRequiredError (-32042) by * displaying the URL elicitation to the user, waiting for the completion @@ -2827,6 +2899,8 @@ export async function callMCPToolWithUrlElicitationRetry({ signal, setAppState, onProgress, + imageLimits, + hasResultSizeAnnotation = false, callToolFn = callMCPTool, handleElicitation, }: { @@ -2838,6 +2912,8 @@ export async function callMCPToolWithUrlElicitationRetry({ signal: AbortSignal setAppState: (f: (prev: AppState) => AppState) => void onProgress?: (data: MCPProgress) => void + imageLimits?: ImageLimits + hasResultSizeAnnotation?: boolean /** Injectable for testing. Defaults to callMCPTool. */ callToolFn?: (opts: { client: ConnectedMCPServer @@ -2846,6 +2922,8 @@ export async function callMCPToolWithUrlElicitationRetry({ meta?: Record signal: AbortSignal onProgress?: (data: MCPProgress) => void + imageLimits?: ImageLimits + hasResultSizeAnnotation?: boolean }) => Promise /** Handler for URL elicitations when no hook handles them. * In print/SDK mode, delegates to structuredIO. In REPL, falls back to queue. */ @@ -2865,6 +2943,8 @@ export async function callMCPToolWithUrlElicitationRetry({ meta, signal, onProgress, + imageLimits, + hasResultSizeAnnotation, }) } catch (error) { // The MCP SDK's Protocol creates plain McpError (not UrlElicitationRequiredError) @@ -3041,6 +3121,8 @@ async function callMCPTool({ meta, signal, onProgress, + imageLimits, + hasResultSizeAnnotation = false, }: { client: ConnectedMCPServer tool: string @@ -3048,6 +3130,8 @@ async function callMCPTool({ meta?: Record signal: AbortSignal onProgress?: (data: MCPProgress) => void + imageLimits?: ImageLimits + hasResultSizeAnnotation?: boolean }): Promise<{ content: MCPToolResult _meta?: Record @@ -3176,7 +3260,13 @@ async function callMCPTool({ }) } - const content = await processMCPResult(result, tool, name) + const content = await processMCPResult( + result, + tool, + name, + imageLimits, + hasResultSizeAnnotation, + ) return { content, _meta: result._meta as Record | undefined, diff --git a/src/utils/imageResizer.ts b/src/utils/imageResizer.ts index f7c7347d1..ba5acd4da 100644 --- a/src/utils/imageResizer.ts +++ b/src/utils/imageResizer.ts @@ -162,6 +162,16 @@ interface CompressedImageResult { originalSize: number } +/** + * Per-call image processing limits, overriding the module-level defaults. + */ +export interface ImageLimits { + targetRawSize: number + maxWidth: number + maxHeight: number + maxBase64Size: number +} + /** * Extracted from FileReadTool's readImage function * Resizes image buffer to meet size and dimension constraints @@ -170,7 +180,13 @@ export async function maybeResizeAndDownsampleImageBuffer( imageBuffer: Buffer, originalSize: number, ext: string, + limits?: ImageLimits, ): Promise { + const targetRawSize = limits?.targetRawSize ?? IMAGE_TARGET_RAW_SIZE + const maxWidth = limits?.maxWidth ?? IMAGE_MAX_WIDTH + const maxHeight = limits?.maxHeight ?? IMAGE_MAX_HEIGHT + const maxBase64Size = limits?.maxBase64Size ?? API_IMAGE_MAX_BASE64_SIZE + if (imageBuffer.length === 0) { // Empty buffer would fall through the catch block below (sharp throws // "Unable to determine image format"), and the fallback's size check @@ -189,7 +205,7 @@ export async function maybeResizeAndDownsampleImageBuffer( // If dimensions aren't available from metadata if (!metadata.width || !metadata.height) { - if (originalSize > IMAGE_TARGET_RAW_SIZE) { + if (originalSize > targetRawSize) { // Create fresh sharp instance for compression const compressedBuffer = await sharp(imageBuffer) .jpeg({ quality: 80 }) @@ -210,9 +226,9 @@ export async function maybeResizeAndDownsampleImageBuffer( // Check if the original file just works if ( - originalSize <= IMAGE_TARGET_RAW_SIZE && - width <= IMAGE_MAX_WIDTH && - height <= IMAGE_MAX_HEIGHT + originalSize <= targetRawSize && + width <= maxWidth && + height <= maxHeight ) { return { buffer: imageBuffer, @@ -226,20 +242,19 @@ export async function maybeResizeAndDownsampleImageBuffer( } } - const needsDimensionResize = - width > IMAGE_MAX_WIDTH || height > IMAGE_MAX_HEIGHT + const needsDimensionResize = width > maxWidth || height > maxHeight const isPng = normalizedMediaType === 'png' // If dimensions are within limits but file is too large, try compression first // This preserves full resolution when possible - if (!needsDimensionResize && originalSize > IMAGE_TARGET_RAW_SIZE) { + if (!needsDimensionResize && originalSize > targetRawSize) { // For PNGs, try PNG compression first to preserve transparency if (isPng) { // Create fresh sharp instance for each compression attempt const pngCompressed = await sharp(imageBuffer) .png({ compressionLevel: 9, palette: true }) .toBuffer() - if (pngCompressed.length <= IMAGE_TARGET_RAW_SIZE) { + if (pngCompressed.length <= targetRawSize) { return { buffer: pngCompressed, mediaType: 'png', @@ -258,7 +273,7 @@ export async function maybeResizeAndDownsampleImageBuffer( const compressedBuffer = await sharp(imageBuffer) .jpeg({ quality }) .toBuffer() - if (compressedBuffer.length <= IMAGE_TARGET_RAW_SIZE) { + if (compressedBuffer.length <= targetRawSize) { return { buffer: compressedBuffer, mediaType: 'jpeg', @@ -275,14 +290,14 @@ export async function maybeResizeAndDownsampleImageBuffer( } // Constrain dimensions if needed - if (width > IMAGE_MAX_WIDTH) { - height = Math.round((height * IMAGE_MAX_WIDTH) / width) - width = IMAGE_MAX_WIDTH + if (width > maxWidth) { + height = Math.round((height * maxWidth) / width) + width = maxWidth } - if (height > IMAGE_MAX_HEIGHT) { - width = Math.round((width * IMAGE_MAX_HEIGHT) / height) - height = IMAGE_MAX_HEIGHT + if (height > maxHeight) { + width = Math.round((width * maxHeight) / height) + height = maxHeight } // IMPORTANT: Always create fresh sharp(imageBuffer) instances for each operation. @@ -298,7 +313,7 @@ export async function maybeResizeAndDownsampleImageBuffer( .toBuffer() // If still too large after resize, try compression - if (resizedImageBuffer.length > IMAGE_TARGET_RAW_SIZE) { + if (resizedImageBuffer.length > targetRawSize) { // For PNGs, try PNG compression first to preserve transparency if (isPng) { const pngCompressed = await sharp(imageBuffer) @@ -308,7 +323,7 @@ export async function maybeResizeAndDownsampleImageBuffer( }) .png({ compressionLevel: 9, palette: true }) .toBuffer() - if (pngCompressed.length <= IMAGE_TARGET_RAW_SIZE) { + if (pngCompressed.length <= targetRawSize) { return { buffer: pngCompressed, mediaType: 'png', @@ -331,7 +346,7 @@ export async function maybeResizeAndDownsampleImageBuffer( }) .jpeg({ quality }) .toBuffer() - if (compressedBuffer.length <= IMAGE_TARGET_RAW_SIZE) { + if (compressedBuffer.length <= targetRawSize) { return { buffer: compressedBuffer, mediaType: 'jpeg', @@ -407,11 +422,11 @@ export async function maybeResizeAndDownsampleImageBuffer( imageBuffer[1] === 0x50 && imageBuffer[2] === 0x4e && imageBuffer[3] === 0x47 && - (imageBuffer.readUInt32BE(16) > IMAGE_MAX_WIDTH || - imageBuffer.readUInt32BE(20) > IMAGE_MAX_HEIGHT) + (imageBuffer.readUInt32BE(16) > maxWidth || + imageBuffer.readUInt32BE(20) > maxHeight) // If original image's base64 encoding is within API limit, allow it through uncompressed - if (base64Size <= API_IMAGE_MAX_BASE64_SIZE && !overDim) { + if (base64Size <= maxBase64Size && !overDim) { logEvent('tengu_image_resize_fallback', { original_size_bytes: originalSize, base64_size_bytes: base64Size, @@ -423,7 +438,7 @@ export async function maybeResizeAndDownsampleImageBuffer( // Image is too large and we failed to compress it - fail with user-friendly error throw new ImageResizeError( overDim - ? `Unable to resize image — dimensions exceed the ${IMAGE_MAX_WIDTH}x${IMAGE_MAX_HEIGHT}px limit and image processing failed. ` + + ? `Unable to resize image — dimensions exceed the ${maxWidth}x${maxHeight}px limit and image processing failed. ` + `Please resize the image to reduce its pixel dimensions.` : `Unable to resize image (${formatFileSize(originalSize)} raw, ${formatFileSize(base64Size)} base64). ` + `The image exceeds the 5MB API limit and compression failed. ` +