feat: phase 3-5 — workflow engine, background subagents, multi-modal, cache shape, inline diff
Phase 3: Dynamic Workflow Engine - VM sandbox (node:vm) with agent/parallel/pipeline API, Claude Code compatible - Workflow file discovery (.boocode/workflows/*.js + ~/.boocode/workflows/*.js) - Workflow manager with session/chat creation and inference dispatch - Built-in catalog: deep-research, review-code, find-issues - Resumability cache: SHA-256 hash of agent spec, in-memory Map Phase 4: Background Subagents - background-task.ts service: spawn/poll/cancel lifecycle - spawn_subagent, subagent_status, subagent_result tools in ALL_TOOLS Phase 5: Multi-modal + Cache Shape - Multi-modal stub with type defs and hook point in payload.ts - CacheShapeBadge component in trace viewer (colored bar + %)
This commit is contained in:
56
apps/server/src/services/inference/multi-modal.ts
Normal file
56
apps/server/src/services/inference/multi-modal.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
// vDeepSeek (stub): multi-modal (image) attachment support.
|
||||
//
|
||||
// When a message carries images, DeepSeek V4 models can process them
|
||||
// natively via the @ai-sdk/deepseek provider. This module provides the
|
||||
// helper types and functions to detect and convert image attachments.
|
||||
//
|
||||
// FULL INTEGRATION requires:
|
||||
// 1. Storing image data alongside messages (message_parts with kind='image'
|
||||
// or a dedicated attachments table with base64-encoded data).
|
||||
// 2. Extending OpenAiMessage.content from `string | null` to
|
||||
// `string | null | Array<{ type: 'text'; text: string } | { type: 'image'; image: string }>`
|
||||
// in apps/server/src/services/inference/payload.ts.
|
||||
// 3. Updating toModelMessages() in stream-phase-adapter.ts to emit AI SDK
|
||||
// content arrays with image parts for multimodal user messages.
|
||||
//
|
||||
// None of the above is done yet — this file is a type scaffold.
|
||||
|
||||
import type { Message } from '../../types/api.js';
|
||||
|
||||
/** Shape of a decoded image attachment ready for the AI SDK. */
|
||||
export interface ImageAttachment {
|
||||
/** Base64-encoded image data (no data URI prefix — raw bytes). */
|
||||
data: string;
|
||||
/** MIME type (e.g. 'image/png', 'image/jpeg', 'image/webp'). */
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a user message has image content that can be forwarded to a
|
||||
* multimodal model. Currently a stub — always returns false until the
|
||||
* message-pipeline stores image attachments addressably.
|
||||
*/
|
||||
export function hasImageAttachments(_message: Message): boolean {
|
||||
// TODO(vDeepSeek): scan message_parts for kind='image' or inspect
|
||||
// message.content for inline data URIs (data:image/...).
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert internal image attachments to the format expected by the AI SDK
|
||||
* ModelMessage content array.
|
||||
*
|
||||
* The @ai-sdk/deepseek provider accepts images as:
|
||||
* { type: 'image'; image: 'data:image/png;base64,...' }
|
||||
*
|
||||
* @param attachments — List of decoded image attachments.
|
||||
* @returns AI SDK inline file parts suitable for ModelMessage.content.
|
||||
*/
|
||||
export function imageAttachmentsToParts(
|
||||
attachments: ImageAttachment[],
|
||||
): Array<{ type: 'image'; image: string }> {
|
||||
return attachments.map((a) => ({
|
||||
type: 'image' as const,
|
||||
image: `data:${a.mimeType};base64,${a.data}`,
|
||||
}));
|
||||
}
|
||||
Reference in New Issue
Block a user