indifferentketchup/boocode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: codebase audit cleanup — dead code, dedup, module splits

Browse Source 
Multi-agent audit + aggressive cleanup across server/web/coder/booterm,
delivered behind a DEFER discipline so none of the in-flight files were
touched. Removes dead code/deps/columns, dedups server + coder helpers,
and splits the oversized modules (tools.ts, opencode-server.ts,
sentinel-summaries, turn.ts, TerminalPane.tsx) behind stable contracts.
Adds 78 parity/unit tests (server 587, coder 323); fixes two latent bugs
(ChatPane queue keys, FileViewerOverlay blank-line parity).

Intended tag: v2.7.12-audit-cleanup.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
indifferentketchup
2026-06-02 21:10:06 +00:00
parent e5ce01ae72
commit 8c200216eb
143 changed files with 6729 additions and 6087 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
apps/server/src/services/inference/budget.ts
View File

@@ -1,32 +1,10 @@
import type { Agent } from '../../types/api.js';
import { READ_ONLY_TOOL_NAMES } from '../tools.js';
// v1.8.2: tool-call budget defaults. Resolved per-turn by resolveToolBudget.
// - Agent with explicit max_tool_calls: that value.
// - Agent with read-only-only tools: BUDGET_READ_ONLY (50).
// - Agent with any non-read-only tool: BUDGET_NON_READ_ONLY (10).
// - No agent (raw chat): BUDGET_NO_AGENT (50).
// v1.13.7: bumped BUDGET_NO_AGENT 15→30 to match BUDGET_READ_ONLY. Every tool
// in ALL_TOOLS today is read-only (see services/tools.ts comment at
// READ_ONLY_TOOL_NAMES); the cautious 15-cap was a forward-looking guard for
// write tools that haven't landed yet. No-agent mode gets the same toolset as
// an all-read-only agent at runtime, so they should share the same budget.
// v1.13.12: bumped read-only caps 30→50. Real recon sessions were hitting 30
// with ~3 turns wasted on codecontext parse failures (empty node_modules
// files); legitimate need was ~27, and Architect-class system overviews want
// deeper recon than a 30-cap permits. Headroom of 20 absorbs failure-retry
// turns + deeper exploration without changing the safety floor materially —
// the doom-loop guard (3 identical calls → abort) catches the actual failure
// mode this cap was guarding against.
export const BUDGET_READ_ONLY = 100;
export const BUDGET_NON_READ_ONLY = 100;
export const BUDGET_NO_AGENT = 100;
const READ_ONLY_SET: ReadonlySet<string> = new Set(READ_ONLY_TOOL_NAMES);
// Tool-call budget. All three historical tiers (read-only, non-read-only,
// no-agent) converged to 100 as of v1.13.12, collapsing the tier logic.
// The only remaining override is per-agent max_tool_calls from AGENTS.md
// frontmatter. Flat default of 100; doom-loop guard in sentinels.ts catches
// pathological cases well before the cap is reached.
export function resolveToolBudget(agent: Agent | null): number {
if (agent?.max_tool_calls != null) return agent.max_tool_calls;
if (!agent) return BUDGET_NO_AGENT;
const allReadOnly = agent.tools.every((t) => READ_ONLY_SET.has(t));
return allReadOnly ? BUDGET_READ_ONLY : BUDGET_NON_READ_ONLY;
return agent?.max_tool_calls ?? 100;
}

64
apps/server/src/services/inference/content-flusher.ts Normal file
View File

@@ -0,0 +1,64 @@
// P5: the debounced DB content-flush timer, extracted from the verbatim copy
// that lived in executeStreamPhase + the three sentinel summaries (4 sites).
// Each site streamed deltas into a local `accumulated`/`state.accumulated`
// string and threw an UPDATE at the row at most once per DB_FLUSH_INTERVAL_MS
// to bound write rate under heavy streaming.
//
// The accumulated string stays owned by the caller (stream-phase keeps it on
// the shared StreamPhaseState; the summaries keep a local) — the flusher reads
// it through a `getContent` thunk at fire time, snapshotting the latest value
// exactly as the inline `const snapshot = accumulated` did. No final flush is
// performed on drain (matches the originals): every caller writes the full
// content itself in its terminal UPDATE, so drain only cancels the pending
// timer and awaits whatever write is already chained.
import type { Sql } from '../../db.js';
import { DB_FLUSH_INTERVAL_MS } from './types.js';
export interface ContentFlusher {
// Arm a debounced flush. No-op if one is already pending (the in-flight timer
// will pick up the latest content via getContent when it fires).
scheduleFlush: () => void;
// Cancel any pending timer and await the in-flight write chain. Does NOT
// perform a final flush — the caller's terminal UPDATE owns the final write.
drain: () => Promise<void>;
}
export function createContentFlusher(
sql: Sql,
messageId: string,
getContent: () => string,
intervalMs: number = DB_FLUSH_INTERVAL_MS,
): ContentFlusher {
let pendingFlushTimer: NodeJS.Timeout | null = null;
let flushPromise: Promise<unknown> = Promise.resolve();
const flushNow = () => {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
const snapshot = getContent();
flushPromise = flushPromise.then(() =>
sql`UPDATE messages SET content = ${snapshot} WHERE id = ${messageId}`
);
};
const scheduleFlush = () => {
if (pendingFlushTimer) return;
pendingFlushTimer = setTimeout(() => {
pendingFlushTimer = null;
flushNow();
}, intervalMs);
};
const drain = async () => {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
await flushPromise;
};
return { scheduleFlush, drain };
}

86
apps/server/src/services/inference/error-handler.ts
View File

@@ -10,7 +10,7 @@ import { maybeFlagForCompaction } from './payload.js';
import { insertParts, partsFromAssistantMessage } from './parts.js';
import type { PartInsert } from './parts.js';
import { stripToolMarkup } from './tool-call-parser.js';
import type { InferenceContext, StreamResult, TurnArgs } from './turn.js';
import type { InferenceContext, StreamResult, TurnArgs } from './types.js';
export async function handleAbortOrError(
ctx: InferenceContext,
@@ -95,6 +95,90 @@ export async function handleAbortOrError(
}
}
// P5: the success-finalize atom shared by the wrap-up summaries
// (sentinel-summaries.ts) and the synthesis pass (synthesisPipeline.ts). Both
// previously hand-rolled this exact ceremony — n_ctx lookup, the complete
// UPDATE (content/status/tokens/ctx/ctx_max/finished_at; NO model column), and
// the message_complete frame with the full token fields. Single-sourcing it
// means a message_complete frame-contract change lands in one place instead of
// silently skipping the summary/synthesis paths.
//
// `beforeComplete` runs AFTER the UPDATE and BEFORE the message_complete frame
// — synthesis uses it to write its kind='synthesis' part in the original order
// (UPDATE → insertParts → message_complete), preserving timing exactly.
//
// NOTE: finalizeCompletion does NOT use this — it additionally writes the
// `model` column, the text/reasoning/html_artifact parts, the compaction flag,
// and the session_updated bump, which this atom deliberately omits (the summary
// and synthesis paths handle those — or not — themselves).
export async function finalizeStreamedRow(
ctx: InferenceContext,
opts: {
sessionId: string;
chatId: string;
messageId: string;
model: string;
content: string;
completionTokens: number | null;
promptTokens: number | null;
startedAt: string | null;
beforeComplete?: () => Promise<void>;
},
): Promise<void> {
// v1.11.3: see executeToolPhase for the rationale.
const mctx = await modelContext.getModelContext(opts.model);
const nCtx = mctx?.n_ctx ?? null;
const [updated] = await ctx.sql<
{ tokens_used: number | null; ctx_used: number | null; ctx_max: number | null; finished_at: string | null }[]
>`
UPDATE messages
SET content = ${opts.content},
status = 'complete',
tokens_used = ${opts.completionTokens},
ctx_used = ${opts.promptTokens},
ctx_max = ${nCtx},
finished_at = clock_timestamp()
WHERE id = ${opts.messageId}
RETURNING tokens_used, ctx_used, ctx_max, finished_at
`;
if (opts.beforeComplete) await opts.beforeComplete();
ctx.publish(opts.sessionId, {
type: 'message_complete',
message_id: opts.messageId,
chat_id: opts.chatId,
tokens_used: updated?.tokens_used ?? null,
ctx_used: updated?.ctx_used ?? null,
ctx_max: updated?.ctx_max ?? null,
started_at: opts.startedAt,
finished_at: updated?.finished_at ?? null,
model: opts.model,
});
}
// P5: minimal empty-finalize for the mistake-escalate path. The escalate
// branch in runAssistantTurn stops the turn cap-hit-style; the next assistant
// row is still 'streaming', so it's finalized as an empty complete row (no
// tokens, no parts, no session bump — the escalate branch handles the sentinel
// + chat_status itself). Centralizing the status-column write + message_complete
// frame here keeps it next to the other finalize paths so a status-column
// change is found in one place.
export async function finalizeEmpty(
ctx: InferenceContext,
args: TurnArgs,
): Promise<void> {
const { sessionId, chatId, assistantMessageId } = args;
await ctx.sql`
UPDATE messages
SET content = '', status = 'complete', finished_at = clock_timestamp()
WHERE id = ${assistantMessageId}
`;
ctx.publish(sessionId, {
type: 'message_complete',
message_id: assistantMessageId,
chat_id: chatId,
});
}
export async function finalizeCompletion(
ctx: InferenceContext,
args: TurnArgs,

15
apps/server/src/services/inference/index.ts
View File

@@ -7,26 +7,17 @@
export {
createInferenceRunner,
MAX_STEPS,
runAssistantTurn,
runInference,
} from './turn.js';
// P5: the shared pipeline types moved from turn.ts to types.ts (breaking the
// hub-and-leaf near-cycle). Re-exported here so the public surface is unchanged.
export type {
FramePublisher,
InferenceContext,
InferenceFrame,
StreamResult,
TurnArgs,
} from './turn.js';
} from './types.js';
export type { ToolPhaseResult } from './tool-phase.js';
export { detectDoomLoop, DOOM_LOOP_THRESHOLD } from './sentinels.js';
export {
detectMistakePattern,
freshMistakeState,
recordStep,
MISTAKE_THRESHOLD,
MISTAKE_RECOVERY_NOTE,
} from './mistake-tracker.js';
export type { FailureKind, MistakeState } from './mistake-tracker.js';
export { buildMessagesPayload } from './payload.js';
export { generateToolUseSummary } from './tool-summaries.js';
export type { ToolInfo } from './tool-summaries.js';

9
apps/server/src/services/inference/parts.ts
View File

@@ -1,11 +1,10 @@
import type { Sql } from '../../db.js';
import type { ToolCall, ToolResult } from '../../types/api.js';
// v1.13.0: dual-write helper. Every site that writes the legacy
// messages.tool_calls / messages.tool_results JSON columns calls into here
// to mirror the same data into message_parts rows. Reads still go to the
// JSON columns; the swap to parts-as-source-of-truth happens in a later
// v1.13 dispatch alongside the AI SDK streamText migration.
// v1.13.0: message_parts write helpers. v1.13.20: legacy tool_calls/
// tool_results JSON columns dropped; message_parts is the sole source of
// truth. All writes go through insertParts / partsFromAssistantMessage /
// partsFromToolMessage. Reads use the messages_with_parts view.
// v1.13.13: 'synthesis' added. Schema CHECK constraint is updated in lockstep
// (schema.sql adds 'synthesis' to message_parts_kind_chk on startup). The

7
apps/server/src/services/inference/payload.ts
View File

@@ -10,7 +10,8 @@ import * as compaction from '../compaction.js';
import { buildSystemPromptWithFingerprint } from '../system-prompt.js';
import { isAnySentinel } from './sentinels.js';
import { PRUNE_TRIGGER_TOKENS, prune } from './prune.js';
import type { InferenceContext } from './turn.js';
import type { InferenceContext } from './types.js';
import { INFERENCE_MESSAGE_COLUMNS } from '../message-columns.js';
export interface OpenAiMessage {
role: 'system' | 'user' | 'assistant' | 'tool';
@@ -205,9 +206,7 @@ export async function loadContext(
// v1.13.1-C: also pull reasoning_parts so assistant messages from
// reasoning models can be replayed with their reasoning context preserved.
const history = await sql<Message[]>`
SELECT id, session_id, chat_id, role, content, kind, tool_calls, tool_results, status, last_seq,
tokens_used, ctx_used, ctx_max, started_at, finished_at, created_at, metadata,
reasoning_parts
SELECT ${sql.unsafe(INFERENCE_MESSAGE_COLUMNS)}
FROM messages_with_parts
WHERE chat_id = ${chatId} AND compacted_at IS NULL
ORDER BY created_at ASC, id ASC

719
apps/server/src/services/inference/sentinel-summaries.ts
View File

@@ -5,16 +5,16 @@ import type {
Project,
Session,
} from '../../types/api.js';
import * as modelContext from '../model-context.js';
import { buildMessagesPayload } from './payload.js';
import { DOOM_LOOP_THRESHOLD } from './sentinels.js';
import { streamCompletion } from './stream-phase.js';
import { DB_FLUSH_INTERVAL_MS } from './types.js';
import { streamCompletion, samplerOptsFromAgent } from './stream-phase.js';
import { createContentFlusher } from './content-flusher.js';
import { finalizeStreamedRow } from './error-handler.js';
import type {
InferenceContext,
StreamResult,
TurnArgs,
} from './turn.js';
} from './types.js';
// Synthetic system note appended to the cap-hit summary call. Verbatim from
// the v1.8.2 spec — do not paraphrase: the model is more reliable when the
@@ -25,21 +25,50 @@ const CAP_HIT_SUMMARY_NOTE = (limit: number) =>
const DOOM_LOOP_NOTE = (name: string) =>
`You called ${name} with the same arguments ${DOOM_LOOP_THRESHOLD} times in a row. Stop calling it. Produce the best answer you can with what you have.`;
export async function runCapHitSummary(
// v1.14.0: step-cap wrap-up note. Names the step limit rather than the tool
// budget. The sentinel reuses metadata.kind = 'cap_hit' so the frontend
// CapHitSentinel component renders it without changes.
const STEP_CAP_NOTE = (steps: number, cap: number) =>
`You've reached the step limit (${steps}/${cap} steps). Produce the best answer you can with what you have. Do not call more tools.`;
// P5: the ONE generic wrap-up flow shared by the three sentinel summaries
// (cap-hit, doom-loop, step-cap). Each reuses the in-flight assistant slot to
// stream a short tools-disabled summary, finalizes via the same 3-outcome
// branch (complete / cancelled / failed), bumps the session, then drops a
// sentinel and the chat_status. The three differ only in:
// - `note`: the synthetic system instruction appended to the summary call.
// - `errorText`: the fallback used in the failed-status metadata + error frame.
// - sentinel timing: cap-hit inserts BEFORE the stream (`beforeStream`);
// doom-loop + step-cap insert AFTER the session bump (`afterSession`).
// - `logMsg` / `logFields`: per-kind log line + extra fields.
// All three use error_reason / chat_status reason = 'summary_after_cap_failed'
// (doom-loop reuses it deliberately — the user-visible failure mode is the
// same "model gave up mid-summary"; the ErrorReason union is shared and the UI
// surfaces a generic "summary failed" line for every sentinel path).
interface WrapUpOpts {
note: string;
errorText: string;
logMsg: string;
logFields: Record<string, unknown>;
beforeStream?: () => Promise<void>;
afterSession?: () => Promise<void>;
}
async function runWrapUpSummary(
ctx: InferenceContext,
args: TurnArgs,
session: Session,
project: Project,
history: Message[],
agent: Agent | null,
budget: number,
opts: WrapUpOpts,
): Promise<void> {
const { sessionId, chatId, assistantMessageId, signal } = args;
await insertCapHitSentinel(ctx, sessionId, chatId, agent, budget);
if (opts.beforeStream) await opts.beforeStream();
const messages = await buildMessagesPayload(session, project, history, agent, ctx.log);
messages.push({ role: 'system', content: CAP_HIT_SUMMARY_NOTE(budget) });
messages.push({ role: 'system', content: opts.note });
const startedRow = await ctx.sql<{ started_at: string }[]>`
UPDATE messages
@@ -57,25 +86,7 @@ export async function runCapHitSummary(
});
let accumulated = '';
let pendingFlushTimer: NodeJS.Timeout | null = null;
let flushPromise: Promise<unknown> = Promise.resolve();
const flushNow = () => {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
const snapshot = accumulated;
flushPromise = flushPromise.then(() =>
ctx.sql`UPDATE messages SET content = ${snapshot} WHERE id = ${assistantMessageId}`
);
};
const scheduleFlush = () => {
if (pendingFlushTimer) return;
pendingFlushTimer = setTimeout(() => {
pendingFlushTimer = null;
flushNow();
}, DB_FLUSH_INTERVAL_MS);
};
const flusher = createContentFlusher(ctx.sql, assistantMessageId, () => accumulated);
let summaryOk = false;
let summarySoftCancelled = false;
@@ -86,7 +97,7 @@ export async function runCapHitSummary(
ctx,
session.model,
messages,
{ tools: null, temperature: agent?.temperature, top_p: agent?.top_p ?? undefined, top_k: agent?.top_k ?? undefined, min_p: agent?.min_p ?? undefined, presence_penalty: agent?.presence_penalty ?? undefined, top_n_sigma: agent?.top_n_sigma ?? undefined, dry_multiplier: agent?.dry_multiplier ?? undefined, dry_base: agent?.dry_base ?? undefined, dry_allowed_length: agent?.dry_allowed_length ?? undefined, dry_penalty_last_n: agent?.dry_penalty_last_n ?? undefined },
{ tools: null, ...samplerOptsFromAgent(agent) },
(delta) => {
accumulated += delta;
ctx.publish(sessionId, {
@@ -95,7 +106,7 @@ export async function runCapHitSummary(
chat_id: chatId,
content: delta,
});
scheduleFlush();
flusher.scheduleFlush();
},
undefined,
signal,
@@ -108,44 +119,23 @@ export async function runCapHitSummary(
summaryError = err instanceof Error ? err.message : String(err);
}
} finally {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
await flushPromise;
await flusher.drain();
}
// Finalize the summary message based on the three outcomes. The sentinel
// is inserted regardless so the user always has the Continue affordance —
// even on a partial / failed summary the chat history shows where the
// budget was hit.
// Finalize the summary message based on the three outcomes. The sentinel is
// inserted regardless (before or after, per opts) so the user always has the
// appropriate affordance — even on a partial / failed summary the chat
// history shows where the loop stopped.
if (summaryOk && result) {
// v1.11.3: see executeToolPhase for the rationale.
const mctx = await modelContext.getModelContext(session.model);
const nCtx = mctx?.n_ctx ?? null;
const [updated] = await ctx.sql<
{ tokens_used: number | null; ctx_used: number | null; ctx_max: number | null; finished_at: string | null }[]
>`
UPDATE messages
SET content = ${result.content},
status = 'complete',
tokens_used = ${result.completionTokens},
ctx_used = ${result.promptTokens},
ctx_max = ${nCtx},
finished_at = clock_timestamp()
WHERE id = ${assistantMessageId}
RETURNING tokens_used, ctx_used, ctx_max, finished_at
`;
ctx.publish(sessionId, {
type: 'message_complete',
message_id: assistantMessageId,
chat_id: chatId,
tokens_used: updated?.tokens_used ?? null,
ctx_used: updated?.ctx_used ?? null,
ctx_max: updated?.ctx_max ?? null,
started_at: startedAt,
finished_at: updated?.finished_at ?? null,
await finalizeStreamedRow(ctx, {
sessionId,
chatId,
messageId: assistantMessageId,
model: session.model,
content: result.content,
completionTokens: result.completionTokens,
promptTokens: result.promptTokens,
startedAt,
});
} else if (summarySoftCancelled) {
await ctx.sql`
@@ -164,7 +154,7 @@ export async function runCapHitSummary(
const errMeta: MessageMetadata = {
kind: 'error',
error_reason: 'summary_after_cap_failed',
error_text: summaryError ?? 'summary failed',
error_text: summaryError ?? opts.errorText,
};
await ctx.sql`
UPDATE messages
@@ -178,7 +168,7 @@ export async function runCapHitSummary(
type: 'error',
message_id: assistantMessageId,
chat_id: chatId,
error: summaryError ?? 'summary failed',
error: summaryError ?? opts.errorText,
reason: 'summary_after_cap_failed',
});
}
@@ -197,11 +187,11 @@ export async function runCapHitSummary(
updated_at: sessRow!.updated_at,
});
if (opts.afterSession) await opts.afterSession();
// Status frame fires last so the dot color reflects the terminal state.
// Success → idle, abort → idle (user-driven stop), error → error+reason.
if (summaryOk) {
ctx.publishUser({ type: 'chat_status', chat_id: chatId, status: 'idle', at: new Date().toISOString() });
} else if (summarySoftCancelled) {
if (summaryOk || summarySoftCancelled) {
ctx.publishUser({ type: 'chat_status', chat_id: chatId, status: 'idle', at: new Date().toISOString() });
} else {
ctx.publishUser({
@@ -214,11 +204,113 @@ export async function runCapHitSummary(
}
ctx.log.info(
{ sessionId, chatId, assistantMessageId, budget, summaryOk, summaryCancelled: summarySoftCancelled },
'inference cap-hit summary finished',
{ sessionId, chatId, assistantMessageId, ...opts.logFields, summaryOk, summaryCancelled: summarySoftCancelled },
opts.logMsg,
);
}
// v1.8.2: cap-hit summary flow. Called instead of erroring when the loop hits
// its budget. The cap-hit sentinel is inserted FIRST (before the summary
// stream) so the UI shows the Continue affordance regardless of summary
// outcome.
export async function runCapHitSummary(
ctx: InferenceContext,
args: TurnArgs,
session: Session,
project: Project,
history: Message[],
agent: Agent | null,
budget: number,
): Promise<void> {
await runWrapUpSummary(ctx, args, session, project, history, agent, {
note: CAP_HIT_SUMMARY_NOTE(budget),
errorText: 'summary failed',
logMsg: 'inference cap-hit summary finished',
logFields: { budget },
beforeStream: () => insertCapHitSentinel(ctx, args.sessionId, args.chatId, agent, budget),
});
}
// v1.11.6: doom-loop wrap-up. The doom-loop sentinel is inserted AFTER the
// session bump (no Continue affordance — continuing would re-trigger the loop
// with the same tools available; the user needs to restate or switch agents).
export async function runDoomLoopSummary(
ctx: InferenceContext,
args: TurnArgs,
session: Session,
project: Project,
history: Message[],
agent: Agent | null,
loop: { name: string; args: Record<string, unknown> },
): Promise<void> {
await runWrapUpSummary(ctx, args, session, project, history, agent, {
note: DOOM_LOOP_NOTE(loop.name),
errorText: 'doom-loop summary failed',
logMsg: 'inference doom-loop summary finished',
logFields: { loopedTool: loop.name },
afterSession: () => insertDoomLoopSentinel(ctx, args.sessionId, args.chatId, loop),
});
}
// v1.14.0: step-cap wrap-up. Reuses the cap_hit sentinel (inserted AFTER the
// session bump) so the frontend CapHitSentinel component renders it without
// changes; the content text distinguishes step cap from budget.
export async function runStepCapSummary(
ctx: InferenceContext,
args: TurnArgs,
session: Session,
project: Project,
history: Message[],
agent: Agent | null,
steps: number,
cap: number,
): Promise<void> {
await runWrapUpSummary(ctx, args, session, project, history, agent, {
note: STEP_CAP_NOTE(steps, cap),
errorText: 'step-cap summary failed',
logMsg: 'inference step-cap summary finished',
logFields: { steps, cap },
afterSession: () => insertCapHitSentinel(ctx, args.sessionId, args.chatId, agent, cap),
});
}
// P5: the ONE INSERT + message_started → delta → message_complete frame
// sequence shared by every sentinel inserter. The sentinel row is a
// role='system', status='complete' message; the static content rides the same
// streaming-frame path useSessionStream's reducer uses for assistant messages
// (the delta carries the full text in one chunk).
async function insertSentinel(
ctx: InferenceContext,
sessionId: string,
chatId: string,
metadata: MessageMetadata,
content: string,
): Promise<void> {
const [row] = await ctx.sql<{ id: string }[]>`
INSERT INTO messages (session_id, chat_id, role, content, status, created_at, metadata)
VALUES (${sessionId}, ${chatId}, 'system', ${content}, 'complete', clock_timestamp(), ${ctx.sql.json(metadata as never)})
RETURNING id
`;
ctx.publish(sessionId, {
type: 'message_started',
message_id: row!.id,
chat_id: chatId,
role: 'system',
});
ctx.publish(sessionId, {
type: 'delta',
message_id: row!.id,
chat_id: chatId,
content,
});
ctx.publish(sessionId, {
type: 'message_complete',
message_id: row!.id,
chat_id: chatId,
metadata,
});
}
async function insertCapHitSentinel(
ctx: InferenceContext,
sessionId: string,
@@ -246,430 +338,7 @@ async function insertCapHitSentinel(
can_continue: canContinue,
};
const content = `Reached tool budget (${budget}/${budget}). Continue to extend.`;
const [row] = await ctx.sql<{ id: string }[]>`
INSERT INTO messages (session_id, chat_id, role, content, status, created_at, metadata)
VALUES (${sessionId}, ${chatId}, 'system', ${content}, 'complete', clock_timestamp(), ${ctx.sql.json(metadata as never)})
RETURNING id
`;
// The sentinel content is static, but we still walk the standard frame
// sequence (started → delta → complete) so useSessionStream's reducer
// appends it via the same path it uses for streaming assistant messages.
// The delta carries the full text in one chunk.
ctx.publish(sessionId, {
type: 'message_started',
message_id: row!.id,
chat_id: chatId,
role: 'system',
});
ctx.publish(sessionId, {
type: 'delta',
message_id: row!.id,
chat_id: chatId,
content,
});
ctx.publish(sessionId, {
type: 'message_complete',
message_id: row!.id,
chat_id: chatId,
metadata,
});
}
// v1.11.6: doom-loop wrap-up. Mirrors runCapHitSummary structurally — same
// in-flight-slot reuse, same tools-disabled streaming-summary call, same
// post-finalize sentinel insert + chat_status drop. Differences:
// - synthetic note text comes from DOOM_LOOP_NOTE (names the looping tool)
// - sentinel metadata is { kind: 'doom_loop', tool_name, args, threshold }
// and has no Continue affordance (manual retry would just re-loop)
// - chat_status error path uses reason: 'doom_loop_summary_failed'
// Kept as a clone rather than refactored into a shared helper because the
// two summary paths still differ in error reason + sentinel shape; a third
// sentinel would justify factoring out runWrapUpSummary(opts).
export async function runDoomLoopSummary(
ctx: InferenceContext,
args: TurnArgs,
session: Session,
project: Project,
history: Message[],
agent: Agent | null,
loop: { name: string; args: Record<string, unknown> },
): Promise<void> {
const { sessionId, chatId, assistantMessageId, signal } = args;
const messages = await buildMessagesPayload(session, project, history, agent, ctx.log);
messages.push({ role: 'system', content: DOOM_LOOP_NOTE(loop.name) });
const startedRow = await ctx.sql<{ started_at: string }[]>`
UPDATE messages
SET started_at = clock_timestamp()
WHERE id = ${assistantMessageId}
RETURNING started_at
`;
const startedAt = startedRow[0]?.started_at ?? null;
ctx.publish(sessionId, {
type: 'message_started',
message_id: assistantMessageId,
chat_id: chatId,
role: 'assistant',
});
let accumulated = '';
let pendingFlushTimer: NodeJS.Timeout | null = null;
let flushPromise: Promise<unknown> = Promise.resolve();
const flushNow = () => {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
const snapshot = accumulated;
flushPromise = flushPromise.then(() =>
ctx.sql`UPDATE messages SET content = ${snapshot} WHERE id = ${assistantMessageId}`
);
};
const scheduleFlush = () => {
if (pendingFlushTimer) return;
pendingFlushTimer = setTimeout(() => {
pendingFlushTimer = null;
flushNow();
}, DB_FLUSH_INTERVAL_MS);
};
let summaryOk = false;
let summarySoftCancelled = false;
let summaryError: string | null = null;
let result: StreamResult | null = null;
try {
result = await streamCompletion(
ctx,
session.model,
messages,
{ tools: null, temperature: agent?.temperature, top_p: agent?.top_p ?? undefined, top_k: agent?.top_k ?? undefined, min_p: agent?.min_p ?? undefined, presence_penalty: agent?.presence_penalty ?? undefined, top_n_sigma: agent?.top_n_sigma ?? undefined, dry_multiplier: agent?.dry_multiplier ?? undefined, dry_base: agent?.dry_base ?? undefined, dry_allowed_length: agent?.dry_allowed_length ?? undefined, dry_penalty_last_n: agent?.dry_penalty_last_n ?? undefined },
(delta) => {
accumulated += delta;
ctx.publish(sessionId, {
type: 'delta',
message_id: assistantMessageId,
chat_id: chatId,
content: delta,
});
scheduleFlush();
},
undefined,
signal,
);
summaryOk = true;
} catch (err) {
if (err instanceof Error && err.name === 'AbortError') {
summarySoftCancelled = true;
} else {
summaryError = err instanceof Error ? err.message : String(err);
}
} finally {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
await flushPromise;
}
if (summaryOk && result) {
const mctx = await modelContext.getModelContext(session.model);
const nCtx = mctx?.n_ctx ?? null;
const [updated] = await ctx.sql<
{ tokens_used: number | null; ctx_used: number | null; ctx_max: number | null; finished_at: string | null }[]
>`
UPDATE messages
SET content = ${result.content},
status = 'complete',
tokens_used = ${result.completionTokens},
ctx_used = ${result.promptTokens},
ctx_max = ${nCtx},
finished_at = clock_timestamp()
WHERE id = ${assistantMessageId}
RETURNING tokens_used, ctx_used, ctx_max, finished_at
`;
ctx.publish(sessionId, {
type: 'message_complete',
message_id: assistantMessageId,
chat_id: chatId,
tokens_used: updated?.tokens_used ?? null,
ctx_used: updated?.ctx_used ?? null,
ctx_max: updated?.ctx_max ?? null,
started_at: startedAt,
finished_at: updated?.finished_at ?? null,
model: session.model,
});
} else if (summarySoftCancelled) {
await ctx.sql`
UPDATE messages
SET content = ${accumulated},
status = 'cancelled',
finished_at = clock_timestamp()
WHERE id = ${assistantMessageId}
`;
ctx.publish(sessionId, {
type: 'message_complete',
message_id: assistantMessageId,
chat_id: chatId,
});
} else {
// Doom-loop summary failure reuses the existing summary_after_cap_failed
// error reason — the ErrorReason union is shared between sentinel paths
// and the UI surfaces a generic "summary failed" line for both. We don't
// add a new reason code because the user-visible failure mode is the
// same (model gave up mid-summary). Sentinel below still fires.
const errMeta: MessageMetadata = {
kind: 'error',
error_reason: 'summary_after_cap_failed',
error_text: summaryError ?? 'doom-loop summary failed',
};
await ctx.sql`
UPDATE messages
SET content = ${accumulated},
status = 'failed',
finished_at = clock_timestamp(),
metadata = ${ctx.sql.json(errMeta as never)}
WHERE id = ${assistantMessageId}
`;
ctx.publish(sessionId, {
type: 'error',
message_id: assistantMessageId,
chat_id: chatId,
error: summaryError ?? 'doom-loop summary failed',
reason: 'summary_after_cap_failed',
});
}
const [sessRow] = await ctx.sql<{ project_id: string; name: string; updated_at: string }[]>`
UPDATE sessions SET updated_at = clock_timestamp()
WHERE id = ${sessionId}
RETURNING project_id, name, updated_at
`;
ctx.publishUser({
type: 'session_updated',
session_id: sessionId,
project_id: sessRow!.project_id,
name: sessRow!.name,
updated_at: sessRow!.updated_at,
});
await insertDoomLoopSentinel(ctx, sessionId, chatId, loop);
if (summaryOk || summarySoftCancelled) {
ctx.publishUser({ type: 'chat_status', chat_id: chatId, status: 'idle', at: new Date().toISOString() });
} else {
ctx.publishUser({
type: 'chat_status',
chat_id: chatId,
status: 'error',
at: new Date().toISOString(),
reason: 'summary_after_cap_failed',
});
}
ctx.log.info(
{ sessionId, chatId, assistantMessageId, loopedTool: loop.name, summaryOk, summaryCancelled: summarySoftCancelled },
'inference doom-loop summary finished',
);
}
// v1.14.0: step-cap wrap-up. Mirrors runCapHitSummary structurally — same
// in-flight-slot reuse, same tools-disabled streaming-summary call, same
// post-finalize sentinel insert + chat_status drop. Difference: the note
// text names the step limit rather than the tool budget. Sentinel reuses
// metadata.kind = 'cap_hit' so the frontend CapHitSentinel component
// renders it without changes.
const STEP_CAP_NOTE = (steps: number, cap: number) =>
`You've reached the step limit (${steps}/${cap} steps). Produce the best answer you can with what you have. Do not call more tools.`;
export async function runStepCapSummary(
ctx: InferenceContext,
args: TurnArgs,
session: Session,
project: Project,
history: Message[],
agent: Agent | null,
steps: number,
cap: number,
): Promise<void> {
const { sessionId, chatId, assistantMessageId, signal } = args;
const messages = await buildMessagesPayload(session, project, history, agent, ctx.log);
messages.push({ role: 'system', content: STEP_CAP_NOTE(steps, cap) });
const startedRow = await ctx.sql<{ started_at: string }[]>`
UPDATE messages
SET started_at = clock_timestamp()
WHERE id = ${assistantMessageId}
RETURNING started_at
`;
const startedAt = startedRow[0]?.started_at ?? null;
ctx.publish(sessionId, {
type: 'message_started',
message_id: assistantMessageId,
chat_id: chatId,
role: 'assistant',
});
let accumulated = '';
let pendingFlushTimer: NodeJS.Timeout | null = null;
let flushPromise: Promise<unknown> = Promise.resolve();
const flushNow = () => {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
const snapshot = accumulated;
flushPromise = flushPromise.then(() =>
ctx.sql`UPDATE messages SET content = ${snapshot} WHERE id = ${assistantMessageId}`
);
};
const scheduleFlush = () => {
if (pendingFlushTimer) return;
pendingFlushTimer = setTimeout(() => {
pendingFlushTimer = null;
flushNow();
}, DB_FLUSH_INTERVAL_MS);
};
let summaryOk = false;
let summarySoftCancelled = false;
let summaryError: string | null = null;
let result: StreamResult | null = null;
try {
result = await streamCompletion(
ctx,
session.model,
messages,
{ tools: null, temperature: agent?.temperature, top_p: agent?.top_p ?? undefined, top_k: agent?.top_k ?? undefined, min_p: agent?.min_p ?? undefined, presence_penalty: agent?.presence_penalty ?? undefined, top_n_sigma: agent?.top_n_sigma ?? undefined, dry_multiplier: agent?.dry_multiplier ?? undefined, dry_base: agent?.dry_base ?? undefined, dry_allowed_length: agent?.dry_allowed_length ?? undefined, dry_penalty_last_n: agent?.dry_penalty_last_n ?? undefined },
(delta) => {
accumulated += delta;
ctx.publish(sessionId, {
type: 'delta',
message_id: assistantMessageId,
chat_id: chatId,
content: delta,
});
scheduleFlush();
},
undefined,
signal,
);
summaryOk = true;
} catch (err) {
if (err instanceof Error && err.name === 'AbortError') {
summarySoftCancelled = true;
} else {
summaryError = err instanceof Error ? err.message : String(err);
}
} finally {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
await flushPromise;
}
if (summaryOk && result) {
const mctx = await modelContext.getModelContext(session.model);
const nCtx = mctx?.n_ctx ?? null;
const [updated] = await ctx.sql<
{ tokens_used: number | null; ctx_used: number | null; ctx_max: number | null; finished_at: string | null }[]
>`
UPDATE messages
SET content = ${result.content},
status = 'complete',
tokens_used = ${result.completionTokens},
ctx_used = ${result.promptTokens},
ctx_max = ${nCtx},
finished_at = clock_timestamp()
WHERE id = ${assistantMessageId}
RETURNING tokens_used, ctx_used, ctx_max, finished_at
`;
ctx.publish(sessionId, {
type: 'message_complete',
message_id: assistantMessageId,
chat_id: chatId,
tokens_used: updated?.tokens_used ?? null,
ctx_used: updated?.ctx_used ?? null,
ctx_max: updated?.ctx_max ?? null,
started_at: startedAt,
finished_at: updated?.finished_at ?? null,
model: session.model,
});
} else if (summarySoftCancelled) {
await ctx.sql`
UPDATE messages
SET content = ${accumulated},
status = 'cancelled',
finished_at = clock_timestamp()
WHERE id = ${assistantMessageId}
`;
ctx.publish(sessionId, {
type: 'message_complete',
message_id: assistantMessageId,
chat_id: chatId,
});
} else {
const errMeta: MessageMetadata = {
kind: 'error',
error_reason: 'summary_after_cap_failed',
error_text: summaryError ?? 'step-cap summary failed',
};
await ctx.sql`
UPDATE messages
SET content = ${accumulated},
status = 'failed',
finished_at = clock_timestamp(),
metadata = ${ctx.sql.json(errMeta as never)}
WHERE id = ${assistantMessageId}
`;
ctx.publish(sessionId, {
type: 'error',
message_id: assistantMessageId,
chat_id: chatId,
error: summaryError ?? 'step-cap summary failed',
reason: 'summary_after_cap_failed',
});
}
const [sessRow] = await ctx.sql<{ project_id: string; name: string; updated_at: string }[]>`
UPDATE sessions SET updated_at = clock_timestamp()
WHERE id = ${sessionId}
RETURNING project_id, name, updated_at
`;
ctx.publishUser({
type: 'session_updated',
session_id: sessionId,
project_id: sessRow!.project_id,
name: sessRow!.name,
updated_at: sessRow!.updated_at,
});
// Reuse cap_hit sentinel so the frontend CapHitSentinel component renders
// it without changes. The content text distinguishes step cap from budget.
await insertCapHitSentinel(ctx, sessionId, chatId, agent, cap);
if (summaryOk || summarySoftCancelled) {
ctx.publishUser({ type: 'chat_status', chat_id: chatId, status: 'idle', at: new Date().toISOString() });
} else {
ctx.publishUser({
type: 'chat_status',
chat_id: chatId,
status: 'error',
at: new Date().toISOString(),
reason: 'summary_after_cap_failed',
});
}
ctx.log.info(
{ sessionId, chatId, assistantMessageId, steps, cap, summaryOk, summaryCancelled: summarySoftCancelled },
'inference step-cap summary finished',
);
await insertSentinel(ctx, sessionId, chatId, metadata, content);
}
async function insertDoomLoopSentinel(
@@ -689,39 +358,12 @@ async function insertDoomLoopSentinel(
threshold: DOOM_LOOP_THRESHOLD,
};
const content = `Detected ${DOOM_LOOP_THRESHOLD} identical calls to ${loop.name}. Stopping the tool-call loop. Produce the best answer you can with what you have.`;
const [row] = await ctx.sql<{ id: string }[]>`
INSERT INTO messages (session_id, chat_id, role, content, status, created_at, metadata)
VALUES (${sessionId}, ${chatId}, 'system', ${content}, 'complete', clock_timestamp(), ${ctx.sql.json(metadata as never)})
RETURNING id
`;
// Standard frame sequence — same as cap-hit sentinel — so
// useSessionStream's reducer appends the row via the existing path.
ctx.publish(sessionId, {
type: 'message_started',
message_id: row!.id,
chat_id: chatId,
role: 'system',
});
ctx.publish(sessionId, {
type: 'delta',
message_id: row!.id,
chat_id: chatId,
content,
});
ctx.publish(sessionId, {
type: 'message_complete',
message_id: row!.id,
chat_id: chatId,
metadata,
});
await insertSentinel(ctx, sessionId, chatId, metadata, content);
}
// #12 MistakeTracker: heterogeneous-failure recovery sentinel. Mirrors
// insertDoomLoopSentinel structurally — a role='system', status='complete' row
// firing the standard message_started → delta → message_complete frame
// sequence. Two variants distinguished by `escalated`:
// #12 MistakeTracker: heterogeneous-failure recovery sentinel. A role='system',
// status='complete' row firing the standard sentinel frame sequence. Two
// variants distinguished by `escalated`:
// - escalated:false → a nudge fired; recovery guidance was injected into the
// model's next step and the loop continued. can_continue is true (the turn
// is still live).
@@ -744,30 +386,5 @@ export async function insertMistakeRecoverySentinel(
const content = opts.escalated
? `Repeated different errors persisted after a recovery nudge (${opts.count} in a row). Stopping the tool-call loop.`
: `Hit ${opts.count} different errors in a row. Injected recovery guidance and continuing.`;
const [row] = await ctx.sql<{ id: string }[]>`
INSERT INTO messages (session_id, chat_id, role, content, status, created_at, metadata)
VALUES (${sessionId}, ${chatId}, 'system', ${content}, 'complete', clock_timestamp(), ${ctx.sql.json(metadata as never)})
RETURNING id
`;
// Standard frame sequence — same as cap-hit / doom-loop sentinels.
ctx.publish(sessionId, {
type: 'message_started',
message_id: row!.id,
chat_id: chatId,
role: 'system',
});
ctx.publish(sessionId, {
type: 'delta',
message_id: row!.id,
chat_id: chatId,
content,
});
ctx.publish(sessionId, {
type: 'message_complete',
message_id: row!.id,
chat_id: chatId,
metadata,
});
await insertSentinel(ctx, sessionId, chatId, metadata, content);
}

11
apps/server/src/services/inference/sentinels.ts
View File

@@ -27,6 +27,10 @@ export function detectDoomLoop(
return { name: ref.name, args: ref.args };
}
// All sentinel kinds. isAnySentinel and compaction.ts's local predicate both
// consume this set — single source so a new kind can't be missed in one.
export const SENTINEL_KINDS = new Set(['cap_hit', 'doom_loop', 'mistake_recovery']);
export function isCapHitSentinel(m: Message): boolean {
return (
m.role === 'system' &&
@@ -61,5 +65,10 @@ export function isMistakeRecoverySentinel(m: Message): boolean {
}
export function isAnySentinel(m: Message): boolean {
return isCapHitSentinel(m) || isDoomLoopSentinel(m) || isMistakeRecoverySentinel(m);
return (
m.role === 'system' &&
m.metadata !== null &&
typeof m.metadata === 'object' &&
SENTINEL_KINDS.has((m.metadata as { kind?: unknown }).kind as string)
);
}

47
apps/server/src/services/inference/step-decision.ts Normal file
View File

@@ -0,0 +1,47 @@
// P5 (SPLIT SKETCH 5): pure step-decision helpers for the runAssistantTurn
// loop. These COMPOSE the existing decision predicates (detectDoomLoop,
// detectMistakePattern) — they do not reimplement them — so the loop body in
// turn.ts becomes a thin driver and the branch logic is unit-testable without
// a DB, broker, or stream.
import type { ToolCall } from '../../types/api.js';
import { detectDoomLoop } from './sentinels.js';
import { detectMistakePattern, type MistakeState } from './mistake-tracker.js';
import type { ToolPhaseResult } from './tool-phase.js';
// Top-of-loop gate, evaluated before the stream phase. Order matters and
// matches the original inline checks exactly: doom-loop first (identical-repeat
// guard), then the cumulative tool-call budget, otherwise proceed to stream.
export type PreStepDecision =
| { kind: 'doom'; loop: { name: string; args: Record<string, unknown> } }
| { kind: 'budget' }
| { kind: 'stream' };
export function decideStep(input: {
recentToolCalls: ToolCall[];
toolsUsed: number;
budget: number;
}): PreStepDecision {
const loop = detectDoomLoop(input.recentToolCalls);
if (loop) return { kind: 'doom', loop };
if (input.toolsUsed >= input.budget) return { kind: 'budget' };
return { kind: 'stream' };
}
// Post-tool-phase decision, evaluated after the tool phase returns. 'stop'
// covers the tool-phase's own non-'continue' actions ('paused' for user input,
// 'synthesis_done'); on 'continue' the mistake-tracker pattern gates the
// nudge/escalate/continue choice (detectMistakePattern is only consulted on the
// 'continue' path, exactly as the original loop did).
export type PostToolDecision = 'continue' | 'nudge' | 'escalate' | 'stop';
export function decidePostToolAction(
action: ToolPhaseResult['action'],
mistakeTracker: MistakeState,
): PostToolDecision {
if (action !== 'continue') return 'stop';
const mistake = detectMistakePattern(mistakeTracker);
if (mistake === 'nudge') return 'nudge';
if (mistake === 'escalate') return 'escalate';
return 'continue';
}

405
apps/server/src/services/inference/stream-phase-adapter.ts Normal file
View File

@@ -0,0 +1,405 @@
// P5 (SPLIT SKETCH): the generic AI-SDK adapter, split out of stream-phase.ts.
// This module is the v1.13.1-A streamText adapter and nothing else — it has NO
// SQL, broker, or BooCode persistence dependencies (its only `ctx` access is
// config + log), so it can be unit-tested without standing up a DB or broker.
// stream-phase.ts (the I/O layer) re-exports the public names below so existing
// importers (`./stream-phase.js`) are unchanged.
import type { FastifyBaseLogger } from 'fastify';
import type { Config } from '../../config.js';
import type { Agent, ToolCall } from '../../types/api.js';
import type { ToolJsonSchema } from '../tools.js';
import type { OpenAiMessage } from './payload.js';
import { extractToolCallBlocks } from './tool-call-parser.js';
import type { StreamResult } from './types.js';
import { upstreamModel } from './provider.js';
import {
jsonSchema,
streamText,
tool,
type JSONValue,
type ModelMessage,
type ToolCallRepairFunction,
} from 'ai';
// The slice of InferenceContext the adapter actually needs. Narrowing it here
// (instead of taking the full InferenceContext) keeps the adapter free of the
// SQL/broker/publish surface. InferenceContext structurally satisfies this, so
// callers pass their ctx unchanged.
export interface StreamAdapterContext {
config: Config;
log: FastifyBaseLogger;
}
export interface StreamOptions {
// null = omit tools entirely (compact phase); [] = caller stripped all tools
// (rare; we still omit from the request body to avoid OpenAI 400).
tools: ToolJsonSchema[] | null;
temperature?: number;
top_p?: number | null;
top_k?: number | null;
min_p?: number | null;
presence_penalty?: number | null;
// v2.6 sampling-streamjson-tokens (#11): llama.cpp sampler extensions. These
// are NOT standard AI-SDK streamText options and are NOT serialized by the
// openai-compatible provider's standardized-settings path (topK is even
// explicitly dropped with an "unsupported feature: topK" warning). They reach
// llama-server only via providerOptions.openaiCompatible (see buildSamplerProviderOptions).
top_n_sigma?: number | null;
dry_multiplier?: number | null;
dry_base?: number | null;
dry_allowed_length?: number | null;
dry_penalty_last_n?: number | null;
}
// P5: the 10-field sampler-options literal that was copy-pasted at 4 sites
// (the three sentinel summaries + executeStreamPhase). Builds the StreamOptions
// sampler subset from an agent's frontmatter knobs. `temperature` is
// `agent?.temperature` (already number|undefined); the nullable fields strip
// null → undefined so they're omitted from the request body when unset. Keep
// this in lockstep with the StreamOptions sampler fields — a new sampler knob
// (the v2.7.3 dry_* family did this) is added here once instead of at 4 sites.
export type SamplerOpts = Omit<StreamOptions, 'tools'>;
export function samplerOptsFromAgent(agent: Agent | null): SamplerOpts {
return {
temperature: agent?.temperature,
top_p: agent?.top_p ?? undefined,
top_k: agent?.top_k ?? undefined,
min_p: agent?.min_p ?? undefined,
presence_penalty: agent?.presence_penalty ?? undefined,
top_n_sigma: agent?.top_n_sigma ?? undefined,
dry_multiplier: agent?.dry_multiplier ?? undefined,
dry_base: agent?.dry_base ?? undefined,
dry_allowed_length: agent?.dry_allowed_length ?? undefined,
dry_penalty_last_n: agent?.dry_penalty_last_n ?? undefined,
};
}
// v2.6 #11: build the providerOptions.openaiCompatible extraBody object for the
// llama.cpp sampler extensions. @ai-sdk/openai-compatible (2.0.47) merges every
// non-reserved key under providerOptions.openaiCompatible straight into the
// chat-completion request body (see its getArgs: the Object.fromEntries spread
// filtered against openaiCompatibleLanguageModelChatOptions.shape). This is the
// ONLY working passthrough for these params:
// - top_k / min_p were latently dropped before this: top_k was passed as the
// AI-SDK `topK` setting which the openai-compatible provider rejects as
// unsupported; min_p was never passed to streamText at all.
// - top_n_sigma + the dry_* family have no AI-SDK equivalent.
// Keys use llama-server's snake_case body names so they land verbatim.
function buildSamplerProviderOptions(opts: StreamOptions): Record<string, number> | undefined {
const body: Record<string, number> = {};
if (typeof opts.top_k === 'number') body.top_k = opts.top_k;
if (typeof opts.min_p === 'number') body.min_p = opts.min_p;
if (typeof opts.top_n_sigma === 'number') body.top_n_sigma = opts.top_n_sigma;
if (typeof opts.dry_multiplier === 'number') body.dry_multiplier = opts.dry_multiplier;
if (typeof opts.dry_base === 'number') body.dry_base = opts.dry_base;
if (typeof opts.dry_allowed_length === 'number') body.dry_allowed_length = opts.dry_allowed_length;
if (typeof opts.dry_penalty_last_n === 'number') body.dry_penalty_last_n = opts.dry_penalty_last_n;
return Object.keys(body).length > 0 ? body : undefined;
}
// v1.13.1-A: convert BooCode's OpenAI-shaped history into AI SDK
// ModelMessage[]. Tool result messages need a `toolName` field that the
// OpenAI shape doesn't carry; we look it up by scanning earlier assistant
// `tool_calls` entries for a matching id.
function toModelMessages(messages: OpenAiMessage[]): ModelMessage[] {
const toolNameById = new Map<string, string>();
for (const m of messages) {
if (m.role === 'assistant' && m.tool_calls) {
for (const tc of m.tool_calls) {
toolNameById.set(tc.id, tc.function.name);
}
}
}
const out: ModelMessage[] = [];
for (const m of messages) {
if (m.role === 'system' || m.role === 'user') {
out.push({ role: m.role, content: m.content ?? '' });
continue;
}
if (m.role === 'assistant') {
const hasTools = m.tool_calls && m.tool_calls.length > 0;
const hasReasoning = typeof m.reasoning === 'string' && m.reasoning.length > 0;
if (!hasTools && !hasReasoning) {
// Bare text assistant (string content). null content + no tool_calls
// is degenerate but harmless to forward.
out.push({ role: 'assistant', content: m.content ?? '' });
continue;
}
// v1.13.1-C: AI SDK ReasoningPart precedes text + tool-calls in the
// assistant content array. Reasoning models (qwen3.6) consume their
// prior reasoning context to resume mid-thought across tool boundaries.
const parts: Array<
| { type: 'reasoning'; text: string }
| { type: 'text'; text: string }
| { type: 'tool-call'; toolCallId: string; toolName: string; input: unknown }
> = [];
if (hasReasoning) {
parts.push({ type: 'reasoning', text: m.reasoning! });
}
if (m.content && m.content.length > 0) {
parts.push({ type: 'text', text: m.content });
}
for (const tc of m.tool_calls ?? []) {
let input: unknown = {};
try {
input = tc.function.arguments.length > 0 ? JSON.parse(tc.function.arguments) : {};
} catch {
// Malformed args from a prior turn: pass through as a raw blob so
// the model sees the same shape it emitted. Wraps the string under
// _raw to match the buildMessagesPayload upstream convention.
input = { _raw: tc.function.arguments };
}
parts.push({ type: 'tool-call', toolCallId: tc.id, toolName: tc.function.name, input });
}
out.push({ role: 'assistant', content: parts });
continue;
}
if (m.role === 'tool') {
const toolCallId = m.tool_call_id ?? '';
const toolName = toolNameById.get(toolCallId) ?? 'unknown';
const raw = m.content ?? '';
let output: { type: 'text'; value: string } | { type: 'json'; value: JSONValue };
try {
// JSON.parse returns `any`; cast to JSONValue since the upstream
// tool_results column is already JSON-serializable by construction.
output = { type: 'json', value: JSON.parse(raw) as JSONValue };
} catch {
output = { type: 'text', value: raw };
}
out.push({
role: 'tool',
content: [{ type: 'tool-result', toolCallId, toolName, output }],
});
continue;
}
}
return out;
}
// Build the AI SDK tools record from BooCode's JSON-schema tool definitions.
// No `execute` field: BooCode runs tools itself in tool-phase.ts; streamText
// surfaces the tool-call parts via fullStream and we capture them for the
// outer loop to dispatch.
function buildAiTools(schemas: ToolJsonSchema[]): Record<string, ReturnType<typeof tool>> {
const out: Record<string, ReturnType<typeof tool>> = {};
for (const s of schemas) {
out[s.function.name] = tool({
description: s.function.description,
inputSchema: jsonSchema(s.function.parameters),
});
}
return out;
}
// v1.10.5 Qwen-coder XML fallback. Some local models (notably qwen3-coder via
// llama-swap) emit tool calls as inline XML inside delta.content rather than
// the structured tool_calls field. We extract them out of the streamed text
// before flushing it to the client.
//
// Qwen shape:
// <tool_call>
// <function=NAME>
// <parameter=KEY>VALUE</parameter>
// ...
// </function>
// </tool_call>
//
// v1.13.16: also recognize Anthropic <invoke> markup that qwen3.6-35b-a3b-mxfp4
// drifts to (training-data residue from Claude Code documentation):
// <invoke name="NAME">
// <parameter name="KEY">VALUE</parameter>
// </invoke>
// Both formats share the synthetic xml_call_${idx} ID space; the counter
// increments across whichever opener appears first. Multiple blocks may
// appear back-to-back in either format and they never nest.
export async function streamCompletion(
ctx: StreamAdapterContext,
model: string,
messages: OpenAiMessage[],
opts: StreamOptions,
onDelta: (content: string) => void,
onUsage: ((prompt: number | null, completion: number | null) => void) | undefined,
signal?: AbortSignal,
agent?: Agent | null,
): Promise<StreamResult> {
const aiMessages = toModelMessages(messages);
const hasTools = opts.tools !== null && opts.tools.length > 0;
const aiTools = hasTools ? buildAiTools(opts.tools!) : undefined;
const startedAt = Date.now();
// v1.13.1-C: accumulate reasoning text across reasoning-delta parts.
// qwen3.6 emits these on a separate channel from text content; we capture
// them per stream so finalizeCompletion can dual-write a 'reasoning' part.
// Replaces the v1.13.1-A counter-only diagnostic.
let reasoningAccumulated = '';
// v1.13.3: experimental_repairToolCall keeps the stream alive when the
// model emits a malformed tool call (bad JSON args, unknown name, etc.).
// Without a repair function streamText throws and the WHOLE stream dies;
// with one, the SDK invokes us and we route the bad call through normally.
// Strategy: pass through unmodified. executeToolPhase's existing error
// path (unknown tool name → "unknown tool: X" result; zod-reject → tool
// 'X' rejected — fieldname: required) already gives the model a clean
// recovery surface on the next turn. Logging gives us visibility into
// how often qwen3.6 actually emits broken calls.
const repairToolCall: ToolCallRepairFunction<NonNullable<typeof aiTools>> = async ({
toolCall,
error,
}) => {
ctx.log.warn(
{
toolCallId: toolCall.toolCallId,
toolName: toolCall.toolName,
error: error.message,
},
'malformed tool call surfaced via repairToolCall',
);
return toolCall;
};
// v2.6 #11: llama.cpp sampler extensions (top_k, min_p, top_n_sigma, dry_*)
// ride providerOptions.openaiCompatible — they are NOT standardized streamText
// settings. NB: top_k used to be passed below as the AI-SDK `topK` setting;
// the openai-compatible provider dropped it with an "unsupported feature: topK"
// warning and min_p was never wired at all, so both were dead on the wire
// before this. They now go through the same extraBody path as the new params.
const samplerBody = buildSamplerProviderOptions(opts);
const result = streamText({
model: upstreamModel(ctx.config, model, agent ?? null),
messages: aiMessages,
...(aiTools
? { tools: aiTools, toolChoice: 'auto' as const, experimental_repairToolCall: repairToolCall }
: {}),
...(typeof opts.temperature === 'number' ? { temperature: opts.temperature } : {}),
...(typeof opts.top_p === 'number' ? { topP: opts.top_p } : {}),
...(typeof opts.presence_penalty === 'number' ? { presencePenalty: opts.presence_penalty } : {}),
...(samplerBody ? { providerOptions: { openaiCompatible: samplerBody } } : {}),
abortSignal: signal,
});
let content = '';
let pendingBuffer = '';
let finishReason: string | null = null;
// v1.13.1-A: AI SDK emits one `tool-call` part per fully-aggregated call,
// so we no longer need the OpenAI-index reassembly map the manual SSE
// parser used. XML tool calls extracted from text content go into the
// same flat list and keep the v1.10.5 synthetic id convention.
const toolCalls: ToolCall[] = [];
for await (const part of result.fullStream) {
switch (part.type) {
case 'text-delta': {
pendingBuffer += part.text;
// v1.13.16: unified extraction. The helper finds the earliest-opening
// complete <tool_call> or <invoke> block, flushes prose between/around
// them, holds any partial opener for the next chunk, and silently
// drops blocks that fail to parse (matches pre-v1.13.16 behavior).
const extracted = extractToolCallBlocks(pendingBuffer);
if (extracted.flushed.length > 0) {
content += extracted.flushed;
onDelta(extracted.flushed);
}
for (const call of extracted.calls) {
const synthIdx = toolCalls.length;
toolCalls.push({
id: `xml_call_${synthIdx}`,
name: call.name,
args: call.args,
});
}
pendingBuffer = extracted.remaining;
break;
}
case 'tool-call': {
// AI SDK has already parsed the input into an object. Match the
// ToolCall shape BooCode passes around in toolCallsBuffer downstream.
toolCalls.push({
id: part.toolCallId,
name: part.toolName,
args: (part.input ?? {}) as Record<string, unknown>,
});
break;
}
case 'reasoning-delta': {
// v1.13.1-C: accumulate; finalizeCompletion / executeToolPhase
// dual-write the resulting text as a kind='reasoning' part.
if (typeof part.text === 'string') {
reasoningAccumulated += part.text;
}
break;
}
case 'finish': {
if (typeof part.finishReason === 'string') {
finishReason = part.finishReason;
}
break;
}
case 'error': {
const err = part.error;
throw err instanceof Error ? err : new Error(String(err));
}
// Intentional no-op: start, start-step, text-start, text-end,
// reasoning-start, reasoning-end, source, file, tool-input-start,
// tool-input-delta, tool-input-end, tool-result, tool-error,
// finish-step, raw. We only care about the aggregated tool-call and
// text-delta paths above; the rest are AI SDK lifecycle/streaming
// breadcrumbs that don't change BooCode's persistence or WS contract.
default:
break;
}
}
// v1.13.1-A: drain any buffered partial XML opener as plain text. The
// pre-AI-SDK path did this on stream end too — better to leak `<tool_c`
// than vanish the text.
if (pendingBuffer.length > 0) {
content += pendingBuffer;
onDelta(pendingBuffer);
pendingBuffer = '';
}
// AI SDK v6 fullStream returns normally on abort; check signal explicitly.
// Without this throw the row would land as status='complete' with partial
// content instead of going through handleAbortOrError → status='cancelled'.
// Smoke D caught this in v1.13.1-A — don't refactor it away.
if (signal?.aborted) {
const abortErr = new Error('aborted');
abortErr.name = 'AbortError';
throw abortErr;
}
// Usage lands as a promise on the result; awaiting after fullStream is
// drained is safe. AI SDK v6 names: `inputTokens` / `outputTokens`.
let promptTokens: number | null = null;
let completionTokens: number | null = null;
try {
const usage = await result.usage;
if (typeof usage.inputTokens === 'number') promptTokens = usage.inputTokens;
if (typeof usage.outputTokens === 'number') completionTokens = usage.outputTokens;
} catch {
// Some providers omit usage on partial streams; leave both null.
}
if (onUsage && (promptTokens !== null || completionTokens !== null)) {
onUsage(promptTokens, completionTokens);
}
if (reasoningAccumulated.length > 0) {
ctx.log.debug(
{ reasoningChars: reasoningAccumulated.length, model, elapsed_ms: Date.now() - startedAt },
'streamCompletion: captured reasoning',
);
}
return {
finishReason,
content,
toolCalls,
promptTokens,
completionTokens,
reasoning: reasoningAccumulated,
};
}

439
apps/server/src/services/inference/stream-phase.ts
View File

@@ -1,377 +1,34 @@
import type {
Agent,
Session,
ToolCall,
} from '../../types/api.js';
// P5 (SPLIT SKETCH): stream-phase.ts is now the BooCode I/O layer for the
// stream phase — `executeStreamPhase` owns the row UPDATE, message_started
// frame, debounced content flush, throttled usage publish, model-context
// lookup, and tool-whitelist filter. The generic AI-SDK adapter
// (streamCompletion / toModelMessages / buildAiTools / sampler helpers) moved
// to ./stream-phase-adapter.ts, which has no SQL/broker/publish deps and is
// unit-testable on its own. The adapter's public names are re-exported below so
// existing importers of './stream-phase.js' (sentinel-summaries, synthesis
// pipeline, the helper tests) keep working unchanged.
import type { Agent, Session } from '../../types/api.js';
import * as modelContext from '../model-context.js';
import { toolJsonSchemas, type ToolJsonSchema } from '../tools.js';
import { matchToolGlob } from '../agents.js';
import type { OpenAiMessage } from './payload.js';
import { extractToolCallBlocks } from './tool-call-parser.js';
import { DB_FLUSH_INTERVAL_MS, type StreamPhaseState } from './types.js';
import { createContentFlusher } from './content-flusher.js';
import type {
StreamPhaseState,
InferenceContext,
StreamResult,
TurnArgs,
} from './turn.js';
import { upstreamModel } from './provider.js';
import {
jsonSchema,
streamText,
tool,
type JSONValue,
type ModelMessage,
type ToolCallRepairFunction,
} from 'ai';
} from './types.js';
import { streamCompletion, samplerOptsFromAgent } from './stream-phase-adapter.js';
interface StreamOptions {
// null = omit tools entirely (compact phase); [] = caller stripped all tools
// (rare; we still omit from the request body to avoid OpenAI 400).
tools: ToolJsonSchema[] | null;
temperature?: number;
top_p?: number | null;
top_k?: number | null;
min_p?: number | null;
presence_penalty?: number | null;
// v2.6 sampling-streamjson-tokens (#11): llama.cpp sampler extensions. These
// are NOT standard AI-SDK streamText options and are NOT serialized by the
// openai-compatible provider's standardized-settings path (topK is even
// explicitly dropped with an "unsupported feature: topK" warning). They reach
// llama-server only via providerOptions.openaiCompatible (see buildSamplerProviderOptions).
top_n_sigma?: number | null;
dry_multiplier?: number | null;
dry_base?: number | null;
dry_allowed_length?: number | null;
dry_penalty_last_n?: number | null;
}
// v2.6 #11: build the providerOptions.openaiCompatible extraBody object for the
// llama.cpp sampler extensions. @ai-sdk/openai-compatible (2.0.47) merges every
// non-reserved key under providerOptions.openaiCompatible straight into the
// chat-completion request body (see its getArgs: the Object.fromEntries spread
// filtered against openaiCompatibleLanguageModelChatOptions.shape). This is the
// ONLY working passthrough for these params:
// - top_k / min_p were latently dropped before this: top_k was passed as the
// AI-SDK `topK` setting which the openai-compatible provider rejects as
// unsupported; min_p was never passed to streamText at all.
// - top_n_sigma + the dry_* family have no AI-SDK equivalent.
// Keys use llama-server's snake_case body names so they land verbatim.
function buildSamplerProviderOptions(opts: StreamOptions): Record<string, number> | undefined {
const body: Record<string, number> = {};
if (typeof opts.top_k === 'number') body.top_k = opts.top_k;
if (typeof opts.min_p === 'number') body.min_p = opts.min_p;
if (typeof opts.top_n_sigma === 'number') body.top_n_sigma = opts.top_n_sigma;
if (typeof opts.dry_multiplier === 'number') body.dry_multiplier = opts.dry_multiplier;
if (typeof opts.dry_base === 'number') body.dry_base = opts.dry_base;
if (typeof opts.dry_allowed_length === 'number') body.dry_allowed_length = opts.dry_allowed_length;
if (typeof opts.dry_penalty_last_n === 'number') body.dry_penalty_last_n = opts.dry_penalty_last_n;
return Object.keys(body).length > 0 ? body : undefined;
}
// v1.13.1-A: convert BooCode's OpenAI-shaped history into AI SDK
// ModelMessage[]. Tool result messages need a `toolName` field that the
// OpenAI shape doesn't carry; we look it up by scanning earlier assistant
// `tool_calls` entries for a matching id.
function toModelMessages(messages: OpenAiMessage[]): ModelMessage[] {
const toolNameById = new Map<string, string>();
for (const m of messages) {
if (m.role === 'assistant' && m.tool_calls) {
for (const tc of m.tool_calls) {
toolNameById.set(tc.id, tc.function.name);
}
}
}
const out: ModelMessage[] = [];
for (const m of messages) {
if (m.role === 'system' || m.role === 'user') {
out.push({ role: m.role, content: m.content ?? '' });
continue;
}
if (m.role === 'assistant') {
const hasTools = m.tool_calls && m.tool_calls.length > 0;
const hasReasoning = typeof m.reasoning === 'string' && m.reasoning.length > 0;
if (!hasTools && !hasReasoning) {
// Bare text assistant (string content). null content + no tool_calls
// is degenerate but harmless to forward.
out.push({ role: 'assistant', content: m.content ?? '' });
continue;
}
// v1.13.1-C: AI SDK ReasoningPart precedes text + tool-calls in the
// assistant content array. Reasoning models (qwen3.6) consume their
// prior reasoning context to resume mid-thought across tool boundaries.
const parts: Array<
| { type: 'reasoning'; text: string }
| { type: 'text'; text: string }
| { type: 'tool-call'; toolCallId: string; toolName: string; input: unknown }
> = [];
if (hasReasoning) {
parts.push({ type: 'reasoning', text: m.reasoning! });
}
if (m.content && m.content.length > 0) {
parts.push({ type: 'text', text: m.content });
}
for (const tc of m.tool_calls ?? []) {
let input: unknown = {};
try {
input = tc.function.arguments.length > 0 ? JSON.parse(tc.function.arguments) : {};
} catch {
// Malformed args from a prior turn: pass through as a raw blob so
// the model sees the same shape it emitted. Wraps the string under
// _raw to match the buildMessagesPayload upstream convention.
input = { _raw: tc.function.arguments };
}
parts.push({ type: 'tool-call', toolCallId: tc.id, toolName: tc.function.name, input });
}
out.push({ role: 'assistant', content: parts });
continue;
}
if (m.role === 'tool') {
const toolCallId = m.tool_call_id ?? '';
const toolName = toolNameById.get(toolCallId) ?? 'unknown';
const raw = m.content ?? '';
let output: { type: 'text'; value: string } | { type: 'json'; value: JSONValue };
try {
// JSON.parse returns `any`; cast to JSONValue since the upstream
// tool_results column is already JSON-serializable by construction.
output = { type: 'json', value: JSON.parse(raw) as JSONValue };
} catch {
output = { type: 'text', value: raw };
}
out.push({
role: 'tool',
content: [{ type: 'tool-result', toolCallId, toolName, output }],
});
continue;
}
}
return out;
}
// Build the AI SDK tools record from BooCode's JSON-schema tool definitions.
// No `execute` field: BooCode runs tools itself in tool-phase.ts; streamText
// surfaces the tool-call parts via fullStream and we capture them for the
// outer loop to dispatch.
function buildAiTools(schemas: ToolJsonSchema[]): Record<string, ReturnType<typeof tool>> {
const out: Record<string, ReturnType<typeof tool>> = {};
for (const s of schemas) {
out[s.function.name] = tool({
description: s.function.description,
inputSchema: jsonSchema(s.function.parameters),
});
}
return out;
}
// v1.10.5 Qwen-coder XML fallback. Some local models (notably qwen3-coder via
// llama-swap) emit tool calls as inline XML inside delta.content rather than
// the structured tool_calls field. We extract them out of the streamed text
// before flushing it to the client.
//
// Qwen shape:
// <tool_call>
// <function=NAME>
// <parameter=KEY>VALUE</parameter>
// ...
// </function>
// </tool_call>
//
// v1.13.16: also recognize Anthropic <invoke> markup that qwen3.6-35b-a3b-mxfp4
// drifts to (training-data residue from Claude Code documentation):
// <invoke name="NAME">
// <parameter name="KEY">VALUE</parameter>
// </invoke>
// Both formats share the synthetic xml_call_${idx} ID space; the counter
// increments across whichever opener appears first. Multiple blocks may
// appear back-to-back in either format and they never nest.
export async function streamCompletion(
ctx: InferenceContext,
model: string,
messages: OpenAiMessage[],
opts: StreamOptions,
onDelta: (content: string) => void,
onUsage: ((prompt: number | null, completion: number | null) => void) | undefined,
signal?: AbortSignal,
agent?: Agent | null,
): Promise<StreamResult> {
const aiMessages = toModelMessages(messages);
const hasTools = opts.tools !== null && opts.tools.length > 0;
const aiTools = hasTools ? buildAiTools(opts.tools!) : undefined;
const startedAt = Date.now();
// v1.13.1-C: accumulate reasoning text across reasoning-delta parts.
// qwen3.6 emits these on a separate channel from text content; we capture
// them per stream so finalizeCompletion can dual-write a 'reasoning' part.
// Replaces the v1.13.1-A counter-only diagnostic.
let reasoningAccumulated = '';
// v1.13.3: experimental_repairToolCall keeps the stream alive when the
// model emits a malformed tool call (bad JSON args, unknown name, etc.).
// Without a repair function streamText throws and the WHOLE stream dies;
// with one, the SDK invokes us and we route the bad call through normally.
// Strategy: pass through unmodified. executeToolPhase's existing error
// path (unknown tool name → "unknown tool: X" result; zod-reject → tool
// 'X' rejected — fieldname: required) already gives the model a clean
// recovery surface on the next turn. Logging gives us visibility into
// how often qwen3.6 actually emits broken calls.
const repairToolCall: ToolCallRepairFunction<NonNullable<typeof aiTools>> = async ({
toolCall,
error,
}) => {
ctx.log.warn(
{
toolCallId: toolCall.toolCallId,
toolName: toolCall.toolName,
error: error.message,
},
'malformed tool call surfaced via repairToolCall',
);
return toolCall;
};
// v2.6 #11: llama.cpp sampler extensions (top_k, min_p, top_n_sigma, dry_*)
// ride providerOptions.openaiCompatible — they are NOT standardized streamText
// settings. NB: top_k used to be passed below as the AI-SDK `topK` setting;
// the openai-compatible provider dropped it with an "unsupported feature: topK"
// warning and min_p was never wired at all, so both were dead on the wire
// before this. They now go through the same extraBody path as the new params.
const samplerBody = buildSamplerProviderOptions(opts);
const result = streamText({
model: upstreamModel(ctx.config, model, agent ?? null),
messages: aiMessages,
...(aiTools
? { tools: aiTools, toolChoice: 'auto' as const, experimental_repairToolCall: repairToolCall }
: {}),
...(typeof opts.temperature === 'number' ? { temperature: opts.temperature } : {}),
...(typeof opts.top_p === 'number' ? { topP: opts.top_p } : {}),
...(typeof opts.presence_penalty === 'number' ? { presencePenalty: opts.presence_penalty } : {}),
...(samplerBody ? { providerOptions: { openaiCompatible: samplerBody } } : {}),
abortSignal: signal,
});
let content = '';
let pendingBuffer = '';
let finishReason: string | null = null;
// v1.13.1-A: AI SDK emits one `tool-call` part per fully-aggregated call,
// so we no longer need the OpenAI-index reassembly map the manual SSE
// parser used. XML tool calls extracted from text content go into the
// same flat list and keep the v1.10.5 synthetic id convention.
const toolCalls: ToolCall[] = [];
for await (const part of result.fullStream) {
switch (part.type) {
case 'text-delta': {
pendingBuffer += part.text;
// v1.13.16: unified extraction. The helper finds the earliest-opening
// complete <tool_call> or <invoke> block, flushes prose between/around
// them, holds any partial opener for the next chunk, and silently
// drops blocks that fail to parse (matches pre-v1.13.16 behavior).
const extracted = extractToolCallBlocks(pendingBuffer);
if (extracted.flushed.length > 0) {
content += extracted.flushed;
onDelta(extracted.flushed);
}
for (const call of extracted.calls) {
const synthIdx = toolCalls.length;
toolCalls.push({
id: `xml_call_${synthIdx}`,
name: call.name,
args: call.args,
});
}
pendingBuffer = extracted.remaining;
break;
}
case 'tool-call': {
// AI SDK has already parsed the input into an object. Match the
// ToolCall shape BooCode passes around in toolCallsBuffer downstream.
toolCalls.push({
id: part.toolCallId,
name: part.toolName,
args: (part.input ?? {}) as Record<string, unknown>,
});
break;
}
case 'reasoning-delta': {
// v1.13.1-C: accumulate; finalizeCompletion / executeToolPhase
// dual-write the resulting text as a kind='reasoning' part.
if (typeof part.text === 'string') {
reasoningAccumulated += part.text;
}
break;
}
case 'finish': {
if (typeof part.finishReason === 'string') {
finishReason = part.finishReason;
}
break;
}
case 'error': {
const err = part.error;
throw err instanceof Error ? err : new Error(String(err));
}
// Intentional no-op: start, start-step, text-start, text-end,
// reasoning-start, reasoning-end, source, file, tool-input-start,
// tool-input-delta, tool-input-end, tool-result, tool-error,
// finish-step, raw. We only care about the aggregated tool-call and
// text-delta paths above; the rest are AI SDK lifecycle/streaming
// breadcrumbs that don't change BooCode's persistence or WS contract.
default:
break;
}
}
// v1.13.1-A: drain any buffered partial XML opener as plain text. The
// pre-AI-SDK path did this on stream end too — better to leak `<tool_c`
// than vanish the text.
if (pendingBuffer.length > 0) {
content += pendingBuffer;
onDelta(pendingBuffer);
pendingBuffer = '';
}
// AI SDK v6 fullStream returns normally on abort; check signal explicitly.
// Without this throw the row would land as status='complete' with partial
// content instead of going through handleAbortOrError → status='cancelled'.
// Smoke D caught this in v1.13.1-A — don't refactor it away.
if (signal?.aborted) {
const abortErr = new Error('aborted');
abortErr.name = 'AbortError';
throw abortErr;
}
// Usage lands as a promise on the result; awaiting after fullStream is
// drained is safe. AI SDK v6 names: `inputTokens` / `outputTokens`.
let promptTokens: number | null = null;
let completionTokens: number | null = null;
try {
const usage = await result.usage;
if (typeof usage.inputTokens === 'number') promptTokens = usage.inputTokens;
if (typeof usage.outputTokens === 'number') completionTokens = usage.outputTokens;
} catch {
// Some providers omit usage on partial streams; leave both null.
}
if (onUsage && (promptTokens !== null || completionTokens !== null)) {
onUsage(promptTokens, completionTokens);
}
if (reasoningAccumulated.length > 0) {
ctx.log.debug(
{ reasoningChars: reasoningAccumulated.length, model, elapsed_ms: Date.now() - startedAt },
'streamCompletion: captured reasoning',
);
}
return {
finishReason,
content,
toolCalls,
promptTokens,
completionTokens,
reasoning: reasoningAccumulated,
};
}
export {
streamCompletion,
samplerOptsFromAgent,
type StreamOptions,
type SamplerOpts,
type StreamAdapterContext,
} from './stream-phase-adapter.js';
export async function executeStreamPhase(
ctx: InferenceContext,
@@ -401,27 +58,7 @@ export async function executeStreamPhase(
role: 'assistant',
});
let pendingFlushTimer: NodeJS.Timeout | null = null;
let flushPromise: Promise<unknown> = Promise.resolve();
const flushNow = () => {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
const snapshot = state.accumulated;
flushPromise = flushPromise.then(() =>
ctx.sql`UPDATE messages SET content = ${snapshot} WHERE id = ${assistantMessageId}`
);
};
const scheduleFlush = () => {
if (pendingFlushTimer) return;
pendingFlushTimer = setTimeout(() => {
pendingFlushTimer = null;
flushNow();
}, DB_FLUSH_INTERVAL_MS);
};
const flusher = createContentFlusher(ctx.sql, assistantMessageId, () => state.accumulated);
// Tool whitelist: if an agent is set, filter the global tool list to only the
// tool names it allows. v1.15.0-mcp-multi: uses matchToolGlob for glob
@@ -434,17 +71,6 @@ export async function executeStreamPhase(
? toolJsonSchemas().filter((t) => matchToolGlob(t.function.name, agent.tools))
: toolJsonSchemas()
).filter((t) => webToolsEnabled || !WEB_TOOL_NAMES.has(t.function.name));
const effectiveTemperature = agent?.temperature;
const effectiveTopP = agent?.top_p ?? undefined;
const effectiveTopK = agent?.top_k ?? undefined;
const effectiveMinP = agent?.min_p ?? undefined;
const effectivePresencePenalty = agent?.presence_penalty ?? undefined;
// v2.6 #11: llama.cpp sampler extensions, threaded the same way as top_k/min_p.
const effectiveTopNSigma = agent?.top_n_sigma ?? undefined;
const effectiveDryMultiplier = agent?.dry_multiplier ?? undefined;
const effectiveDryBase = agent?.dry_base ?? undefined;
const effectiveDryAllowedLength = agent?.dry_allowed_length ?? undefined;
const effectiveDryPenaltyLastN = agent?.dry_penalty_last_n ?? undefined;
// v1.12.2: ctx_max lookup is cached after the first hit per model, so this
// is a Map probe in steady state. We capture nCtx once at the top of the
@@ -484,16 +110,7 @@ export async function executeStreamPhase(
messages,
{
tools: effectiveTools,
temperature: effectiveTemperature,
top_p: effectiveTopP,
top_k: effectiveTopK,
min_p: effectiveMinP,
presence_penalty: effectivePresencePenalty,
top_n_sigma: effectiveTopNSigma,
dry_multiplier: effectiveDryMultiplier,
dry_base: effectiveDryBase,
dry_allowed_length: effectiveDryAllowedLength,
dry_penalty_last_n: effectiveDryPenaltyLastN,
...samplerOptsFromAgent(agent),
},
(delta) => {
state.accumulated += delta;
@@ -504,7 +121,7 @@ export async function executeStreamPhase(
content: delta,
});
ctx.log.debug({ sessionId, delta }, 'inference delta');
scheduleFlush();
flusher.scheduleFlush();
},
(prompt, completion) => {
pendingUsage = { p: prompt, c: completion };
@@ -522,14 +139,10 @@ export async function executeStreamPhase(
agent,
);
} finally {
if (pendingFlushTimer) {
clearTimeout(pendingFlushTimer);
pendingFlushTimer = null;
}
if (usageTimer) {
clearTimeout(usageTimer);
usageTimer = null;
}
await flushPromise;
await flusher.drain();
}
}

2
apps/server/src/services/inference/tool-phase.ts
View File

@@ -22,7 +22,7 @@ import type {
InferenceContext,
StreamResult,
TurnArgs,
} from './turn.js';
} from './types.js';
// v1.13.13: synthesis pipeline — replaces the immediate recursive turn when
// any of this batch's tool calls is in SYNTHESIS_TOOLS. Falls through to
// recursion on synthesis failure (timeout / model error). See module header

81
apps/server/src/services/inference/tool-summaries.ts
View File

@@ -1,81 +0,0 @@
/**
* v2.0.5: Tool-use summary generation.
*
* After a batch of tool calls completes, fire a cheap LLM call to generate
* a "git-commit-subject-style" one-liner label describing what the tools
* accomplished. Ported from the Qwen Code source recon.
*/
import type { FastifyBaseLogger } from 'fastify';
const TOOL_SUMMARY_SYSTEM_PROMPT = `Write a short summary label describing what these tool calls accomplished. Think git-commit-subject, not sentence. Past tense, most distinctive noun. Max 30 characters. Output ONLY the label.
Examples:
- Searched in auth/
- Fixed NPE in UserService
- Created signup endpoint
- Read config.json
- Ran failing tests`;
const INPUT_TRUNCATE = 300;
const MAX_SUMMARY_LENGTH = 100;
export interface ToolInfo {
name: string;
input: string;
output: string;
}
export async function generateToolUseSummary(opts: {
tools: ToolInfo[];
llamaSwapUrl: string;
model: string;
log: FastifyBaseLogger;
signal?: AbortSignal;
}): Promise<string | null> {
const { tools, llamaSwapUrl, model, log, signal } = opts;
if (tools.length === 0) return null;
if (signal?.aborted) return null;
const toolText = tools
.map(t => `Tool: ${t.name}\nInput: ${t.input.slice(0, INPUT_TRUNCATE)}\nOutput: ${t.output.slice(0, INPUT_TRUNCATE)}`)
.join('\n\n');
try {
const res = await fetch(`${llamaSwapUrl}/v1/chat/completions`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
messages: [
{ role: 'system', content: TOOL_SUMMARY_SYSTEM_PROMPT },
{ role: 'user', content: toolText },
],
max_tokens: 30,
temperature: 0.2,
stream: false,
chat_template_kwargs: { enable_thinking: false },
}),
signal,
});
if (!res.ok) {
log.debug({ status: res.status }, 'tool-summary: LLM request failed');
return null;
}
const data = await res.json() as { choices?: Array<{ message?: { content?: string } }> };
const raw = data.choices?.[0]?.message?.content?.trim() ?? '';
if (!raw) return null;
// Clean: strip quotes, "Label:" prefix, cap length
let cleaned = raw.split('\n')[0]?.trim() ?? '';
cleaned = cleaned
.replace(/^[-*•]\s+/, '')
.replace(/^["'`‘’“”]|["'`‘’“”]$/g, '')
.replace(/^(label|summary)\s*:\s*/i, '')
.trim();
return cleaned.length > MAX_SUMMARY_LENGTH
? cleaned.slice(0, MAX_SUMMARY_LENGTH).trim()
: cleaned || null;
} catch (err) {
log.debug({ err: err instanceof Error ? err.message : String(err) }, 'tool-summary: error');
return null;
}
}

33
apps/server/src/services/inference/turn-config.ts Normal file
View File

@@ -0,0 +1,33 @@
// P5 (SPLIT SKETCH 5): pure per-turn configuration resolved once at the top of
// runAssistantTurn. No I/O — just the cap math + budget lookup so it can be
// unit-tested without a DB or broker.
import type { Agent } from '../../types/api.js';
import { resolveToolBudget } from './budget.js';
// v1.14.0: hard ceiling on the number of stream-and-tool iterations per
// user-message turn. Per-agent cap via agent.steps is the primary knob;
// MAX_STEPS is the safety ceiling. 200 is 4x the effective budget ceiling
// (50 tool calls) — in practice budget fires first unless the model makes
// many 0-tool-call iterations (which exit the loop via the non-tool finish
// path anyway).
export const MAX_STEPS = 200;
export interface TurnConfig {
// min(agent.steps ?? Infinity, MAX_STEPS). The while loop runs while
// stepNumber < effectiveCap.
effectiveCap: number;
// cumulative tool-call budget for the turn (resolveToolBudget).
budget: number;
// effectiveCap === 0 → the model responds text-only (no tool execution).
isTextOnly: boolean;
}
export function resolveTurnConfig(agent: Agent | null): TurnConfig {
const budget = resolveToolBudget(agent);
// v1.14.0: effectiveCap = min(agent.steps ?? Infinity, MAX_STEPS).
// steps: 0 means "no tool calls allowed" — the first stream phase runs but
// any tool calls it emits are not executed (finalize as text-only).
const effectiveCap = Math.min(agent?.steps ?? Infinity, MAX_STEPS);
return { effectiveCap, budget, isTextOnly: effectiveCap === 0 };
}

200
apps/server/src/services/inference/turn.ts
View File

@@ -1,33 +1,21 @@
import type { FastifyBaseLogger } from 'fastify';
import type { Sql } from '../../db.js';
import type { Config } from '../../config.js';
import type {
Agent,
ErrorReason,
Message,
MessageMetadata,
Project,
Session,
ToolCall,
UserStreamFrame,
} from '../../types/api.js';
import { ALL_TOOLS } from '../tools.js';
import { resolveProjectRoot } from '../path_guard.js';
import { maybeAutoNameChat } from '../auto_name.js';
import { rewriteSearchQuery } from '../task-search-rewrite.js';
import { getAgentById } from '../agents.js';
import * as compaction from '../compaction.js';
import type { Broker } from '../broker.js';
import { resolveToolBudget } from './budget.js';
import { resolveTurnConfig } from './turn-config.js';
import { decideStep, decidePostToolAction } from './step-decision.js';
import {
detectDoomLoop,
} from './sentinels.js';
import {
detectMistakePattern,
freshMistakeState,
recordStep,
MISTAKE_RECOVERY_NOTE,
type MistakeState,
} from './mistake-tracker.js';
import {
buildMessagesPayload,
@@ -35,13 +23,19 @@ import {
} from './payload.js';
import {
finalizeCompletion,
finalizeEmpty,
handleAbortOrError,
} from './error-handler.js';
import {
executeStreamPhase,
} from './stream-phase.js';
import { executeToolPhase, type ToolPhaseResult } from './tool-phase.js';
import type { StreamPhaseState } from './types.js';
import type {
InferenceContext,
StreamPhaseState,
StreamResult,
TurnArgs,
} from './types.js';
import {
runCapHitSummary,
runDoomLoopSummary,
@@ -49,121 +43,24 @@ import {
insertMistakeRecoverySentinel,
} from './sentinel-summaries.js';
// v1.14.0: hard ceiling on the number of stream-and-tool iterations per
// user-message turn. Per-agent cap via agent.steps is the primary knob;
// MAX_STEPS is the safety ceiling. 200 is 4x the effective budget ceiling
// (50 tool calls) — in practice budget fires first unless the model makes
// many 0-tool-call iterations (which exit the loop via the non-tool finish
// path anyway).
export const MAX_STEPS = 200;
// P5: MAX_STEPS moved to ./turn-config.ts (with resolveTurnConfig). Re-exported
// here so the public surface (index.ts → './turn.js') is unchanged.
export { MAX_STEPS } from './turn-config.js';
// v1.12.4: re-exported so external callers (tests, future consumers) keep
// importing from services/inference.js as the public surface.
export { detectDoomLoop, DOOM_LOOP_THRESHOLD } from './sentinels.js';
export { buildMessagesPayload } from './payload.js';
export interface InferenceFrame {
type:
| 'message_started'
| 'delta'
| 'tool_call'
| 'tool_result'
| 'message_complete'
| 'usage'
| 'messages_deleted'
| 'session_renamed'
| 'chat_renamed'
| 'error';
message_id?: string;
message_ids?: string[];
chat_id?: string;
tool_message_id?: string;
tool_call_id?: string;
// v1.8.2: 'system' added so cap-hit sentinel messages can announce themselves
// through the normal message_started → delta → message_complete sequence.
role?: 'assistant' | 'tool' | 'user' | 'system';
content?: string;
tool_call?: ToolCall;
output?: unknown;
truncated?: boolean;
error?: string;
// v1.8.2: structured error reason. Set on `type: 'error'` so the UI can
// surface a specific message; `error` stays the human-readable text.
reason?: ErrorReason;
// v1.8.2: piggybacks on `message_complete` so static or terminally-resolved
// messages can carry their persisted metadata to the live stream without a
// refetch (sentinels carry { kind: 'cap_hit', ... }; failed messages carry
// { kind: 'error', ... }).
metadata?: MessageMetadata | null;
tokens_used?: number | null;
ctx_used?: number | null;
ctx_max?: number | null;
completion_tokens?: number | null;
started_at?: string | null;
finished_at?: string | null;
model?: string;
session_id?: string;
name?: string;
}
export type FramePublisher = (sessionId: string, frame: InferenceFrame) => void;
export interface InferenceContext {
sql: Sql;
config: Config;
log: FastifyBaseLogger;
publish: FramePublisher;
publishUser: (frame: UserStreamFrame) => void;
// v1.11: passed through so compaction.process can publish 'compacted'
// frames on the same session WS channel useSessionStream subscribes to.
// Compaction is the only path that needs the raw broker handle (regular
// inference goes through `publish`); keeping a separate field avoids
// tempting other code paths into bypassing the session-id binding.
broker: Broker;
}
// v1.12.4: payload assembly extracted to ./inference/payload.ts (tests
// import buildMessagesPayload from this module, so a re-export below
// preserves the public surface). Stream + tool phases extracted to
// ./inference/stream-phase.ts and ./inference/tool-phase.ts.
export interface StreamResult {
finishReason: string | null;
content: string;
toolCalls: ToolCall[];
promptTokens: number | null;
completionTokens: number | null;
// v1.13.1-C: reasoning text accumulated across reasoning-delta parts.
// Empty string when the model doesn't emit reasoning (most cases).
reasoning: string;
}
export interface TurnArgs {
sessionId: string;
chatId: string;
assistantMessageId: string;
// v1.8.2: cumulative tool calls executed this run. Compared against the
// resolved budget at the top of each turn. Replaces the older `depth`
// counter (which counted iterations, not invocations).
toolsUsed: number;
// v1.11.6: ordered tool calls executed in this user-message turn (across
// recursive runAssistantTurn invocations). Reset to [] at user-message
// boundaries by runInference, same as toolsUsed. Doom-loop check at the
// top of runAssistantTurn slices the last DOOM_LOOP_THRESHOLD entries.
recentToolCalls: ToolCall[];
// v#12 MistakeTracker: heterogeneous-failure recovery state. Loop-local,
// reset per runInference (user-message boundary) like recentToolCalls. Folds
// tool-phase outcomes via recordStep each iteration; detectMistakePattern
// gates the nudge/escalate decision.
mistakeTracker: MistakeState;
// v#12: transient model-facing recovery note set when a nudge fires. Consumed
// (appended as a role:'system' message + cleared) on the NEXT payload build.
// Never persisted — mirrors how the cap-hit/doom-loop notes live only inside
// the summary call's messages array.
pendingRecoveryNote?: string;
signal: AbortSignal | undefined;
}
//
// P5: the shared pipeline types (InferenceFrame / FramePublisher /
// InferenceContext / StreamResult / TurnArgs) moved to ./types.js to break the
// turn.ts type-hub-and-leaf near-cycle. They are re-exported from there via
// inference/index.ts for the public surface.
export async function runAssistantTurn(
@@ -184,17 +81,13 @@ export async function runAssistantTurn(
const agent = session.agent_id
? await getAgentById(project.path, session.agent_id)
: null;
const budget = resolveToolBudget(agent);
// v1.14.0: effectiveCap = min(agent.steps ?? Infinity, MAX_STEPS).
// steps: 0 means "no tool calls allowed" — the first stream phase runs
// but if it emits tool calls they are not executed (finalize as text-only).
const effectiveCap = Math.min(agent?.steps ?? Infinity, MAX_STEPS);
// P5: pure per-turn config (budget + cap math + text-only flag).
const { effectiveCap, budget, isTextOnly } = resolveTurnConfig(agent);
// steps: 0 special case — model responds text-only. The while loop would
// never enter (effectiveCap === 0), so we handle it explicitly before the
// loop. The model always gets at least one chance to respond with text.
if (effectiveCap === 0) {
if (isTextOnly) {
const loaded = await loadContext(ctx.sql, sessionId, chatId);
if (loaded) {
await runTextOnlyTurn(ctx, args, loaded.session, loaded.project, loaded.history, agent);
@@ -214,20 +107,18 @@ export async function runAssistantTurn(
let pendingRecoveryNote: string | undefined = args.pendingRecoveryNote;
while (stepNumber < effectiveCap) {
// ---- doom-loop check (moved from top-of-function) ----
const loop = detectDoomLoop(recentToolCalls);
if (loop) {
// ---- top-of-loop gate: doom-loop, then budget (pure decision) ----
const decision = decideStep({ recentToolCalls, toolsUsed, budget });
if (decision.kind === 'doom') {
// Need fresh history for the summary.
const loaded = await loadContext(ctx.sql, sessionId, chatId);
if (loaded) {
const iterArgs: TurnArgs = { sessionId, chatId, assistantMessageId, toolsUsed, recentToolCalls, mistakeTracker, signal };
await runDoomLoopSummary(ctx, iterArgs, loaded.session, loaded.project, loaded.history, agent, loop);
await runDoomLoopSummary(ctx, iterArgs, loaded.session, loaded.project, loaded.history, agent, decision.loop);
}
break;
}
// ---- budget check (moved from top-of-function) ----
if (toolsUsed >= budget) {
if (decision.kind === 'budget') {
const loaded = await loadContext(ctx.sql, sessionId, chatId);
if (loaded) {
const iterArgs: TurnArgs = { sessionId, chatId, assistantMessageId, toolsUsed, recentToolCalls, mistakeTracker, signal };
@@ -235,6 +126,7 @@ export async function runAssistantTurn(
}
break;
}
// decision.kind === 'stream' → proceed with compaction + stream + tools.
// ---- compaction check ----
// v1.11: if the prior turn flagged this chat for compaction, run it
@@ -345,19 +237,17 @@ export async function runAssistantTurn(
recordStep(mistakeTracker, o);
}
if (toolPhaseResult.action !== 'continue') {
// 'paused' (user input) or 'synthesis_done' — stop the loop. The turn is
// already ending, so neither a nudge nor an escalate would change the
// control flow; we skip the mistake decision here.
// v#12 MistakeTracker: post-tool decision (pure). 'stop' = the tool phase
// returned a non-'continue' action ('paused' for user input, or
// 'synthesis_done') — neither a nudge nor an escalate would change the
// control flow, so the mistake check is skipped. On 'continue' the
// heterogeneous-failure pattern gates nudge/escalate/continue. Complements
// the doom-loop gate above, which only catches *identical* repeats.
const post = decidePostToolAction(toolPhaseResult.action, mistakeTracker);
if (post === 'stop') {
break;
}
// v#12 MistakeTracker: heterogeneous-failure decision. Only evaluated on
// the 'continue' path (the only case where the loop would otherwise
// proceed to another step). Complements the doom-loop check above, which
// only catches *identical* repeats.
const mistake = detectMistakePattern(mistakeTracker);
if (mistake === 'nudge') {
if (post === 'nudge') {
// Soft intervention: inject model-facing recovery guidance into the NEXT
// step's payload, drop a UI sentinel, bump nudges, reset the streak, and
// continue. The note is consumed (and cleared) at the top of the next
@@ -379,23 +269,16 @@ export async function runAssistantTurn(
assistantMessageId = toolPhaseResult.nextAssistantId!;
continue;
}
if (mistake === 'escalate') {
if (post === 'escalate') {
// The nudge didn't break the failure run — stop the turn (cap-hit-style)
// to avoid burning the whole step budget on heterogeneous failures. The
// next assistant row is still 'streaming'; finalize it as a short note so
// the slot doesn't dangle, then drop the escalate sentinel.
// next assistant row is still 'streaming'; finalize it as an empty
// complete row so the slot doesn't dangle, then drop the escalate
// sentinel.
const failureKinds = [...mistakeTracker.run];
assistantMessageId = toolPhaseResult.nextAssistantId!;
await ctx.sql`
UPDATE messages
SET content = '', status = 'complete', finished_at = clock_timestamp()
WHERE id = ${assistantMessageId}
`;
ctx.publish(sessionId, {
type: 'message_complete',
message_id: assistantMessageId,
chat_id: chatId,
});
const escalateArgs: TurnArgs = { sessionId, chatId, assistantMessageId, toolsUsed, recentToolCalls, mistakeTracker, signal };
await finalizeEmpty(ctx, escalateArgs);
await insertMistakeRecoverySentinel(ctx, sessionId, chatId, {
failureKinds,
count: failureKinds.length,
@@ -562,4 +445,3 @@ export function createInferenceRunner(
};
}
export const _toolNames = ALL_TOOLS.map((t) => t.name);

116
apps/server/src/services/inference/types.ts
View File

@@ -1,6 +1,25 @@
// v1.12.4: shared inter-phase types/constants for the extracted phase files.
// Lives here so stream-phase, tool-phase, and the summary functions still in
// inference.ts can all reference the same definitions without circular imports.
//
// P5: the shared pipeline types (InferenceContext / TurnArgs / StreamResult /
// InferenceFrame / FramePublisher) moved here from turn.ts. turn.ts was both the
// type hub (every phase imported these from './turn.js') AND the orchestration
// leaf (it imports functions back from payload/stream-phase/tool-phase/
// error-handler/sentinel-summaries) — a hub-and-leaf near-cycle. Hosting the
// shared types here (this module imports no inference functions) breaks it.
import type { FastifyBaseLogger } from 'fastify';
import type { Sql } from '../../db.js';
import type { Config } from '../../config.js';
import type {
ErrorReason,
MessageMetadata,
ToolCall,
UserStreamFrame,
} from '../../types/api.js';
import type { Broker } from '../broker.js';
import type { MistakeState } from './mistake-tracker.js';
export interface StreamPhaseState {
accumulated: string;
@@ -11,3 +30,100 @@ export interface StreamPhaseState {
// executeStreamPhase, runCapHitSummary, and runDoomLoopSummary — every site
// that does a debounced content flush during streaming.
export const DB_FLUSH_INTERVAL_MS = 500;
export interface InferenceFrame {
type:
| 'message_started'
| 'delta'
| 'tool_call'
| 'tool_result'
| 'message_complete'
| 'usage'
| 'messages_deleted'
| 'session_renamed'
| 'chat_renamed'
| 'error';
message_id?: string;
message_ids?: string[];
chat_id?: string;
tool_message_id?: string;
tool_call_id?: string;
// v1.8.2: 'system' added so cap-hit sentinel messages can announce themselves
// through the normal message_started → delta → message_complete sequence.
role?: 'assistant' | 'tool' | 'user' | 'system';
content?: string;
tool_call?: ToolCall;
output?: unknown;
truncated?: boolean;
error?: string;
// v1.8.2: structured error reason. Set on `type: 'error'` so the UI can
// surface a specific message; `error` stays the human-readable text.
reason?: ErrorReason;
// v1.8.2: piggybacks on `message_complete` so static or terminally-resolved
// messages can carry their persisted metadata to the live stream without a
// refetch (sentinels carry { kind: 'cap_hit', ... }; failed messages carry
// { kind: 'error', ... }).
metadata?: MessageMetadata | null;
tokens_used?: number | null;
ctx_used?: number | null;
ctx_max?: number | null;
completion_tokens?: number | null;
started_at?: string | null;
finished_at?: string | null;
model?: string;
session_id?: string;
name?: string;
}
export type FramePublisher = (sessionId: string, frame: InferenceFrame) => void;
export interface InferenceContext {
sql: Sql;
config: Config;
log: FastifyBaseLogger;
publish: FramePublisher;
publishUser: (frame: UserStreamFrame) => void;
// v1.11: passed through so compaction.process can publish 'compacted'
// frames on the same session WS channel useSessionStream subscribes to.
// Compaction is the only path that needs the raw broker handle (regular
// inference goes through `publish`); keeping a separate field avoids
// tempting other code paths into bypassing the session-id binding.
broker: Broker;
}
export interface StreamResult {
finishReason: string | null;
content: string;
toolCalls: ToolCall[];
promptTokens: number | null;
completionTokens: number | null;
// v1.13.1-C: reasoning text accumulated across reasoning-delta parts.
// Empty string when the model doesn't emit reasoning (most cases).
reasoning: string;
}
export interface TurnArgs {
sessionId: string;
chatId: string;
assistantMessageId: string;
// v1.8.2: cumulative tool calls executed this run. Compared against the
// resolved budget at the top of each turn. Replaces the older `depth`
// counter (which counted iterations, not invocations).
toolsUsed: number;
// v1.11.6: ordered tool calls executed in this user-message turn (across
// recursive runAssistantTurn invocations). Reset to [] at user-message
// boundaries by runInference, same as toolsUsed. Doom-loop check at the
// top of runAssistantTurn slices the last DOOM_LOOP_THRESHOLD entries.
recentToolCalls: ToolCall[];
// v#12 MistakeTracker: heterogeneous-failure recovery state. Loop-local,
// reset per runInference (user-message boundary) like recentToolCalls. Folds
// tool-phase outcomes via recordStep each iteration; detectMistakePattern
// gates the nudge/escalate decision.
mistakeTracker: MistakeState;
// v#12: transient model-facing recovery note set when a nudge fires. Consumed
// (appended as a role:'system' message + cleared) on the NEXT payload build.
// Never persisted — mirrors how the cap-hit/doom-loop notes live only inside
// the summary call's messages array.
pendingRecoveryNote?: string;
signal: AbortSignal | undefined;
}