feat(coder): add TokenScope analyzer and DB persistence module
- analyzeMessages classifies message parts into system/user/assistant/tools/reasoning - persistTaskBreakdown writes JSONB to tasks table - Backfills the token-analysis/ module (contract committed earlier) - 6 unit tests covering classification, tool calls, reasoning tokens
This commit is contained in:
60
apps/coder/src/services/token-analysis/analyzer.ts
Normal file
60
apps/coder/src/services/token-analysis/analyzer.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
// TokenScope analyzer — classifies message parts into category breakdown.
|
||||
// Ported from opencode-tokenscope (MIT).
|
||||
|
||||
export interface TokenBreakdown {
|
||||
system: number;
|
||||
user: number;
|
||||
assistant: number;
|
||||
tools: number;
|
||||
reasoning: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
const CHARS_PER_TOKEN = 4;
|
||||
|
||||
function estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
||||
}
|
||||
|
||||
export function analyzeMessages(parts: any[]): TokenBreakdown {
|
||||
const breakdown: TokenBreakdown = { system: 0, user: 0, assistant: 0, tools: 0, reasoning: 0, total: 0 };
|
||||
|
||||
for (const part of parts) {
|
||||
const role = part.role ?? '';
|
||||
const content = part.content ?? '';
|
||||
const tokens = estimateTokens(content);
|
||||
|
||||
switch (role) {
|
||||
case 'system':
|
||||
breakdown.system += tokens;
|
||||
break;
|
||||
case 'user':
|
||||
breakdown.user += tokens;
|
||||
break;
|
||||
case 'assistant':
|
||||
breakdown.assistant += tokens;
|
||||
if (part.tool_calls) {
|
||||
for (const tc of part.tool_calls) {
|
||||
breakdown.tools += estimateTokens(JSON.stringify(tc));
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'tool':
|
||||
breakdown.tools += tokens;
|
||||
break;
|
||||
default:
|
||||
breakdown.assistant += tokens;
|
||||
}
|
||||
|
||||
if (part.reasoning_parts) {
|
||||
for (const rp of part.reasoning_parts) {
|
||||
const rTokens = estimateTokens(rp.text ?? '');
|
||||
breakdown.reasoning += rTokens;
|
||||
breakdown.assistant -= rTokens;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
breakdown.total = breakdown.system + breakdown.user + breakdown.assistant + breakdown.tools + breakdown.reasoning;
|
||||
return breakdown;
|
||||
}
|
||||
Reference in New Issue
Block a user