boocode/apps/server/src/services/__tests__/tool-call-parser.test.ts

import { describe, expect, it } from 'vitest';
import {
  parseXmlToolCall,
  parseInvokeToolCall,
  partialXmlOpenerStart,
  extractToolCallBlocks,
  parseToolCallsFromText,
  stripToolMarkup,
  hasToolSignal,
  XML_TOOL_OPEN,
  XML_TOOL_CLOSE,
  INVOKE_TOOL_OPEN,
  INVOKE_TOOL_CLOSE,
  TOOL_XML_SIGNALS,
  BUDGET_EXHAUSTED_NUDGE,
  DUPLICATE_CALL_NUDGE,
  TOOL_ERROR_NUDGE,
  TOOL_ERROR_PREFIXES,
} from '../inference/tool-call-parser.js';

// ── Ported from xml-parser.test.ts ───────────────────────────────────────

describe('parseXmlToolCall (Qwen/Hermes <tool_call>)', () => {
  it('parses a well-formed single-parameter call', () => {
    const block = '<tool_call><function=view_file><parameter=path>/tmp/foo</parameter></function></tool_call>';
    expect(parseXmlToolCall(block)).toEqual({
      name: 'view_file',
      args: { path: '/tmp/foo' },
    });
  });

  it('parses multi-parameter call', () => {
    const block = '<tool_call><function=grep><parameter=pattern>foo</parameter><parameter=path>src/</parameter></function></tool_call>';
    expect(parseXmlToolCall(block)).toEqual({
      name: 'grep',
      args: { pattern: 'foo', path: 'src/' },
    });
  });

  it('JSON-parses numeric parameter values', () => {
    const block = '<tool_call><function=foo><parameter=count>42</parameter></function></tool_call>';
    expect(parseXmlToolCall(block)).toEqual({ name: 'foo', args: { count: 42 } });
  });

  it('tolerates whitespace around = in function (v1.13.16 tightening)', () => {
    const block = '<tool_call><function = view_file><parameter=path>/tmp/foo</parameter></function></tool_call>';
    expect(parseXmlToolCall(block)).toEqual({
      name: 'view_file',
      args: { path: '/tmp/foo' },
    });
  });

  it('tolerates whitespace around = in parameter (v1.13.16 tightening)', () => {
    const block = '<tool_call><function=view_file><parameter = path>/tmp/foo</parameter></function></tool_call>';
    expect(parseXmlToolCall(block)).toEqual({
      name: 'view_file',
      args: { path: '/tmp/foo' },
    });
  });

  it('returns null when function name is missing', () => {
    const block = '<tool_call><parameter=path>/tmp/foo</parameter></tool_call>';
    expect(parseXmlToolCall(block)).toBeNull();
  });
});

describe('parseInvokeToolCall (Anthropic <invoke>) — v1.13.16', () => {
  it('parses a well-formed single-parameter call (spec case 1)', () => {
    const block = '<invoke name="view_file"><parameter name="path">/tmp/foo</parameter></invoke>';
    expect(parseInvokeToolCall(block)).toEqual({
      name: 'view_file',
      args: { path: '/tmp/foo' },
    });
  });

  it('parses a multi-parameter call (spec case 2)', () => {
    const block = '<invoke name="grep"><parameter name="pattern">foo</parameter><parameter name="path">src/</parameter></invoke>';
    expect(parseInvokeToolCall(block)).toEqual({
      name: 'grep',
      args: { pattern: 'foo', path: 'src/' },
    });
  });

  it('tolerates newlines and spaces in attributes (spec case 3)', () => {
    const block = `<invoke
      name="view_file"
    >
      <parameter
        name="path"
      >/tmp/foo</parameter>
    </invoke>`;
    expect(parseInvokeToolCall(block)).toEqual({
      name: 'view_file',
      args: { path: '/tmp/foo' },
    });
  });

  it('parses a call whose name is not a registered BooCode tool (spec case 4)', () => {
    const block = '<invoke name="read_file"><parameter name="path">/tmp/foo</parameter></invoke>';
    expect(parseInvokeToolCall(block)).toEqual({
      name: 'read_file',
      args: { path: '/tmp/foo' },
    });
  });

  it('supports single-quoted attribute values', () => {
    const block = "<invoke name='view_file'><parameter name='path'>/tmp/foo</parameter></invoke>";
    expect(parseInvokeToolCall(block)).toEqual({
      name: 'view_file',
      args: { path: '/tmp/foo' },
    });
  });

  it('JSON-parses numeric parameter values', () => {
    const block = '<invoke name="foo"><parameter name="count">42</parameter></invoke>';
    expect(parseInvokeToolCall(block)).toEqual({ name: 'foo', args: { count: 42 } });
  });

  it('tolerates spaces around = inside name attribute', () => {
    const block = '<invoke name = "view_file"><parameter name = "path">/tmp/foo</parameter></invoke>';
    expect(parseInvokeToolCall(block)).toEqual({
      name: 'view_file',
      args: { path: '/tmp/foo' },
    });
  });

  it('returns null when name attribute is missing', () => {
    const block = '<invoke><parameter name="path">/tmp/foo</parameter></invoke>';
    expect(parseInvokeToolCall(block)).toBeNull();
  });

  it('returns null when name attribute is empty', () => {
    const block = '<invoke name=""><parameter name="path">/tmp/foo</parameter></invoke>';
    expect(parseInvokeToolCall(block)).toBeNull();
  });

  it('exports the expected delimiters', () => {
    expect(INVOKE_TOOL_OPEN).toBe('<invoke');
    expect(INVOKE_TOOL_CLOSE).toBe('</invoke>');
    expect(XML_TOOL_OPEN).toBe('<tool_call>');
    expect(XML_TOOL_CLOSE).toBe('</tool_call>');
  });
});

describe('partialXmlOpenerStart (v1.13.16 — both flavors)', () => {
  it('returns -1 when the buffer is empty', () => {
    expect(partialXmlOpenerStart('')).toBe(-1);
  });

  it('returns -1 when the buffer has no openers', () => {
    expect(partialXmlOpenerStart('plain prose, no markup')).toBe(-1);
  });

  it('returns the index of a complete <tool_call> opener (existing)', () => {
    expect(partialXmlOpenerStart('prose <tool_call>more')).toBe(6);
  });

  it('returns the index of a complete <invoke opener (v1.13.16)', () => {
    expect(partialXmlOpenerStart('prose <invoke name=')).toBe(6);
  });

  it('holds a partial <tool_ prefix at end of buffer', () => {
    expect(partialXmlOpenerStart('text <tool_')).toBe(5);
  });

  it('holds a partial <invo prefix at end of buffer (v1.13.16)', () => {
    expect(partialXmlOpenerStart('text <invo')).toBe(5);
  });

  it('holds a bare < at end of buffer', () => {
    expect(partialXmlOpenerStart('text <')).toBe(5);
  });

  it('returns -1 when < is followed by non-opener text', () => {
    expect(partialXmlOpenerStart('text <unknown>')).toBe(-1);
  });

  it('returns the earliest opener when both flavors are present', () => {
    expect(partialXmlOpenerStart('xxx <tool_call>YYY <invoke>')).toBe(4);
    expect(partialXmlOpenerStart('xxx <invoke>YYY <tool_call>')).toBe(4);
  });
});

describe('extractToolCallBlocks (v1.13.16 — unified extraction)', () => {
  it('extracts a single <invoke> block (spec case 1)', () => {
    const input = '<invoke name="view_file"><parameter name="path">/tmp/foo</parameter></invoke>';
    const result = extractToolCallBlocks(input);
    expect(result.calls).toEqual([{ name: 'view_file', args: { path: '/tmp/foo' } }]);
    expect(result.flushed).toBe('');
    expect(result.remaining).toBe('');
  });

  it('holds the partial <invoke> chunk when the closer has not arrived (spec case 5, first chunk)', () => {
    const firstChunk = '<invoke name="view_file"><parameter name="path">/tmp/foo</parameter>';
    const result = extractToolCallBlocks(firstChunk);
    expect(result.calls).toEqual([]);
    expect(result.flushed).toBe('');
    expect(result.remaining).toBe(firstChunk);
  });

  it('extracts the block once the closer arrives in a later chunk (spec case 5, completion)', () => {
    const firstChunk = '<invoke name="view_file"><parameter name="path">/tmp/foo</parameter>';
    const r1 = extractToolCallBlocks(firstChunk);
    const combined = r1.remaining + '</invoke>';
    const r2 = extractToolCallBlocks(combined);
    expect(r2.calls).toEqual([{ name: 'view_file', args: { path: '/tmp/foo' } }]);
    expect(r2.flushed).toBe('');
    expect(r2.remaining).toBe('');
  });

  it('flushes prose around a recognized block but not the markup itself (spec case 6)', () => {
    const input = 'I will read the file.\n<invoke name="view_file"><parameter name="path">/tmp/foo</parameter></invoke>\nThanks.';
    const result = extractToolCallBlocks(input);
    expect(result.calls).toEqual([{ name: 'view_file', args: { path: '/tmp/foo' } }]);
    expect(result.flushed).toBe('I will read the file.\n\nThanks.');
    expect(result.remaining).toBe('');
  });

  it('extracts a <tool_call> Qwen block alongside the new code path (spec case 7 regression)', () => {
    const input = '<tool_call><function=view_file><parameter=path>/tmp/foo</parameter></function></tool_call>';
    const result = extractToolCallBlocks(input);
    expect(result.calls).toEqual([{ name: 'view_file', args: { path: '/tmp/foo' } }]);
    expect(result.flushed).toBe('');
    expect(result.remaining).toBe('');
  });

  it('extracts mixed-format blocks in source order (hand-back: shared counter)', () => {
    const input =
      '<invoke name="view_file"><parameter name="path">/a</parameter></invoke>' +
      ' middle ' +
      '<tool_call><function=grep><parameter=pattern>foo</parameter></function></tool_call>';
    const result = extractToolCallBlocks(input);
    expect(result.calls).toEqual([
      { name: 'view_file', args: { path: '/a' } },
      { name: 'grep', args: { pattern: 'foo' } },
    ]);
    expect(result.flushed).toBe(' middle ');
    expect(result.remaining).toBe('');
  });

  it('drops a malformed <invoke> block silently (matches existing <tool_call> behavior)', () => {
    const input = 'prose <invoke><parameter name="path">/a</parameter></invoke> trailing';
    const result = extractToolCallBlocks(input);
    expect(result.calls).toEqual([]);
    expect(result.flushed).toBe('prose  trailing');
    expect(result.remaining).toBe('');
  });

  it('holds a tail with a fresh partial opener after extracting earlier complete blocks', () => {
    const input = '<invoke name="view_file"><parameter name="path">/a</parameter></invoke> next: <tool_';
    const result = extractToolCallBlocks(input);
    expect(result.calls).toEqual([{ name: 'view_file', args: { path: '/a' } }]);
    expect(result.flushed).toBe(' next: ');
    expect(result.remaining).toBe('<tool_');
  });

  it('passes plain prose straight through when no markup is present', () => {
    const input = 'just some text with a < character but no opener';
    const result = extractToolCallBlocks(input);
    expect(result.calls).toEqual([]);
    expect(result.flushed).toBe(input);
    expect(result.remaining).toBe('');
  });

  describe('placeholder arg rejection (qwen3.6 answer-then-spurious-tools)', () => {
    it('rejects <invoke> with path "..." — 0 calls, block in flushed', () => {
      const block = '<invoke name="view_file"><parameter name="path">...</parameter></invoke>';
      const result = extractToolCallBlocks(`Answer text.\n${block}`);
      expect(result.calls).toEqual([]);
      expect(result.flushed).toContain('Answer text.');
      expect(result.flushed).toContain(block);
      expect(result.remaining).toBe('');
    });

    it('rejects <invoke> with empty path — 0 calls, block in flushed', () => {
      const block = '<invoke name="view_file"><parameter name="path"></parameter></invoke>';
      const result = extractToolCallBlocks(block);
      expect(result.calls).toEqual([]);
      expect(result.flushed).toBe(block);
      expect(result.remaining).toBe('');
    });

    it('rejects <invoke> with path "<path>" — 0 calls', () => {
      const block = '<invoke name="view_file"><parameter name="path"><path></parameter></invoke>';
      const result = extractToolCallBlocks(block);
      expect(result.calls).toEqual([]);
      expect(result.flushed).toBe(block);
    });

    it('returns 1 valid call and flushes placeholder block when mixed in same buffer', () => {
      const valid =
        '<invoke name="view_file"><parameter name="path">/opt/boocode/README.md</parameter></invoke>';
      const placeholder =
        '<invoke name="view_file"><parameter name="path">...</parameter></invoke>';
      const result = extractToolCallBlocks(`${valid} tail ${placeholder}`);
      expect(result.calls).toEqual([{ name: 'view_file', args: { path: '/opt/boocode/README.md' } }]);
      expect(result.flushed).toContain('tail');
      expect(result.flushed).toContain(placeholder);
      expect(result.remaining).toBe('');
    });
  });
});

// ── New tests: Unsloth-ported functions ──────────────────────────────────

describe('hasToolSignal', () => {
  it('returns true for <tool_call>', () => {
    expect(hasToolSignal('prefix <tool_call> suffix')).toBe(true);
  });

  it('returns true for <function=', () => {
    expect(hasToolSignal('prefix <function=view_file> suffix')).toBe(true);
  });

  it('returns true for <invoke', () => {
    expect(hasToolSignal('prefix <invoke name="x"> suffix')).toBe(true);
  });

  it('returns false for near-miss <tool>', () => {
    expect(hasToolSignal('prefix <tool> suffix')).toBe(false);
  });

  it('returns false for near-miss <function>', () => {
    expect(hasToolSignal('prefix <function> suffix')).toBe(false);
  });

  it('returns false for near-miss <tool_call_thing>', () => {
    expect(hasToolSignal('<tool_call_thing>')).toBe(false);
  });

  it('returns false for plain text', () => {
    expect(hasToolSignal('just some text')).toBe(false);
  });
});

describe('stripToolMarkup', () => {
  it('strips closed <tool_call> blocks', () => {
    const input = 'before <tool_call>{"name":"x"}</tool_call> after';
    expect(stripToolMarkup(input)).toBe('before  after');
  });

  it('strips closed <function=...> blocks', () => {
    const input = 'before <function=x><parameter=y>z</parameter></function> after';
    expect(stripToolMarkup(input)).toBe('before  after');
  });

  it('strips closed <invoke> blocks', () => {
    const input = 'before <invoke name="x"><parameter name="y">z</parameter></invoke> after';
    expect(stripToolMarkup(input)).toBe('before  after');
  });

  it('leaves trailing unclosed block when final=false', () => {
    const input = 'text <tool_call>{"name":"x"';
    expect(stripToolMarkup(input)).toBe('text <tool_call>{"name":"x"');
  });

  it('strips trailing unclosed <tool_call> when final=true', () => {
    const input = 'text <tool_call>{"name":"x"';
    expect(stripToolMarkup(input, { final: true })).toBe('text');
  });

  it('strips trailing unclosed <function= when final=true', () => {
    const input = 'text <function=run_bash><parameter=command>ls';
    expect(stripToolMarkup(input, { final: true })).toBe('text');
  });

  it('strips trailing unclosed <invoke when final=true', () => {
    const input = 'text <invoke name="x"><parameter name="y">val';
    expect(stripToolMarkup(input, { final: true })).toBe('text');
  });

  it('trims whitespace when final=true', () => {
    const input = '  text  <tool_call>partial';
    expect(stripToolMarkup(input, { final: true })).toBe('text');
  });

  it('strips multiple closed blocks', () => {
    const input = '<tool_call>a</tool_call> mid <tool_call>b</tool_call>';
    expect(stripToolMarkup(input)).toBe(' mid ');
  });
});

describe('parseToolCallsFromText', () => {
  describe('pattern 1: <tool_call>{json}</tool_call>', () => {
    it('parses a well-formed JSON tool call', () => {
      const input = '<tool_call>{"name":"web_search","arguments":{"query":"hello"}}</tool_call>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(calls[0]!.id).toBe('call_0');
      expect(calls[0]!.type).toBe('function');
      expect(calls[0]!.function.name).toBe('web_search');
      expect(JSON.parse(calls[0]!.function.arguments)).toEqual({ query: 'hello' });
    });

    it('handles string arguments field', () => {
      const input = '<tool_call>{"name":"x","arguments":"already a string"}</tool_call>';
      const calls = parseToolCallsFromText(input);
      expect(calls[0]!.function.arguments).toBe('already a string');
    });

    it('handles balanced braces inside JSON strings', () => {
      const input = '<tool_call>{"name":"x","arguments":{"q":"} { extra "}}</tool_call>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      const parsed = JSON.parse(calls[0]!.function.arguments);
      expect(parsed.q).toBe('} { extra ');
    });

    it('respects idOffset', () => {
      const input = '<tool_call>{"name":"a","arguments":{}}</tool_call>';
      const calls = parseToolCallsFromText(input, { idOffset: 5 });
      expect(calls[0]!.id).toBe('call_5');
    });

    it('parses multiple JSON tool calls', () => {
      const input =
        '<tool_call>{"name":"a","arguments":{}}</tool_call>' +
        '<tool_call>{"name":"b","arguments":{}}</tool_call>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(2);
      expect(calls[0]!.id).toBe('call_0');
      expect(calls[1]!.id).toBe('call_1');
    });

    it('skips malformed JSON', () => {
      const input = '<tool_call>{not json}</tool_call>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(0);
    });

    it('handles missing closing tag', () => {
      const input = '<tool_call>{"name":"x","arguments":{"q":"hello"}}';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(calls[0]!.function.name).toBe('x');
    });
  });

  describe('pattern 2: <function=name><parameter=key>value', () => {
    it('parses a single-parameter function call', () => {
      const input = '<function=view_file><parameter=path>/tmp/foo</parameter></function>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(calls[0]!.function.name).toBe('view_file');
      expect(JSON.parse(calls[0]!.function.arguments)).toEqual({ path: '/tmp/foo' });
    });

    it('single-param fast path preserves embedded </parameter>', () => {
      const input = '<function=run_bash><parameter=command>echo "</parameter>"</parameter></function>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(JSON.parse(calls[0]!.function.arguments).command).toBe('echo "</parameter>"');
    });

    it('multi-param: value of first stops at start of second', () => {
      const input = '<function=grep><parameter=pattern>foo</parameter><parameter=path>src/</parameter></function>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      const args = JSON.parse(calls[0]!.function.arguments);
      expect(args.pattern).toBe('foo');
      expect(args.path).toBe('src/');
    });

    it('tolerates missing closing tags', () => {
      const input = '<function=view_file><parameter=path>/tmp/foo';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(calls[0]!.function.name).toBe('view_file');
      expect(JSON.parse(calls[0]!.function.arguments)).toEqual({ path: '/tmp/foo' });
    });

    it('does not fire when pattern 1 found results', () => {
      const input = '<tool_call>{"name":"a","arguments":{}}</tool_call><function=b><parameter=x>y</parameter></function>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(calls[0]!.function.name).toBe('a');
    });
  });

  describe('pattern 3: <invoke name="..."><parameter name="...">value (Anthropic)', () => {
    it('parses a single-parameter invoke call', () => {
      const input = '<invoke name="view_file"><parameter name="path">/tmp/foo</parameter></invoke>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(calls[0]!.function.name).toBe('view_file');
      expect(JSON.parse(calls[0]!.function.arguments)).toEqual({ path: '/tmp/foo' });
    });

    it('parses multi-parameter invoke call', () => {
      const input = '<invoke name="grep"><parameter name="pattern">foo</parameter><parameter name="path">src/</parameter></invoke>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      const args = JSON.parse(calls[0]!.function.arguments);
      expect(args.pattern).toBe('foo');
      expect(args.path).toBe('src/');
    });

    it('does not fire when pattern 1 found results', () => {
      const input = '<tool_call>{"name":"a","arguments":{}}</tool_call><invoke name="b"><parameter name="x">y</parameter></invoke>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(calls[0]!.function.name).toBe('a');
    });

    it('does not fire when pattern 2 found results', () => {
      const input = '<function=a><parameter=x>y</parameter></function><invoke name="b"><parameter name="x">y</parameter></invoke>';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(calls[0]!.function.name).toBe('a');
    });

    it('tolerates missing closing tags', () => {
      const input = '<invoke name="view_file"><parameter name="path">/tmp/foo';
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(JSON.parse(calls[0]!.function.arguments)).toEqual({ path: '/tmp/foo' });
    });

    it('supports single-quoted attributes', () => {
      const input = "<invoke name='view_file'><parameter name='path'>/tmp/foo</parameter></invoke>";
      const calls = parseToolCallsFromText(input);
      expect(calls).toHaveLength(1);
      expect(calls[0]!.function.name).toBe('view_file');
    });
  });
});

describe('constants', () => {
  it('TOOL_XML_SIGNALS includes all three signal prefixes', () => {
    expect(TOOL_XML_SIGNALS).toContain('<tool_call>');
    expect(TOOL_XML_SIGNALS).toContain('<function=');
    expect(TOOL_XML_SIGNALS).toContain('<invoke');
  });

  it('nudge constants are non-empty strings', () => {
    expect(BUDGET_EXHAUSTED_NUDGE.length).toBeGreaterThan(0);
    expect(DUPLICATE_CALL_NUDGE.length).toBeGreaterThan(0);
    expect(TOOL_ERROR_NUDGE.length).toBeGreaterThan(0);
  });

  it('TOOL_ERROR_PREFIXES is a non-empty tuple', () => {
    expect(TOOL_ERROR_PREFIXES.length).toBeGreaterThan(0);
    expect(TOOL_ERROR_PREFIXES).toContain('Error');
  });
});