v1.11.8: web_search + web_fetch tools via SearXNG

Adds two new tools registered through the existing ALL_TOOLS registry:
  - web_search hits SearXNG's JSON API (Fathom, internal Tailscale URL,
    no auth) and returns top results
  - web_fetch retrieves a URL's text content, gated by isPublicUrl
    (url_guard.ts) which blocks loopback / RFC1918 / Tailscale CGNAT /
    link-local / .local / .internal / non-http schemes

Both tools are opt-in via the existing session.web_search_enabled flag
(plumbed in v1.9, activated here). Default off. UI labels updated to
"Enable web search and fetch" / "Web search and fetch" since fetch joins
the same store. Counts against the v1.8.2 per-turn budget; covered by
the v1.11.6 doom-loop guard.

Native Node 20 fetch — no new prod dep. HTML stripping via regex (script
and style content elided wholesale). 5MB body cap, 15s fetch timeout,
8000-char default output, 32000-char cap.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

apps/server/src/services/__tests__/web_tools.test.ts

@@ -0,0 +1,300 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { executeWebSearch } from '../web_search.js';
+import { executeWebFetch } from '../web_fetch.js';
+import { isPublicUrl } from '../url_guard.js';
+
+const TEST_SEARXNG = 'http://searxng.test:8888';
+
+function mockResponse(
+  body: unknown,
+  init: { status?: number; contentType?: string; contentLength?: number } = {},
+): Response {
+  const status = init.status ?? 200;
+  const headers: Record<string, string> = {};
+  if (init.contentType) headers['content-type'] = init.contentType;
+  if (init.contentLength !== undefined) headers['content-length'] = String(init.contentLength);
+  const stringBody = typeof body === 'string' ? body : JSON.stringify(body);
+  return new Response(stringBody, { status, headers });
+}
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+// ============================================================================
+// url_guard — SSRF protection
+// ============================================================================
+
+describe('isPublicUrl', () => {
+  it('blocks http://localhost', () => {
+    expect(isPublicUrl('http://localhost').ok).toBe(false);
+  });
+
+  it('blocks http://127.0.0.1:3000', () => {
+    const r = isPublicUrl('http://127.0.0.1:3000');
+    expect(r.ok).toBe(false);
+    expect(r.reason).toMatch(/loopback/);
+  });
+
+  it('blocks RFC1918 192.168.x.x', () => {
+    expect(isPublicUrl('http://192.168.1.1').ok).toBe(false);
+  });
+
+  it('blocks RFC1918 10.x.x.x', () => {
+    expect(isPublicUrl('http://10.0.0.5').ok).toBe(false);
+  });
+
+  it('blocks RFC1918 172.16-31.x.x', () => {
+    expect(isPublicUrl('http://172.20.0.1').ok).toBe(false);
+    // Boundary: 172.15 is public; 172.16 is private; 172.31 is private; 172.32 is public.
+    expect(isPublicUrl('http://172.15.0.1').ok).toBe(true);
+    expect(isPublicUrl('http://172.31.255.255').ok).toBe(false);
+    expect(isPublicUrl('http://172.32.0.1').ok).toBe(true);
+  });
+
+  it('blocks Tailscale CGNAT 100.64.0.0/10', () => {
+    const r = isPublicUrl('http://100.114.205.53');
+    expect(r.ok).toBe(false);
+    expect(r.reason).toMatch(/cgnat/);
+  });
+
+  it('allows 100.x outside CGNAT range', () => {
+    // 100.63 is public (one below CGNAT lower bound).
+    expect(isPublicUrl('http://100.63.0.1').ok).toBe(true);
+    // 100.128 is public (one above CGNAT upper bound).
+    expect(isPublicUrl('http://100.128.0.1').ok).toBe(true);
+  });
+
+  it('blocks ftp:// (non-http protocol)', () => {
+    const r = isPublicUrl('ftp://example.com');
+    expect(r.ok).toBe(false);
+    expect(r.reason).toMatch(/unsupported_protocol/);
+  });
+
+  it('blocks file:///etc/passwd', () => {
+    expect(isPublicUrl('file:///etc/passwd').ok).toBe(false);
+  });
+
+  it('blocks anything.local (mDNS suffix)', () => {
+    const r = isPublicUrl('http://anything.local');
+    expect(r.ok).toBe(false);
+    expect(r.reason).toMatch(/private_suffix/);
+  });
+
+  it('blocks anything.internal', () => {
+    expect(isPublicUrl('http://service.internal').ok).toBe(false);
+  });
+
+  it('blocks 169.254.x.x link-local (covers AWS/GCP IMDS)', () => {
+    expect(isPublicUrl('http://169.254.169.254').ok).toBe(false);
+  });
+
+  it('allows https://example.com', () => {
+    expect(isPublicUrl('https://example.com').ok).toBe(true);
+  });
+
+  it('rejects malformed URLs', () => {
+    const r = isPublicUrl('not a url');
+    expect(r.ok).toBe(false);
+    expect(r.reason).toBe('invalid_url');
+  });
+});
+
+// ============================================================================
+// web_search
+// ============================================================================
+
+describe('executeWebSearch', () => {
+  it('returns top N results, mapped to {title,url,snippet}', async () => {
+    const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
+      mockResponse(
+        {
+          results: [
+            { title: 'A', url: 'https://a.example/', content: 'snippet a' },
+            { title: 'B', url: 'https://b.example/', content: 'snippet b' },
+            { title: 'C', url: 'https://c.example/', content: 'snippet c' },
+          ],
+        },
+        { contentType: 'application/json' },
+      ),
+    );
+    const out = await executeWebSearch({ query: 'foo', max_results: 2 }, TEST_SEARXNG);
+    expect(out.results).toHaveLength(2);
+    expect(out.results[0]).toEqual({ title: 'A', url: 'https://a.example/', snippet: 'snippet a' });
+    // URL-encodes the query and hits /search?...&format=json.
+    expect(fetchSpy).toHaveBeenCalledExactlyOnceWith(
+      `${TEST_SEARXNG}/search?q=foo&format=json`,
+      expect.objectContaining({ signal: expect.any(AbortSignal) }),
+    );
+  });
+
+  it('caps max_results at 10 even if a larger value is requested', async () => {
+    const many = Array.from({ length: 20 }, (_, i) => ({
+      title: `t${i}`,
+      url: `https://${i}.example/`,
+      content: `c${i}`,
+    }));
+    vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
+      mockResponse({ results: many }, { contentType: 'application/json' }),
+    );
+    const out = await executeWebSearch({ query: 'x', max_results: 999 }, TEST_SEARXNG);
+    expect(out.results).toHaveLength(10);
+  });
+
+  it('throws on non-200 from SearXNG (executeToolCall surfaces the error to the LLM)', async () => {
+    vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
+      new Response('boom', { status: 503 }),
+    );
+    await expect(
+      executeWebSearch({ query: 'x' }, TEST_SEARXNG),
+    ).rejects.toThrow(/SearXNG returned 503/);
+  });
+
+  it('returns empty results cleanly when SearXNG has no matches', async () => {
+    vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
+      mockResponse({ results: [] }, { contentType: 'application/json' }),
+    );
+    const out = await executeWebSearch({ query: 'xyz' }, TEST_SEARXNG);
+    expect(out.results).toEqual([]);
+    expect(out.total).toBe(0);
+  });
+
+  it('drops result entries with missing url (defensive)', async () => {
+    vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
+      mockResponse(
+        { results: [{ title: 'no url', content: 'orphan' }, { url: 'https://ok/', title: 't', content: 's' }] },
+        { contentType: 'application/json' },
+      ),
+    );
+    const out = await executeWebSearch({ query: 'x' }, TEST_SEARXNG);
+    expect(out.results).toHaveLength(1);
+    expect(out.results[0]!.url).toBe('https://ok/');
+  });
+});
+
+// ============================================================================
+// web_fetch
+// ============================================================================
+
+describe('executeWebFetch — URL-guard short-circuit', () => {
+  it('returns blocked_by_url_guard for ftp://', async () => {
+    const result = await executeWebFetch({ url: 'ftp://example.com' });
+    expect('error' in result && result.error).toBe('blocked_by_url_guard');
+  });
+
+  it('returns blocked_by_url_guard for file:///', async () => {
+    const result = await executeWebFetch({ url: 'file:///etc/passwd' });
+    expect('error' in result && result.error).toBe('blocked_by_url_guard');
+  });
+
+  it('returns blocked_by_url_guard for Tailscale CGNAT', async () => {
+    const result = await executeWebFetch({ url: 'http://100.114.205.53/admin' });
+    expect('error' in result && result.error).toBe('blocked_by_url_guard');
+  });
+});
+
+describe('executeWebFetch — content-type handling', () => {
+  it('strips HTML tags and returns plain text + title', async () => {
+    const html = `<html><head><title>  Hello World  </title></head>
+      <body><script>alert('xss')</script><h1>Heading</h1><p>Body text</p></body></html>`;
+    const fakeFetch = vi.fn().mockResolvedValue(
+      mockResponse(html, { contentType: 'text/html; charset=utf-8' }),
+    );
+    const result = await executeWebFetch(
+      { url: 'https://example.com/page' },
+      fakeFetch as unknown as typeof fetch,
+    );
+    expect('content' in result).toBe(true);
+    if ('content' in result) {
+      expect(result.title).toBe('Hello World');
+      // Script CONTENT must not leak through — the regex stripper deletes
+      // the whole <script>...</script> block, not just the tags.
+      expect(result.content).not.toContain('alert(');
+      expect(result.content).toContain('Heading');
+      expect(result.content).toContain('Body text');
+    }
+  });
+
+  it('returns JSON content as-is (no stripping)', async () => {
+    const json = '{"foo": "bar"}';
+    const fakeFetch = vi.fn().mockResolvedValue(
+      mockResponse(json, { contentType: 'application/json' }),
+    );
+    const result = await executeWebFetch(
+      { url: 'https://example.com/api' },
+      fakeFetch as unknown as typeof fetch,
+    );
+    expect('content' in result && result.content).toBe(json);
+  });
+
+  it('returns plain text as-is', async () => {
+    const txt = 'just\nplain\ntext';
+    const fakeFetch = vi.fn().mockResolvedValue(
+      mockResponse(txt, { contentType: 'text/plain' }),
+    );
+    const result = await executeWebFetch(
+      { url: 'https://example.com/file.txt' },
+      fakeFetch as unknown as typeof fetch,
+    );
+    expect('content' in result && result.content).toBe(txt);
+  });
+
+  it('returns unsupported_content_type for binary content', async () => {
+    const fakeFetch = vi.fn().mockResolvedValue(
+      mockResponse('binary garbage', { contentType: 'application/octet-stream' }),
+    );
+    const result = await executeWebFetch(
+      { url: 'https://example.com/blob' },
+      fakeFetch as unknown as typeof fetch,
+    );
+    expect('error' in result && result.error).toBe('unsupported_content_type');
+  });
+});
+
+describe('executeWebFetch — size + truncation', () => {
+  it('rejects responses whose Content-Length exceeds 5MB', async () => {
+    const fakeFetch = vi.fn().mockResolvedValue(
+      new Response('small body', {
+        status: 200,
+        headers: {
+          'content-type': 'text/plain',
+          'content-length': String(6 * 1024 * 1024),
+        },
+      }),
+    );
+    const result = await executeWebFetch(
+      { url: 'https://example.com/huge' },
+      fakeFetch as unknown as typeof fetch,
+    );
+    expect('error' in result && result.error).toBe('response_too_large');
+  });
+
+  it('truncates output to max_chars and appends a marker', async () => {
+    const big = 'A'.repeat(50_000);
+    const fakeFetch = vi.fn().mockResolvedValue(
+      mockResponse(big, { contentType: 'text/plain' }),
+    );
+    const result = await executeWebFetch(
+      { url: 'https://example.com/big', max_chars: 200 },
+      fakeFetch as unknown as typeof fetch,
+    );
+    expect('content' in result).toBe(true);
+    if ('content' in result) {
+      expect(result.truncated).toBe(true);
+      expect(result.content).toContain('[truncated');
+      // First 200 chars + the marker line.
+      expect(result.content.startsWith('A'.repeat(200))).toBe(true);
+    }
+  });
+
+  it('does NOT mark short content as truncated', async () => {
+    const fakeFetch = vi.fn().mockResolvedValue(
+      mockResponse('short', { contentType: 'text/plain' }),
+    );
+    const result = await executeWebFetch(
+      { url: 'https://example.com/tiny' },
+      fakeFetch as unknown as typeof fetch,
+    );
+    expect('content' in result && result.truncated).toBe(false);
+  });
+});