batch3 T4: file_ops + file_index services; UI endpoints; tools refactor
- services/file_ops.ts: shared listDir/viewFile/grep/findFiles core - services/file_index.ts: per-project flat file list cached on mtime of project root + .git/HEAD + .git/index (rg --files honors .gitignore) - services/tools.ts: tools delegate to file_ops, output format unchanged - routes/projects.ts: GET /list_dir, /view_file, /files endpoints - web client: api.projects.listDir/viewFile/files + mirrored types Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
49
apps/server/src/services/file_index.ts
Normal file
49
apps/server/src/services/file_index.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { execFile } from 'node:child_process';
|
||||
|
||||
interface MtimeSnap {
|
||||
root: number;
|
||||
gitHead: number | null;
|
||||
gitIndex: number | null;
|
||||
}
|
||||
|
||||
interface CacheEntry {
|
||||
files: string[];
|
||||
mtimes: MtimeSnap;
|
||||
}
|
||||
|
||||
const cache = new Map<string, CacheEntry>(); // keyed by projectId
|
||||
|
||||
export async function getProjectFiles(projectId: string, projectRoot: string): Promise<string[]> {
|
||||
const current = await snapMtimes(projectRoot);
|
||||
const cached = cache.get(projectId);
|
||||
if (cached && eqMtimes(cached.mtimes, current)) {
|
||||
return cached.files;
|
||||
}
|
||||
const files = await runRgFiles(projectRoot);
|
||||
cache.set(projectId, { files, mtimes: current });
|
||||
return files;
|
||||
}
|
||||
|
||||
async function snapMtimes(root: string): Promise<MtimeSnap> {
|
||||
const rootStat = await fs.stat(root);
|
||||
let gitHead: number | null = null;
|
||||
let gitIndex: number | null = null;
|
||||
try { gitHead = (await fs.stat(path.join(root, '.git', 'HEAD'))).mtimeMs; } catch {}
|
||||
try { gitIndex = (await fs.stat(path.join(root, '.git', 'index'))).mtimeMs; } catch {}
|
||||
return { root: rootStat.mtimeMs, gitHead, gitIndex };
|
||||
}
|
||||
|
||||
function eqMtimes(a: MtimeSnap, b: MtimeSnap): boolean {
|
||||
return a.root === b.root && a.gitHead === b.gitHead && a.gitIndex === b.gitIndex;
|
||||
}
|
||||
|
||||
function runRgFiles(root: string): Promise<string[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile('rg', ['--files'], { cwd: root, maxBuffer: 32 * 1024 * 1024 }, (err, stdout) => {
|
||||
if (err) return reject(err);
|
||||
resolve(stdout.split('\n').filter(Boolean));
|
||||
});
|
||||
});
|
||||
}
|
||||
250
apps/server/src/services/file_ops.ts
Normal file
250
apps/server/src/services/file_ops.ts
Normal file
@@ -0,0 +1,250 @@
|
||||
import { readFile, readdir, stat } from 'node:fs/promises';
|
||||
import { resolve, basename, relative } from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
import type { Stats } from 'node:fs';
|
||||
import { pathGuard, PathScopeError } from './path_guard.js';
|
||||
|
||||
const MAX_FILE_BYTES = 5 * 1024 * 1024;
|
||||
const DEFAULT_VIEW_LINES = 200;
|
||||
const MAX_GREP_RESULTS = 200;
|
||||
const DEFAULT_GREP_RESULTS = 100;
|
||||
const MAX_FIND_RESULTS = 1000;
|
||||
const DEFAULT_FIND_RESULTS = 100;
|
||||
const MAX_DIR_ENTRIES = 500;
|
||||
|
||||
export interface FileEntry {
|
||||
name: string;
|
||||
kind: 'file' | 'dir';
|
||||
size?: number;
|
||||
}
|
||||
|
||||
export interface ListDirResult {
|
||||
entries: FileEntry[];
|
||||
truncated: boolean;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface ViewFileResult {
|
||||
content: string;
|
||||
truncated: boolean;
|
||||
total_bytes: number;
|
||||
bytes_returned: number;
|
||||
}
|
||||
|
||||
export interface GrepMatch {
|
||||
path: string;
|
||||
line: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface GrepResult {
|
||||
matches: GrepMatch[];
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export interface FindFilesResult {
|
||||
files: string[];
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
// Suppress unused import warning — Stats is part of the public API surface
|
||||
void (undefined as unknown as Stats);
|
||||
|
||||
export async function listDir(projectRoot: string, relPath: string): Promise<ListDirResult> {
|
||||
const real = await pathGuard(projectRoot, relPath);
|
||||
const s = await stat(real);
|
||||
if (!s.isDirectory()) {
|
||||
throw new PathScopeError(`not a directory: ${relPath}`);
|
||||
}
|
||||
const entries = await readdir(real, { withFileTypes: true });
|
||||
const total = entries.length;
|
||||
const slice = entries.slice(0, MAX_DIR_ENTRIES);
|
||||
const out: FileEntry[] = await Promise.all(
|
||||
slice.map(async (e) => {
|
||||
const child = resolve(real, e.name);
|
||||
let size: number | undefined;
|
||||
if (e.isFile()) {
|
||||
try {
|
||||
const cs = await stat(child);
|
||||
size = cs.size;
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
return {
|
||||
name: e.name,
|
||||
kind: e.isDirectory() ? ('dir' as const) : ('file' as const),
|
||||
...(size != null ? { size } : {}),
|
||||
};
|
||||
})
|
||||
);
|
||||
return {
|
||||
entries: out,
|
||||
total,
|
||||
truncated: total > MAX_DIR_ENTRIES,
|
||||
};
|
||||
}
|
||||
|
||||
export async function viewFile(projectRoot: string, relPath: string): Promise<ViewFileResult> {
|
||||
const real = await pathGuard(projectRoot, relPath);
|
||||
const s = await stat(real);
|
||||
if (!s.isFile()) {
|
||||
throw new PathScopeError(`not a file: ${relPath}`);
|
||||
}
|
||||
if (s.size > MAX_FILE_BYTES) {
|
||||
throw new Error(`file too large (${s.size} bytes, max ${MAX_FILE_BYTES})`);
|
||||
}
|
||||
const raw = await readFile(real, 'utf8');
|
||||
const lines = raw.split('\n');
|
||||
const total = lines.length;
|
||||
const end = Math.min(total, DEFAULT_VIEW_LINES);
|
||||
const slice = lines.slice(0, end);
|
||||
const content = slice.join('\n');
|
||||
const truncated = total > end;
|
||||
const bytes_returned = Buffer.byteLength(content, 'utf8');
|
||||
return {
|
||||
content,
|
||||
truncated,
|
||||
total_bytes: s.size,
|
||||
bytes_returned,
|
||||
};
|
||||
}
|
||||
|
||||
interface RipgrepMatch {
|
||||
type: string;
|
||||
data?: {
|
||||
path?: { text?: string };
|
||||
line_number?: number;
|
||||
lines?: { text?: string };
|
||||
};
|
||||
}
|
||||
|
||||
export async function grep(
|
||||
projectRoot: string,
|
||||
pattern: string,
|
||||
opts?: { path?: string; max_matches?: number; case_sensitive?: boolean; hidden?: boolean }
|
||||
): Promise<GrepResult> {
|
||||
const targetPath = opts?.path ?? projectRoot;
|
||||
const target = await pathGuard(projectRoot, targetPath);
|
||||
const limit = Math.min(
|
||||
Math.max(opts?.max_matches ?? DEFAULT_GREP_RESULTS, 1),
|
||||
MAX_GREP_RESULTS
|
||||
);
|
||||
const args = [
|
||||
'--json',
|
||||
'--max-count',
|
||||
String(limit),
|
||||
'--max-columns',
|
||||
'300',
|
||||
];
|
||||
if (!opts?.case_sensitive) args.push('--ignore-case');
|
||||
if (opts?.hidden) args.push('--hidden');
|
||||
args.push('--', pattern, target);
|
||||
|
||||
return new Promise((resolveP, rejectP) => {
|
||||
const child = spawn('rg', args, { cwd: projectRoot });
|
||||
const matches: GrepMatch[] = [];
|
||||
let buf = '';
|
||||
let stderr = '';
|
||||
child.stdout.setEncoding('utf8');
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stdout.on('data', (chunk: string) => {
|
||||
buf += chunk;
|
||||
let idx;
|
||||
while ((idx = buf.indexOf('\n')) >= 0) {
|
||||
const line = buf.slice(0, idx);
|
||||
buf = buf.slice(idx + 1);
|
||||
if (!line) continue;
|
||||
if (matches.length >= limit) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line) as RipgrepMatch;
|
||||
if (parsed.type !== 'match' || !parsed.data) continue;
|
||||
const filePath = parsed.data.path?.text ?? '';
|
||||
const lineNumber = parsed.data.line_number ?? 0;
|
||||
const content = parsed.data.lines?.text ?? '';
|
||||
matches.push({
|
||||
path: relative(projectRoot, filePath) || filePath,
|
||||
line: lineNumber,
|
||||
text: content.replace(/\n$/, ''),
|
||||
});
|
||||
} catch {
|
||||
/* ignore non-json */
|
||||
}
|
||||
}
|
||||
if (matches.length >= limit) {
|
||||
child.kill();
|
||||
}
|
||||
});
|
||||
child.stderr.on('data', (chunk: string) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.on('error', (err) => rejectP(err));
|
||||
child.on('close', (code) => {
|
||||
if (code === 2 && matches.length === 0) {
|
||||
rejectP(new Error(`ripgrep failed: ${stderr.slice(0, 300)}`));
|
||||
return;
|
||||
}
|
||||
resolveP({
|
||||
matches,
|
||||
truncated: matches.length >= limit,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function findFiles(
|
||||
projectRoot: string,
|
||||
pattern?: string,
|
||||
opts?: { type?: 'file' | 'dir'; max_results?: number }
|
||||
): Promise<FindFilesResult> {
|
||||
const limit = Math.min(
|
||||
Math.max(opts?.max_results ?? DEFAULT_FIND_RESULTS, 1),
|
||||
MAX_FIND_RESULTS
|
||||
);
|
||||
const args = ['--files'];
|
||||
if (pattern) args.push('--glob', pattern);
|
||||
args.push(projectRoot);
|
||||
|
||||
return new Promise((resolveP, rejectP) => {
|
||||
const child = spawn('rg', args, { cwd: projectRoot });
|
||||
const files: string[] = [];
|
||||
let total = 0;
|
||||
let buf = '';
|
||||
let stderr = '';
|
||||
child.stdout.setEncoding('utf8');
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stdout.on('data', (chunk: string) => {
|
||||
buf += chunk;
|
||||
let idx;
|
||||
while ((idx = buf.indexOf('\n')) >= 0) {
|
||||
const line = buf.slice(0, idx);
|
||||
buf = buf.slice(idx + 1);
|
||||
if (!line) continue;
|
||||
total++;
|
||||
if (files.length < limit) {
|
||||
files.push(relative(projectRoot, line) || line);
|
||||
}
|
||||
}
|
||||
});
|
||||
child.stderr.on('data', (chunk: string) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.on('error', (err) => rejectP(err));
|
||||
child.on('close', (code) => {
|
||||
if (code === 2) {
|
||||
rejectP(new Error(`ripgrep failed: ${stderr.slice(0, 300)}`));
|
||||
return;
|
||||
}
|
||||
if (buf.length > 0) {
|
||||
total++;
|
||||
if (files.length < limit) {
|
||||
files.push(relative(projectRoot, buf) || buf);
|
||||
}
|
||||
}
|
||||
resolveP({
|
||||
files,
|
||||
truncated: total > files.length,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { resolve, basename, relative } from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { z } from 'zod';
|
||||
import { pathGuard, PathScopeError } from './path_guard.js';
|
||||
import { grep as fileOpsGrep } from './file_ops.js';
|
||||
|
||||
const MAX_FILE_BYTES = 5 * 1024 * 1024;
|
||||
const DEFAULT_VIEW_LINES = 200;
|
||||
@@ -168,15 +169,6 @@ const GrepInput = z.object({
|
||||
});
|
||||
type GrepInputT = z.infer<typeof GrepInput>;
|
||||
|
||||
interface RipgrepMatch {
|
||||
type: string;
|
||||
data?: {
|
||||
path?: { text?: string };
|
||||
line_number?: number;
|
||||
lines?: { text?: string };
|
||||
};
|
||||
}
|
||||
|
||||
export const grep: ToolDef<GrepInputT> = {
|
||||
name: 'grep',
|
||||
description:
|
||||
@@ -203,73 +195,27 @@ export const grep: ToolDef<GrepInputT> = {
|
||||
},
|
||||
},
|
||||
async execute(input, projectRoot) {
|
||||
const target = await pathGuard(projectRoot, input.path ?? projectRoot);
|
||||
const limit = Math.min(
|
||||
Math.max(input.max_results ?? DEFAULT_GREP_RESULTS, 1),
|
||||
MAX_GREP_RESULTS
|
||||
);
|
||||
const args = [
|
||||
'--json',
|
||||
'--max-count',
|
||||
String(limit),
|
||||
'--max-columns',
|
||||
'300',
|
||||
];
|
||||
if (!input.case_sensitive) args.push('--ignore-case');
|
||||
if (input.hidden) args.push('--hidden');
|
||||
args.push('--', input.pattern, target);
|
||||
|
||||
return await new Promise((resolveP, rejectP) => {
|
||||
const child = spawn('rg', args, { cwd: projectRoot });
|
||||
const matches: Array<{ path: string; line: number; content: string }> = [];
|
||||
let buf = '';
|
||||
let stderr = '';
|
||||
child.stdout.setEncoding('utf8');
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stdout.on('data', (chunk: string) => {
|
||||
buf += chunk;
|
||||
let idx;
|
||||
while ((idx = buf.indexOf('\n')) >= 0) {
|
||||
const line = buf.slice(0, idx);
|
||||
buf = buf.slice(idx + 1);
|
||||
if (!line) continue;
|
||||
if (matches.length >= limit) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line) as RipgrepMatch;
|
||||
if (parsed.type !== 'match' || !parsed.data) continue;
|
||||
const path = parsed.data.path?.text ?? '';
|
||||
const lineNumber = parsed.data.line_number ?? 0;
|
||||
const content = parsed.data.lines?.text ?? '';
|
||||
matches.push({
|
||||
path: relative(projectRoot, path) || path,
|
||||
line: lineNumber,
|
||||
content: content.replace(/\n$/, ''),
|
||||
});
|
||||
} catch {
|
||||
/* ignore non-json */
|
||||
}
|
||||
}
|
||||
if (matches.length >= limit) {
|
||||
child.kill();
|
||||
}
|
||||
});
|
||||
child.stderr.on('data', (chunk: string) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.on('error', (err) => rejectP(err));
|
||||
child.on('close', (code) => {
|
||||
// rg exits 1 when no matches, 2 on real error
|
||||
if (code === 2 && matches.length === 0) {
|
||||
rejectP(new Error(`ripgrep failed: ${stderr.slice(0, 300)}`));
|
||||
return;
|
||||
}
|
||||
resolveP({
|
||||
matches,
|
||||
total: matches.length,
|
||||
truncated: matches.length >= limit,
|
||||
});
|
||||
});
|
||||
// Delegate to file_ops.grep; reshape match objects to preserve LLM output format
|
||||
// (file_ops uses {path, line, text}; tool output uses {path, line, content})
|
||||
const result = await fileOpsGrep(projectRoot, input.pattern, {
|
||||
path: input.path,
|
||||
max_matches: limit,
|
||||
case_sensitive: input.case_sensitive,
|
||||
hidden: input.hidden,
|
||||
});
|
||||
return {
|
||||
matches: result.matches.map((m) => ({
|
||||
path: m.path,
|
||||
line: m.line,
|
||||
content: m.text,
|
||||
})),
|
||||
total: result.matches.length,
|
||||
truncated: result.truncated,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user