From 02bb355a092c44c3c89f6fe5fb43bf44da8c7c19 Mon Sep 17 00:00:00 2001 From: indifferentketchup Date: Sun, 7 Jun 2026 17:57:44 +0000 Subject: [PATCH] feat(server): add institutional memory recall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - File-based memory under .boocode/memory/ (project/user/reference topics) - Hierarchical 4-scope scan: global → home → project → session - Keyword/tag relevance matching for query-based recall - Injected as block in system prompt at assembly - v1 recall-only (extract/dream deferred to v2) --- .../services/memory/__tests__/entries.test.ts | 31 ++++++++ .../services/memory/__tests__/paths.test.ts | 14 ++++ .../services/memory/__tests__/prompt.test.ts | 15 ++++ .../services/memory/__tests__/recall.test.ts | 15 ++++ apps/server/src/services/memory/entries.ts | 54 ++++++++++++++ apps/server/src/services/memory/index.ts | 6 ++ apps/server/src/services/memory/paths.ts | 17 +++++ apps/server/src/services/memory/prompt.ts | 5 ++ apps/server/src/services/memory/recall.ts | 44 ++++++++++++ apps/server/src/services/memory/scan.ts | 72 +++++++++++++++++++ apps/server/src/services/memory/store.ts | 35 +++++++++ apps/server/src/services/system-prompt.ts | 8 ++- 12 files changed, 315 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/services/memory/__tests__/entries.test.ts create mode 100644 apps/server/src/services/memory/__tests__/paths.test.ts create mode 100644 apps/server/src/services/memory/__tests__/prompt.test.ts create mode 100644 apps/server/src/services/memory/__tests__/recall.test.ts create mode 100644 apps/server/src/services/memory/entries.ts create mode 100644 apps/server/src/services/memory/index.ts create mode 100644 apps/server/src/services/memory/paths.ts create mode 100644 apps/server/src/services/memory/prompt.ts create mode 100644 apps/server/src/services/memory/recall.ts create mode 100644 apps/server/src/services/memory/scan.ts create mode 100644 apps/server/src/services/memory/store.ts diff --git a/apps/server/src/services/memory/__tests__/entries.test.ts b/apps/server/src/services/memory/__tests__/entries.test.ts new file mode 100644 index 0000000..a72588b --- /dev/null +++ b/apps/server/src/services/memory/__tests__/entries.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest'; +import { parseMemoryEntries } from '../entries.js'; + +describe('parseMemoryEntries', () => { + it('parses a single entry with tags', () => { + const md = '## project: Indentation\n> tags: style\n\nUse two-space indentation\n'; + const entries = parseMemoryEntries('style.md', md); + expect(entries).toHaveLength(1); + expect(entries[0].title).toBe('Indentation'); + expect(entries[0].topic).toBe('project'); + expect(entries[0].tags).toEqual(['style']); + expect(entries[0].content).toContain('two-space'); + }); + + it('parses multiple entries', () => { + const md = [ + '## project: Style', + '', + 'Use tab indentation', + '', + '## user: Preference', + '', + 'Prefer pnpm', + '', + ].join('\n'); + const entries = parseMemoryEntries('mem.md', md); + expect(entries).toHaveLength(2); + expect(entries[0].topic).toBe('project'); + expect(entries[1].topic).toBe('user'); + }); +}); diff --git a/apps/server/src/services/memory/__tests__/paths.test.ts b/apps/server/src/services/memory/__tests__/paths.test.ts new file mode 100644 index 0000000..45fc5c7 --- /dev/null +++ b/apps/server/src/services/memory/__tests__/paths.test.ts @@ -0,0 +1,14 @@ +import { describe, it, expect } from 'vitest'; +import { getMemoryRoot, getTopicDir } from '../paths.js'; + +describe('getMemoryRoot', () => { + it('returns .boocode/memory under project root', () => { + expect(getMemoryRoot('/proj')).toBe('/proj/.boocode/memory'); + }); +}); + +describe('getTopicDir', () => { + it('returns project/ under memory root', () => { + expect(getTopicDir('/r/.boocode/memory', 'project')).toBe('/r/.boocode/memory/project'); + }); +}); diff --git a/apps/server/src/services/memory/__tests__/prompt.test.ts b/apps/server/src/services/memory/__tests__/prompt.test.ts new file mode 100644 index 0000000..59a72a7 --- /dev/null +++ b/apps/server/src/services/memory/__tests__/prompt.test.ts @@ -0,0 +1,15 @@ +import { describe, it, expect } from 'vitest'; +import { formatMemoryBlock } from '../prompt.js'; + +describe('formatMemoryBlock', () => { + it('wraps entries in boocode-memory tags', () => { + const block = formatMemoryBlock(['Use pnpm', 'Tests in vitest']); + expect(block).toContain(''); + expect(block).toContain('Use pnpm'); + expect(block).toContain(''); + }); + + it('returns empty string for no entries', () => { + expect(formatMemoryBlock([])).toBe(''); + }); +}); diff --git a/apps/server/src/services/memory/__tests__/recall.test.ts b/apps/server/src/services/memory/__tests__/recall.test.ts new file mode 100644 index 0000000..bc345e0 --- /dev/null +++ b/apps/server/src/services/memory/__tests__/recall.test.ts @@ -0,0 +1,15 @@ +import { describe, it, expect } from 'vitest'; +import { rankByRelevance } from '../recall.js'; +import type { MemoryEntry } from '../entries.js'; + +describe('rankByRelevance', () => { + it('returns entries matching query keywords', () => { + const entries: MemoryEntry[] = [ + { id: '1', topic: 'project', title: 'Style', content: 'Use two-space indentation', tags: ['style'] }, + { id: '2', topic: 'project', title: 'Tests', content: 'Use vitest for testing', tags: ['testing'] }, + ]; + const result = rankByRelevance('what indentation?', entries); + expect(result).toHaveLength(1); + expect(result[0].title).toBe('Style'); + }); +}); diff --git a/apps/server/src/services/memory/entries.ts b/apps/server/src/services/memory/entries.ts new file mode 100644 index 0000000..0e8bd66 --- /dev/null +++ b/apps/server/src/services/memory/entries.ts @@ -0,0 +1,54 @@ +export interface MemoryEntry { + id: string; + topic: string; + title: string; + content: string; + tags: string[]; +} + +export function parseMemoryEntries(fileName: string, markdown: string): MemoryEntry[] { + const entries: MemoryEntry[] = []; + const lines = markdown.split('\n'); + let currentEntry: Partial | null = null; + let currentContent: string[] = []; + + for (const line of lines) { + const headingMatch = line.match(/^##\s+(.+):\s+(.+)$/); + if (headingMatch && headingMatch[1] && headingMatch[2]) { + if (currentEntry && currentEntry.title) { + entries.push({ + id: `${fileName}-${entries.length}`, + topic: currentEntry.topic ?? '', + title: currentEntry.title, + content: currentContent.join('\n').trim(), + tags: currentEntry.tags ?? [], + }); + } + currentEntry = { topic: headingMatch[1].trim(), title: headingMatch[2].trim(), tags: [] }; + currentContent = []; + continue; + } + + const tagsMatch = line.match(/^>\s*tags:\s*(.+)$/i); + if (tagsMatch && tagsMatch[1] && currentEntry) { + currentEntry.tags = tagsMatch[1].split(',').map((t) => t.trim()); + continue; + } + + if (currentEntry) { + currentContent.push(line); + } + } + + if (currentEntry && currentEntry.title) { + entries.push({ + id: `${fileName}-${entries.length}`, + topic: currentEntry.topic ?? '', + title: currentEntry.title, + content: currentContent.join('\n').trim(), + tags: currentEntry.tags ?? [], + }); + } + + return entries; +} diff --git a/apps/server/src/services/memory/index.ts b/apps/server/src/services/memory/index.ts new file mode 100644 index 0000000..9b1fae1 --- /dev/null +++ b/apps/server/src/services/memory/index.ts @@ -0,0 +1,6 @@ +export { loadMemoryForSession } from './recall.js'; +export { formatMemoryBlock } from './prompt.js'; +export { scanMemoryScopes } from './scan.js'; +export { parseMemoryEntries } from './entries.js'; +export { ensureMemoryScaffold, getMemoryRoot } from './paths.js'; +export type { MemoryEntry } from './entries.js'; diff --git a/apps/server/src/services/memory/paths.ts b/apps/server/src/services/memory/paths.ts new file mode 100644 index 0000000..026ab47 --- /dev/null +++ b/apps/server/src/services/memory/paths.ts @@ -0,0 +1,17 @@ +import { join } from 'node:path'; +import { mkdir } from 'node:fs/promises'; + +const TOPICS = ['project', 'user', 'reference'] as const; +export type MemoryTopic = (typeof TOPICS)[number]; + +export function getMemoryRoot(projectRoot: string): string { + return join(projectRoot, '.boocode', 'memory'); +} + +export function getTopicDir(root: string, topic: MemoryTopic): string { + return join(root, topic); +} + +export async function ensureMemoryScaffold(root: string): Promise { + await Promise.all(TOPICS.map((t) => mkdir(join(root, t), { recursive: true }))); +} diff --git a/apps/server/src/services/memory/prompt.ts b/apps/server/src/services/memory/prompt.ts new file mode 100644 index 0000000..d565936 --- /dev/null +++ b/apps/server/src/services/memory/prompt.ts @@ -0,0 +1,5 @@ +export function formatMemoryBlock(entries: string[]): string { + if (entries.length === 0) return ''; + const body = entries.map((e) => `- ${e}`).join('\n'); + return `\n${body}\n`; +} diff --git a/apps/server/src/services/memory/recall.ts b/apps/server/src/services/memory/recall.ts new file mode 100644 index 0000000..bd50114 --- /dev/null +++ b/apps/server/src/services/memory/recall.ts @@ -0,0 +1,44 @@ +import type { MemoryEntry } from './entries.js'; +import { scanProjectMemory } from './scan.js'; + +function extractKeywords(query: string): string[] { + return query + .toLowerCase() + .replace(/[^a-z0-9\s]/g, '') + .split(/\s+/) + .filter((w) => w.length > 2); +} + +export function rankByRelevance(query: string, entries: MemoryEntry[]): MemoryEntry[] { + const keywords = extractKeywords(query); + if (keywords.length === 0) return entries.slice(0, 5); + + const scored = entries.map((entry) => { + let score = 0; + const searchText = `${entry.title} ${entry.content} ${entry.tags.join(' ')}`.toLowerCase(); + for (const kw of keywords) { + if (entry.title.toLowerCase().includes(kw)) score += 3; + if (entry.tags.some((t) => t.toLowerCase().includes(kw))) score += 2; + if (entry.content.toLowerCase().includes(kw)) score += 1; + } + return { entry, score }; + }); + + return scored + .filter((s) => s.score > 0) + .sort((a, b) => b.score - a.score) + .slice(0, 10) + .map((s) => s.entry); +} + +export async function loadMemoryForSession( + projectRoot: string, + _sessionId?: string, + query?: string, +): Promise { + const entries = await scanProjectMemory(projectRoot); + if (entries.length === 0) return []; + + const relevant = query ? rankByRelevance(query, entries) : entries.slice(0, 5); + return relevant.map((e) => `[${e.topic}] ${e.title}: ${e.content}`); +} diff --git a/apps/server/src/services/memory/scan.ts b/apps/server/src/services/memory/scan.ts new file mode 100644 index 0000000..15ede25 --- /dev/null +++ b/apps/server/src/services/memory/scan.ts @@ -0,0 +1,72 @@ +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { readFile, readdir } from 'node:fs/promises'; +import type { MemoryEntry } from './entries.js'; +import { parseMemoryEntries } from './entries.js'; +import { getMemoryRoot } from './paths.js'; + +export interface MemoryScope { + projectRoot: string; + sessionDir?: string; + homeDir?: string; +} + +async function scanDirectory(dir: string): Promise { + const entries: MemoryEntry[] = []; + try { + const files = await readdir(dir, { withFileTypes: true }); + for (const file of files) { + if (file.isFile() && file.name.endsWith('.md')) { + const content = await readFile(join(dir, file.name), 'utf8'); + entries.push(...parseMemoryEntries(file.name, content)); + } + } + } catch { + // Directory doesn't exist + } + return entries; +} + +const MEMORY_TOPICS = ['project', 'user', 'reference'] as const; + +async function scanTopicDirs(root: string): Promise { + const entries: MemoryEntry[] = []; + for (const topic of MEMORY_TOPICS) { + entries.push(...(await scanDirectory(join(root, topic)))); + } + return entries; +} + +export async function scanMemoryScopes(scope: MemoryScope): Promise { + const allEntries: MemoryEntry[] = []; + + // 1. Global (~/.boocode/memory/) - lowest priority + allEntries.push(...(await scanTopicDirs(getMemoryRoot(homedir())))); + + // 2. Home ($HOME/.boocode/memory) + const homeDir = scope.homeDir ?? homedir(); + const homeRoot = getMemoryRoot(homeDir); + if (homeRoot !== getMemoryRoot(homedir())) { + allEntries.push(...(await scanTopicDirs(homeRoot))); + } + + // 3. Project (.boocode/memory/ under project root) + allEntries.push(...(await scanTopicDirs(getMemoryRoot(scope.projectRoot)))); + + // 4. Session (.boocode/sessions//memory.md) - highest priority + if (scope.sessionDir) { + try { + const sessionFile = join(scope.sessionDir, 'memory.md'); + const content = await readFile(sessionFile, 'utf8'); + allEntries.push(...parseMemoryEntries('session-memory', content)); + } catch { + // No session memory file + } + } + + return allEntries; +} + +export async function scanProjectMemory(projectRoot: string): Promise { + return scanMemoryScopes({ projectRoot }); +} diff --git a/apps/server/src/services/memory/store.ts b/apps/server/src/services/memory/store.ts new file mode 100644 index 0000000..9347dde --- /dev/null +++ b/apps/server/src/services/memory/store.ts @@ -0,0 +1,35 @@ +import { readFile, writeFile, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import type { MemoryTopic } from './paths.js'; +import { getTopicDir } from './paths.js'; + +export async function readTopicFiles(root: string, topic: MemoryTopic): Promise> { + const dir = getTopicDir(root, topic); + const files = new Map(); + try { + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isFile() && entry.name.endsWith('.md')) { + const content = await readFile(join(dir, entry.name), 'utf8'); + files.set(entry.name, content); + } + } + } catch { + // Directory doesn't exist yet + } + return files; +} + +export async function writeEntry( + root: string, + topic: MemoryTopic, + title: string, + content: string, + tags: string[], +): Promise { + const dir = getTopicDir(root, topic); + const tagLine = tags.length > 0 ? `> tags: ${tags.join(', ')}\n\n` : '\n'; + const entry = `## ${topic}: ${title}\n${tagLine}${content}\n`; + const filename = title.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '') + '.md'; + await writeFile(join(dir, filename), entry, 'utf8'); +} diff --git a/apps/server/src/services/system-prompt.ts b/apps/server/src/services/system-prompt.ts index b9a21d5..a1dde52 100644 --- a/apps/server/src/services/system-prompt.ts +++ b/apps/server/src/services/system-prompt.ts @@ -22,6 +22,8 @@ import { readFile, stat } from 'node:fs/promises'; import type { Agent, Project, Session } from '../types/api.js'; import { getAgentsMtimes } from './agents.js'; import { resolveRoute } from './inference/provider.js'; +import { loadMemoryForSession } from './memory/recall.js'; +import { formatMemoryBlock } from './memory/prompt.js'; const BASE_SYSTEM_PROMPT = (projectPath: string) => `You are BooCode Chat, a code investigation assistant. The user is working on a project located at ${projectPath}. Use the file-read tools (view_file, list_dir, grep, find_files) to investigate code when needed. Be concise. Cite file paths and line numbers when discussing code. Do not hallucinate file contents — read the file first. Tool results may be truncated; if so, narrow your query rather than guessing.`; @@ -164,7 +166,11 @@ export async function buildSystemPromptWithFingerprint( let out = BASE_SYSTEM_PROMPT(project.path); const guidance = await getContainerGuidance(); if (guidance) { - out += `\n\n--- Container guidance ---\n${guidance}\n--- end container guidance ---\n`; + out += '\n\n--- Container guidance ---\n' + guidance + '\n--- end container guidance ---\n'; + } + const memory = await loadMemoryForSession(project.path, session.id).catch(() => []); + if (memory.length > 0) { + out += '\n\n' + formatMemoryBlock(memory); } if (agent && agent.system_prompt.trim().length > 0) { out += '\n\n' + agent.system_prompt.trim();