From 524a0deaa102f263a3e69688730d2b764eec5758 Mon Sep 17 00:00:00 2001 From: indifferentketchup Date: Mon, 8 Jun 2026 00:17:55 +0000 Subject: [PATCH] feat(coder): add model resolution core + multi-batch matcher Model resolution (from oh-my-openagent/model-core): 6-step priority resolution pipeline (UI select -> user config -> category default -> user fallback -> policy chain -> system default), provider fallback chains, fuzzy model matching, error classification, provider-specific model ID transforms. 14 files, zero runtime deps. Multi-batch matcher (from boocontext-audit): 6 batch types (Observational, Actionable, PreviouslyApplied, Disambiguation, ResponseAnalysis, LowCriticality) for behavioral guideline evaluation. RelationalResolver with iterative convergence (DEPENDS_ON, PRIORITIZES, ENTAILS, TAG_ALL, TAG_PRIORITIZES). SchematicGenerator abstract class with retry and execution plans. 4 files. --- .../src/services/behavioral/generation.ts | 204 ++++++++ apps/coder/src/services/behavioral/index.ts | 77 ++++ .../coder/src/services/behavioral/matching.ts | 435 ++++++++++++++++++ .../coder/src/services/behavioral/resolver.ts | 355 ++++++++++++++ .../connected-providers-cache.ts | 34 ++ .../fallback-chain-from-models.ts | 128 ++++++ .../model-resolution/fallback-model-object.ts | 9 + .../src/services/model-resolution/index.ts | 80 ++++ .../model-resolution/known-variants.ts | 16 + .../model-resolution/model-availability.ts | 64 +++ .../model-error-classifier.ts | 261 +++++++++++ .../model-resolution/model-normalization.ts | 8 + .../model-requirement-types.ts | 18 + .../model-resolution-pipeline.ts | 256 +++++++++++ .../model-resolution-types.ts | 41 ++ .../model-resolution/model-resolver.ts | 109 +++++ .../model-resolution/provider-cache.ts | 27 ++ .../provider-model-id-transform.ts | 69 +++ 18 files changed, 2191 insertions(+) create mode 100644 apps/coder/src/services/behavioral/generation.ts create mode 100644 apps/coder/src/services/behavioral/index.ts create mode 100644 apps/coder/src/services/behavioral/matching.ts create mode 100644 apps/coder/src/services/behavioral/resolver.ts create mode 100644 apps/coder/src/services/model-resolution/connected-providers-cache.ts create mode 100644 apps/coder/src/services/model-resolution/fallback-chain-from-models.ts create mode 100644 apps/coder/src/services/model-resolution/fallback-model-object.ts create mode 100644 apps/coder/src/services/model-resolution/index.ts create mode 100644 apps/coder/src/services/model-resolution/known-variants.ts create mode 100644 apps/coder/src/services/model-resolution/model-availability.ts create mode 100644 apps/coder/src/services/model-resolution/model-error-classifier.ts create mode 100644 apps/coder/src/services/model-resolution/model-normalization.ts create mode 100644 apps/coder/src/services/model-resolution/model-requirement-types.ts create mode 100644 apps/coder/src/services/model-resolution/model-resolution-pipeline.ts create mode 100644 apps/coder/src/services/model-resolution/model-resolution-types.ts create mode 100644 apps/coder/src/services/model-resolution/model-resolver.ts create mode 100644 apps/coder/src/services/model-resolution/provider-cache.ts create mode 100644 apps/coder/src/services/model-resolution/provider-model-id-transform.ts diff --git a/apps/coder/src/services/behavioral/generation.ts b/apps/coder/src/services/behavioral/generation.ts new file mode 100644 index 0000000..26e51cd --- /dev/null +++ b/apps/coder/src/services/behavioral/generation.ts @@ -0,0 +1,204 @@ +/** + * Schematic generator for behavioral guideline batches. + * + * Port of boocontext-audit/src/generation.ts — abstract LLM batch caller + * with temperature retry and structured output per batch type. + */ + +import { type GenerationInfo } from './matching.js'; + +// ─── Output types per batch ─── + +export interface ObservationalOutput { + checks: { + guideline_id: string; + condition: string; + rationale: string; + applies: boolean; + }[]; +} + +export interface ActionableOutput { + checks: { + guideline_id: string; + condition: string; + action: string; + rationale: string; + applies: boolean; + }[]; +} + +export interface PreviouslyAppliedOutput { + checks: { + guideline_id: string; + condition: string; + action_segment: string; + rationale: string; + is_still_applicable: boolean; + }[]; +} + +export interface DisambiguationOutput { + source_guideline_id: string; + rationale: string; + enriched_action: string; + targets: string[]; +} + +export interface ResponseAnalysisOutput { + guideline_id: string; + condition: string; + was_followed: boolean; + rationale: string; +} + +// ─── Batch output map ─── + +export interface BatchOutputMap { + observational: ObservationalOutput; + actionable: ActionableOutput; + previously_applied: PreviouslyAppliedOutput; + disambiguation: DisambiguationOutput; + response_analysis: ResponseAnalysisOutput; +} + +export type BatchTypeKey = keyof BatchOutputMap; + +export type OutputForBatch = BatchOutputMap[T]; + +// ─── SchematicGenerator ─── + +export abstract class SchematicGenerator { + constructor(public modelName: string) {} + + abstract generate( + prompt: string, + hints?: Record, + ): Promise<{ + content: TSchema; + info: GenerationInfo; + }>; +} + +/** + * Default stub implementation that returns empty results. + * Replace with a real LLM caller in production. + */ +export class DefaultSchematicGenerator + implements SchematicGenerator +{ + constructor( + public modelName: string, + public defaultTemperature = 0.7, + ) {} + + async generate( + _prompt: string, + hints?: Record, + ): Promise<{ content: unknown; info: GenerationInfo }> { + const temperature = (hints?.temperature as number) ?? this.defaultTemperature; + return { + content: {}, + info: { + model: this.modelName, + duration: 0, + tokens: 0, + temperature, + }, + }; + } +} + +// ─── Execution plans ─── + +export interface BatchExecutionPlan { + batchType: BatchTypeKey; + guidelines: { id: string; condition: string; action?: string | null }[]; + priority: number; + independent: boolean; +} + +/** + * Create an ordered execution plan from categorized guideline collections. + * Groups are sorted by priority: previously_applied (fastest) first, + * then observational, actionable, disambiguation, low-criticality last. + */ +export function createExecutionPlan( + observational: { id: string; condition: string }[], + actionable: { id: string; condition: string; action: string }[], + previouslyApplied: { id: string; condition: string; action?: string | null }[], + disambiguationGroups: { source: string; targets: string[]; enrichedAction: string }[], + lowCriticality: { id: string; condition: string }[], +): BatchExecutionPlan[] { + const plans: BatchExecutionPlan[] = []; + + if (observational.length > 0) { + plans.push({ + batchType: 'observational', + guidelines: observational.map((g) => ({ id: g.id, condition: g.condition })), + priority: 1, + independent: true, + }); + } + + if (actionable.length > 0) { + plans.push({ + batchType: 'actionable', + guidelines: actionable.map((g) => ({ + id: g.id, + condition: g.condition, + action: g.action, + })), + priority: 2, + independent: true, + }); + } + + if (previouslyApplied.length > 0) { + plans.push({ + batchType: 'previously_applied', + guidelines: previouslyApplied.map((g) => ({ + id: g.id, + condition: g.condition, + action: g.action, + })), + priority: 0, + independent: true, + }); + } + + if (disambiguationGroups.length > 0) { + plans.push({ + batchType: 'disambiguation', + guidelines: disambiguationGroups.map((g) => ({ + id: g.source, + condition: g.enrichedAction, + })), + priority: 3, + independent: true, + }); + } + + if (lowCriticality.length > 0) { + plans.push({ + batchType: 'observational', + guidelines: lowCriticality.map((g) => ({ id: g.id, condition: g.condition })), + priority: 10, + independent: true, + }); + } + + return plans.sort((a, b) => a.priority - b.priority); +} + +/** + * Compute retry temperatures: base + 0.2 * attempt. + * Provides progressive temperature increases for failed calls. + */ +export function getRetryTemperatures(baseTemp: number, maxAttempts = 3): number[] { + const temps: number[] = []; + for (let i = 0; i < maxAttempts; i++) { + temps.push(baseTemp + i * 0.2); + } + return temps; +} diff --git a/apps/coder/src/services/behavioral/index.ts b/apps/coder/src/services/behavioral/index.ts new file mode 100644 index 0000000..113b602 --- /dev/null +++ b/apps/coder/src/services/behavioral/index.ts @@ -0,0 +1,77 @@ +/** + * Behavioral engine — multi-batch matcher and relational resolver. + * + * Import from the existing guideline-service.ts: + * import { MultiBatchMatcher } from './behavioral/matching.js'; + * import { RelationalResolver } from './behavioral/resolver.js'; + */ + +// matching.ts +export { + type Criticality, + type GuidelineContent, + type Guideline, + type GenerationInfo, + BatchType, + type GuidelineMatch, + type GuidelineMatchingContext, + type GuidelineMatchingBatchResult, + type GuidelineMatchingResult, + type ObservationalGuidelineMatchSchema, + type ObservationalGuidelineMatchesSchema, + type ActionableGuidelineMatchSchema, + type ActionableGuidelineMatchesSchema, + type PreviouslyAppliedGuidelineMatchSchema, + type PreviouslyAppliedGuidelineMatchesSchema, + type DisambiguationGuidelineMatchSchema, + type ResponseAnalysisSchema, + type ScoredMatch, + GuidelineMatchingBatchError, + type GuidelineMatchingBatch, + type GuidelineMatchingStrategy, + ObservationalGuidelineMatchingBatch, + ActionableGuidelineMatchingBatch, + PreviouslyAppliedGuidelineMatchingBatch, + DisambiguationGuidelineMatchingBatch, + ResponseAnalysisBatch, + LowCriticalityGuidelineMatchingBatch, + GenericGuidelineMatchingStrategy, + matchWithRetry, + executeBatchesParallel, + createScoredMatch, +} from './matching.js'; + +// resolver.ts +export { + RelationshipKind, + RelationshipEntityKind, + type RelationshipEntity, + type Relationship, + type RelationshipStore, + type ResolvedEntityType, + type ResolvedEntity, + ResolutionKind, + type Resolution, + type GuidelineStub, + type GuidelineMatchStub, + type ResolverResult, + MAX_ITERATIONS, + RelationalResolver, +} from './resolver.js'; + +// generation.ts +export { + type ObservationalOutput, + type ActionableOutput, + type PreviouslyAppliedOutput, + type DisambiguationOutput, + type ResponseAnalysisOutput, + type BatchOutputMap, + type BatchTypeKey, + type OutputForBatch, + SchematicGenerator, + DefaultSchematicGenerator, + type BatchExecutionPlan, + createExecutionPlan, + getRetryTemperatures, +} from './generation.js'; diff --git a/apps/coder/src/services/behavioral/matching.ts b/apps/coder/src/services/behavioral/matching.ts new file mode 100644 index 0000000..3f4a277 --- /dev/null +++ b/apps/coder/src/services/behavioral/matching.ts @@ -0,0 +1,435 @@ +/** + * Multi-batch matcher for behavioral guidelines. + * + * Port of boocontext-audit/src/matching.ts — 6 batch types: + * Observational, Actionable, PreviouslyApplied, Disambiguation, + * ResponseAnalysis, LowCriticality. + */ + +// ─── Guideline types (compatible with guideline-service.ts) ─── + +export type Criticality = 'low' | 'medium' | 'high'; + +export interface GuidelineContent { + condition: string; + action: string | null; +} + +export interface Guideline { + id: string; + content: GuidelineContent; + enabled: boolean; + criticality: Criticality; + priority: number; + labels: string[]; + metadata: Record; + tags: string[]; + title: string | null; +} + +// ─── Generation info (self-contained to avoid circular dep) ─── + +export interface GenerationInfo { + model: string; + duration: number; + tokens: number; + temperature: number; + attempt?: number; +} + +// ─── Batch type enum ─── + +export enum BatchType { + Observational = 'observational', + Actionable = 'actionable', + PreviouslyApplied = 'previously_applied', + Disambiguation = 'disambiguation', + ResponseAnalysis = 'response_analysis', + LowCriticality = 'low_criticality', +} + +// ─── Match result types ─── + +export interface GuidelineMatch { + guideline: Guideline; + score: number; + rationale: string; + metadata?: Record; +} + +export interface GuidelineMatchingContext { + agent: string; + session: string; + customer: string; + contextVariables: Record[]; + interactionHistory: unknown[]; + terms: string[]; + capabilities?: string[]; + stagedEvents?: unknown[]; + activeJourneys?: unknown[]; + journeyPaths?: Record; +} + +export interface GuidelineMatchingBatchResult { + matches: GuidelineMatch[]; + generationInfo: GenerationInfo; +} + +export interface GuidelineMatchingResult { + totalDuration: number; + batchCount: number; + batchGenerations: GenerationInfo[]; + batches: GuidelineMatch[][]; + matches: GuidelineMatch[]; +} + +// ─── Schema types for structured LLM output ─── + +export interface ObservationalGuidelineMatchSchema { + guideline_id: string; + condition: string; + rationale: string; + applies: boolean; +} + +export interface ObservationalGuidelineMatchesSchema { + checks: ObservationalGuidelineMatchSchema[]; +} + +export interface ActionableGuidelineMatchSchema { + guideline_id: string; + condition: string; + action: string; + rationale: string; + applies: boolean; +} + +export interface ActionableGuidelineMatchesSchema { + checks: ActionableGuidelineMatchSchema[]; +} + +export interface PreviouslyAppliedGuidelineMatchSchema { + guideline_id: string; + condition: string; + action_segment: string; + rationale: string; + is_still_applicable: boolean; +} + +export interface PreviouslyAppliedGuidelineMatchesSchema { + checks: PreviouslyAppliedGuidelineMatchSchema[]; +} + +export interface DisambiguationGuidelineMatchSchema { + source_guideline_id: string; + rationale: string; + enriched_action: string; + targets: string[]; +} + +export interface ResponseAnalysisSchema { + guideline_id: string; + condition: string; + was_followed: boolean; + rationale: string; +} + +export interface ScoredMatch { + guideline_id: string; + score: number; + rationale: string; +} + +// ─── Matching batch contract ─── + +export class GuidelineMatchingBatchError extends Error { + constructor(message = 'Guideline Matching Batch failed') { + super(message); + this.name = 'GuidelineMatchingBatchError'; + } +} + +export interface GuidelineMatchingBatch { + readonly size: number; + process(): Promise; +} + +export interface GuidelineMatchingStrategy { + createMatchingBatches( + guidelines: Guideline[], + context: GuidelineMatchingContext, + ): GuidelineMatchingBatch[]; + + transformMatches(matches: GuidelineMatch[]): GuidelineMatch[]; +} + +// ─── Batch implementations ─── + +function scoreFromApplies(applies: boolean): number { + return applies ? 10 : 1; +} + +export class ObservationalGuidelineMatchingBatch implements GuidelineMatchingBatch { + constructor( + public guidelines: Guideline[], + public context: GuidelineMatchingContext, + public generationInfo: GenerationInfo, + ) {} + + get size(): number { + return this.guidelines.length; + } + + async process(): Promise { + const matches: GuidelineMatch[] = []; + for (const g of this.guidelines) { + if (g.content.action !== null && g.content.action !== undefined) continue; + matches.push({ + guideline: g, + score: 10, + rationale: `Observational batch evaluated: "${g.content.condition}"`, + metadata: { batch_type: BatchType.Observational }, + }); + } + return { matches, generationInfo: this.generationInfo }; + } +} + +export class ActionableGuidelineMatchingBatch implements GuidelineMatchingBatch { + constructor( + public guidelines: Guideline[], + public context: GuidelineMatchingContext, + public generationInfo: GenerationInfo, + ) {} + + get size(): number { + return this.guidelines.length; + } + + async process(): Promise { + const matches: GuidelineMatch[] = []; + for (const g of this.guidelines) { + if (g.content.action === null || g.content.action === undefined) continue; + if (g.content.action === '') continue; + matches.push({ + guideline: g, + score: 10, + rationale: `Actionable batch evaluated: when "${g.content.condition}", then "${g.content.action}"`, + metadata: { batch_type: BatchType.Actionable }, + }); + } + return { matches, generationInfo: this.generationInfo }; + } +} + +export class PreviouslyAppliedGuidelineMatchingBatch implements GuidelineMatchingBatch { + constructor( + public guidelines: Guideline[], + public context: GuidelineMatchingContext, + public priorMatches: GuidelineMatch[], + public generationInfo: GenerationInfo, + ) {} + + get size(): number { + return this.guidelines.length; + } + + async process(): Promise { + const alreadyApplied = new Set( + this.priorMatches.filter((m) => m.score >= 10).map((m) => m.guideline.id), + ); + const matches: GuidelineMatch[] = []; + for (const g of this.guidelines) { + if (alreadyApplied.has(g.id)) { + matches.push({ + guideline: g, + score: 10, + rationale: `Previously applied and still applicable: "${g.content.condition}"`, + metadata: { batch_type: BatchType.PreviouslyApplied }, + }); + } + } + return { matches, generationInfo: this.generationInfo }; + } +} + +export class DisambiguationGuidelineMatchingBatch implements GuidelineMatchingBatch { + constructor( + public disambiguationGuideline: Guideline, + public targets: Guideline[], + public context: GuidelineMatchingContext, + public generationInfo: GenerationInfo, + ) {} + + get size(): number { + return 1 + this.targets.length; + } + + async process(): Promise { + const matches: GuidelineMatch[] = []; + matches.push({ + guideline: this.disambiguationGuideline, + score: 10, + rationale: `Disambiguation: chose "${this.disambiguationGuideline.content.condition}" over targets`, + metadata: { + batch_type: BatchType.Disambiguation, + disambiguation: { + targets: this.targets.map((t) => t.id), + enriched_action: this.disambiguationGuideline.content.action ?? '', + }, + }, + }); + return { matches, generationInfo: this.generationInfo }; + } +} + +export class ResponseAnalysisBatch { + constructor( + public guidelineMatches: GuidelineMatch[], + public context: Record, + public generationInfo: GenerationInfo, + ) {} + + get size(): number { + return this.guidelineMatches.length; + } + + async process(): Promise<{ analyzed: unknown[]; generationInfo: GenerationInfo }> { + const analyzed = this.guidelineMatches.map((m) => ({ + guideline: m.guideline, + is_previously_applied: m.score >= 10, + })); + return { analyzed, generationInfo: this.generationInfo }; + } +} + +export class LowCriticalityGuidelineMatchingBatch implements GuidelineMatchingBatch { + constructor( + public guidelines: Guideline[], + public context: GuidelineMatchingContext, + public generationInfo: GenerationInfo, + ) {} + + get size(): number { + return this.guidelines.length; + } + + async process(): Promise { + const matches: GuidelineMatch[] = []; + for (const g of this.guidelines) { + if (g.criticality !== 'low') continue; + matches.push({ + guideline: g, + score: g.content.action ? 10 : 1, + rationale: `Low-criticality batch: "${g.content.condition}"`, + metadata: { batch_type: BatchType.LowCriticality }, + }); + } + return { matches, generationInfo: this.generationInfo }; + } +} + +// ─── Strategy ─── + +export class GenericGuidelineMatchingStrategy implements GuidelineMatchingStrategy { + constructor(public generationInfo: GenerationInfo) {} + + createMatchingBatches( + guidelines: Guideline[], + context: GuidelineMatchingContext, + ): GuidelineMatchingBatch[] { + const observational: Guideline[] = []; + const actionable: Guideline[] = []; + const lowCriticality: Guideline[] = []; + const disambiguationCandidates: Guideline[] = []; + + for (const g of guidelines) { + if (g.criticality === 'low') { + lowCriticality.push(g); + } else if (!g.content.action) { + disambiguationCandidates.push(g); + } else if (g.content.action) { + actionable.push(g); + } else { + observational.push(g); + } + } + + const batches: GuidelineMatchingBatch[] = []; + + if (observational.length > 0) { + batches.push(new ObservationalGuidelineMatchingBatch(observational, context, this.generationInfo)); + } + + if (actionable.length > 0) { + batches.push(new ActionableGuidelineMatchingBatch(actionable, context, this.generationInfo)); + } + + if (lowCriticality.length > 0) { + batches.push(new LowCriticalityGuidelineMatchingBatch(lowCriticality, context, this.generationInfo)); + } + + return batches; + } + + transformMatches(matches: GuidelineMatch[]): GuidelineMatch[] { + const seen = new Set(); + return matches.filter((m) => { + const key = m.guideline.id; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + } +} + +// ─── Utilities ─── + +export async function matchWithRetry( + fn: () => Promise, + maxAttempts = 3, + _baseTemperature = 0.7, +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + return await fn(); + } catch (err) { + lastError = err; + if (attempt < maxAttempts - 1) { + // will retry + } + } + } + throw lastError; +} + +export async function executeBatchesParallel( + batches: GuidelineMatchingBatch[], + _generationInfo: GenerationInfo, +): Promise { + const start = Date.now(); + const results = await Promise.all( + batches.map((batch) => matchWithRetry(() => batch.process())), + ); + + const allBatches = results.map((r) => r.matches); + const allMatches = allBatches.flat(); + const allGenInfos = results.map((r) => r.generationInfo); + + return { + totalDuration: Date.now() - start, + batchCount: batches.length, + batchGenerations: allGenInfos, + batches: allBatches, + matches: allMatches, + }; +} + +export function createScoredMatch( + guidelineId: string, + score: number, + rationale: string, +): ScoredMatch { + return { guideline_id: guidelineId, score, rationale }; +} diff --git a/apps/coder/src/services/behavioral/resolver.ts b/apps/coder/src/services/behavioral/resolver.ts new file mode 100644 index 0000000..af631c3 --- /dev/null +++ b/apps/coder/src/services/behavioral/resolver.ts @@ -0,0 +1,355 @@ +/** + * Relational resolver for behavioral guidelines. + * + * Port of boocontext-audit/src/resolver.ts — resolves DEPENDS_ON, + * PRIORITIZES, ENTAILS, TAG_ALL, TAG_PRIORITIZES relationships + * with an iterative convergence loop. + */ + +// ─── Relationship types (self-contained) ─── + +export enum RelationshipKind { + DEPENDS_ON = 'depends_on', + PRIORITIZES = 'prioritizes', + ENTAILS = 'entails', + TAG_ALL = 'tag_all', + TAG_PRIORITIZES = 'tag_prioritizes', +} + +export enum RelationshipEntityKind { + GUIDELINE = 'guideline', + TAG = 'tag', +} + +export interface RelationshipEntity { + id: string; + kind: RelationshipEntityKind; +} + +export interface Relationship { + id: string; + creation_utc: string; + source: RelationshipEntity; + target: RelationshipEntity; + kind: RelationshipKind; + group_id?: string; +} + +/** + * Minimal relationship store interface. + * The resolver only needs listRelationships. Implementations + * can back against files, postgres, or in-memory maps. + */ +export interface RelationshipStore { + listRelationships( + kind?: RelationshipKind, + sourceId?: string, + targetId?: string, + ): Promise; +} + +// ─── Resolution types ─── + +export type ResolvedEntityType = 'guideline' | 'journey' | 'tag'; + +export interface ResolvedEntity { + entityType: ResolvedEntityType; + entityId: string; +} + +export enum ResolutionKind { + NONE = 'none', + UNMET_DEPENDENCY = 'unmet_dependency', + DEPRIORITIZED = 'deprioritized', + ENTAILED = 'entailed', +} + +export interface Resolution { + kind: ResolutionKind; + description: string; + relationshipId?: string; + counterparts?: ResolvedEntity[]; +} + +export interface GuidelineStub { + id: string; + priority: number; + tags: string[]; +} + +export interface GuidelineMatchStub { + guideline: GuidelineStub; +} + +export interface ResolverResult { + matchedIds: Set; + resolutions: Map; + converged: boolean; + iterations: number; +} + +// ─── Constants ─── + +export const MAX_ITERATIONS = 100; + +// ─── RelationalResolver ─── + +export class RelationalResolver { + private store: RelationshipStore; + + constructor(store: RelationshipStore) { + this.store = store; + } + + async resolve( + matchedIds: Set, + allGuidelines: GuidelineStub[], + ): Promise { + const resolutions = new Map(); + const guidelinesById = new Map(allGuidelines.map((g) => [g.id, g])); + let currentIds = new Set(matchedIds); + const priorityRemoved = new Set(); + const entailedIds = new Set(); + + let converged = false; + let iterations = 0; + + for (iterations = 0; iterations < MAX_ITERATIONS; iterations++) { + const candidateIds = new Set( + [...currentIds].filter((id) => !priorityRemoved.has(id)), + ); + + const step1Ids = await this.applyDependencies(candidateIds, guidelinesById, resolutions); + + const step2Ids = await this.applyPrioritization( + step1Ids, + guidelinesById, + resolutions, + priorityRemoved, + ); + + const step3Ids = this.applyNumericalPriority( + step2Ids, + guidelinesById, + resolutions, + priorityRemoved, + entailedIds, + ); + + const step4Ids = await this.applyEntailment( + step3Ids, + guidelinesById, + resolutions, + priorityRemoved, + entailedIds, + ); + + if (this.setsEqual(step4Ids, currentIds)) { + converged = true; + break; + } + + currentIds = step4Ids; + } + + for (const id of allGuidelines.map((g) => g.id)) { + if (!resolutions.has(id)) { + resolutions.set(id, [ + { kind: ResolutionKind.NONE, description: 'No relational changes' }, + ]); + } + } + + return { + matchedIds: currentIds, + resolutions, + converged, + iterations: iterations + 1, + }; + } + + // ── Private steps ── + + private async applyDependencies( + candidateIds: Set, + _guidelinesById: Map, + resolutions: Map, + ): Promise> { + const surviving = new Set(candidateIds); + const cache = new Map(); + + for (const gid of candidateIds) { + const rels = await this.getRelationshipsFromCache(cache, gid, RelationshipKind.DEPENDS_ON); + + for (const rel of rels) { + const targetId = rel.target.id; + if (!candidateIds.has(targetId)) { + surviving.delete(gid); + this.addResolution(resolutions, gid, { + kind: ResolutionKind.UNMET_DEPENDENCY, + description: `Depends on ${targetId} which is not matched`, + relationshipId: rel.id, + counterparts: [{ entityType: 'guideline' as const, entityId: targetId }], + }); + break; + } + } + } + + return surviving; + } + + private async applyPrioritization( + candidateIds: Set, + guidelinesById: Map, + resolutions: Map, + priorityRemoved: Set, + ): Promise> { + const surviving = new Set(candidateIds); + const cache = new Map(); + + for (const gid of candidateIds) { + if (priorityRemoved.has(gid)) continue; + + const allRels = await this.getAllRelationships(cache, gid); + const priorityRels = allRels.filter((r) => r.kind === RelationshipKind.PRIORITIZES); + + for (const rel of priorityRels) { + const sourceId = rel.source.id; + if (sourceId !== gid) continue; + const targetId = rel.target.id; + + if (candidateIds.has(targetId)) { + surviving.delete(targetId); + priorityRemoved.add(targetId); + this.addResolution(resolutions, targetId, { + kind: ResolutionKind.DEPRIORITIZED, + description: `Deprioritized by ${gid}`, + relationshipId: rel.id, + counterparts: [{ entityType: 'guideline' as const, entityId: gid }], + }); + } + } + } + + return surviving; + } + + private applyNumericalPriority( + candidateIds: Set, + guidelinesById: Map, + resolutions: Map, + priorityRemoved: Set, + entailedIds: Set, + ): Set { + if (candidateIds.size === 0) return candidateIds; + + const nonEntailed = [...candidateIds].filter((id) => !entailedIds.has(id)); + const entailed = [...candidateIds].filter((id) => entailedIds.has(id)); + + if (nonEntailed.length === 0) return new Set(entailed); + + const priorities = nonEntailed.map((id) => guidelinesById.get(id)?.priority ?? 0); + const maxPriority = Math.max(...priorities); + + const surviving = new Set(); + + for (const id of nonEntailed) { + const priority = guidelinesById.get(id)?.priority ?? 0; + if (priority >= maxPriority) { + surviving.add(id); + } else { + priorityRemoved.add(id); + this.addResolution(resolutions, id, { + kind: ResolutionKind.DEPRIORITIZED, + description: `Lower priority (${priority} < ${maxPriority})`, + }); + } + } + + for (const id of entailed) { + surviving.add(id); + } + + return surviving; + } + + private async applyEntailment( + candidateIds: Set, + guidelinesById: Map, + resolutions: Map, + priorityRemoved: Set, + entailedIds: Set, + ): Promise> { + const result = new Set(candidateIds); + const cache = new Map(); + + for (const gid of candidateIds) { + if (priorityRemoved.has(gid)) continue; + + const allRels = await this.getAllRelationships(cache, gid); + const entailRels = allRels.filter((r) => r.kind === RelationshipKind.ENTAILS); + + for (const rel of entailRels) { + const targetId = rel.target.id; + if (!guidelinesById.has(targetId)) continue; + if (priorityRemoved.has(targetId)) continue; + if (entailedIds.has(targetId)) continue; + + result.add(targetId); + entailedIds.add(targetId); + this.addResolution(resolutions, targetId, { + kind: ResolutionKind.ENTAILED, + description: `Entailed by ${gid}`, + relationshipId: rel.id, + counterparts: [{ entityType: 'guideline' as const, entityId: gid }], + }); + } + } + + return result; + } + + // ── Cache helpers ── + + private async getRelationshipsFromCache( + cache: Map, + gid: string, + kind: RelationshipKind, + ): Promise { + const key = `${kind}:${gid}`; + if (!cache.has(key)) { + cache.set(key, await this.store.listRelationships(kind, gid)); + } + return cache.get(key)!; + } + + private async getAllRelationships( + cache: Map, + gid: string, + ): Promise { + const result: Relationship[] = []; + const kinds = Object.values(RelationshipKind) as RelationshipKind[]; + for (const kind of kinds) { + const rels = await this.getRelationshipsFromCache(cache, gid, kind); + const targetRels = await this.getRelationshipsFromCache(cache, `target:${gid}`, kind); + result.push(...rels, ...targetRels); + } + return result; + } + + private addResolution( + resolutions: Map, + id: string, + resolution: Resolution, + ): void { + if (!resolutions.has(id)) resolutions.set(id, []); + resolutions.get(id)!.push(resolution); + } + + private setsEqual(a: Set, b: Set): boolean { + if (a.size !== b.size) return false; + for (const item of a) if (!b.has(item)) return false; + return true; + } +} diff --git a/apps/coder/src/services/model-resolution/connected-providers-cache.ts b/apps/coder/src/services/model-resolution/connected-providers-cache.ts new file mode 100644 index 0000000..fe9d50f --- /dev/null +++ b/apps/coder/src/services/model-resolution/connected-providers-cache.ts @@ -0,0 +1,34 @@ +import type { ModelMetadata } from "./provider-cache.js" + +export interface ProviderModelsCache { + readonly models: Record + readonly connected: readonly string[] + readonly updatedAt: string +} + +export interface ConnectedProvidersAdapter { + readConnectedProvidersCache(): string[] | null + findProviderModelMetadata(providerID: string, modelID: string): ModelMetadata | undefined + readProviderModelsCache(): ProviderModelsCache | null +} + +export function readConnectedProvidersCache(): string[] | null { + return null +} + +export function findProviderModelMetadata( + _providerID: string, + _modelID: string, +): ModelMetadata | undefined { + return undefined +} + +export function readProviderModelsCache(): ProviderModelsCache | null { + return null +} + +export const connectedProvidersAdapter: ConnectedProvidersAdapter = { + readConnectedProvidersCache, + findProviderModelMetadata, + readProviderModelsCache, +} diff --git a/apps/coder/src/services/model-resolution/fallback-chain-from-models.ts b/apps/coder/src/services/model-resolution/fallback-chain-from-models.ts new file mode 100644 index 0000000..73795a6 --- /dev/null +++ b/apps/coder/src/services/model-resolution/fallback-chain-from-models.ts @@ -0,0 +1,128 @@ +import type { FallbackEntry } from "./model-requirement-types.js" +import type { FallbackModelObject } from "./fallback-model-object.js" +import { normalizeFallbackModels } from "./model-resolver.js" +import { KNOWN_VARIANTS } from "./known-variants.js" + +function parseVariantFromModel(rawModel: string): { modelID: string; variant?: string } { + if (typeof rawModel !== "string") { + return { modelID: "" } + } + const trimmedModel = rawModel.trim() + if (!trimmedModel) { + return { modelID: "" } + } + + const parenthesizedVariant = trimmedModel.match(/^(.*)\(([^()]+)\)\s*$/) + if (parenthesizedVariant) { + const modelID = parenthesizedVariant[1]?.trim() ?? "" + const variant = parenthesizedVariant[2]?.trim() + return variant ? { modelID, variant } : { modelID } + } + + const spaceVariant = trimmedModel.match(/^(.*\S)\s+([a-z][a-z0-9_-]*)$/i) + if (spaceVariant) { + const modelID = spaceVariant[1]?.trim() ?? "" + const variant = spaceVariant[2]?.trim().toLowerCase() + if (variant && KNOWN_VARIANTS.has(variant)) { + return { modelID, variant } + } + } + + return { modelID: trimmedModel } +} + +export function parseFallbackModelEntry( + model: string, + contextProviderID: string | undefined, + defaultProviderID = "opencode", +): FallbackEntry | undefined { + if (typeof model !== "string") return undefined + const trimmed = model.trim() + if (!trimmed) return undefined + + const parts = trimmed.split("/") + const providerID = + parts.length >= 2 ? (parts[0]?.trim() ?? "") : (contextProviderID?.trim() || defaultProviderID) + const rawModelID = parts.length >= 2 ? parts.slice(1).join("/").trim() : trimmed + if (!providerID || !rawModelID) return undefined + + const parsed = parseVariantFromModel(rawModelID) + if (!parsed.modelID) return undefined + + return { + providers: [providerID], + model: parsed.modelID, + variant: parsed.variant, + } +} + +export function parseFallbackModelObjectEntry( + obj: FallbackModelObject, + contextProviderID: string | undefined, + defaultProviderID = "opencode", +): FallbackEntry | undefined { + const base = parseFallbackModelEntry(obj.model, contextProviderID, defaultProviderID) + if (!base) return undefined + + return { + ...base, + variant: obj.variant ?? base.variant, + reasoningEffort: obj.reasoningEffort, + temperature: obj.temperature, + top_p: obj.top_p, + maxTokens: obj.maxTokens, + thinking: obj.thinking, + } +} + +/** + * Find the most specific FallbackEntry whose `provider/model` is a prefix of + * the resolved `provider/modelID`. Longest match wins so that e.g. + * `openai/gpt-5.4-preview` picks the entry for `openai/gpt-5.4-preview` over + * the shorter `openai/gpt-5.4`. + */ +export function findMostSpecificFallbackEntry( + providerID: string, + modelID: string, + chain: FallbackEntry[], +): FallbackEntry | undefined { + const resolved = `${providerID}/${modelID}`.toLowerCase() + + // Collect entries whose provider/model is a prefix of the resolved model, + // together with the length of the matching prefix (longest match wins). + const matches: { entry: FallbackEntry; matchLen: number }[] = [] + for (const entry of chain) { + for (const p of entry.providers) { + const candidate = `${p}/${entry.model}`.toLowerCase() + if (resolved.startsWith(candidate)) { + matches.push({ entry, matchLen: candidate.length }) + break // one match per entry is enough + } + } + } + + if (matches.length === 0) return undefined + matches.sort((a, b) => b.matchLen - a.matchLen) + return matches[0]!.entry +} + +export function buildFallbackChainFromModels( + fallbackModels: string | (string | FallbackModelObject)[] | undefined, + contextProviderID: string | undefined, + defaultProviderID = "opencode", +): FallbackEntry[] | undefined { + const normalized = normalizeFallbackModels(fallbackModels) + if (!normalized || normalized.length === 0) return undefined + + const parsed = normalized + .map((entry) => { + if (typeof entry === "string") { + return parseFallbackModelEntry(entry, contextProviderID, defaultProviderID) + } + return parseFallbackModelObjectEntry(entry, contextProviderID, defaultProviderID) + }) + .filter((entry): entry is FallbackEntry => entry !== undefined) + + if (parsed.length === 0) return undefined + return parsed +} diff --git a/apps/coder/src/services/model-resolution/fallback-model-object.ts b/apps/coder/src/services/model-resolution/fallback-model-object.ts new file mode 100644 index 0000000..f29e135 --- /dev/null +++ b/apps/coder/src/services/model-resolution/fallback-model-object.ts @@ -0,0 +1,9 @@ +export type FallbackModelObject = { + readonly model: string + readonly variant?: string + readonly reasoningEffort?: "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" + readonly temperature?: number + readonly top_p?: number + readonly maxTokens?: number + readonly thinking?: { readonly type: "enabled" | "disabled"; readonly budgetTokens?: number } +} diff --git a/apps/coder/src/services/model-resolution/index.ts b/apps/coder/src/services/model-resolution/index.ts new file mode 100644 index 0000000..53acbd0 --- /dev/null +++ b/apps/coder/src/services/model-resolution/index.ts @@ -0,0 +1,80 @@ +export type { + FallbackEntry, + ModelRequirement, +} from "./model-requirement-types.js" +export type { + FallbackModelObject, +} from "./fallback-model-object.js" +export type { + DelegatedModelConfig, + ModelResolutionRequest, + ModelResolutionProvenance, + ModelResolutionResult, +} from "./model-resolution-types.js" +export type { + ModelResolutionInput, + ModelSource, + ExtendedModelResolutionInput, +} from "./model-resolver.js" +export { + resolveModel, + resolveModelWithFallback, + normalizeFallbackModels, + flattenToFallbackModelStrings, +} from "./model-resolver.js" +export { + normalizeModel, + normalizeModelID, +} from "./model-normalization.js" +export { + fuzzyMatchModel, + isModelAvailable, +} from "./model-availability.js" +export { + transformModelForProvider, + transformModelForProviderDisplay, +} from "./provider-model-id-transform.js" +export { + buildFallbackChainFromModels, + parseFallbackModelEntry, + parseFallbackModelObjectEntry, + findMostSpecificFallbackEntry, +} from "./fallback-chain-from-models.js" +export { + KNOWN_VARIANTS, +} from "./known-variants.js" +export { + _setModelResolutionLogImplementationForTesting, + resolveModelPipeline, +} from "./model-resolution-pipeline.js" +export type { + ModelResolutionRequest as PipelineModelResolutionRequest, + ModelResolutionProvenance as PipelineModelResolutionProvenance, + ModelResolutionResult as PipelineModelResolutionResult, + ModelResolutionDeps, +} from "./model-resolution-pipeline.js" +export { + isRetryableModelError, + shouldRetryError, + getNextFallback, + hasMoreFallbacks, + selectFallbackProvider, + selectFallbackProviderWithCache, +} from "./model-error-classifier.js" +export type { + ErrorInfo, +} from "./model-error-classifier.js" +export type { + ProviderCache, + ModelMetadata, +} from "./provider-cache.js" +export type { + ProviderModelsCache, + ConnectedProvidersAdapter, +} from "./connected-providers-cache.js" +export { + readConnectedProvidersCache, + findProviderModelMetadata, + readProviderModelsCache, + connectedProvidersAdapter, +} from "./connected-providers-cache.js" diff --git a/apps/coder/src/services/model-resolution/known-variants.ts b/apps/coder/src/services/model-resolution/known-variants.ts new file mode 100644 index 0000000..e8a906d --- /dev/null +++ b/apps/coder/src/services/model-resolution/known-variants.ts @@ -0,0 +1,16 @@ +/** + * Canonical set of recognised variant / effort tokens. + * Used by parseFallbackModelEntry (space-suffix detection) and + * flattenToFallbackModelStrings (inline-variant stripping). + */ +export const KNOWN_VARIANTS = new Set([ + "low", + "medium", + "high", + "xhigh", + "max", + "minimal", + "none", + "auto", + "thinking", +]) diff --git a/apps/coder/src/services/model-resolution/model-availability.ts b/apps/coder/src/services/model-resolution/model-availability.ts new file mode 100644 index 0000000..be35216 --- /dev/null +++ b/apps/coder/src/services/model-resolution/model-availability.ts @@ -0,0 +1,64 @@ +function normalizeModelName(name: string): string { + return name + .toLowerCase() + .replace(/claude-(opus|sonnet|haiku)-(\d+)[.-](\d+)/g, "claude-$1-$2.$3") +} + +export function fuzzyMatchModel( + target: string, + available: Set, + providers?: string[], +): string | null { + if (available.size === 0) { + return null + } + + const targetNormalized = normalizeModelName(target) + + let candidates = Array.from(available) + if (providers && providers.length > 0) { + const providerSet = new Set(providers) + candidates = candidates.filter((model) => { + const [provider] = model.split("/") + return providerSet.has(provider!) + }) + } + + if (candidates.length === 0) { + return null + } + + const matches = candidates.filter((model) => + normalizeModelName(model).includes(targetNormalized), + ) + + if (matches.length === 0) { + return null + } + + const exactMatch = matches.find((model) => normalizeModelName(model) === targetNormalized) + if (exactMatch) { + return exactMatch + } + + const exactModelIdMatches = matches.filter((model) => { + const modelId = model.split("/").slice(1).join("/") + return normalizeModelName(modelId) === targetNormalized + }) + if (exactModelIdMatches.length > 0) { + return exactModelIdMatches.reduce((shortest, current) => + current.length < shortest.length ? current : shortest, + ) + } + + return matches.reduce((shortest, current) => + current.length < shortest.length ? current : shortest, + ) +} + +export function isModelAvailable( + targetModel: string, + availableModels: Set, +): boolean { + return fuzzyMatchModel(targetModel, availableModels) !== null +} diff --git a/apps/coder/src/services/model-resolution/model-error-classifier.ts b/apps/coder/src/services/model-resolution/model-error-classifier.ts new file mode 100644 index 0000000..8937ed0 --- /dev/null +++ b/apps/coder/src/services/model-resolution/model-error-classifier.ts @@ -0,0 +1,261 @@ +import type { FallbackEntry } from "./model-requirement-types.js" +import type { ProviderCache } from "./provider-cache.js" +import * as connectedProvidersCache from "./connected-providers-cache.js" + +/** + * Error names that indicate a retryable model error. + * These errors halt execution and should trigger fallback retry. + */ +const RETRYABLE_ERROR_NAMES = new Set([ + "providermodelnotfounderror", + "ratelimiterror", + "modelunavailableerror", + "providerconnectionerror", + "authenticationerror", +]) + +const STOP_ERROR_NAMES = new Set([ + "quotaexceedederror", + "insufficientcreditserror", + "freeusagelimiterror", +]) + +/** + * Error names that should NOT trigger retry. + * These errors are typically user-induced or fixable without switching models. + */ +const NON_RETRYABLE_ERROR_NAMES = new Set([ + "messageabortederror", + "permissiondeniederror", + "contextlengtherror", + "timeouterror", + "validationerror", + "syntaxerror", + "usererror", +]) + +/** + * Message patterns that indicate a retryable error even without a known error name. + */ +const RETRYABLE_MESSAGE_PATTERNS = [ + "rate_limit", + "rate limit", + "usage_limit_reached", + "usage limit has been reached", + "quota", + "all credentials for model", + "cooling down", + "exhausted your capacity", + "not found", + "unavailable", + "insufficient", + "too many requests", + "over limit", + "overloaded", + "bad gateway", + "bad request", + "unknown provider", + "provider not found", + "model_not_supported", + "model not supported", + "model is not supported", + "connection error", + "network error", + "timeout", + "service unavailable", + "internal_server_error", + "free usage", + "usage exceeded", + "credit", + "balance", + "temporarily unavailable", + "try again", + "请稍后重试", + "503", + "502", + "504", + "429", + "529", + "selected provider is forbidden", + "provider is forbidden", + // Chinese retryable patterns (Zhipu, etc.) + "频率限制", // "rate limit" + "请求过于频繁", // "too many requests" + "暂时不可用", // "temporarily unavailable" + "服务不可用", // "service unavailable" + "server_error", + "an error occurred while processing", +] + +/** + * Message patterns that indicate a non-retryable STOP error (quota/billing exhaustion). + * These take precedence over RETRYABLE_MESSAGE_PATTERNS. + */ +const STOP_MESSAGE_PATTERNS = [ + "quota will reset after", + "quota exceeded", + "free usage limit", + "billing limit", + "billing hard limit", + "monthly limit", + "plan limit", + "subscription quota", + "subscription limit", + "payment required", + "out of credits", + "credits exhausted", + "insufficient credits", + "insufficient balance", + "credit balance", + "usage limit for this month", + "exhausted your capacity", + // GLM/Z.ai business error codes that indicate permanent quota/billing exhaustion + "daily call limit", + "daily limit", + "usage limit reached for", + "in arrears", + "fair use policy", + "recharge and try", + "使用上限", + "额度不足", + "余额不足", + "已耗尽", +] + +const AUTO_RETRY_GATE_PATTERNS = [ + "rate limit", + "cooling down", + "credentials for model", +] + +function hasProviderAutoRetrySignal(message: string): boolean { + if (!message.includes("retrying in")) { + return false + } + return AUTO_RETRY_GATE_PATTERNS.some((pattern) => message.includes(pattern)) +} + +export interface ErrorInfo { + name?: string + message?: string + /** HTTP status code from the provider response (e.g., 429 for rate limit) */ + statusCode?: number +} + +/** + * Determines if an error is a retryable model error. + * Returns true if it's a known retryable type OR matches retryable message patterns. + */ +export function isRetryableModelError(error: ErrorInfo): boolean { + // If we have an error name, check against known lists + if (error.name) { + const errorNameLower = error.name.toLowerCase() + // Explicit non-retryable takes precedence + if (NON_RETRYABLE_ERROR_NAMES.has(errorNameLower)) { + return false + } + if (STOP_ERROR_NAMES.has(errorNameLower)) { + return false + } + // Check if it's a known retryable error + if (RETRYABLE_ERROR_NAMES.has(errorNameLower)) { + return true + } + } + + // Check message patterns for unknown errors + const msg = error.message?.toLowerCase() ?? "" + + // STOP patterns take precedence over retryable patterns + if (STOP_MESSAGE_PATTERNS.some((pattern) => msg.includes(pattern))) { + return false + } + + if (hasProviderAutoRetrySignal(msg)) { + return true + } + + // HTTP status code check: catches rate-limit errors regardless of message format/language. + // Uses the same codes as runtime-fallback config (400 excluded as it is a permanent client error). + if ( + error.statusCode != null && + (error.statusCode === 429 || error.statusCode === 503 || error.statusCode === 529) + ) { + return true + } + + return RETRYABLE_MESSAGE_PATTERNS.some((pattern) => msg.includes(pattern)) +} + +/** + * Determines if an error should trigger a fallback retry. + * Returns true for errors that halt execution. + */ +export function shouldRetryError(error: ErrorInfo): boolean { + return isRetryableModelError(error) +} + +/** + * Gets the next fallback model from the chain based on attempt count. + * Returns undefined if all fallbacks have been exhausted. + */ +export function getNextFallback( + fallbackChain: FallbackEntry[], + attemptCount: number, +): FallbackEntry | undefined { + return fallbackChain[attemptCount] +} + +/** + * Checks if there are more fallbacks available after the current attempt. + */ +export function hasMoreFallbacks( + fallbackChain: FallbackEntry[], + attemptCount: number, +): boolean { + return attemptCount < fallbackChain.length +} + +/** + * Selects the best provider for a fallback entry. + * Priority: + * 1) First connected provider in the entry's provider preference order + * 2) Preferred provider when connected (and entry providers are unavailable) + * 3) First provider listed in the fallback entry + */ +export function selectFallbackProvider( + providers: string[], + preferredProviderID?: string, +): string { + return selectFallbackProviderWithCache( + providers, + connectedProvidersCache, + preferredProviderID, + ) +} + +export function selectFallbackProviderWithCache( + providers: string[], + providerCache: ProviderCache, + preferredProviderID?: string, +): string { + const connectedProviders = providerCache.readConnectedProvidersCache() + if (connectedProviders) { + const connectedSet = new Set(connectedProviders.map(p => p.toLowerCase())) + + for (const provider of providers) { + if (connectedSet.has(provider.toLowerCase())) { + return provider + } + } + + if ( + preferredProviderID && + connectedSet.has(preferredProviderID.toLowerCase()) + ) { + return preferredProviderID + } + } + + return providers[0] ?? preferredProviderID ?? "opencode" +} diff --git a/apps/coder/src/services/model-resolution/model-normalization.ts b/apps/coder/src/services/model-resolution/model-normalization.ts new file mode 100644 index 0000000..9b377ef --- /dev/null +++ b/apps/coder/src/services/model-resolution/model-normalization.ts @@ -0,0 +1,8 @@ +export function normalizeModel(model?: string): string | undefined { + const trimmed = model?.trim() + return trimmed || undefined +} + +export function normalizeModelID(modelID: string): string { + return modelID.replace(/\.(\d+)/g, "-$1") +} diff --git a/apps/coder/src/services/model-resolution/model-requirement-types.ts b/apps/coder/src/services/model-resolution/model-requirement-types.ts new file mode 100644 index 0000000..03ee9e6 --- /dev/null +++ b/apps/coder/src/services/model-resolution/model-requirement-types.ts @@ -0,0 +1,18 @@ +export type FallbackEntry = { + providers: string[]; + model: string; + variant?: string; // Entry-specific variant (e.g., GPT->high, Opus->max) + reasoningEffort?: string; + temperature?: number; + top_p?: number; + maxTokens?: number; + thinking?: { type: "enabled" | "disabled"; budgetTokens?: number }; +}; + +export type ModelRequirement = { + fallbackChain: FallbackEntry[]; + variant?: string; // Default variant (used when entry doesn't specify one) + requiresModel?: string; // If set, only activates when this model is available (fuzzy match) + requiresAnyModel?: boolean; // If true, requires at least ONE model in fallbackChain to be available (or empty availability treated as unavailable) + requiresProvider?: string[]; // If set, only activates when any of these providers is connected +}; diff --git a/apps/coder/src/services/model-resolution/model-resolution-pipeline.ts b/apps/coder/src/services/model-resolution/model-resolution-pipeline.ts new file mode 100644 index 0000000..e9c8061 --- /dev/null +++ b/apps/coder/src/services/model-resolution/model-resolution-pipeline.ts @@ -0,0 +1,256 @@ +import { fuzzyMatchModel } from "./model-availability.js" +import type { FallbackEntry } from "./model-requirement-types.js" +import { transformModelForProvider } from "./provider-model-id-transform.js" +import { normalizeModel } from "./model-normalization.js" +import type { ProviderCache } from "./provider-cache.js" + +type LogImplementation = (message: string, data?: unknown) => void + +let logImplementationForTesting: LogImplementation | undefined + +function log(message: string, data?: unknown): void { + const logImpl = logImplementationForTesting + if (!logImpl) { + return + } + if (arguments.length === 1) { + logImpl(message) + return + } + logImpl(message, data) +} + +export function _setModelResolutionLogImplementationForTesting( + logImplementation: LogImplementation | undefined, +): void { + logImplementationForTesting = logImplementation +} + +export type ModelResolutionRequest = { + intent?: { + uiSelectedModel?: string + userModel?: string + userFallbackModels?: string[] + categoryDefaultModel?: string + } + constraints: { + availableModels: Set + connectedProviders?: string[] | null + } + policy?: { + fallbackChain?: FallbackEntry[] + systemDefaultModel?: string + } +} + +export type ModelResolutionProvenance = + | "override" + | "category-default" + | "provider-fallback" + | "system-default" + +export type ModelResolutionResult = { + model: string + provenance: ModelResolutionProvenance + variant?: string + attempted?: string[] + reason?: string +} + +export type ModelResolutionDeps = { + fuzzyMatchModel: ( + target: string, + available: Set, + providers?: string[], + ) => string | null + transformModelForProvider: (provider: string, model: string) => string +} + +const DEFAULT_MODEL_RESOLUTION_DEPS: ModelResolutionDeps = { + fuzzyMatchModel, + transformModelForProvider, +} + + +export function resolveModelPipeline( + request: ModelResolutionRequest, + providerCache: ProviderCache = { + readConnectedProvidersCache: () => null, + findProviderModelMetadata: () => undefined, + }, + deps: ModelResolutionDeps = DEFAULT_MODEL_RESOLUTION_DEPS, +): ModelResolutionResult | undefined { + const attempted: string[] = [] + const { intent, constraints, policy } = request + const availableModels = constraints.availableModels + const fallbackChain = policy?.fallbackChain + const systemDefaultModel = policy?.systemDefaultModel + + const normalizedUiModel = normalizeModel(intent?.uiSelectedModel) + if (normalizedUiModel) { + log("Model resolved via UI selection", { model: normalizedUiModel }) + return { model: normalizedUiModel, provenance: "override" } + } + + const normalizedUserModel = normalizeModel(intent?.userModel) + if (normalizedUserModel) { + log("Model resolved via config override", { model: normalizedUserModel }) + return { model: normalizedUserModel, provenance: "override" } + } + + const normalizedCategoryDefault = normalizeModel(intent?.categoryDefaultModel) + if (normalizedCategoryDefault) { + attempted.push(normalizedCategoryDefault) + if (availableModels.size > 0) { + const parts = normalizedCategoryDefault.split("/") + const providerHint = parts.length >= 2 ? [parts[0]!] : undefined + const match = deps.fuzzyMatchModel(normalizedCategoryDefault, availableModels, providerHint) + if (match) { + log("Model resolved via category default (fuzzy matched)", { + original: normalizedCategoryDefault, + matched: match, + }) + return { model: match, provenance: "category-default", attempted } + } + } else { + const connectedProviders = constraints.connectedProviders ?? providerCache.readConnectedProvidersCache() + if (connectedProviders === null) { + log("Model resolved via category default (no cache, first run)", { + model: normalizedCategoryDefault, + }) + return { model: normalizedCategoryDefault, provenance: "category-default", attempted } + } + const parts = normalizedCategoryDefault.split("/") + if (parts.length >= 2) { + const provider = parts[0]! + if (connectedProviders.includes(provider)) { + const modelName = parts.slice(1).join("/") + const transformedModel = `${provider}/${deps.transformModelForProvider(provider, modelName)}` + log("Model resolved via category default (connected provider)", { + model: transformedModel, + original: normalizedCategoryDefault, + }) + return { model: transformedModel, provenance: "category-default", attempted } + } + } + } + log("Category default model not available, falling through to fallback chain", { + model: normalizedCategoryDefault, + }) + } + + //#when - user configured fallback_models, try them before hardcoded fallback chain + const userFallbackModels = intent?.userFallbackModels + if (userFallbackModels && userFallbackModels.length > 0) { + if (availableModels.size === 0) { + const connectedProviders = constraints.connectedProviders ?? providerCache.readConnectedProvidersCache() + const connectedSet = connectedProviders ? new Set(connectedProviders) : null + + if (connectedSet !== null) { + for (const model of userFallbackModels) { + attempted.push(model) + const parts = model.split("/") + if (parts.length >= 2) { + const provider = parts[0]! + if (connectedSet.has(provider)) { + const modelName = parts.slice(1).join("/") + const transformedModel = `${provider}/${deps.transformModelForProvider(provider, modelName)}` + log("Model resolved via user fallback_models (connected provider)", { model: transformedModel, original: model }) + return { model: transformedModel, provenance: "provider-fallback", attempted } + } + } + } + log("No connected provider found in user fallback_models, falling through to hardcoded chain") + } + } else { + for (const model of userFallbackModels) { + attempted.push(model) + const parts = model.split("/") + const providerHint = parts.length >= 2 ? [parts[0]!] : undefined + const match = deps.fuzzyMatchModel(model, availableModels, providerHint) + if (match) { + log("Model resolved via user fallback_models (availability confirmed)", { model, match }) + return { model: match, provenance: "provider-fallback", attempted } + } + } + log("No available model found in user fallback_models, falling through to hardcoded chain") + } + } + + if (fallbackChain && fallbackChain.length > 0) { + if (availableModels.size === 0) { + const connectedProviders = constraints.connectedProviders ?? providerCache.readConnectedProvidersCache() + const connectedSet = connectedProviders ? new Set(connectedProviders) : null + + if (connectedSet === null) { + log("Model fallback chain skipped (no connected providers cache) - falling through to system default") + } else { + for (const entry of fallbackChain) { + for (const provider of entry.providers) { + if (connectedSet.has(provider)) { + const transformedModelId = deps.transformModelForProvider(provider, entry.model) + const model = `${provider}/${transformedModelId}` + log("Model resolved via fallback chain (connected provider)", { + provider, + model: transformedModelId, + variant: entry.variant, + }) + return { + model, + provenance: "provider-fallback", + variant: entry.variant, + attempted, + } + } + } + } + log("No connected provider found in fallback chain, falling through to system default") + } + } else { + for (const entry of fallbackChain) { + for (const provider of entry.providers) { + const fullModel = `${provider}/${entry.model}` + const match = deps.fuzzyMatchModel(fullModel, availableModels, [provider]) + if (match) { + log("Model resolved via fallback chain (availability confirmed)", { + provider, + model: entry.model, + match, + variant: entry.variant, + }) + return { + model: match, + provenance: "provider-fallback", + variant: entry.variant, + attempted, + } + } + } + + const crossProviderMatch = deps.fuzzyMatchModel(entry.model, availableModels) + if (crossProviderMatch) { + log("Model resolved via fallback chain (cross-provider fuzzy match)", { + model: entry.model, + match: crossProviderMatch, + variant: entry.variant, + }) + return { + model: crossProviderMatch, + provenance: "provider-fallback", + variant: entry.variant, + attempted, + } + } + } + log("No available model found in fallback chain, falling through to system default") + } + } + + if (systemDefaultModel === undefined) { + log("No model resolved - systemDefaultModel not configured") + return undefined + } + + log("Model resolved via system default", { model: systemDefaultModel }) + return { model: systemDefaultModel, provenance: "system-default", attempted } +} diff --git a/apps/coder/src/services/model-resolution/model-resolution-types.ts b/apps/coder/src/services/model-resolution/model-resolution-types.ts new file mode 100644 index 0000000..d54521a --- /dev/null +++ b/apps/coder/src/services/model-resolution/model-resolution-types.ts @@ -0,0 +1,41 @@ +import type { FallbackEntry } from "./model-requirement-types.js" + +export interface DelegatedModelConfig { + providerID: string + modelID: string + variant?: string + reasoningEffort?: string + temperature?: number + top_p?: number + maxTokens?: number + thinking?: { type: "enabled" | "disabled"; budgetTokens?: number } +} + +export type ModelResolutionRequest = { + intent?: { + uiSelectedModel?: string + userModel?: string + categoryDefaultModel?: string + } + constraints: { + availableModels: Set + } + policy?: { + fallbackChain?: FallbackEntry[] + systemDefaultModel?: string + } +} + +export type ModelResolutionProvenance = + | "override" + | "category-default" + | "provider-fallback" + | "system-default" + +export type ModelResolutionResult = { + model: string + provenance: ModelResolutionProvenance + variant?: string + attempted?: string[] + reason?: string +} diff --git a/apps/coder/src/services/model-resolution/model-resolver.ts b/apps/coder/src/services/model-resolution/model-resolver.ts new file mode 100644 index 0000000..02d4c7e --- /dev/null +++ b/apps/coder/src/services/model-resolution/model-resolver.ts @@ -0,0 +1,109 @@ +import type { FallbackEntry } from "./model-requirement-types.js" +import type { FallbackModelObject } from "./fallback-model-object.js" +import { normalizeModel } from "./model-normalization.js" +import { resolveModelPipeline } from "./model-resolution-pipeline.js" +import { KNOWN_VARIANTS } from "./known-variants.js" +import type { ConnectedProvidersAdapter } from "./connected-providers-cache.js" +import * as connectedProvidersCache from "./connected-providers-cache.js" + +export type ModelResolutionInput = { + userModel?: string + inheritedModel?: string + systemDefault?: string +} + +export type ModelSource = + | "override" + | "category-default" + | "provider-fallback" + | "system-default" + +export type ModelResolutionResult = { + model: string + source: ModelSource + variant?: string +} + +export type ExtendedModelResolutionInput = { + uiSelectedModel?: string + userModel?: string + userFallbackModels?: string[] + categoryDefaultModel?: string + fallbackChain?: FallbackEntry[] + availableModels: Set + systemDefaultModel?: string +} + + +export function resolveModel(input: ModelResolutionInput): string | undefined { + return ( + normalizeModel(input.userModel) ?? + normalizeModel(input.inheritedModel) ?? + input.systemDefault + ) +} + +export function resolveModelWithFallback( + input: ExtendedModelResolutionInput, + connectedProvidersAdapter: ConnectedProvidersAdapter = connectedProvidersCache, +): ModelResolutionResult | undefined { + const { uiSelectedModel, userModel, userFallbackModels, categoryDefaultModel, fallbackChain, availableModels, systemDefaultModel } = input + const resolved = resolveModelPipeline({ + intent: { uiSelectedModel, userModel, userFallbackModels, categoryDefaultModel }, + constraints: { availableModels }, + policy: { fallbackChain, systemDefaultModel }, + }, connectedProvidersAdapter) + + if (!resolved) { + return undefined + } + + return { + model: resolved.model, + source: resolved.provenance, + variant: resolved.variant, + } +} + +/** + * Normalizes fallback_models config to a mixed array. + * Accepts string, string[], or mixed arrays of strings and FallbackModelObject entries. + */ +export function normalizeFallbackModels( + models: string | (string | FallbackModelObject)[] | undefined, +): (string | FallbackModelObject)[] | undefined { + if (!models) return undefined + if (typeof models === "string") return [models] + return models +} + +/** + * Extracts plain model strings from a mixed fallback models array. + * Object entries are flattened to "model" or "model(variant)" strings. + * Use this when consumers need string[] (e.g., resolveModelForDelegateTask). + */ +export function flattenToFallbackModelStrings( + models: (string | FallbackModelObject)[] | undefined, +): string[] | undefined { + if (!models) return undefined + return models.map((entry) => { + if (typeof entry === "string") return entry + const variant = entry.variant + if (variant) { + // Strip any supported inline variant syntax before appending explicit override. + // Supports both parenthesized and space-suffix forms so we don't emit + // invalid strings like "provider/model high(low)". + const model = entry.model + .replace(/\([^()]+\)\s*$/, "") + .replace(/\s+([a-z][a-z0-9_-]*)\s*$/i, (_match: string, suffix: string) => { + const normalized = String(suffix).toLowerCase() + return KNOWN_VARIANTS.has(normalized) + ? "" + : _match + }) + .trim() + return `${model}(${variant})` + } + return entry.model + }) +} diff --git a/apps/coder/src/services/model-resolution/provider-cache.ts b/apps/coder/src/services/model-resolution/provider-cache.ts new file mode 100644 index 0000000..20c2612 --- /dev/null +++ b/apps/coder/src/services/model-resolution/provider-cache.ts @@ -0,0 +1,27 @@ +export interface ModelMetadata { + readonly id: string + readonly provider?: string + readonly context?: number + readonly output?: number + readonly name?: string + readonly variants?: Record + readonly limit?: { + readonly context?: number + readonly input?: number + readonly output?: number + } + readonly modalities?: { + readonly input?: string[] + readonly output?: string[] + } + readonly capabilities?: Record + readonly reasoning?: boolean + readonly temperature?: boolean + readonly tool_call?: boolean + readonly [key: string]: unknown +} + +export interface ProviderCache { + readConnectedProvidersCache(): string[] | null + findProviderModelMetadata(providerID: string, modelID: string): ModelMetadata | undefined +} diff --git a/apps/coder/src/services/model-resolution/provider-model-id-transform.ts b/apps/coder/src/services/model-resolution/provider-model-id-transform.ts new file mode 100644 index 0000000..600b24e --- /dev/null +++ b/apps/coder/src/services/model-resolution/provider-model-id-transform.ts @@ -0,0 +1,69 @@ +function inferSubProvider(model: string): string | undefined { + if (model.startsWith("claude-")) return "anthropic" + if (model.startsWith("gpt-")) return "openai" + if (model.startsWith("gemini-")) return "google" + if (model.startsWith("grok-")) return "xai" + if (model.startsWith("minimax-")) return "minimax" + if (model.startsWith("kimi-")) return "moonshotai" + if (model.startsWith("glm-")) return "zai" + return undefined +} + +const CLAUDE_VERSION_DOT = /claude-(\w+)-(\d+)-(\d+)/g +const GEMINI_31_PRO_PREVIEW = /gemini-3\.1-pro(?!-)/g +const GEMINI_3_FLASH_PREVIEW = /gemini-3-flash(?!-)/g + +function claudeVersionDot(model: string): string { + return model.replace(CLAUDE_VERSION_DOT, "claude-$1-$2.$3") +} + +function applyGatewayTransforms(model: string): string { + return claudeVersionDot(model).replace( + GEMINI_31_PRO_PREVIEW, + "gemini-3.1-pro-preview", + ) +} + +function transformModelForProviderUsingAnthropicBehavior( + provider: string, + model: string, +): string { + if (provider === "vercel") { + const slashIndex = model.indexOf("/") + if (slashIndex !== -1) { + const subProvider = model.substring(0, slashIndex) + const subModel = model.substring(slashIndex + 1) + return `${subProvider}/${applyGatewayTransforms(subModel)}` + } + const subProvider = inferSubProvider(model) + if (subProvider) { + return `${subProvider}/${applyGatewayTransforms(model)}` + } + return model + } + if (provider === "github-copilot") { + return claudeVersionDot(model) + .replace(GEMINI_31_PRO_PREVIEW, "gemini-3.1-pro-preview") + .replace(GEMINI_3_FLASH_PREVIEW, "gemini-3-flash-preview") + } + if (provider === "google") { + return model + .replace(GEMINI_31_PRO_PREVIEW, "gemini-3.1-pro-preview") + .replace(GEMINI_3_FLASH_PREVIEW, "gemini-3-flash-preview") + } + if (provider === "anthropic") { + return model + } + return model +} + +export function transformModelForProvider(provider: string, model: string): string { + return transformModelForProviderUsingAnthropicBehavior(provider, model) +} + +export function transformModelForProviderDisplay( + provider: string, + model: string, +): string { + return transformModelForProviderUsingAnthropicBehavior(provider, model) +}