diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 32f6c5333..8ce164699 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -303,7 +303,7 @@ interface NormalizedOptions { /** Removed: the run directory always uses index.jsonl */ readonly outputFormat?: string; readonly graderTarget?: string; - /** Config-level fallback grader target name, from `.agentv/config.yaml`'s `defaults.grader`. */ + /** Config-level fallback grader provider name, from `.agentv/config.yaml`'s `defaults.grader`. */ readonly defaultGraderTarget?: string; readonly model?: string; readonly outputMessages: number | 'all'; @@ -1405,7 +1405,12 @@ async function prepareFileMetadata(params: { : suite.tests; const testIds = testCases.map((value) => value.id); const suiteTargetSpec = suite.targetSpec; + const suiteDefaults = suite.defaults; const suiteTargets = suiteTargetSpec ? [suiteTargetSpec.name] : suite.targets; + const fileOptions = + suiteDefaults?.grader && !effectiveOptions.graderTarget + ? { ...effectiveOptions, defaultGraderTarget: suiteDefaults.grader } + : effectiveOptions; const defaultBudgetUsd = effectiveOptions.cliBudgetUsd === undefined ? (effectiveOptions.budgetUsd ?? suite.budgetUsd) @@ -1414,11 +1419,11 @@ async function prepareFileMetadata(params: { if (testCases.length === 0) { return { - options: effectiveOptions, + options: fileOptions, testIds, testCases, selections: [], - trialsConfig: effectiveOptions.experimentTrialsConfig, + trialsConfig: fileOptions.experimentTrialsConfig, suiteTargets, yamlCache: suite.cacheConfig?.enabled, yamlCachePath: suite.cacheConfig?.cachePath, @@ -1432,7 +1437,7 @@ async function prepareFileMetadata(params: { let selections: { selection: TargetSelection; inlineTargetLabel: string }[]; - if (effectiveOptions.transcript) { + if (fileOptions.transcript) { // --transcript mode: bypass target resolution entirely. // Create a synthetic TargetSelection for the transcript provider. const transcriptSelection: TargetSelection = { @@ -1444,15 +1449,15 @@ async function prepareFileMetadata(params: { }, targetName: 'transcript', targetSource: 'cli', - targetsFilePath: effectiveOptions.transcript, + targetsFilePath: fileOptions.transcript, }; selections = [ { selection: transcriptSelection, - inlineTargetLabel: `transcript (${path.basename(effectiveOptions.transcript)})`, + inlineTargetLabel: `transcript (${path.basename(fileOptions.transcript)})`, }, ]; - } else if (suite.inlineTarget && effectiveOptions.cliTargets.length === 0) { + } else if (suite.inlineTarget && fileOptions.cliTargets.length === 0) { const targetDefinition = suite.inlineTarget; const resolvedTarget = resolveProviderDefinition(targetDefinition, process.env, testFilePath, { emitDeprecationWarnings: false, @@ -1469,7 +1474,7 @@ async function prepareFileMetadata(params: { inlineTargetLabel: resolveTargetLabel(targetDefinition.name, resolvedTarget.name), }, ]; - } else if (suite.providerFactory && effectiveOptions.cliTargets.length === 0) { + } else if (suite.providerFactory && fileOptions.cliTargets.length === 0) { const taskTarget: ResolvedProviderBackend = { kind: 'mock', name: 'custom-task', @@ -1490,12 +1495,12 @@ async function prepareFileMetadata(params: { ]; } else { // Determine provider labels: CLI --provider flags override YAML - const cliTargets = effectiveOptions.cliTargets; - const experimentTargets = effectiveOptions.experimentTargets ?? []; + const cliTargets = fileOptions.cliTargets; + const experimentTargets = fileOptions.experimentTargets ?? []; const suiteTargetSpec = suite.targetSpec; const suiteTargets = suiteTargetSpec ? [suiteTargetSpec.name] : suite.targets; const suiteTargetRefs = suite.targetRefs; - const experimentTargetRefs = effectiveOptions.experimentTargetRefs; + const experimentTargetRefs = fileOptions.experimentTargetRefs; // Resolve which target names to use (precedence: CLI/experiment > suite YAML targets > default) let targetNames: readonly string[]; @@ -1511,6 +1516,9 @@ async function prepareFileMetadata(params: { } else if (suiteTargets && suiteTargets.length > 0) { targetNames = suiteTargets; targetRefs = suiteTargetRefs; + } else if (suiteDefaults?.provider) { + targetNames = [suiteDefaults.provider]; + targetRefs = undefined; } else { targetNames = []; targetRefs = undefined; @@ -1526,12 +1534,12 @@ async function prepareFileMetadata(params: { providerDefinitions, providerDefinitionsSource, requireExplicitProviderCatalog: true, - allowLegacyTargetFiles: effectiveOptions.allowLegacyTargetFiles, + allowLegacyTargetFiles: fileOptions.allowLegacyTargetFiles, env: process.env, targetNames, targetRefs, targetSource, - modelOverride: effectiveOptions.targetModelOverride, + modelOverride: fileOptions.targetModelOverride, }); selections = multiSelections.map((sel) => ({ @@ -1549,18 +1557,18 @@ async function prepareFileMetadata(params: { providerDefinitions, providerDefinitionsSource, requireExplicitProviderCatalog: true, - allowLegacyTargetFiles: effectiveOptions.allowLegacyTargetFiles, + allowLegacyTargetFiles: fileOptions.allowLegacyTargetFiles, cliTargetName: targetSource === 'cli' ? targetNames.length === 1 ? targetNames[0] - : effectiveOptions.target - : effectiveOptions.target, + : fileOptions.target + : fileOptions.target, fileTargetName: targetSource === 'test-file' && targetNames.length === 1 ? targetNames[0] : undefined, fileTargetSpec: targetSource === 'test-file' && targetNames.length === 1 ? suiteTargetSpec : undefined, - modelOverride: effectiveOptions.targetModelOverride, + modelOverride: fileOptions.targetModelOverride, env: process.env, }); @@ -1587,11 +1595,11 @@ async function prepareFileMetadata(params: { } return { - options: effectiveOptions, + options: fileOptions, testIds, testCases, selections, - trialsConfig: effectiveOptions.experimentTrialsConfig, + trialsConfig: fileOptions.experimentTrialsConfig, suiteTargets, yamlCache: suite.cacheConfig?.enabled, yamlCachePath: suite.cacheConfig?.cachePath, @@ -1951,9 +1959,9 @@ export async function runEvalCommand( process.env.AGENTV_EXPERIMENT = normalizedExperiment; } - // Validate --grader-target / --model combinations + // Validate --grader-provider / --model combinations if (options.graderTarget === 'agentv' && !options.model) { - throw new Error('--grader-target agentv requires --model (e.g., --model openai:gpt-5-mini)'); + throw new Error('--grader-provider agentv requires --model (e.g., --model openai:gpt-5-mini)'); } if (options.removedOut) { diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 6a8dda903..47c3c2852 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -636,12 +636,81 @@ function bundledEvalFileName(evalFilePath: string): string { function uniqueTargetDefinitions( selections: readonly TaskBundleTargetSelection[], + tests: readonly EvalTest[] = [], ): readonly ProviderDefinition[] { const selected: ProviderDefinition[] = []; const seen = new Set(); + function addDefinitions( + names: readonly string[], + definitions: readonly ProviderDefinition[], + ): void { + for (const name of names) { + for (const definition of selectTargetDefinitions(name, definitions)) { + if (seen.has(definition.name)) { + continue; + } + seen.add(definition.name); + selected.push(definition); + } + } + } + + const graderTargetNames = collectGraderTargetNames(tests); for (const selection of selections) { - for (const definition of selectTargetDefinitions(selection.targetName, selection.definitions)) { + addDefinitions([selection.targetName, ...graderTargetNames], selection.definitions); + } + + return selected; +} + +function collectGraderTargetNames(tests: readonly EvalTest[]): readonly string[] { + const names: string[] = []; + const seen = new Set(); + + function collect(value: unknown, key?: string): void { + if (key === 'target' && typeof value === 'string') { + const target = value.trim(); + if (target.length > 0 && !target.includes('${{') && !seen.has(target)) { + seen.add(target); + names.push(target); + } + return; + } + + if (Array.isArray(value)) { + for (const item of value) { + collect(item); + } + return; + } + + if (isRecord(value)) { + for (const [childKey, childValue] of Object.entries(value)) { + collect(childValue, childKey); + } + } + } + + for (const test of tests) { + for (const grader of test.source?.graderDefinitions ?? []) { + collect(grader.definition); + } + } + + return names; +} + +function selectTaskBundleTargetDefinitions( + targetName: string, + definitions: readonly ProviderDefinition[], + tests: readonly EvalTest[], +): readonly ProviderDefinition[] { + const selected: ProviderDefinition[] = []; + const seen = new Set(); + + for (const name of [targetName, ...collectGraderTargetNames(tests)]) { + for (const definition of selectTargetDefinitions(name, definitions)) { if (seen.has(definition.name)) { continue; } @@ -1133,7 +1202,11 @@ export async function materializeTaskBundle( return undefined; } - const targetDefinitions = selectTargetDefinitions(options.targetName, options.targetDefinitions); + const targetDefinitions = selectTaskBundleTargetDefinitions( + options.targetName, + options.targetDefinitions, + [options.test], + ); if (targetDefinitions.length === 0) { return undefined; } @@ -1228,7 +1301,7 @@ export async function materializeEvalBundle( }); await writeYamlFile( providersPath, - serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)), + serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections, options.tests)), ); await mkdir(path.dirname(configPath), { recursive: true }); await writeYamlFile(configPath, { providers: `file://../${BUNDLE_PROVIDERS_FILENAME}` }); diff --git a/apps/cli/src/commands/grade/index.ts b/apps/cli/src/commands/grade/index.ts index 7793f8dee..a8fbe1f05 100644 --- a/apps/cli/src/commands/grade/index.ts +++ b/apps/cli/src/commands/grade/index.ts @@ -657,7 +657,7 @@ export const gradeCommand = command({ model: option({ type: optional(string), long: 'model', - description: 'Override model for the grader target (e.g., "openai:gpt-5-mini")', + description: 'Override model for the grader provider (e.g., "openai:gpt-5-mini")', }), threshold: option({ type: optional(number), diff --git a/apps/cli/test/commands/eval/task-bundle.test.ts b/apps/cli/test/commands/eval/task-bundle.test.ts index 19a15a8a9..670c6b1dd 100644 --- a/apps/cli/test/commands/eval/task-bundle.test.ts +++ b/apps/cli/test/commands/eval/task-bundle.test.ts @@ -55,6 +55,7 @@ describe('materializeTaskBundle', () => { name: 'quality', type: 'llm-grader', prompt: 'file://graders/prompt.md', + target: 'judge', command: ['bun', scriptPath, '--token', 'literal-secret'], }, }, @@ -97,6 +98,11 @@ describe('materializeTaskBundle', () => { provider: 'mock', api_key: 'literal-secret', }, + { + name: 'judge', + provider: 'mock', + api_key: '${{ JUDGE_API_KEY }}', + }, ], outputDir: path.join(tempDir, 'out'), cwd: tempDir, @@ -132,8 +138,11 @@ describe('materializeTaskBundle', () => { 'file://files/fixtures/input.txt', ); expect(assertion.prompt).toBe('file://graders/graders/prompt.md'); + expect(assertion.target).toBe('judge'); expect(assertion.command).toEqual(['bun', 'graders/graders/check.ts', '--token', '[redacted]']); expect(taskProviders).toContain('api_key: ${{ MOCK_API_KEY }}'); + expect(taskProviders).toContain('label: judge'); + expect(taskProviders).toContain('api_key: ${{ JUDGE_API_KEY }}'); expect(taskProviders).toContain('api_key: "[redacted]"'); expect(taskEval).not.toContain('literal-secret'); expect(taskProviders).not.toContain('literal-secret'); diff --git a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx index 959975e47..273050425 100644 --- a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx @@ -162,10 +162,27 @@ tests: ## Per-Grader Provider -By default, an `llm-rubric` uses `defaults.grader` from the resolved config graph. -Override it per assertion when you need multiple grader models in one run: +By default, an `llm-rubric` uses `tests[].options.provider`, then +`default_test.options.provider`, then `defaults.grader` from the resolved config +graph. Each value selects from the same `providers` pool used for candidate +providers; AgentV does not infer a grader from the first candidate provider. +Override the provider per assertion when you need multiple grader models in one +run: ```yaml +default_test: + options: + provider: grader_gpt_5_mini + +tests: + - id: strict-case + options: + provider: grader_claude_haiku + assert: + - name: semantic_quality + type: llm-rubric + value: The answer is correct and concise. + assert: - name: grader-gpt type: llm-rubric diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index 40ea6bf4c..18b5b34c3 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -63,8 +63,11 @@ settings belong under `config`. Process-backed coding-agent providers use A grader is not a separate kind of entity — it is a provider selected for a grading role, either through `defaults.grader` (shown above) or an -assertion-level `provider` override. There is no separate `graders:` list; -authoring one is a hard error telling you to move each entry into `providers`. +`options.provider` / assertion-level `provider` override. `default_test.options.provider` +sets a shared grader fallback for tests, `tests[].options.provider` overrides it +for one test, and an assertion-level `provider` wins for that assertion. There +is no separate `graders:` list; authoring one is a hard error telling you to +move each entry into `providers`. ## Runtime Modes @@ -306,8 +309,9 @@ providers: reasoning_effort: high ``` -Use `defaults.grader` for the project default grader. A specific evaluator can -still choose its own grader provider when the evaluator supports that override. +Use `defaults.grader` for the project default grader. `default_test.options.provider` +and `tests[].options.provider` can choose a grader provider for LLM-backed +assertions before an assertion-level `provider` override takes final precedence. ### Environment And Lifecycle Extensions diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index f29f4f1f9..d806781c8 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -227,6 +227,7 @@ function assertSupportedPromptfooType(type: string, evalId: string, name?: strin export async function parseGraders( rawEvalCase: JsonObject & { readonly execution?: JsonValue; + readonly options?: JsonValue; readonly assert?: JsonValue; }, globalExecution: JsonObject | undefined, @@ -245,6 +246,7 @@ export async function parseGraders( // Root-level default graders. const skipDefaults = executionObject?.skip_defaults === true; const rootEvaluators = skipDefaults ? undefined : globalExecution?.assert; + const inheritedGraderTarget = readOptionsProvider(rawEvalCase.options, evalId); // Parse case-level evaluators const parsedCase = await parseGraderList( @@ -253,6 +255,8 @@ export async function parseGraders( evalId, defaultPreprocessors, defaultRubricPrompt, + undefined, + inheritedGraderTarget, ); // Parse root-level evaluators (appended after case-level) const parsedRoot = await parseGraderList( @@ -261,6 +265,8 @@ export async function parseGraders( evalId, defaultPreprocessors, defaultRubricPrompt, + undefined, + inheritedGraderTarget, ); if (!parsedCase && !parsedRoot) { @@ -273,6 +279,24 @@ export async function parseGraders( return evaluators.length > 0 ? evaluators : undefined; } +function readOptionsProvider(options: JsonValue | undefined, evalId: string): string | undefined { + if (options === undefined || options === null) { + return undefined; + } + if (!isJsonObject(options)) { + return undefined; + } + const provider = options.provider; + if (provider === undefined || provider === null) { + return undefined; + } + if (typeof provider === 'string' && provider.trim().length > 0) { + return provider.trim(); + } + logWarning(`Skipping options.provider for '${evalId}': provider must be a non-empty string`); + return undefined; +} + interface IncludeContext { readonly depth: number; readonly chain: readonly string[]; @@ -548,6 +572,7 @@ async function parseGraderList( defaultPreprocessors?: readonly ContentPreprocessorConfig[], defaultRubricPrompt?: JsonValue, inheritedAssertionConfig?: JsonObject, + inheritedGraderTarget?: string, ): Promise { const expandedEvaluators = await expandGraderEntries(candidateEvaluators, searchRoots, evalId); if (!expandedEvaluators) { @@ -749,6 +774,7 @@ async function parseGraderList( defaultPreprocessors, defaultRubricPrompt, config, + inheritedGraderTarget, ); if (!parsedMembers || parsedMembers.length === 0) { logWarning( @@ -1673,12 +1699,14 @@ async function parseGraderList( let graderTargetName: string | undefined; if (graderTarget !== undefined) { if (typeof graderTarget === 'string' && graderTarget.trim().length > 0) { - graderTargetName = graderTarget; + graderTargetName = graderTarget.trim(); } else { logWarning( `Skipping provider override for llm-grader evaluator '${name}' in '${evalId}': provider must be a non-empty string`, ); } + } else { + graderTargetName = inheritedGraderTarget; } // Parse prompt field - can be string (text template) or object (executable script) diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 23aae2461..123c2462a 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -41,11 +41,7 @@ import type { ProviderResponse, ProviderStreamCallbacks, } from './providers/types.js'; -import { - LLM_GRADER_CAPABLE_KINDS, - extractLastAssistantContent, - isAgentProvider, -} from './providers/types.js'; +import { extractLastAssistantContent, isAgentProvider } from './providers/types.js'; import { createBuiltinRegistry, discoverAssertions, discoverGraders } from './registry/index.js'; import { type ReplayRecordingOptions, @@ -79,6 +75,7 @@ import type { GraderResult, JsonObject, JsonValue, + LlmBackedGraderConfig, LlmGraderConfig, TestMessage, TestMessageRole, @@ -140,6 +137,26 @@ function usesFileReferencePrompt(provider: Provider): boolean { return isAgentProvider(provider) || provider.kind === 'cli'; } +function isLlmBackedGraderConfig(config: GraderConfig): config is LlmBackedGraderConfig { + return ( + config.type === 'llm-grader' || config.type === 'llm-rubric' || config.type === 'agent-rubric' + ); +} + +function caseNeedsFallbackGraderProvider(evalCase: EvalTest): boolean { + if (evalCase.assertions && evalCase.assertions.length > 0) { + return evalCase.assertions.some( + (assertion) => isLlmBackedGraderConfig(assertion) && !assertion.target, + ); + } + + if (evalCase.evaluator === 'llm-grader' || evalCase.evaluator === 'llm-rubric') { + return true; + } + + return (evalCase.preprocessors?.length ?? 0) > 0; +} + function extractProviderRawLogPath(response: ProviderResponse): string | undefined { const raw = response.raw; if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { @@ -185,7 +202,7 @@ interface EvaluationRuntimeOptions { readonly providerFactory?: (target: ResolvedProviderBackend) => Provider; readonly evalFilePath?: string; readonly graderTarget?: string; - /** Config-level fallback grader target name (`.agentv/config.yaml`'s `defaults.grader`), used when a target has no `grader_target` and no CLI `--grader-target` override is given. */ + /** Config-level fallback grader provider name (`.agentv/config.yaml`'s `defaults.grader`), used when no assertion/test-level provider and no CLI override is given. */ readonly defaultGraderTarget?: string; readonly model?: string; } @@ -248,34 +265,34 @@ function createEvaluationRuntime(options: EvaluationRuntimeOptions): EvaluationR const resolveGraderProvider = async ( targetContext: ResolvedProviderBackend, ): Promise => { - // CLI --grader-target takes highest priority. + // CLI --grader-provider takes highest priority. if (cliGraderTarget) { if (cliGraderTarget === 'agentv') { if (!cliModel) { - throw new Error('--grader-target "agentv" requires --model (e.g., "openai:gpt-5-mini")'); + throw new Error( + '--grader-provider "agentv" requires --model (e.g., "openai:gpt-5-mini")', + ); } const { AgentvProvider } = await import('./providers/agentv-provider.js'); return new AgentvProvider('agentv', { model: cliModel, temperature: 0 }); } const overrideTarget = resolveTargetByName(cliGraderTarget); if (!overrideTarget) { - throw new Error(`--grader-target "${cliGraderTarget}" not found in targets`); + throw new Error(`--grader-provider "${cliGraderTarget}" not found in providers`); } return getOrCreateProvider(overrideTarget); } - // TODO: When --model is provided without --grader-target, override the model of - // whichever grader target is resolved. For now, --model only works with --grader-target agentv. + // TODO: When --model is provided without --grader-provider, override the model of + // whichever grader provider is resolved. For now, --model only works with --grader-provider agentv. - const graderName = targetContext.graderTarget ?? defaultGraderTarget ?? targetContext.name; + const graderName = targetContext.graderTarget ?? defaultGraderTarget; + if (!graderName) { + return undefined; + } const resolvedGrader = resolveTargetByName(graderName); if (!resolvedGrader) { - // Only use the eval target as its own grader if it can return structured JSON. - // Agent providers, transcript, cli, and replay cannot grade. - if (!LLM_GRADER_CAPABLE_KINDS.includes(targetContext.kind)) { - return undefined; - } - return getOrCreateProvider(targetContext); + throw new Error(`Grader provider "${graderName}" not found in configured providers`); } return getOrCreateProvider(resolvedGrader); }; @@ -545,11 +562,11 @@ export interface RunEvaluationOptions { readonly retainOnSuccess?: 'keep' | 'cleanup'; /** Retention policy override for failed cases */ readonly retainOnFailure?: 'keep' | 'cleanup'; - /** CLI override: grader target name (e.g., "agentv" or a target from targets.yaml) */ + /** CLI override: grader provider name (e.g., "agentv" or a provider label from providers.yaml) */ readonly graderTarget?: string; - /** Config-level fallback grader target name (`.agentv/config.yaml`'s `defaults.grader`), used when a target has no `grader_target` and no CLI `--grader-target` override is given. */ + /** Config-level fallback grader provider name (`.agentv/config.yaml`'s `defaults.grader`). */ readonly defaultGraderTarget?: string; - /** CLI override: model for grader target (e.g., "openai:gpt-5-mini") */ + /** CLI override: model for grader provider (e.g., "openai:gpt-5-mini") */ readonly model?: string; /** Per-test score threshold for pass/fail (default: 0.8) */ readonly threshold?: number; @@ -868,6 +885,7 @@ export async function runEvaluation( } return []; } + validateDependencyGraph(filteredEvalCases); const runtime = createEvaluationRuntime({ target, @@ -881,19 +899,32 @@ export async function runEvaluation( }); const { getOrCreateProvider, resolveGraderProvider, targetResolver, availableTargets } = runtime; - // Validate grader_target: error if an agent provider would be used as grader. - // Agent providers can't return structured JSON for grading — they respond with - // tool calls and markdown, causing silent score-0 failures. - // CLI --grader-target override or config-level `defaults.grader` also satisfy - // this requirement. + const usesBuiltinLlmGrader = evaluators?.['llm-grader'] === undefined; + const needsFallbackGraderProvider = + usesBuiltinLlmGrader && filteredEvalCases.some(caseNeedsFallbackGraderProvider); + + // Agent providers cannot grade themselves with structured JSON. This keeps + // the error specific for agent candidates when they lack an explicit fallback. if ( isAgentProvider(getOrCreateProvider(target)) && !target.graderTarget && !cliGraderTarget && - !defaultGraderTarget + !defaultGraderTarget && + needsFallbackGraderProvider + ) { + throw new Error( + `Provider "${target.name}" is an agent provider ("${target.kind}") with LLM-backed assertions but no grader provider. Set assertion provider, tests[].options.provider, default_test.options.provider, pass --grader-provider, or set defaults.grader in .agentv/config.yaml to a grader-capable provider.`, + ); + } + + if ( + !target.graderTarget && + !cliGraderTarget && + !defaultGraderTarget && + needsFallbackGraderProvider ) { throw new Error( - `Target "${target.name}" is an agent provider ("${target.kind}") with no grader_target — agent providers cannot return structured JSON for grading. Set grader_target on the target, pass --grader-target, or set defaults.grader in .agentv/config.yaml to an LLM provider (e.g., azure-llm).`, + 'No grader provider configured for LLM-backed assertions. Set assertion provider, tests[].options.provider, default_test.options.provider, pass --grader-provider, or set defaults.grader in .agentv/config.yaml to a grader-capable provider.', ); } @@ -1038,8 +1069,7 @@ export async function runEvaluation( // fail_on_error tracking (best-effort under concurrency > 1, matching budgetExhausted semantics) let failOnErrorTriggered = false; - // --- Validate dependency graph and compute execution waves --- - validateDependencyGraph(filteredEvalCases); + // --- Compute dependency-aware execution waves --- const waves = computeWaves(filteredEvalCases); // Track completed test results for dependency injection diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 3e6d4730d..b33835873 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -25,6 +25,7 @@ import { loadCasesFromDirectory, loadCasesFromFile, } from './loaders/case-file-loader.js'; +import type { ConfigDefaults } from './loaders/config-graph.js'; import { type ReferenceMap, extractBudgetUsd, @@ -207,6 +208,7 @@ type RawTestSuite = JsonObject & { readonly budget_usd?: JsonValue; readonly threshold?: JsonValue; readonly default_test?: JsonValue; + readonly defaults?: JsonValue; readonly environment?: JsonValue; readonly workspace?: JsonValue; readonly assert?: JsonValue; @@ -1298,6 +1300,8 @@ export type EvalSuiteResult = { readonly threshold?: number; /** Preferred inherited per-test defaults from default_test. */ readonly defaultTest?: EvalDefaultTestDefaults; + /** Eval-authored default provider selections from top-level defaults. */ + readonly defaults?: ConfigDefaults; /** Internal normalized run controls derived from flat eval YAML. */ readonly experimentConfig?: ExperimentConfig; /** Inline target definition from a TS eval config. */ @@ -1906,6 +1910,7 @@ function buildEvalSuiteResult( : undefined; const experimentConfig = normalizeSuiteExperimentConfig(parsed); const tags = extractSuiteTagMap(parsed); + const defaults = extractSuiteDefaults(parsed); return { tests, @@ -1919,11 +1924,46 @@ function buildEvalSuiteResult( ...(failOnError !== undefined && { failOnError }), ...(threshold !== undefined && { threshold }), ...(defaultTest !== undefined && { defaultTest }), + ...(defaults !== undefined && { defaults }), ...(experimentConfig !== undefined && { experimentConfig }), ...(tags !== undefined && { tags }), }; } +function readOptionalDefaultSelection(value: unknown, location: string): string | undefined { + if (value === undefined || value === null) { + return undefined; + } + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`Invalid ${location}: expected a non-empty string.`); + } + return value.trim(); +} + +function extractSuiteDefaults(parsed: JsonObject): ConfigDefaults | undefined { + const rawDefaults = parsed.defaults; + if (rawDefaults === undefined || rawDefaults === null) { + return undefined; + } + if (!isJsonObject(rawDefaults)) { + throw new Error('Invalid defaults: expected an object.'); + } + if (rawDefaults.target !== undefined) { + throw new Error( + 'Invalid defaults.target: defaults.target has been removed. Use defaults.provider.', + ); + } + const provider = readOptionalDefaultSelection(rawDefaults.provider, 'defaults.provider'); + const grader = readOptionalDefaultSelection(rawDefaults.grader, 'defaults.grader'); + if (!provider && !grader) { + return undefined; + } + return { + ...(provider ? { provider } : {}), + ...(grader ? { grader } : {}), + }; +} + /** * Extract the promptfoo-shaped suite tags map when `tags:` (or `metadata.tags:`) * is authored as a `Record`. The list/string form is a selection diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 90c48b5a9..20ccc8801 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -115,6 +115,35 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(suite.experimentConfig?.threshold).toBe(0.9); }); + it('parses top-level provider and grader defaults', async () => { + const evalPath = path.join(tempDir, 'suite-defaults.eval.yaml'); + await writeFile( + evalPath, + [ + 'name: suite-defaults', + 'providers:', + ' - openai-candidate', + 'defaults:', + ' provider: openai-candidate', + ' grader: openai-grader', + 'prompts:', + ' - "{{ input }}"', + 'tests:', + ' - id: one', + ' criteria: ok', + ' vars:', + ' input: hello', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + + expect(suite.defaults).toEqual({ + provider: 'openai-candidate', + grader: 'openai-grader', + }); + }); + it('loads default_test from a repo-root env file reference', async () => { const agentvDir = path.join(tempDir, '.agentv'); await mkdir(agentvDir, { recursive: true }); diff --git a/packages/core/test/evaluation/grader-provider-selection.test.ts b/packages/core/test/evaluation/grader-provider-selection.test.ts new file mode 100644 index 000000000..00e379885 --- /dev/null +++ b/packages/core/test/evaluation/grader-provider-selection.test.ts @@ -0,0 +1,326 @@ +import { describe, expect, it } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { runEvaluation } from '../../src/evaluation/orchestrator.js'; +import { + type ResolvedProviderBackend, + normalizeProviderDefinition, +} from '../../src/evaluation/providers/targets.js'; +import type { + Provider, + ProviderRequest, + ProviderResponse, +} from '../../src/evaluation/providers/types.js'; +import type { EvalTest } from '../../src/evaluation/types.js'; +import { loadTests } from '../../src/evaluation/yaml-parser.js'; + +class SequenceProvider implements Provider { + readonly id: string; + readonly kind = 'mock' as const; + readonly targetName: string; + lastRequest?: ProviderRequest; + + private readonly responses: ProviderResponse[]; + + constructor(targetName: string, responses: readonly ProviderResponse[]) { + this.id = `mock:${targetName}`; + this.targetName = targetName; + this.responses = [...responses]; + } + + async invoke(request: ProviderRequest): Promise { + this.lastRequest = request; + const response = this.responses.shift(); + if (!response) { + throw new Error(`No response configured for ${this.targetName}`); + } + return response; + } +} + +class AgentLikeProvider implements Provider { + readonly id: string; + readonly kind = 'codex-cli' as const; + readonly targetName: string; + lastRequest?: ProviderRequest; + + constructor( + targetName: string, + private readonly response: ProviderResponse, + ) { + this.id = `agent:${targetName}`; + this.targetName = targetName; + } + + async invoke(request: ProviderRequest): Promise { + this.lastRequest = request; + return this.response; + } +} + +const answerTarget: ResolvedProviderBackend = { + name: 'answer', + kind: 'mock', + config: { response: 'answer' }, +}; + +const agentTarget = { + name: 'agent-answer', + kind: 'codex-cli', + config: {}, +} as ResolvedProviderBackend; + +function tempDir(prefix: string): string { + return mkdtempSync(path.join(tmpdir(), prefix)); +} + +function graderResponse(score: number): ProviderResponse { + return { + output: [ + { + role: 'assistant', + content: JSON.stringify({ score, assertions: [{ text: 'graded', passed: score >= 0.5 }] }), + }, + ], + }; +} + +function baseCase(overrides: Partial = {}): EvalTest { + return { + id: 'case-1', + suite: 'provider-grader-selection', + question: 'Answer the prompt', + input: [{ role: 'user', content: 'Answer the prompt' }], + expected_output: [], + reference_answer: '', + file_paths: [], + criteria: '', + assertions: [{ name: 'quality', type: 'llm-rubric', value: 'The answer is good.' }], + ...overrides, + }; +} + +describe('provider-based grader selection', () => { + it('loads default_test and test options.provider into LLM-backed assertion providers', async () => { + const dir = tempDir('agentv-grader-provider-parser-'); + try { + const evalPath = path.join(dir, 'suite.eval.yaml'); + writeFileSync( + evalPath, + ` +prompts: + - "{{ input }}" +default_test: + vars: + input: say hi + options: + provider: inherited-grader +tests: + - id: inherited + assert: + - type: llm-rubric + value: good + - id: test-override + options: + provider: test-grader + assert: + - type: llm-rubric + value: good + - id: assertion-override + options: + provider: test-grader + assert: + - type: llm-rubric + provider: assertion-grader + value: good +`, + 'utf8', + ); + + const tests = await loadTests(evalPath, dir); + + expect(tests[0]?.assertions?.[0]).toMatchObject({ target: 'inherited-grader' }); + expect(tests[1]?.assertions?.[0]).toMatchObject({ target: 'test-grader' }); + expect(tests[2]?.assertions?.[0]).toMatchObject({ target: 'assertion-grader' }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('uses defaults.grader labels from the shared providers pool without inferring from the candidate', async () => { + const answerProvider = new SequenceProvider('answer', [ + { output: [{ role: 'assistant', content: 'candidate answer' }] }, + ]); + const graderProvider = new SequenceProvider('grader-label', [graderResponse(1)]); + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: answerTarget, + targets: [ + { name: 'answer', provider: 'mock' }, + normalizeProviderDefinition({ id: 'mock', label: 'grader-label' }), + ], + defaultGraderTarget: 'grader-label', + providerFactory: (target) => + target.name === 'grader-label' ? graderProvider : answerProvider, + evalCases: [baseCase()], + }); + + expect(results[0]?.score).toBe(1); + expect(answerProvider.lastRequest).toBeDefined(); + expect(graderProvider.lastRequest).toBeDefined(); + }); + + it('uses defaults.grader ids from the shared providers pool when no label is supplied', async () => { + const answerProvider = new SequenceProvider('answer', [ + { output: [{ role: 'assistant', content: 'candidate answer' }] }, + ]); + const graderProvider = new SequenceProvider('mock', [graderResponse(1)]); + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: answerTarget, + targets: [{ name: 'answer', provider: 'mock' }, normalizeProviderDefinition({ id: 'mock' })], + defaultGraderTarget: 'mock', + providerFactory: (target) => (target.name === 'mock' ? graderProvider : answerProvider), + evalCases: [baseCase()], + }); + + expect(results[0]?.score).toBe(1); + expect(graderProvider.lastRequest).toBeDefined(); + }); + + it('lets test options.provider override defaults.grader for grader-required assertions', async () => { + const answerProvider = new SequenceProvider('answer', [ + { output: [{ role: 'assistant', content: 'candidate answer' }] }, + ]); + const defaultGrader = new SequenceProvider('default-grader', [graderResponse(0)]); + const testGrader = new SequenceProvider('test-grader', [graderResponse(1)]); + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: answerTarget, + targets: [ + { name: 'answer', provider: 'mock' }, + { name: 'default-grader', provider: 'mock' }, + { name: 'test-grader', provider: 'mock' }, + ], + defaultGraderTarget: 'default-grader', + providerFactory: (target) => { + if (target.name === 'default-grader') return defaultGrader; + if (target.name === 'test-grader') return testGrader; + return answerProvider; + }, + evalCases: [ + baseCase({ + assertions: [ + { + name: 'quality', + type: 'llm-rubric', + target: 'test-grader', + value: 'The answer is good.', + }, + ], + }), + ], + }); + + expect(results[0]?.score).toBe(1); + expect(testGrader.lastRequest).toBeDefined(); + expect(defaultGrader.lastRequest).toBeUndefined(); + }); + + it('does not fall back to grading with the candidate provider', async () => { + const answerProvider = new SequenceProvider('answer', [ + { output: [{ role: 'assistant', content: 'candidate answer' }] }, + ]); + + await expect( + runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: answerTarget, + targets: [{ name: 'answer', provider: 'mock' }], + providerFactory: () => answerProvider, + evalCases: [baseCase()], + }), + ).rejects.toThrow(/no grader provider configured/i); + expect(answerProvider.lastRequest).toBeUndefined(); + }); + + it('fails agent providers clearly when no usable grader provider is configured', async () => { + const agentProvider = new AgentLikeProvider('agent-answer', { + output: [{ role: 'assistant', content: 'candidate answer' }], + }); + + await expect( + runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: agentTarget, + targets: [{ name: 'agent-answer', provider: 'codex-cli' }], + providerFactory: () => agentProvider, + evalCases: [baseCase()], + }), + ).rejects.toThrow(/agent provider.*no grader provider/i); + expect(agentProvider.lastRequest).toBeUndefined(); + }); + + it('allows assertion-level provider overrides for agent candidates', async () => { + const agentProvider = new AgentLikeProvider('agent-answer', { + output: [{ role: 'assistant', content: 'candidate answer' }], + }); + const graderProvider = new SequenceProvider('grader', [graderResponse(1)]); + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: agentTarget, + targets: [ + { name: 'agent-answer', provider: 'codex-cli' }, + { name: 'grader', provider: 'mock' }, + ], + providerFactory: (target) => (target.name === 'grader' ? graderProvider : agentProvider), + evalCases: [ + baseCase({ + assertions: [ + { + name: 'quality', + type: 'llm-rubric', + target: 'grader', + value: 'The answer is good.', + }, + ], + }), + ], + }); + + expect(results[0]?.score).toBe(1); + expect(agentProvider.lastRequest).toBeDefined(); + expect(graderProvider.lastRequest).toBeDefined(); + }); + + it('allows the same configured provider to run as candidate and grader', async () => { + const sharedProvider = new SequenceProvider('shared', [ + { output: [{ role: 'assistant', content: 'candidate answer' }] }, + graderResponse(1), + ]); + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: { ...answerTarget, name: 'shared' }, + targets: [{ name: 'shared', provider: 'mock' }], + defaultGraderTarget: 'shared', + providerFactory: () => sharedProvider, + evalCases: [baseCase()], + }); + + expect(results[0]?.score).toBe(1); + }); +});