Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 | 1x 1x 1x 1x 1x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 4x 4x 4x 4x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 5x 2x 2x 2x 2x 2x 2x 2x 2x 2x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 5x 3x 5x 1x 1x 1x 1x 2x 2x 2x 2x 2x 2x 2x 2x 2x | import { getApiUrl } from '@api/constants';
import type {
EvaluationMethodConnector,
UserDataStorageConnector,
} from '@api/types/connector';
import type { AppContext } from '@api/types/hono';
import { resolveSystemSettingsModel } from '@api/utils/evaluation-model-resolver';
import { warn } from '@shared/console-logging';
import type { Skill } from '@shared/types/data';
import type { SkillOptimizationEvaluationCreateParams } from '@shared/types/data/skill-optimization-evaluation';
import type { EvaluationMethodName } from '@shared/types/evaluations';
import OpenAI from 'openai';
import type { ParsedChatCompletion } from 'openai/resources/chat/completions.mjs';
import z from 'zod';
function getSetupEvaluationSystemPrompt() {
const systemPrompt = `You are an AI assistant that configures evaluation methods for AI agent training systems.
Your role is to generate evaluation METHOD PARAMETERS, not to create evaluation prompts or judge agents yourself.
The evaluation system has two stages:
1. EXTRACTION: An AI extracts the "task" and "outcome" from each request/response
2. VERDICT: An AI judges if the outcome matches the task
You are configuring WHAT TASK to extract (stage 1), not creating the judge (stage 2).`;
return systemPrompt;
}
function getSetupEvaluationFirstMessage(
agentDescription: string,
skillDescription: string,
evaluationConnector: EvaluationMethodConnector,
examples?: string[],
) {
const details = evaluationConnector.getDetails();
const evaluationName = details.method;
const evaluationDescription = details.description;
let firstMessage = `
I am building an AI agent with the following purpose:
Agent Description:
'''
${agentDescription}
'''
This agent has a specific skill that it needs to perform:
Skill Description:
'''
${skillDescription}
'''
Configure the evaluation method: ${evaluationName} (${evaluationDescription}).
Your task: Generate parameters that define WHAT TASK the extraction AI should look for in request/response pairs.
IMPORTANT: You are NOT creating judge prompts or evaluation criteria. You are defining:
- The "task" field: A clear, concise description of what task the AI agent is trying to accomplish
Example: "Generate a calendar event JSON object from plain text with only real participant names"
This task description will be used by a separate extraction AI to identify what the user was trying to accomplish in each request.
Focus on being specific and actionable. The task should be something measurable and observable in the agent's outputs.
`;
// If we have examples from actual requests, include them
if (examples && examples.length > 0) {
firstMessage += `
CONTEXT: Below are examples of actual requests and responses from this skill in production.
'''
${examples.join('\n\n---\n\n')}
'''
Based on these examples, refine your task description to be:
1. SPECIFIC to what you observe the agent doing (e.g., "Generate JSON with fields X, Y, Z" not just "Generate JSON")
2. CLEAR about any constraints or requirements (e.g., "Only use real person names" if you see that pattern)
3. FOCUSED on the core deliverable (what the user actually receives)
Pay attention to:
- Any "Request Constraints" sections showing JSON schemas, tools, or output formats
- Patterns in what the agent produces (structured data formats, specific fields, constraints)
- Consistent requirements across multiple examples
Your task description will guide the extraction AI to understand what users are asking for.
`;
}
return firstMessage;
}
/**
* Whether the method has any parameter for a model to fill in.
*
* Most methods declare an empty AI schema: they are configured entirely by
* their own defaults, and `task_completion` is the only one that asks for
* anything. Sending the model a schema with no properties spends a request to
* be told `{}`, and a provider that does not enforce the schema -- a
* self-hosted model, most of all -- answers with whatever it thought was
* wanted. That reply used to be stored verbatim, and the method's own
* parameter schema then rejected it at evaluation time, long after the fact:
*
* [REALTIME_EVAL] Failed to evaluate log ... with method tool_correctness:
* Unrecognized key: "task"
*/
function hasGeneratableParameters(schema: z.ZodType): boolean {
if (schema instanceof z.ZodObject) {
return Object.keys(schema.shape).length > 0;
}
return true;
}
export async function generateEvaluationCreateParams(
c: AppContext,
skill: Skill,
evaluationConnector: EvaluationMethodConnector,
method: EvaluationMethodName,
agentDescription: string,
connector: UserDataStorageConnector,
examples?: string[],
): Promise<SkillOptimizationEvaluationCreateParams> {
// Resolve evaluation generation model from system settings
const modelConfig = await resolveSystemSettingsModel(
c,
'evaluation_generation',
connector,
);
if (!modelConfig) {
warn(
'[OPTIMIZER] No evaluation generation model configured in system settings',
);
throw new Error(
'No evaluation generation model configured in system settings',
);
}
// Create OpenAI client pointing to local Super Agents API
const client = new OpenAI({
apiKey: '',
baseURL: `${getApiUrl(c)}/v1`,
});
const saConfig = {
targets: [
{
provider: modelConfig.provider,
model: modelConfig.model,
...(modelConfig.apiKey ? { api_key: modelConfig.apiKey } : {}),
...(modelConfig.customHost
? { custom_host: modelConfig.customHost }
: {}),
},
],
agent_name: 'super-agents',
skill_name: 'create-evaluations',
};
const schema = evaluationConnector.getAIParameterSchema;
// If the evaluation method doesn't use AI for parameter generation,
// use default parameters from the parameter schema
if (!schema || !hasGeneratableParameters(schema)) {
const params: SkillOptimizationEvaluationCreateParams = {
agent_id: skill.agent_id,
skill_id: skill.id,
evaluation_method: method,
params: {}, // Use default parameters from schema
weight: 1.0,
};
return params;
}
const jsonSchema = z.toJSONSchema(schema);
const systemPrompt = getSetupEvaluationSystemPrompt();
const firstMessage = getSetupEvaluationFirstMessage(
agentDescription,
skill.description,
evaluationConnector,
examples,
);
const response: ParsedChatCompletion<typeof schema> = await client
.withOptions({
defaultHeaders: {
'sa-config': JSON.stringify(saConfig),
},
})
.chat.completions.parse({
model: modelConfig.model,
messages: [
{ role: 'system', content: systemPrompt },
{
role: 'user',
content: firstMessage,
},
],
// This is a custom zodTextFormat to make it work with zod v4
response_format: {
type: 'json_schema',
json_schema: {
name: 'params',
strict: true,
schema: jsonSchema,
},
},
});
const structuredOutputResponse = response.choices[0].message.parsed;
if (!structuredOutputResponse) {
throw new Error(
`[OPTIMIZER] can't generate evaluations for skill - No response found`,
);
}
// The model's answer is checked against the schema it was given rather than
// trusted: only a provider that enforces `response_format` guarantees the
// shape, and these parameters are read back much later, by the evaluation
// that runs on every request. Parsing also fills in the defaults the model
// left out, such as `task_completion`'s threshold.
const validated = schema.safeParse(structuredOutputResponse);
if (!validated.success) {
throw new Error(
`[OPTIMIZER] can't generate evaluations for skill - the ${method} parameters the model returned do not match its schema: ${validated.error.message}`,
);
}
const params: SkillOptimizationEvaluationCreateParams = {
agent_id: skill.agent_id,
skill_id: skill.id,
evaluation_method: method,
params: validated.data as Record<string, unknown>,
weight: 1.0, // Default weight - can be adjusted by user
};
return params;
}
/**
* Regenerates evaluations for a skill using real request examples.
* This should be called after the skill has been used in production to create
* evaluations that are better aligned with actual usage patterns.
*/
export async function regenerateEvaluationsWithExamples(
c: AppContext,
skill: Skill,
agentDescription: string,
examples: string[],
evaluationConnectors: Record<string, EvaluationMethodConnector>,
existingEvaluationMethods: EvaluationMethodName[],
connector: UserDataStorageConnector,
): Promise<SkillOptimizationEvaluationCreateParams[]> {
const regeneratePromises = existingEvaluationMethods.map(async (method) => {
const evaluationConnector = evaluationConnectors[method];
if (!evaluationConnector) {
throw new Error(`Evaluation connector not found for method ${method}`);
}
return await generateEvaluationCreateParams(
c,
skill,
evaluationConnector,
method,
agentDescription,
connector,
examples,
);
});
return await Promise.all(regeneratePromises);
}
|