Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 | 1x 1x 1x 1x 1x 1x 1x 1x | import { getApiUrl } from '@api/constants';
import type { UserDataStorageConnector } from '@api/types/connector';
import type { AppContext } from '@api/types/hono';
import { resolveSystemSettingsModel } from '@api/utils/evaluation-model-resolver';
import { warn } from '@shared/console-logging';
import type { Skill } from '@shared/types/data';
import OpenAI from 'openai';
import type { ParsedChatCompletion } from 'openai/resources/chat/completions.mjs';
import z from 'zod';
const StructuredOutputResponse = z.object({
system_prompt: z.string(),
});
type StructuredOutputResponse = z.infer<typeof StructuredOutputResponse>;
/**
* Shared function to generate the template variables section for prompts.
* Only returns content if variables are provided (assumes user defined them for a reason).
*/
function getTemplateVariablesSection(
allowedTemplateVariables?: string[],
): string {
if (!allowedTemplateVariables || allowedTemplateVariables.length === 0) {
return '';
}
return `
**Template Variables:**
You MUST use template variables in the system prompt using double curly braces.
Required variables:
${allowedTemplateVariables.map((v) => `- {{ ${v} }}`).join('\n')}
These variables will be provided by the user at runtime via system_prompt_variables.`;
}
/**
* Shared function to generate the response format section for prompts.
*/
function getResponseFormatSection(responseFormat?: unknown): string {
if (!responseFormat) {
return '';
}
return `
CRITICAL: This agent must produce output conforming to the following JSON schema:
${JSON.stringify(responseFormat, null, 2)}
The system prompt MUST be designed around this schema. Include explicit instructions for every required field, specify exact data types and formats, explain constraints, and guide the assistant on how to structure its output to match this schema perfectly.`;
}
/**
* Shared function to generate the examples section for prompts.
*/
function getExamplesSection(examples: string[]): string {
if (examples.length === 0) {
return '';
}
return `
Here are real examples of inputs and outputs to inform the system prompt:
${examples.join('\n\n---\n\n')}
Analyze these examples to understand the task better and incorporate any patterns or domain-specific knowledge into the system prompt.`;
}
/**
* Shared function to create and configure OpenAI client for system prompt generation.
*/
async function createSystemPromptClient(
c: AppContext,
skillName: string,
connector: UserDataStorageConnector,
) {
// Resolve system prompt reflection model from system settings
const modelConfig = await resolveSystemSettingsModel(
c,
'system_prompt_reflection',
connector,
);
if (!modelConfig) {
warn(
'[OPTIMIZER] No system prompt reflection model configured in system settings',
);
throw new Error(
'No system prompt reflection model configured in system settings',
);
}
const client = new OpenAI({
apiKey: '',
baseURL: `${getApiUrl(c)}/v1`,
});
const saConfig = {
targets: [
{
provider: modelConfig.provider,
model: modelConfig.model,
...(modelConfig.apiKey ? { api_key: modelConfig.apiKey } : {}),
...(modelConfig.customHost
? { custom_host: modelConfig.customHost }
: {}),
},
],
agent_name: 'super-agents',
skill_name: skillName,
};
return { client, saConfig, model: modelConfig.model };
}
/**
* Shared function to call OpenAI with structured output for system prompt generation.
*/
async function callSystemPromptAPI(
client: OpenAI,
saConfig: unknown,
model: string,
systemPrompt: string,
userMessage: string,
): Promise<string> {
const response: ParsedChatCompletion<StructuredOutputResponse> = await client
.withOptions({
defaultHeaders: {
'sa-config': JSON.stringify(saConfig),
},
})
.chat.completions.parse({
model,
messages: [
{ role: 'system', content: systemPrompt },
{
role: 'user',
content: userMessage,
},
],
response_format: {
type: 'json_schema',
json_schema: {
name: 'system_prompt_generator',
strict: true,
schema: z.toJSONSchema(StructuredOutputResponse),
},
},
});
const structuredOutputResponse = response.choices[0].message.parsed;
if (!structuredOutputResponse) {
throw new Error(
`[OPTIMIZER] can't generate system prompt - No response found`,
);
}
// Only a provider that enforces `response_format` guarantees the shape, so
// the reply is checked rather than trusted -- otherwise a model that answered
// in some other shape stores `undefined` as the arm's system prompt.
const validated = StructuredOutputResponse.safeParse(
structuredOutputResponse,
);
if (!validated.success) {
throw new Error(
`[OPTIMIZER] can't generate system prompt - the response does not match the schema: ${validated.error.message}`,
);
}
return validated.data.system_prompt;
}
function getSeederSystemPrompt() {
const systemPrompt =
'You are an AI assistant in charge of training AI agents to do specific tasks.';
return systemPrompt;
}
function getSeederFirstMessage(description: string) {
const firstMessage = `
Given the following description of an AI agent, generate a system prompt for the AI agent so that it can produce high-quality responses:
${description}
`;
return firstMessage;
}
function getSeederWithContextFirstMessage(
agentDescription: string,
skillDescription: string,
examples: string[],
responseFormat?: unknown,
allowedTemplateVariables?: string[],
) {
const responseFormatSection = getResponseFormatSection(responseFormat);
const examplesSection = getExamplesSection(examples);
const templateVariablesSection = getTemplateVariablesSection(
allowedTemplateVariables,
);
const firstMessage = `
I am building an AI agent with the following purpose:
Agent Description:
${agentDescription}
The agent has a specific skill it needs to perform:
Skill Description:
${skillDescription}
${responseFormatSection}
${examplesSection}
${templateVariablesSection}
Generate a comprehensive system prompt that will enable the assistant to perform this skill effectively. The system prompt should be clear, detailed, and actionable.
Return the system prompt in a JSON object.`;
return firstMessage;
}
export async function generateSeedSystemPromptForSkill(
c: AppContext,
skill: Skill,
connector: UserDataStorageConnector,
) {
const { client, saConfig, model } = await createSystemPromptClient(
c,
'system-prompt-seeding',
connector,
);
const systemPrompt = getSeederSystemPrompt();
const userMessage = getSeederFirstMessage(skill.description);
return await callSystemPromptAPI(
client,
saConfig,
model,
systemPrompt,
userMessage,
);
}
export async function generateSeedSystemPromptWithContext(
c: AppContext,
agentDescription: string,
skillDescription: string,
examples: string[],
connector: UserDataStorageConnector,
responseFormat?: unknown,
allowedTemplateVariables?: string[],
) {
const { client, saConfig, model } = await createSystemPromptClient(
c,
'system-prompt-seeding-with-context',
connector,
);
const systemPrompt = getSeederSystemPrompt();
const userMessage = getSeederWithContextFirstMessage(
agentDescription,
skillDescription,
examples,
responseFormat,
allowedTemplateVariables,
);
return await callSystemPromptAPI(
client,
saConfig,
model,
systemPrompt,
userMessage,
);
}
function getReflectorSystemPrompt() {
const systemPrompt = `You are an expert at refining AI system prompts based on performance feedback. You will receive:
1. The current system prompt
2. The best example it ever produced (what to preserve)
3. Recent failures with evaluation results explaining what went wrong (what to fix)
Your task: Generate an improved system prompt that maintains the strengths while fixing the specific issues identified in the evaluation feedback.`;
return systemPrompt;
}
function getReflectorFirstMessage(
currentSystemPrompt: string,
bestExamples: string[],
worstExamples: string[],
agentDescription: string,
skillDescription: string,
allowedTemplateVariables: string[],
) {
const firstMessage = `# Context
Agent: ${agentDescription}
Skill: ${skillDescription}
# Current System Prompt
'''
${currentSystemPrompt}
'''
# Performance Analysis
## Best Example (Peak Performance - Preserve These Qualities)
'''
${bestExamples.join('\n\n---\n\n')}
'''
## Recent Failures (Fix These Issues)
Each failure below includes evaluation results that explain exactly what went wrong. Use this feedback to identify and fix specific problems.
'''
${worstExamples.join('\n\n---\n\n')}
'''
# Instructions
Generate an improved system prompt that:
1. **Preserves** the qualities that led to the best example
2. **Fixes** the specific issues identified in the evaluation results
3. **Handles** any "Request Constraints" (JSON schemas, tools, etc.) shown in the examples
Key points:
- If examples show structured output (response_format), design the prompt around that schema
- Include explicit instructions for every required field, data type, and constraint
- Extract and include any domain-specific knowledge or strategies from the examples
- Make the prompt clear, actionable, and focused on the actual task requirements
${getTemplateVariablesSection(allowedTemplateVariables)}
Return the new system prompt as JSON.`;
return firstMessage;
}
export async function generateReflectiveSystemPromptForSkill(
c: AppContext,
currentSystemPrompt: string,
bestExamples: string[],
worstExamples: string[],
agentDescription: string,
skillDescription: string,
allowedTemplateVariables: string[],
connector: UserDataStorageConnector,
) {
const { client, saConfig, model } = await createSystemPromptClient(
c,
'system-prompt-reflection',
connector,
);
const systemPrompt = getReflectorSystemPrompt();
const userMessage = getReflectorFirstMessage(
currentSystemPrompt,
bestExamples,
worstExamples,
agentDescription,
skillDescription,
allowedTemplateVariables,
);
return await callSystemPromptAPI(
client,
saConfig,
model,
systemPrompt,
userMessage,
);
}
|