Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | 1x 1x 1x 1x 1x 1x 1x 1x 1x | import type { TaskCompletionEvaluationParameters } from '@api/connectors/evaluations/task-completion/types';
import { getApiUrl } from '@api/constants';
import type { UserDataStorageConnector } from '@api/types/connector';
import type { AppContext } from '@api/types/hono';
import { resolveSystemSettingsModel } from '@api/utils/evaluation-model-resolver';
import { warn } from '@shared/console-logging';
import OpenAI from 'openai';
import type { ParsedChatCompletion } from 'openai/resources/chat/completions.mjs';
import z from 'zod';
const StructuredOutputResponse = z.object({
task: z.string(),
outcome: z.string(),
});
type StructuredOutputResponse = z.infer<typeof StructuredOutputResponse>;
function getSystemPrompt(task?: string) {
const systemPrompt = `You are an expert at analyzing AI system interactions to extract task objectives and factual outcomes.
${task ? `The TASK: ${task}\n` : 'Your job is to analyze the provided input, tools used, and output to determine the task. What was the user trying to accomplish?\n'}
Your job is to analyze the provided input, tools used, and output to determine the outcome. What actually happened or was produced?
Be precise and factual. Focus on the concrete task and measurable outcome.`;
return systemPrompt;
}
function getFirstMessage(input: string, output: string) {
const firstMessage = `Analyze this interaction and extract the task and outcome:
INPUT:
${input}
ACTUAL OUTPUT:
${output}
Extract the TASK and OUTCOME from this interaction.`;
return firstMessage;
}
export async function extractTaskAndOutcome(
c: AppContext,
params: TaskCompletionEvaluationParameters,
input: string,
output: string,
connector: UserDataStorageConnector,
) {
// Resolve judge model from system settings for task extraction
const modelConfig = await resolveSystemSettingsModel(c, 'judge', connector);
if (!modelConfig) {
warn('[OPTIMIZER] No judge model configured in system settings');
throw new Error('No judge model configured in system settings');
}
const client = new OpenAI({
apiKey: '',
baseURL: `${getApiUrl(c)}/v1`,
});
const saConfig = {
targets: [
{
provider: modelConfig.provider,
model: modelConfig.model,
...(modelConfig.apiKey ? { api_key: modelConfig.apiKey } : {}),
// Same reason as the other internal skills: without this a self-hosted
// provider is sent to its vendor default.
...(modelConfig.customHost
? { custom_host: modelConfig.customHost }
: {}),
},
],
agent_name: 'super-agents',
skill_name: 'extract-task-and-outcome',
};
const systemPrompt = getSystemPrompt(params.task);
const firstMessage = getFirstMessage(input, output);
const response: ParsedChatCompletion<StructuredOutputResponse> = await client
.withOptions({
defaultHeaders: {
'sa-config': JSON.stringify(saConfig),
},
})
.chat.completions.parse({
model: modelConfig.model,
messages: [
{ role: 'system', content: systemPrompt },
{
role: 'user',
content: firstMessage,
},
],
// This is a custom zodTextFormat to make it work with zod v4
response_format: {
type: 'json_schema',
json_schema: {
name: 'event',
strict: true,
schema: z.toJSONSchema(StructuredOutputResponse),
},
},
});
const structuredOutputResponse = response.choices[0].message.parsed;
if (!structuredOutputResponse) {
throw new Error(
`[OPTIMIZER] can't extract task and outcome - No response found`,
);
}
// Only a provider that enforces `response_format` guarantees the shape, so
// the reply is checked rather than trusted -- a missing outcome would
// otherwise reach the judge as `undefined` and be scored as if it were an
// answer.
const validated = StructuredOutputResponse.safeParse(
structuredOutputResponse,
);
if (!validated.success) {
throw new Error(
`[OPTIMIZER] can't extract task and outcome - the response does not match the schema: ${validated.error.message}`,
);
}
return validated.data;
}
|