Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | 1x 1x 1x 1x 1x 1x 1x 1x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x | import type { KnowledgeRetentionEvaluationParameters } from '@api/connectors/evaluations/knowledge-retention/types';
import { createLLMJudge } from '@api/evaluations/llm-judge';
import type { UserDataStorageConnector } from '@api/types/connector';
import type { AppContext } from '@api/types/hono';
import { resolveEvaluationModelConfig } from '@api/utils/evaluation-model-resolver';
import { formatMessagesForExtraction } from '@api/utils/messages';
import { extractMessagesFromRequestData } from '@api/utils/super-agents/requests';
import { extractOutputFromResponseBody } from '@api/utils/super-agents/responses';
import type {
ChatCompletionRequestData,
ResponsesRequestData,
StreamChatCompletionRequestData,
} from '@shared/types/api/request';
import { SuperAgentsResponseBody } from '@shared/types/api/response';
import type {
SkillOptimizationEvaluation,
SkillOptimizationEvaluationResult,
} from '@shared/types/data';
import type { Log } from '@shared/types/data/log';
import { EvaluationMethodName } from '@shared/types/evaluations';
import { produceSuperAgentsRequestData } from '@shared/utils/sa-request-data';
export async function evaluateLog(
c: AppContext,
evaluation: SkillOptimizationEvaluation,
log: Log,
storageConnector: UserDataStorageConnector,
): Promise<SkillOptimizationEvaluationResult> {
const params = evaluation.params as KnowledgeRetentionEvaluationParameters;
// Resolve model configuration from evaluation.model_id or system settings
const modelConfig = await resolveEvaluationModelConfig(
c,
evaluation,
storageConnector,
);
const llmJudge = createLLMJudge(
c,
{
temperature: params.temperature,
max_tokens: params.max_tokens,
},
modelConfig ?? undefined,
);
const start_time = Date.now();
// Extract messages and outputs using standard utilities
const saRequestData = produceSuperAgentsRequestData(
log.ai_provider_request_log.method,
log.ai_provider_request_log.request_url,
{},
log.ai_provider_request_log.request_body,
);
const responseBody = SuperAgentsResponseBody.parse(
log.ai_provider_request_log.response_body,
);
const messages = extractMessagesFromRequestData(
saRequestData as
| ChatCompletionRequestData
| StreamChatCompletionRequestData
| ResponsesRequestData,
);
const input = formatMessagesForExtraction(messages);
const output = extractOutputFromResponseBody(responseBody);
// Create evaluation prompt
const evaluationText = `Analyze the following conversation for knowledge retention quality. CONVERSATION: ${input} ASSISTANT RESPONSE: ${output} Consider how well the assistant retains and recalls information provided by the user throughout the conversation. Look for: Knowledge retention vs. knowledge attrition patterns, consistency in recalling previously mentioned information, ability to maintain context across multiple turns, and specific instances where information was retained or lost. For single-turn conversations, assess if the assistant would be able to retain the information for future reference. Provide a score between 0 and 1 with detailed reasoning for your analysis.`;
// Evaluate using LLM judge
const result = await llmJudge.evaluate({
text: evaluationText,
outputFormat: 'json',
});
const execution_time = Date.now() - start_time;
const judgeModelName = modelConfig?.model ?? null;
const judgeModelProvider = modelConfig?.provider ?? null;
const evaluationResult: SkillOptimizationEvaluationResult = {
evaluation_id: evaluation.id,
method: EvaluationMethodName.KNOWLEDGE_RETENTION,
score: result.score,
extra_data: {
reasoning: result.reasoning,
knowledgeRetention: result.metadata?.knowledgeRetention,
metadata: result.metadata,
execution_time,
execution_time_ms: execution_time,
evaluated_at: new Date().toISOString(),
},
display_info: [
{
label: 'Reasoning',
content: result.reasoning,
},
...(result.metadata?.knowledgeRetention
? [
{
label: 'Knowledge Retention Analysis',
content: JSON.stringify(
result.metadata.knowledgeRetention,
null,
2,
),
},
]
: []),
],
judge_model_name: judgeModelName,
judge_model_provider: judgeModelProvider,
};
return evaluationResult;
}
|