Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | 1x 1x 1x 1x 1x 1x 1x 1x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x | import type {
ConversationCompletenessEvaluationParameters,
ConversationCompletenessResult,
} from '@api/connectors/evaluations/conversation-completeness/types';
import {
createLLMJudge,
type LLMJudgeModelConfig,
} from '@api/evaluations/llm-judge';
import type { UserDataStorageConnector } from '@api/types/connector';
import type { AppContext } from '@api/types/hono';
import { resolveEvaluationModelConfig } from '@api/utils/evaluation-model-resolver';
import { formatMessagesForExtraction } from '@api/utils/messages';
import { extractMessagesFromRequestData } from '@api/utils/super-agents/requests';
import { extractOutputFromResponseBody } from '@api/utils/super-agents/responses';
import type {
ChatCompletionRequestData,
ResponsesRequestData,
StreamChatCompletionRequestData,
} from '@shared/types/api/request';
import { SuperAgentsResponseBody } from '@shared/types/api/response';
import type {
SkillOptimizationEvaluation,
SkillOptimizationEvaluationResult,
} from '@shared/types/data';
import type { Log } from '@shared/types/data/log';
import { EvaluationMethodName } from '@shared/types/evaluations';
import { produceSuperAgentsRequestData } from '@shared/utils/sa-request-data';
/**
* Evaluate conversation completeness for a single log
*/
export async function evaluateConversationCompleteness(
c: AppContext,
log: Log,
params: ConversationCompletenessEvaluationParameters,
modelConfig?: LLMJudgeModelConfig | null,
): Promise<ConversationCompletenessResult> {
// Create LLM judge instance with resolved model config
const llmJudge = createLLMJudge(
c,
{
temperature: params.temperature,
max_tokens: params.max_tokens,
},
modelConfig ?? undefined,
);
// Extract messages and outputs using standard utilities
const saRequestData = produceSuperAgentsRequestData(
log.ai_provider_request_log.method,
log.ai_provider_request_log.request_url,
{},
log.ai_provider_request_log.request_body,
);
const responseBody = SuperAgentsResponseBody.parse(
log.ai_provider_request_log.response_body,
);
const messages = extractMessagesFromRequestData(
saRequestData as
| ChatCompletionRequestData
| StreamChatCompletionRequestData
| ResponsesRequestData,
);
const input = formatMessagesForExtraction(messages);
const output = extractOutputFromResponseBody(responseBody);
// Create a simple evaluation prompt that won't trigger template-based evaluation
const evaluationText = `Analyze the following conversation for completeness quality. CONVERSATION: ${input} ASSISTANT RESPONSE: ${output} Consider how well the assistant completes the conversation by satisfying user needs. Look for: Whether all user intentions were identified and addressed, if the conversation feels complete and resolved, whether there are any unresolved user requests, and the overall satisfaction of user needs throughout the conversation. Provide a score between 0 and 1 with detailed reasoning for your analysis.`;
// Evaluate using LLM judge with conversation completeness criteria
const result = await llmJudge.evaluate({
text: evaluationText,
evaluationCriteria: {
criteria: [
'Extract all user intentions from the conversation',
'Identify what the user is trying to accomplish',
'Assess whether each user intention was satisfied by the assistant',
'Evaluate the completeness of the conversation in addressing user needs',
'Check for unresolved user requests or incomplete responses',
'Calculate the conversation completeness score based on the formula: (Number of Satisfied User Intentions) / (Total Number of User Intentions)',
],
},
});
return {
score: result.score,
reasoning: result.reasoning,
metadata: result.metadata,
};
}
export async function evaluateLog(
c: AppContext,
evaluation: SkillOptimizationEvaluation,
log: Log,
storageConnector: UserDataStorageConnector,
): Promise<SkillOptimizationEvaluationResult> {
const params =
evaluation.params as ConversationCompletenessEvaluationParameters;
const start_time = Date.now();
// Resolve model configuration from evaluation.model_id or system settings
const modelConfig = await resolveEvaluationModelConfig(
c,
evaluation,
storageConnector,
);
// Evaluate the log using the existing function
const result = await evaluateConversationCompleteness(
c,
log,
params,
modelConfig,
);
const execution_time = Date.now() - start_time;
const judgeModelName = modelConfig?.model ?? null;
const judgeModelProvider = modelConfig?.provider ?? null;
const evaluationResult: SkillOptimizationEvaluationResult = {
evaluation_id: evaluation.id,
method: EvaluationMethodName.CONVERSATION_COMPLETENESS,
score: result.score,
extra_data: {
reasoning: result.reasoning,
metadata: result.metadata,
execution_time,
execution_time_ms: execution_time,
evaluated_at: new Date().toISOString(),
},
display_info: [
{
label: 'Reasoning',
content: result.reasoning,
},
...(result.metadata
? [
{
label: 'Analysis Details',
content: JSON.stringify(result.metadata, null, 2),
},
]
: []),
],
judge_model_name: judgeModelName,
judge_model_provider: judgeModelProvider,
};
return evaluationResult;
}
|