Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 | 1x 1x | import type {
EvaluationMethodConnector,
UserDataStorageConnector,
} from '@api/types/connector';
import type { AppContext } from '@api/types/hono';
import { error } from '@shared/console-logging';
import type {
Log,
SkillOptimizationEvaluation,
SkillOptimizationEvaluationResult,
} from '@shared/types/data';
import type { EvaluationMethodName } from '@shared/types/evaluations';
/**
* Run realtime evaluations for a single log using skill optimization evaluations
*/
export async function runEvaluationsForLog(
c: AppContext,
log: Log,
skillOptimizationEvaluations: SkillOptimizationEvaluation[],
evaluationConnectorsMap: Partial<
Record<EvaluationMethodName, EvaluationMethodConnector>
>,
storageConnector: UserDataStorageConnector,
): Promise<SkillOptimizationEvaluationResult[]> {
if (skillOptimizationEvaluations.length === 0) {
return [];
}
// Execute all evaluations in parallel for better performance
const evaluationPromises = skillOptimizationEvaluations.map(
async (evaluation) => {
try {
const connector = evaluationConnectorsMap[evaluation.evaluation_method];
if (!connector || !connector.evaluateLog) {
error(
`[REALTIME_EVAL] No connector found for evaluation method: ${evaluation.evaluation_method}`,
);
return;
}
// Use the evaluation ID for skill optimization evaluations
return await connector.evaluateLog(
c,
evaluation,
log,
storageConnector,
);
} catch (err) {
// Don't throw - we want other evaluations to continue even if one fails
error(
`[REALTIME_EVAL] Failed to evaluate log ${log.id} with method ${evaluation.evaluation_method}:`,
err instanceof Error ? err.message : String(err),
);
}
},
);
try {
return (await Promise.allSettled(evaluationPromises))
.filter((result) => result.status === 'fulfilled')
.map((result) => result.value)
.filter(
(value): value is SkillOptimizationEvaluationResult =>
value !== undefined,
);
} catch (e) {
if (e instanceof Error) {
throw new Error('Error in skill optimization evaluations batch:', e);
}
throw new Error(
'Error in skill optimization evaluations batch: unknown error',
);
}
}
/**
* Check if a request should trigger realtime evaluations
*/
export function shouldTriggerRealtimeEvaluation(
status: number,
url: URL,
): boolean {
// Only trigger on successful responses
if (status !== 200) {
return false;
}
// Only evaluate actual AI provider calls, not internal Super Agents API calls
if (!url.pathname.startsWith('/v1/')) {
return false;
}
// Don't evaluate Super Agents internal API calls
if (url.pathname.startsWith('/v1/super-agents')) {
return false;
}
return true;
}
|