All files / api/src/connectors/evaluations/role-adherence/service role-adherence-criteria.ts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
import { getRoleAdherenceMainTemplate } from '@api/connectors/evaluations/role-adherence/templates/main';
import type { RoleAdherenceMetadata } from '@api/connectors/evaluations/role-adherence/types';
import { RoleAdherenceResultSchema } from '@api/connectors/evaluations/role-adherence/types';
 
import { createLLMJudge } from '@api/evaluations/llm-judge';
import type {
  GenericEvaluationInput,
  GenericEvaluator,
} from '@api/types/evaluations/generic';
import type { LLMJudgeResult } from '@api/types/evaluations/llm-judge';
import type { AppContext } from '@api/types/hono';
import { z } from 'zod';
 
export const RoleAdherenceCriteriaSchema = z.object({
  description: z.string().optional(),
  strict_mode: z.boolean().optional().default(false),
  verbose_mode: z.boolean().optional().default(true),
  include_reason: z.boolean().optional().default(true),
});
export type RoleAdherenceCriteria = z.infer<typeof RoleAdherenceCriteriaSchema>;
 
export const RoleAdherenceInputSchema = z.object({
  role_definition: z.string(),
  assistant_output: z.string(),
  instructions: z.string().optional(),
  criteria: RoleAdherenceCriteriaSchema.optional(),
});
export type RoleAdherenceInput = z.infer<typeof RoleAdherenceInputSchema>;
 
function parseRoleAdherenceResult(response: LLMJudgeResult): LLMJudgeResult {
  if (response.reasoning.trim().startsWith('{')) {
    try {
      const parsed = JSON.parse(response.reasoning);
      const validated = RoleAdherenceResultSchema.parse(parsed);
      return {
        score: validated.score,
        reasoning: validated.reasoning || response.reasoning,
        metadata: {
          ...response.metadata,
          criteria: validated.criteria,
          overall_success: validated.overall_success,
          parsed_with_schema: true,
        },
      };
    } catch {
      // ignore parse errors, fall through
    }
  }
  return {
    ...response,
    metadata: { ...response.metadata, parsed_with_schema: false },
  };
}
 
async function evaluateRoleAdherenceWithJudge(
  c: AppContext,
  input: RoleAdherenceInput,
): Promise<LLMJudgeResult> {
  const llmJudge = createLLMJudge(c);
  const tpl = getRoleAdherenceMainTemplate({
    role_definition: input.role_definition,
    assistant_output: input.assistant_output,
    instructions: input.instructions,
    strict_mode: input.criteria?.strict_mode || false,
    verbose_mode: input.criteria?.verbose_mode ?? true,
    include_reason: input.criteria?.include_reason ?? true,
  });
 
  const result = await llmJudge.evaluate({
    text: `${tpl.systemPrompt}\n\n${tpl.userPrompt}`,
    evaluationCriteria: input.criteria?.description
      ? { criteria: [input.criteria.description] }
      : undefined,
  });
 
  return parseRoleAdherenceResult(result);
}
 
export function createRoleAdherenceEvaluator(
  c: AppContext,
): GenericEvaluator<RoleAdherenceMetadata> & {
  evaluateRoleAdherence: (input: RoleAdherenceInput) => Promise<LLMJudgeResult>;
} {
  const llmJudge = createLLMJudge(c);
  return {
    evaluate: async (input: GenericEvaluationInput<RoleAdherenceMetadata>) => {
      const role_definition = input.metadata?.role_definition || '';
      const assistant_output = input.metadata?.assistant_output || '';
      const instructions = input.metadata?.instructions || undefined;
      const criteria = input.metadata?.criteria
        ? {
            description: input.metadata.criteria.description,
            strict_mode: input.metadata.criteria.strict_mode || false,
            verbose_mode: input.metadata.criteria.verbose_mode ?? true,
            include_reason: input.metadata.criteria.include_reason ?? true,
          }
        : undefined;
 
      const tpl = getRoleAdherenceMainTemplate({
        role_definition,
        assistant_output,
        instructions,
        strict_mode: criteria?.strict_mode || false,
        verbose_mode: criteria?.verbose_mode ?? true,
        include_reason: criteria?.include_reason ?? true,
      });
 
      const result = await llmJudge.evaluate({
        text: `${tpl.systemPrompt}\n\n${tpl.userPrompt}`,
        evaluationCriteria: criteria?.description
          ? { criteria: [criteria.description] }
          : undefined,
      });
 
      const parsed = parseRoleAdherenceResult(result);
      return {
        score: parsed.score,
        reasoning: parsed.reasoning,
        metadata: parsed.metadata as RoleAdherenceMetadata,
      };
    },
    evaluateRoleAdherence: async (input: RoleAdherenceInput) => {
      return await evaluateRoleAdherenceWithJudge(c, input);
    },
    config: llmJudge.config,
  };
}
 
export async function evaluateRoleAdherence(
  c: AppContext,
  input: RoleAdherenceInput,
): Promise<LLMJudgeResult> {
  return await evaluateRoleAdherenceWithJudge(c, input);
}