Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | import { z } from 'zod';
// Template data for role adherence evaluation
export const RoleAdherenceTemplateDataSchema = z.object({
role_definition: z.string(),
assistant_output: z.string(),
instructions: z.string().optional(),
strict_mode: z.boolean().optional().default(false),
verbose_mode: z.boolean().optional().default(true),
include_reason: z.boolean().optional().default(true),
});
export type RoleAdherenceTemplateData = z.infer<
typeof RoleAdherenceTemplateDataSchema
>;
// Template config for prompts
export const RoleAdherenceTemplateConfigSchema = z.object({
systemPrompt: z.string(),
userPrompt: z.string(),
outputFormat: z.literal('json'),
});
export type RoleAdherenceTemplateConfig = z.infer<
typeof RoleAdherenceTemplateConfigSchema
>;
// Result schema returned by LLM
export const RoleAdherenceResultSchema = z.object({
criteria: z.object({
adhered_to_role: z.boolean(),
adherence_level: z.number().min(0).max(1),
violations: z.array(z.string()).optional().default([]),
}),
score: z.number().min(0).max(1),
reasoning: z.string().optional(),
overall_success: z.boolean().optional(),
});
export type RoleAdherenceResult = z.infer<typeof RoleAdherenceResultSchema>;
// Metadata captured in outputs
export type RoleAdherenceMetadata = {
role_definition?: string;
assistant_output?: string;
instructions?: string;
criteria?: {
description?: string;
strict_mode?: boolean;
verbose_mode?: boolean;
include_reason?: boolean;
};
parsed_with_schema?: boolean;
};
// Scoring guidelines specific to role adherence
export const roleAdherenceScoringGuidelines = {
perfect: '1.0: Perfect adherence — fully aligned with role and constraints',
excellent: '0.9: Excellent adherence — minor non-impacting deviations',
good: '0.7-0.8: Good adherence — mostly aligned with some issues',
adequate: '0.5-0.6: Adequate — mixed adherence; notable issues present',
poor: '0.3-0.4: Poor adherence — frequent deviations from role',
failed: '0.0-0.2: Failed — clear disregard of role or unsafe behavior',
} as const;
export const roleAdherenceScoringText = Object.values(
roleAdherenceScoringGuidelines,
).join('\n- ');
// Average result type for dataset evaluations
export interface RoleAdherenceAverageResult {
average_score: number;
total_logs: number;
passed_count: number;
failed_count: number;
threshold_used: number;
evaluation_run_id: string;
}
// AI-modifiable parameters - none needed, evaluation works automatically
export const RoleAdherenceEvaluationAIParameters = z.object({});
// Full parameters including user-modifiable settings
// Note: model is configured via evaluation.model_id, not in parameters
export const RoleAdherenceEvaluationParameters =
RoleAdherenceEvaluationAIParameters.extend({
threshold: z.number().min(0).max(1).default(0.7),
include_reason: z.boolean().default(true),
strict_mode: z.boolean().default(false),
async_mode: z.boolean().default(false),
verbose_mode: z.boolean().default(false),
temperature: z.number().min(0).max(2).default(0.1),
max_tokens: z.number().positive().default(1000),
batch_size: z.number().positive().default(1000),
role_definition: z.string().optional(),
assistant_output: z.string().optional(),
instructions: z.string().optional(),
});
export type RoleAdherenceEvaluationParameters = z.infer<
typeof RoleAdherenceEvaluationParameters
>;
|