All files / api/src/ai-providers/triton api.ts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
import type { InternalProviderAPIConfig } from '@shared/types/ai-providers/config';
import { FunctionName } from '@shared/types/api/request';
 
const tritonAPIConfig: InternalProviderAPIConfig = {
  getBaseURL: ({ saTarget }) => {
    // Use custom host if provided, otherwise default to localhost with standard Triton port
    if (saTarget.custom_host) {
      return saTarget.custom_host;
    }
 
    // Default Triton HTTP port is 8000
    return 'http://localhost:8000';
  },
 
  headers: ({ saTarget }) => {
    const headers: Record<string, string> = {
      'Content-Type': 'application/json',
    };
 
    // Add API key authentication if provided
    if (saTarget.api_key) {
      headers.Authorization = `Bearer ${saTarget.api_key}`;
    }
 
    return headers;
  },
 
  getEndpoint: ({ saRequestData }) => {
    // Extract model name from request body for KServe v2 endpoints
    const model =
      ((saRequestData.requestBody as Record<string, unknown>)?.model as
        | string
        | undefined) || 'default';
    const encodedModel = encodeURIComponent(model);
 
    switch (saRequestData.functionName) {
      // Core inference endpoints using KServe v2 protocol
      case FunctionName.COMPLETE:
      case FunctionName.CHAT_COMPLETE:
        return `/v2/models/${encodedModel}/infer`;
 
      case FunctionName.STREAM_COMPLETE:
      case FunctionName.STREAM_CHAT_COMPLETE:
        // Triton handles streaming via parameters in the same endpoint
        return `/v2/models/${encodedModel}/infer`;
 
      // Embeddings
      case FunctionName.EMBED:
        return `/v2/models/${encodedModel}/infer`;
 
      // Model management endpoints
      case FunctionName.RETRIEVE_FILE: // Model metadata
        return `/v2/models/${encodedModel}`;
 
      case FunctionName.GET_BATCH_OUTPUT: // Model stats
        return `/v2/models/${encodedModel}/stats`;
 
      // Health endpoints
      case FunctionName.CREATE_SPEECH: // Server liveness
        return '/v2/health/live';
 
      case FunctionName.CREATE_TRANSCRIPTION: // Server readiness
        return '/v2/health/ready';
 
      case FunctionName.CREATE_TRANSLATION: // Model readiness
        return `/v2/models/${encodedModel}/ready`;
 
      // Server metadata
      case FunctionName.LIST_BATCHES: // Server info
        return '/v2';
 
      // Repository management
      case FunctionName.LIST_FILES: // List models
        return '/v2/repository/index';
 
      default:
        return '';
    }
  },
};
 
export default tritonAPIConfig;