
    Cj                        S r SSKrSSKrSSKJr  SSKJrJrJrJ	r	J
r
Jr  SSKrSSKrSSKJr  SSKJr  SSKJr  SSKJr  SS	KJrJrJrJrJrJrJrJrJ r J!r!J"r"J#r#J$r$  SS
K%J&r&  SSK'J(r(J)r)  \" 5       r*Sr+\)       S5S\\,\4   S\	\\,\4      S\
\,   S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\4S jj5       r/\)       S5S\\,\4   S\	\\,\4      S\
\,   S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\\\\\4   4   4S jj5       r0\)         S6S\
\1   S\
\,   S\
\,   S\
\,   S\
\,   S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\4S jj5       r2\)         S6S\
\1   S\
\,   S\
\,   S\
\,   S\
\,   S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\\\\\4   4   4S jj5       r3\)    S7S \,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\4S! jj5       r4\)    S7S \,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\\\\\4   4   4S" jj5       r5\)       S5S \,S\
\,   S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\4S# jj5       r6\)       S5S \,S\
\,   S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\\\\\4   4   4S$ jj5       r7\)    S7S \,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\4S% jj5       r8\)    S7S \,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\\\\\4   4   4S& jj5       r9\)    S7S \,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\4S' jj5       r:\)    S7S \,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\\\\\4   4   4S( jj5       r;\)       S5S \,S)\\,\4   S\
\,   S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\"4S* jj5       r<\)       S5S \,S)\\,\4   S\
\,   S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\"\\\\"4   4   4S+ jj5       r=\)        S8S \,S\
\1   S\
\,   S\
\,   S\
\,   S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\!4S, jj5       r>\)        S8S \,S\
\1   S\
\,   S\
\,   S\
\,   S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\!\\\\!4   4   4S- jj5       r?\)    S7S \,S.\,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\"4S/ jj5       r@\)    S7S \,S.\,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\"\\\\"4   4   4S0 jj5       rA\)    S7S \,S.\,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\4S1 jj5       rB\)    S7S \,S.\,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\\\\\4   4   4S2 jj5       rC\)    S7S \,S.\,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\#4S3 jj5       rD\)    S7S \,S.\,S\
\\,\4      S\
\\,\4      S\
\\-\R\                  4      S\
\,   S\\#\\\\#4   4   4S4 jj5       rEg)9zw
Main entry point for Evals API operations
Provides create, list, get, update, delete, and cancel operations for evals
    N)partial)Any	CoroutineDictListOptionalUnion)request_timeout)Logging)BaseEvalsAPIConfig)BaseLLMHTTPHandler)CancelEvalResponseCancelRunResponseCreateEvalRequestCreateRunRequestDeleteEvalResponseEvalListEvalsParamsListEvalsResponseListRunsParamsListRunsResponseRunRunDeleteResponseUpdateEvalRequest)GenericLiteLLMParams)ProviderConfigManagerclientzhttps://api.openai.comdata_source_configtesting_criterianamemetadataextra_headersextra_query
extra_bodytimeoutcustom_llm_providerreturnc	                   #    [        5       n
 [        R                  " 5       nSU	S'   [        [        4U UUUUUUUUS.	U	D6n[
        R                  " 5       n[        UR                  U5      nUR                  SU5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUU
U	S9eSnAff = f7f)a)  
Async: Create a new evaluation

Args:
    data_source_config: Configuration for the data source
    testing_criteria: List of graders for all eval runs
    name: Optional name for the evaluation
    metadata: Optional additional metadata (max 16 key-value pairs)
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    extra_body: Additional body parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Eval object
Tacreate_eval)	r   r   r    r!   r"   r#   r$   r%   r&   Nmodelr&   original_exceptioncompletion_kwargsextra_kwargs)localsasyncioget_event_loopr   create_evalcontextvarscopy_contextrunrun_in_executoriscoroutine	Exceptionlitellmexception_type)r   r   r    r!   r"   r#   r$   r%   r&   kwargs
local_varsloopfuncctxfunc_with_contextinit_responseresponsees                     d/home/rurouni/.local/share/pipx/venvs/strix-agent/lib/python3.13/site-packages/litellm/evals/main.pyr)   r)   )   s    > J"
%%'!%~
1-'#! 3
 
 &&(#CGGT2"2249JKK}--**H  %H L +  
$$ 3 (
 	

Y   C A4B6 B2$B6 'B4(B6 -C .B6 1C 2B6 4B6 6
C CCC c	                    [        5       n
 U	R                  S5      nU	R                  SS5      nU	R                  SS5      SL n[        S0 U	D6nUc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      eU US
.nUb  UUS'   U(       a  UR                  U5        U=(       d    0 nUR                  UUS9nUR                  UUUS9nUR                  =(       d    [        nUR                  USS9nUR                  U	SUSU0US9  [         R#                  UUUUUUUU=(       d    [$        UU	R                  S5      U	R                  S5      S9nU$ ! [&         a  n[        R(                  " SUUU
U	S9eSnAff = f)a"  
Create a new evaluation

Args:
    data_source_config: Configuration for the data source
    testing_criteria: List of graders for all eval runs
    name: Optional name for the evaluation
    metadata: Optional additional metadata (max 16 key-value pairs)
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    extra_body: Additional body parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Eval object
litellm_logging_objlitellm_call_idNr)   FTopenaiproviderz!CREATE eval is not supported for )r   r   r    headerslitellm_params)create_requestrN   rM   evals)api_baseendpointr;   r+   optional_paramsrN   r&   r   shared_sessionurlrequest_bodyevals_api_provider_configr&   rN   logging_objr"   r%   	_is_asyncr   rU   r*    )r/   getpopr   r   get_provider_evals_api_configr9   LlmProviders
ValueErrorupdatevalidate_environmenttransform_create_eval_requestrQ   DEFAULT_OPENAI_API_BASEget_complete_urlupdate_from_kwargsbase_llm_http_handlercreate_eval_handlerr
   r8   r:   )r   r   r    r!   r"   r#   r$   r%   r&   r;   r<   rG   rH   r[   rN   rY   rO   rM   rX   rQ   rW   rB   rC   s                          rD   r2   r2   n   s   > JW
17<Q1R)/4Et)LJJ~u5=	 .77 &"* "?? --.AB 	" %,@AT@UVWW #5 0-
 %)N6" !!*-  %2+@@N A 

 1NN)) O 
 "**E.E'88 9 

 	..(!? !4 	/ 	
 )<<%&? 3)+!.::h'!::&67 = 
  
$$ 3 (
 	

s   EE% %
F/FFlimitafterbeforeorderorder_byc	                   #    [        5       n
 [        R                  " 5       nSU	S'   [        [        4U UUUUUUUUS.	U	D6n[
        R                  " 5       n[        UR                  U5      nUR                  SU5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUU
U	S9eSnAff = f7f)ay  
Async: List all evaluations

Args:
    limit: Number of results to return per page (max 100, default 20)
    after: Cursor for pagination - returns evals after this ID
    before: Cursor for pagination - returns evals before this ID
    order: Sort order ('asc' or 'desc', default 'desc')
    order_by: Field to sort by ('created_at' or 'updated_at', default 'created_at')
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    ListEvalsResponse object
Talist_evals)	rj   rk   rl   rm   rn   r"   r#   r%   r&   Nr*   )r/   r0   r1   r   
list_evalsr3   r4   r5   r6   r7   r8   r9   r:   )rj   rk   rl   rm   rn   r"   r#   r%   r&   r;   r<   r=   r>   r?   r@   rA   rB   rC   s                     rD   rp   rp      s    > J"
%%' $}
'# 3
 
 &&(#CGGT2"2249JKK}--**H  %H L +  
$$ 3 (
 	

rE   c	                    [        5       n
 U	R                  S5      nU	R                  SS5      nU	R                  SS5      SL n[        S0 U	D6nUc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      e0 nU b  U US
'   Ub  UUS'   Ub  UUS'   Ub  UUS'   Ub  UUS'   U(       a  UR                  U5        U=(       d    0 nUR                  UUS9nUR                  UUUS9u  nnUR                  U	SUSU0US9  [        R                  UUUUUUUU=(       d    [        UU	R                  S5      U	R                  S5      S9nU$ ! [          a  n[        R"                  " SUUU
U	S9eSnAff = f)ar  
List all evaluations

Args:
    limit: Number of results to return per page (max 100, default 20)
    after: Cursor for pagination - returns evals after this ID
    before: Cursor for pagination - returns evals before this ID
    order: Sort order ('asc' or 'desc', default 'desc')
    order_by: Field to sort by ('created_at' or 'updated_at', default 'created_at')
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    ListEvalsResponse object
rG   rH   Nrp   FTrI   rJ   z LIST evals is not supported for rj   rk   rl   rm   rn   rL   )list_paramsrN   rM   rS   r   rU   rW   query_paramsrY   r&   rN   rZ   r"   r%   r[   r   rU   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   rb   rc   transform_list_evals_requestrg   rh   list_evals_handlerr
   r8   r:   )rj   rk   rl   rm   rn   r"   r#   r%   r&   r;   r<   rG   rH   r[   rN   rY   rs   rM   rW   ru   rB   rC   s                         rD   rq   rq   -  s   > JV
17<Q1R)/4Et)LJJ}e4<	 .77 &"* "?? --.AB 	" %,?@S?TUVV (*#(K #(K $*K!#(K &.K
# {+  %2+@@N A 

 6RR#) S 
\ 	..(!? !4 	/ 	
 );;%&? 3)+!.::h'!::&67 < 
  
$$ 3 (
 	

s   EE 
F&E>>Feval_idc           	        #    [        5       n [        R                  " 5       nSUS'   [        [        4U UUUUS.UD6n[
        R                  " 5       n	[        U	R                  U5      n
UR                  SU
5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUUUS9eSnAff = f7f)aL  
Async: Get an evaluation by ID

Args:
    eval_id: The ID of the evaluation to fetch
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Eval object
T	aget_evalrx   r"   r#   r%   r&   Nr*   )r/   r0   r1   r   get_evalr3   r4   r5   r6   r7   r8   r9   r:   rx   r"   r#   r%   r&   r;   r<   r=   r>   r?   r@   rA   rB   rC   s                 rD   rz   rz     s     . J
%%'"{
'# 3
 
 &&(#CGGT2"2249JKK}--**H  %H L +  
$$ 3 (
 	

Y   CA0B2 >B.?$B2 #B0$B2 )C*B2 -C.B2 0B2 2
C<CCCc                    [        5       n UR                  S5      nUR                  SS5      nUR                  SS5      SL n	[        S0 UD6n
Uc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      eU=(       d    0 nUR                  XS
9nU
R                  =(       d    [        nUR                  U UU
US9u  pUR                  USSU 0SU0US9  [        R                  UUUU
UUU=(       d    [         U	UR                  S5      UR                  S5      S9
nU$ ! ["         a  n[        R$                  " SUUUUS9eSnAff = f)aE  
Get an evaluation by ID

Args:
    eval_id: The ID of the evaluation to fetch
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Eval object
rG   rH   Nrz   FTrI   rJ   zGET eval is not supported for rL   rx   rQ   rN   rM   rx   rS   r   rU   
rW   rY   r&   rN   rZ   r"   r%   r[   r   rU   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   rc   rQ   re   transform_get_eval_requestrg   rh   get_eval_handlerr
   r8   r:   rx   r"   r#   r%   r&   r;   r<   rG   rH   r[   rN   rY   rM   rQ   rW   rB   rC   s                    rD   r|   r|     s   . JF
17<Q1R)/4Et)LJJ{E2d:	 .77 &"* "?? --.AB 	" %,=>Q=RSTT  %2+@@ A 

 "**E.E0KK)	 L 
 	..&0!? !4 	/ 	
 )99&? 3)+!.::h'!::&67 : 
  
$$ 3 (
 	

   D&D3 3
E=EEc                   #    [        5       n	 [        R                  " 5       n
SUS'   [        [        4U UUUUUUUS.UD6n[
        R                  " 5       n[        UR                  U5      nU
R                  SU5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUU	US9eSnAff = f7f)a  
Async: Update an evaluation

Args:
    eval_id: The ID of the evaluation to update
    name: Updated name
    metadata: Updated metadata
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    extra_body: Additional body parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Eval object
Taupdate_eval)rx   r    r!   r"   r#   r$   r%   r&   Nr*   )r/   r0   r1   r   update_evalr3   r4   r5   r6   r7   r8   r9   r:   )rx   r    r!   r"   r#   r$   r%   r&   r;   r<   r=   r>   r?   r@   rA   rB   rC   s                    rD   r   r   @  s     : J!
%%'!%~
'#! 3
 
 &&(#CGGT2"2249JKK}--**H  %H L +  
$$ 3 (
 	

sY   CA3B5 B1$B5 &B3'B5 ,C-B5 0C1B5 3B5 5
C?CCCc                    [        5       n	 UR                  S5      n
UR                  SS5      nUR                  SS5      SL n[        S0 UD6nUc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      e0 nUb  XS
'   Ub>  1 SknUR                  5        VVs0 s H  u  nnUU;  d  M  UU_M     nnnU(       a  UUS'   U(       a  UR                  U5        U=(       d    0 nUR                  UUS9nUR                  =(       d    [        nUR                  U UUUUS9u  nnnU
R                  USUSU0US9  [         R#                  UUUUUU
UU=(       d    [$        UUR                  S5      UR                  S5      S9nU$ s  snnf ! [&         a  n[        R(                  " SUUU	US9eSnAff = f)a  
Update an evaluation

Args:
    eval_id: The ID of the evaluation to update
    name: Updated name
    metadata: Updated metadata
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    extra_body: Additional body parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Eval object
rG   rH   Nr   FTrI   rJ   z!UPDATE eval is not supported for r    >   rM   rR   
user_agentuser_api_keyuser_api_key_authuser_api_key_hashrequester_metadatauser_api_key_aliasuser_api_key_spendlitellm_api_versionuser_api_key_org_idrequester_ip_addressuser_api_key_team_iduser_api_key_user_iduser_api_key_metadatauser_api_key_max_budgetuser_api_key_team_aliasuser_api_key_team_spenduser_api_key_user_emailuser_api_key_user_spendlitellm_parent_otel_spanuser_api_key_end_user_iduser_api_key_auth_metadatauser_api_key_request_routeglobal_max_parallel_requestsuser_api_end_user_max_budgetuser_api_key_budget_reset_atuser_api_key_team_max_budgetuser_api_key_user_max_budgetuser_api_key_model_max_budgetr!   rL   )rx   update_requestrQ   rN   rM   rS   r   rU   rV   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   itemsrb   rc   rQ   re   transform_update_eval_requestrg   rh   update_eval_handlerr
   r8   r:   )rx   r    r!   r"   r#   r$   r%   r&   r;   r<   rG   rH   r[   rN   rY   r   internal_keyskvfiltered_metadatarM   rQ   rW   rX   rB   rC   s                             rD   r   r     sO   : J
17<Q1R)/4Et)LJJ~u5=	 .77 &"* "?? --.AB 	" %,@AT@UVWW -/%)6" MD "*!1!!1AQm5K1!1  ! !->z* !!*-  %2+@@N A 

 "**E.E
 &CC)) D 
		
 	..(!? !4 	/ 	
 )<<%&? 3)+!.::h'!::&67 = 
 q!r  
$$ 3 (
 	

s1   B#F /F FCF F 
G(G  Gc           	        #    [        5       n [        R                  " 5       nSUS'   [        [        4U UUUUS.UD6n[
        R                  " 5       n	[        U	R                  U5      n
UR                  SU
5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUUUS9eSnAff = f7f)aX  
Async: Delete an evaluation

Args:
    eval_id: The ID of the evaluation to delete
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    DeleteEvalResponse object
Tadelete_evalr{   Nr*   )r/   r0   r1   r   delete_evalr3   r4   r5   r6   r7   r8   r9   r:   r}   s                 rD   r   r   "       . J
%%'!%~
'# 3
 
 &&(#CGGT2"2249JKK}--**H  %H L +  
$$ 3 (
 	

r~   c                    [        5       n UR                  S5      nUR                  SS5      nUR                  SS5      SL n	[        S0 UD6n
Uc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      eU=(       d    0 nUR                  XS
9nU
R                  =(       d    [        nUR                  U UU
US9u  pUR                  USSU 0SU0US9  [        R                  UUUU
UUU=(       d    [         U	UR                  S5      UR                  S5      S9
nU$ ! ["         a  n[        R$                  " SUUUUS9eSnAff = f)aQ  
Delete an evaluation

Args:
    eval_id: The ID of the evaluation to delete
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    DeleteEvalResponse object
rG   rH   Nr   FTrI   rJ   z!DELETE eval is not supported for rL   r   rx   rS   r   rU   r   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   rc   rQ   re   transform_delete_eval_requestrg   rh   delete_eval_handlerr
   r8   r:   r   s                    rD   r   r   [  s   . JF
17<Q1R)/4Et)LJJ~u5=	 .77 &"* "?? --.AB 	" %,@AT@UVWW  %2+@@ A 

 "**E.E0NN)	 O 
 	..&0!? !4 	/ 	
 )<<&? 3)+!.::h'!::&67 = 
  
$$ 3 (
 	

r   c           	        #    [        5       n [        R                  " 5       nSUS'   [        [        4U UUUUS.UD6n[
        R                  " 5       n	[        U	R                  U5      n
UR                  SU
5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUUUS9eSnAff = f7f)a_  
Async: Cancel a running evaluation

Args:
    eval_id: The ID of the evaluation to cancel
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    CancelEvalResponse object
Tacancel_evalr{   Nr*   )r/   r0   r1   r   cancel_evalr3   r4   r5   r6   r7   r8   r9   r:   r}   s                 rD   r   r     r   r~   c                    [        5       n UR                  S5      nUR                  SS5      nUR                  SS5      SL n	[        S0 UD6n
Uc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      eU=(       d    0 nUR                  XS
9nU
R                  =(       d    [        nUR                  U UU
US9u  nnnUR                  USSU 0SU0US9  [        R                  UUUU
UUU=(       d    [         U	UR                  S5      UR                  S5      S9
nU$ ! ["         a  n[        R$                  " SUUUUS9eSnAff = f)aX  
Cancel a running evaluation

Args:
    eval_id: The ID of the evaluation to cancel
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    CancelEvalResponse object
rG   rH   Nr   FTrI   rJ   z!CANCEL eval is not supported for rL   r   rx   rS   r   rU   r   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   rc   rQ   re   transform_cancel_eval_requestrg   rh   cancel_eval_handlerr
   r8   r:   )rx   r"   r#   r%   r&   r;   r<   rG   rH   r[   rN   rY   rM   rQ   rW   rX   rB   rC   s                     rD   r   r     s   . JJ
17<Q1R)/4Et)LJJ~u5=	 .77 &"* "?? --.AB 	" %,@AT@UVWW  %2+@@ A 

 "**E.E
 &CC)	 D 
		
 	..&0!? !4 	/ 	
 )<<&? 3)+!.::h'!::&67 = 
  
$$ 3 (
 	

s   D(D5 5
E?EEdata_sourcec	                   #    [        5       n
 [        R                  " 5       nSU	S'   [        [        4U UUUUUUUUS.	U	D6n[
        R                  " 5       n[        UR                  U5      nUR                  SU5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUU
U	S9eSnAff = f7f)a4  
Async: Create a new run for an evaluation

Args:
    eval_id: The ID of the evaluation to run
    data_source: Data source configuration for the run (can be jsonl, completions, or responses type)
    name: Optional name for the run
    metadata: Optional additional metadata
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    extra_body: Additional body parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Run object
Tacreate_run)	rx   r   r    r!   r"   r#   r$   r%   r&   Nr*   )r/   r0   r1   r   
create_runr3   r4   r5   r6   r7   r8   r9   r:   )rx   r   r    r!   r"   r#   r$   r%   r&   r;   r<   r=   r>   r?   r@   rA   rB   rC   s                     rD   r   r   _  s    > J"
%%' $}
#'#! 3
 
 &&(#CGGT2"2249JKK}--**H  %H L +  
$$ 3 (
 	

rE   c	                 "   [        5       n
 U	R                  S5      nU	R                  SS5      nU	R                  SS5      SL n[        S0 U	D6nUc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      eS
U0nUb  UUS'   U(       a  UR                  U5        U=(       d    0 nUR                  UUS9nUR                  =(       d    [        nUR                  U UUUS9u  nnUR                  U	SUSU0US9  [        R!                  UUUUUUUU=(       d    ["        R$                  " SSS9UU	R                  S5      U	R                  S5      S9nU$ ! [&         a  n[        R(                  " SUUU
U	S9eSnAff = f)aX  
Create a new run for an evaluation

Args:
    eval_id: The ID of the evaluation to run
    data_source: Data source configuration for the run (can be jsonl, completions, or responses type)
    name: Optional name for the run
    metadata: Optional additional metadata
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    extra_body: Additional body parameters
    timeout: Request timeout (default 600s for long-running operations)
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Run object
rG   rH   Nr   FTrI   rJ   z CREATE run is not supported for r   r    rL   )rx   rO   rN   rM   rS   g     @g      @)r%   connectr   rU   rV   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   rb   rc   rQ   re   transform_create_run_requestrg   rh   create_run_handlerhttpxTimeoutr8   r:   )rx   r   r    r!   r"   r#   r$   r%   r&   r;   r<   rG   rH   r[   rN   rY   rO   rM   rQ   rW   rX   rB   rC   s                          rD   r   r     s   > JT
17<Q1R)/4Et)LJJ}e4<	 .77 &"* "?? --.AB 	" %,?@S?TUVV ;,
 %)N6"
 !!*-  %2+@@N A 

 "**E.E5RR))	 S 
\ 	..(!? !4 	/ 	
 );;%&? 3)+!Hu}}UCH::h'!::&67 < 
  
$$ 3 (
 	

s   EE' '
F1F		Fc	                   #    [        5       n
 [        R                  " 5       nSU	S'   [        [        4U UUUUUUUUS.	U	D6n[
        R                  " 5       n[        UR                  U5      nUR                  SU5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUU
U	S9eSnAff = f7f)aS  
Async: List all runs for an evaluation

Args:
    eval_id: The ID of the evaluation
    limit: Number of results to return per page (max 100, default 20)
    after: Cursor for pagination - returns runs after this ID
    before: Cursor for pagination - returns runs before this ID
    order: Sort order ('asc' or 'desc', default 'desc')
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    ListRunsResponse object
T
alist_runs)	rx   rj   rk   rl   rm   r"   r#   r%   r&   Nr*   )r/   r0   r1   r   	list_runsr3   r4   r5   r6   r7   r8   r9   r:   )rx   rj   rk   rl   rm   r"   r#   r%   r&   r;   r<   r=   r>   r?   r@   rA   rB   rC   s                     rD   r   r     s    > J"
%%'#|
'# 3
 
 &&(#CGGT2"2249JKK}--**H  %H L +  
$$ 3 (
 	

rE   c	                    [        5       n
 U	R                  S5      nU	R                  SS5      nU	R                  SS5      SL n[        S0 U	D6nUc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      e0 nUb  UUS
'   Ub  UUS'   Ub  UUS'   Ub  UUS'   U(       a  UR                  U5        U=(       d    0 nUR                  UUS9nUR                  U UUUS9u  nnUR                  U	SSU 0UESU0US9  [        R                  UUUUUUUU=(       d    [        UU	R                  S5      U	R                  S5      S9nU$ ! [          a  n[        R"                  " SUUU
U	S9eSnAff = f)aL  
List all runs for an evaluation

Args:
    eval_id: The ID of the evaluation
    limit: Number of results to return per page (max 100, default 20)
    after: Cursor for pagination - returns runs after this ID
    before: Cursor for pagination - returns runs before this ID
    order: Sort order ('asc' or 'desc', default 'desc')
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    ListRunsResponse object
rG   rH   Nr   FTrI   rJ   zLIST runs is not supported for rj   rk   rl   rm   rL   )rx   rs   rN   rM   rx   rS   r   rU   rt   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   rb   rc   transform_list_runs_requestrg   rh   list_runs_handlerr
   r8   r:   )rx   rj   rk   rl   rm   r"   r#   r%   r&   r;   r<   rG   rH   r[   rN   rY   rs   rM   rW   ru   rB   rC   s                         rD   r   r   `  s   > JU
17<Q1R)/4Et)LJJ|U3t;	 .77 &"* "?? --.AB 	" %,>?R>STUU ')#(K #(K $*K!#(K  {+  %2+@@N A 

 6QQ#)	 R 
\ 	..&@<@!? !4 	/ 	
 )::%&? 3)+!.::h'!::&67 ; 
  
$$ 3 (
 	

s   EE 
F #E;;F run_idc           
        #    [        5       n [        R                  " 5       nSUS'   [        [        4U UUUUUS.UD6n	[
        R                  " 5       n
[        U
R                  U	5      nUR                  SU5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUUUS9eSnAff = f7f)ag  
Async: Get a specific run

Args:
    eval_id: The ID of the evaluation
    run_id: The ID of the run to retrieve
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Run object
Taget_runrx   r   r"   r#   r%   r&   Nr*   )r/   r0   r1   r   get_runr3   r4   r5   r6   r7   r8   r9   r:   rx   r   r"   r#   r%   r&   r;   r<   r=   r>   r?   r@   rA   rB   rC   s                  rD   r   r     s     2 J
%%'!z	
'# 3	
 	
 &&(#CGGT2"2249JKK}--**H  %H L +  
$$ 3 (
 	

Y   CA1B3 ?B/ $B3 $B1%B3 *C+B3 .C/B3 1B3 3
C=CCCc                    [        5       n UR                  S5      nUR                  SS5      n	UR                  SS5      SL n
[        S0 UD6nUc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      eU=(       d    0 nUR                  XS
9nUR                  =(       d    [        nUR                  U UUUUS9u  pUR                  USXS.SU	0US9  [        R                  UUUUUUU=(       d    [         U
UR                  S5      UR                  S5      S9
nU$ ! ["         a  n[        R$                  " SUUUUS9eSnAff = f)a`  
Get a specific run

Args:
    eval_id: The ID of the evaluation
    run_id: The ID of the run to retrieve
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    Run object
rG   rH   Nr   FTrI   rJ   zGET run is not supported for rL   rx   r   rQ   rN   rM   rx   r   rS   r   rU   r   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   rc   rQ   re   transform_get_run_requestrg   rh   get_run_handlerr
   r8   r:   )rx   r   r"   r#   r%   r&   r;   r<   rG   rH   r[   rN   rY   rM   rQ   rW   rB   rC   s                     rD   r   r     s   2 JG
17<Q1R)/4Et)LJJz51T9	 .77 &"* "?? --.AB 	" %,<=P<QRSS  %2+@@ A 

 "**E.E0JJ) K 
 	..(/B!? !4 	/ 	
 )88&? 3)+!.::h'!::&67 9 
  
$$ 3 (
 	

s   D'D4 4
E>EEc           
        #    [        5       n [        R                  " 5       nSUS'   [        [        4U UUUUUS.UD6n	[
        R                  " 5       n
[        U
R                  U	5      nUR                  SU5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUUUS9eSnAff = f7f)au  
Async: Cancel a running run

Args:
    eval_id: The ID of the evaluation
    run_id: The ID of the run to cancel
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    CancelRunResponse object
Tacancel_runr   Nr*   )r/   r0   r1   r   
cancel_runr3   r4   r5   r6   r7   r8   r9   r:   r   s                  rD   r   r   x       2 J
%%' $}	
'# 3	
 	
 &&(#CGGT2"2249JKK}--**H  %H L +  
$$ 3 (
 	

r   c                    [        5       n UR                  S5      nUR                  SS5      n	UR                  SS5      SL n
[        S0 UD6nUc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      eU=(       d    0 nUR                  XS
9nUR                  =(       d    [        nUR                  U UUUUS9u  nnnUR                  USXS.SU	0US9  [        R                  UUUUUUU=(       d    [         U
UR                  S5      UR                  S5      S9
nU$ ! ["         a  n[        R$                  " SUUUUS9eSnAff = f)an  
Cancel a running run

Args:
    eval_id: The ID of the evaluation
    run_id: The ID of the run to cancel
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    CancelRunResponse object
rG   rH   Nr   FTrI   rJ   z CANCEL run is not supported for rL   r   r   rS   r   rU   r   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   rc   rQ   re   transform_cancel_run_requestrg   rh   cancel_run_handlerr
   r8   r:   rx   r   r"   r#   r%   r&   r;   r<   rG   rH   r[   rN   rY   rM   rQ   rW   rX   rB   rC   s                      rD   r   r        2 JK
17<Q1R)/4Et)LJJ}e4<	 .77 &"* "?? --.AB 	" %,?@S?TUVV  %2+@@ A 

 "**E.E
 &BB) C 
		
 	..(/B!? !4 	/ 	
 );;&? 3)+!.::h'!::&67 < 
  
$$ 3 (
 	

   D)D6 6
E EEc           
        #    [        5       n [        R                  " 5       nSUS'   [        [        4U UUUUUS.UD6n	[
        R                  " 5       n
[        U
R                  U	5      nUR                  SU5      I Sh  vN n[        R                  " U5      (       a  UI Sh  vN nU$ UnU$  N/ N! [         a  n[        R                  " SUUUUS9eSnAff = f7f)am  
Async: Delete a run

Args:
    eval_id: The ID of the evaluation
    run_id: The ID of the run to delete
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    RunDeleteResponse object
Tadelete_runr   Nr*   )r/   r0   r1   r   
delete_runr3   r4   r5   r6   r7   r8   r9   r:   r   s                  rD   r   r   !  r   r   c                    [        5       n UR                  S5      nUR                  SS5      n	UR                  SS5      SL n
[        S0 UD6nUc  Sn[        R
                  " [        R                  " U5      S9nUc  [        S	U 35      eU=(       d    0 nUR                  XS
9nUR                  =(       d    [        nUR                  U UUUUS9u  nnnUR                  USXS.SU	0US9  [        R                  UUUUUUU=(       d    [         U
UR                  S5      UR                  S5      S9
nU$ ! ["         a  n[        R$                  " SUUUUS9eSnAff = f)af  
Delete a run

Args:
    eval_id: The ID of the evaluation
    run_id: The ID of the run to delete
    extra_headers: Additional headers for the request
    extra_query: Additional query parameters
    timeout: Request timeout
    custom_llm_provider: Provider name (e.g., 'openai')
    **kwargs: Additional parameters

Returns:
    RunDeleteResponse object
rG   rH   Nr   FTrI   rJ   z DELETE run is not supported for rL   r   r   rS   r   rU   r   r*   r\   )r/   r]   r^   r   r   r_   r9   r`   ra   rc   rQ   re   transform_delete_run_requestrg   rh   delete_run_handlerr
   r8   r:   r   s                      rD   r   r   ]  r   r   )NNNNNNN)	NNNNNNNNN)NNNN)NNNNNNNN)F__doc__r0   r3   	functoolsr   typingr   r   r   r   r   r	   r   r9   litellm.constantsr
   *litellm.litellm_core_utils.litellm_loggingr   LiteLLMLoggingObj*litellm.llms.base_llm.evals.transformationr   *litellm.llms.custom_httpx.llm_http_handlerr   litellm.types.llms.openai_evalsr   r   r   r   r   r   r   r   r   r   r   r   r   litellm.types.routerr   litellm.utilsr   r   rh   re   strfloatr   r)   r2   intrp   rq   rz   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r\       rD   <module>r      sN  
    > >   - S I I    6 7 +, 2   )-.2,0+/59)-A
S#XA
4S>*A
 3-A
 tCH~&	A

 DcN+A
 $sCx.)A
 c3h(A
 eE5==012A
 "#A
 
A
 A
H  )-.2,0+/59)-v
S#Xv
4S>*v
 3-v
 tCH~&	v

 DcN+v
 $sCx.)v
 c3h(v
 eE5==012v
 "#v
 43T>**+v
 v
r  ".2,059)-A
C=A
C=A
 SMA
 C=	A

 smA
 DcN+A
 $sCx.)A
 eE5==012A
 "#A
 A
 A
H  ".2,059)-u
C=u
C=u
 SMu
 C=	u

 smu
 DcN+u
 $sCx.)u
 eE5==012u
 "#u
 iS2C(CDDEu
 u
p  /3,059)-5
5
DcN+5
 $sCx.)5
 eE5==012	5

 "#5
 
5
 5
p  /3,059)-]
]
DcN+]
 $sCx.)]
 eE5==012	]

 "#]
 43T>**+]
 ]
@  )-.2,0+/59)->
>

3->
 tCH~&>
 DcN+	>

 $sCx.)>
 c3h(>
 eE5==012>
 "#>
 
>
 >
B  )-.2,0+/59)-\
\

3-\
 tCH~&\
 DcN+	\

 $sCx.)\
 c3h(\
 eE5==012\
 "#\
 43T>**+\
 \
~  /3,059)-5
5
DcN+5
 $sCx.)5
 eE5==012	5

 "#5
 5
 5
p  /3,059)-]
]
DcN+]
 $sCx.)]
 eE5==012	]

 "#]
 yc3E)EFFG]
 ]
@  /3,059)-5
5
DcN+5
 $sCx.)5
 eE5==012	5

 "#5
 5
 5
p  /3,059)-a
a
DcN+a
 $sCx.)a
 eE5==012	a

 "#a
 yc3E)EFFGa
 a
R  )-.2,0+/59)-A
A
c3hA
 3-A
 tCH~&	A

 DcN+A
 $sCx.)A
 c3h(A
 eE5==012A
 "#A
 	A
 A
H  )-.2,0+/59)-s
s
c3hs
 3-s
 tCH~&	s

 DcN+s
 $sCx.)s
 c3h(s
 eE5==012s
 "#s
 3	#sC-(()s
 s
l    .2,059)-A
A
C=A
 C=A
 SM	A

 C=A
 DcN+A
 $sCx.)A
 eE5==012A
 "#A
 A
 A
H    .2,059)-t
t
C=t
 C=t
 SM	t

 C=t
 DcN+t
 $sCx.)t
 eE5==012t
 "#t
 YsC1A'ABBCt
 t
n  /3,059)-8
8
8
 DcN+8
 $sCx.)	8

 eE5==0128
 "#8
 	8
 8
v  /3,059)-`
`
`
 DcN+`
 $sCx.)	`

 eE5==012`
 "#`
 3	#sC-(()`
 `
F  /3,059)-8
8
8
 DcN+8
 $sCx.)	8

 eE5==0128
 "#8
 8
 8
v  /3,059)-d
d
d
 DcN+d
 $sCx.)	d

 eE5==012d
 "#d
 iS2C(CDDEd
 d
X  /3,059)-8
8
8
 DcN+8
 $sCx.)	8

 eE5==0128
 "#8
 8
 8
v  /3,059)-d
d
d
 DcN+d
 $sCx.)	d

 eE5==012d
 "#d
 iS2C(CDDEd
 d
r   