
    Cj3                      S SK Jr  S SKrS SKrS SKrS SKJrJr  S SKJrJ	r	J
r
JrJrJrJrJrJrJrJrJr  S SKrS SKJrJr  S SKJrJr  S SKJr  S SKJrJr  S S	K J!r!  S S
K"J#r#J$r$  S SK%J&r&J'r'J(r(J)r)  S SK*J+r+  S SK,J-r-  S SK.J/r/  S SK07  S SK0J1r1J2r2  S SK3J4r4  \(       a  S SK5J6r6  O\	r6 " S S\5      r7      SS jr8 SSS.         SS jjjr9SS jr:    S S jr;      S!S jr<S"S jr=g)#    )annotationsN)datetime	timedelta)TYPE_CHECKINGAny	AwaitableCallableDictListLiteralOptionalSequenceTupleUnioncast)print_verboseverbose_logger)validate_rate_limit_categoryvalidate_rate_limit_type)CustomLogger)PrometheusLabelFactoryContext)_get_cached_end_user_id_for_cost_tracking)BoundedPrometheusSeriesTracker) get_litellm_metadata_from_kwargs&get_metadata_variable_name_from_kwargs) LiteLLM_DeletedVerificationTokenLiteLLM_TeamTableLiteLLM_UserTableUserAPIKeyAuth)OrganizationRepository)TeamRepository)UserRepository)*)_sanitize_prometheus_label_name _sanitize_prometheus_label_value)StandardLoggingPayload)AsyncIOSchedulerc                  <   \ rS rSr\SYS j5       rS rSZS jrS[S jr    S\S jr	      S]S jr
S^S jrS_S	 jr    S`S
 jr        SaS jr      SbS jrScS jr    SdS jrSeS jrS r    SfS jr        SgS jr  Sh           SiS jjrS r Sj                   SkS jjr Sj       SlS jjr Sj     SmS jjr  Sn               SoS jjr Sj                   SpS jjr Sj         SqS jjr Sj               SrS jjrS r   Ss       StS jjr  Sj     SuS jjr!     Sv           SwS  jjr" Sj       SxS! jjr#    SyS" jr$SjSzS# jjr%    S{S$ jr&S|S% jr'            S}S& jr( Sj       SlS' jjr)  S~       SS( jjr*          SS) jr+          SS* jr,   Ss           SS+ jjr-  Sn     SS, jjr.          SS- jr/SS. jr0 Sj   SS/ jjr1SS0 jr2\SS1 j5       r3\    SS2 j5       r4      SS3 jr5      SS4 jr6          SS5 jr7        SS6 jr8        SS7 jr9        SS8 jr:          SS9 jr;  SS: jr<      SS; jr=      SS< jr>      SS= jr?S> r@S? rAS@ rBSA rCSB rDSC rESD rF  SSE jrGSSF jrHSSG jrISSH jrJ          SSI jrK            SSJ jrL  SSK jrM    SSL jrN          SSM jrOSSN jrP          SSO jrQ            SSP jrR        SSQ jrS          SSR jrT  SSS jrUSST jrV      SSU jrW\SSV j5       rX\SW 5       rYSXrZg)PrometheusLoggerA   c                 d    SSK n U R                   H  n[        U[        5      (       d  M  Us  $    g)zIFind the PrometheusLogger instance from litellm.callbacks, if registered.r   N)litellm	callbacks
isinstancer)   )r,   cbs     q/home/rurouni/.local/share/pipx/venvs/strix-agent/lib/python3.13/site-packages/litellm/integrations/prometheus.pyget_instancePrometheusLogger.get_instanceD   s/     	##B".//	 $     c                D    SSK JnJnJn  U R	                  5       U l        0 U l        [        R                  nUb  [        U5      O[        U l        [        5       U l        U R                  U5      U l        U R                  U5      U l        U R                  U5      U l        U R                  SSU R%                  S5      S9U l        U R                  SSU R%                  S5      S9U l        U R#                  SS	U R%                  S5      U R                  S
9U l        U R#                  SSU R%                  S5      U R                  S
9U l        U R#                  SSU R%                  S5      U R                  S
9U l        U R                  SSU R%                  S5      S9U l        U R                  SSU R%                  S5      S9U l        U R                  SSU R%                  S5      S9U l        U R                  SSU R%                  S5      S9U l        U R                  SSU R%                  S5      S9U l        U R                  SSU R%                  S5      S9U l        U R                  SSU R%                  S5      S9U l        U R                  SSU R%                  S5      S9U l        U R                  S S!U R%                  S 5      S9U l         U R!                  S"S#U R%                  S"5      S9U l!        U R!                  S$S%U R%                  S$5      S9U l"        U R!                  S&S'U R%                  S&5      S9U l#        U R!                  S(S)U R%                  S(5      S9U l$        U R!                  S*S+U R%                  S*5      S9U l%        U R!                  S,S-U R%                  S,5      S9U l&        U R!                  S.S/U R%                  S.5      S9U l'        U R!                  S0S1U R%                  S05      S9U l(        U R!                  S2S3U R%                  S25      S9U l)        U R!                  S4S5U R%                  S45      S9U l*        U R!                  S6S7U R%                  S65      S9U l+        U R!                  S8S9U R%                  S85      S9U l,        U R!                  S:S;U R%                  S:5      S9U l-        U R!                  S<S=U R%                  S<5      S9U l.        U R!                  S>S?U R%                  S>5      S9U l/        U R!                  S@SAU R%                  S@5      S9U l0        U R#                  SBSCU R%                  SB5      U R                  S
9U l1        U R#                  SDSEU R%                  SD5      U R                  S
9U l2        U R#                  SFSG/ SHQU R                  S
9U l3        U R                  SISJ/ SKQS9U l4        U R                  SLSM/ SNQS9U l5        U R!                  SOSPSQ/S9U l6        U R!                  SRSSU R%                  SR5      S9U l7        U R!                  STSUU R%                  ST5      S9U l8        U R!                  SVSWU R%                  SV5      S9U l9        U R                  SXSYU R%                  SX5      S9U l:        U R                  SZS[U R%                  SZ5      S9U l;        U R                  S\S]U R%                  S\5      S9U l<        U R                  S^S_U R%                  S^5      S9U l=        U R#                  S`SaU R%                  S`5      S9U l>        U R                  SbScU R%                  Sb5      5      U l?        U R                  SdSeU R%                  Sd5      5      U l@        U R                  SfSgSh/S9U lA        U R                  SiSjU R%                  Si5      S9U lB        U R                  SkSlU R%                  Sk5      S9U lC        U R                  SmSnU R%                  Sm5      S9U lD        U R                  SoSpU R%                  So5      S9U lE        U R                  SqSrU R%                  Sq5      S9U lF        U R                  SsStU R%                  Ss5      S9U lG        U R                  SuSvU R%                  Su5      S9U lH        U R!                  SwSx/ S9U lI        U R!                  SySz/ S9U lJ        U R                  S{S|/ S}QS9U lK        U R!                  S~S/ SQS9U lL        U R#                  SSSSQ/[        S
9U lN        U R                  SS/ S}QS9U lO        U R                  SSS/S9U lP        U R!                  SS/ S9U lQ        U R                  SSSSQ/S9U lR        U R                  SSS/S9U lS        U R!                  SS/ S9U lT        g ! [         a  n[        S[        U5       35        UeS nAff = f)Nr   )CounterGauge	Histogram$litellm_proxy_failed_requests_metriczjTotal number of failed responses from proxy - the client did not get a success response from litellm proxy)namedocumentation
labelnames#litellm_proxy_total_requests_metriczXTotal number of requests made to the proxy server - track number of client side requests$litellm_request_total_latency_metricz0Total latency (seconds) for a request to LiteLLM)r;   bucketslitellm_llm_api_latency_metricz1Total latency (seconds) for a models LLM API call*litellm_llm_api_time_to_first_token_metricz-Time to first token for a models LLM API calllitellm_spend_metriczTotal spend on LLM requests)r;   litellm_total_tokens_metricz7Total number of input + output tokens from LLM requestslitellm_input_tokens_metricz.Total number of input tokens from LLM requestslitellm_output_tokens_metricz/Total number of output tokens from LLM requests"litellm_input_cached_tokens_metriczvProvider-side cached input tokens (e.g. OpenAI prompt_tokens_details.cached_tokens, Anthropic cache_read_input_tokens)*litellm_input_cache_creation_tokens_metricz_Provider-side input tokens written to prompt cache (e.g. Anthropic cache_creation_input_tokens)!litellm_input_audio_tokens_metriczAAudio input tokens reported in prompt_tokens_details.audio_tokens&litellm_output_reasoning_tokens_metriczGReasoning tokens reported in completion_tokens_details.reasoning_tokens"litellm_output_audio_tokens_metriczFAudio output tokens reported in completion_tokens_details.audio_tokens$litellm_remaining_team_budget_metriczRemaining budget for teamlitellm_team_max_budget_metriczMaximum budget set for team*litellm_team_budget_remaining_hours_metricz*Remaining days for team budget to be reset#litellm_remaining_org_budget_metriczRemaining budget for orglitellm_org_max_budget_metriczMaximum budget set for org)litellm_org_budget_remaining_hours_metricz*Remaining hours for org budget to be reset'litellm_remaining_api_key_budget_metriczRemaining budget for api key!litellm_api_key_max_budget_metriczMaximum budget set for api key-litellm_api_key_budget_remaining_hours_metricz.Remaining hours for api key budget to be reset$litellm_remaining_user_budget_metriczRemaining budget for userlitellm_user_max_budget_metriczMaximum budget set for user*litellm_user_budget_remaining_hours_metricz+Remaining hours for user budget to be reset,litellm_remaining_api_key_requests_for_modelzLRemaining Requests API Key can make for model (model based rpm limit on key)*litellm_remaining_api_key_tokens_for_modelzJRemaining Tokens API Key can make for model (model based tpm limit on key)!litellm_remaining_requests_metriczWLLM Deployment Analytics - remaining requests for model, returned from LLM API Providerlitellm_remaining_tokens_metricz:remaining tokens for model, returned from LLM API Providerlitellm_overhead_latency_metricz;Latency overhead (milliseconds) added by LiteLLM processing"litellm_request_queue_time_secondsz>Time spent in request queue before processing starts (seconds)!litellm_guardrail_latency_secondsz)Latency (seconds) for guardrail executionguardrail_namestatus
error_type	hook_typelitellm_guardrail_errors_totalz=Total number of errors encountered during guardrail executionr^   r`   ra    litellm_guardrail_requests_totalz%Total number of guardrail invocationsr^   r_   ra   (litellm_provider_remaining_budget_metriczHRemaining budget for provider - used when you set provider budget limitsapi_providerlitellm_deployment_statezlLLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outagelitellm_deployment_tpm_limitz$Deployment TPM limit found in configlitellm_deployment_rpm_limitz$Deployment RPM limit found in configlitellm_deployment_cooled_downzLLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down$litellm_deployment_success_responseszOLLM Deployment Analytics - Total number of successful LLM API calls via litellm$litellm_deployment_failure_responseszLLM Deployment Analytics - Total number of failed LLM API calls for a specific LLM deploymeny. exception_status is the status of the exception from the llm api!litellm_deployment_total_requestszXLLM Deployment Analytics - Total number of LLM API calls via litellm - success + failure+litellm_deployment_latency_per_output_tokenz3LLM Deployment Analytics - Latency per output token'litellm_deployment_successful_fallbackszfLLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model#litellm_deployment_failed_fallbackszbLLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model(litellm_callback_logging_failures_metriczTTotal number of failures when emitting logs to callbacks (e.g. s3_v2, langfuse, etc)callback_name&litellm_llm_api_failed_requests_metricz5deprecated - use litellm_proxy_failed_requests_metriclitellm_requests_metriczdeprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, userlitellm_cache_hits_metricz"Total number of LiteLLM cache hitslitellm_cache_misses_metricz$Total number of LiteLLM cache misseslitellm_cached_tokens_metricz&Total tokens served from LiteLLM cache/litellm_provider_cache_read_input_tokens_metricz`Total prompt/input tokens read from provider prompt cache (e.g. OpenAI/Anthropic/Gemini/Bedrock)3litellm_provider_cache_creation_input_tokens_metriczSTotal prompt/input tokens written to provider prompt cache (e.g. Anthropic/Bedrock)litellm_total_usersz Total number of users in LiteLLMlitellm_teams_countz Total number of teams in LiteLLM#litellm_managed_batch_created_totalz'Total number of managed batches createdmodelrg   user
user_emailapi_key_aliaslitellm_managed_file_size_bytesz[Size of the most recent managed batch file in bytes (last-seen value per label combination)purpose	file_typer   rg   r   &litellm_managed_batch_duration_secondszLDuration of completed managed batches in seconds (completed_at - created_at)r   "litellm_managed_file_created_totalz%Total number of managed files created"litellm_managed_file_deleted_totalz;Total number of managed file deletions (success or blocked)result$litellm_check_batch_cost_jobs_polledzCNumber of unprocessed batches found by the last CheckBatchCost poll-litellm_check_batch_cost_jobs_processed_totalzCTotal number of batches successfully cost-tracked by CheckBatchCost%litellm_check_batch_cost_errors_totalz6Total number of errors in CheckBatchCost by error typer`   +litellm_check_batch_cost_last_run_timestampz1Unix timestamp of the last CheckBatchCost job runz(Got exception on init prometheus client )Xprometheus_clientr5   r6   r7   _parse_prometheus_configlabel_filters_cached_metric_labelsr,   prometheus_latency_bucketstupleLATENCY_BUCKETSlatency_bucketsr   "_bounded_prometheus_series_tracker_create_metric_factory_counter_factory_gauge_factory_histogram_factoryget_labels_for_metricr8   r<   r=   r?   r@   rA   litellm_tokens_metricrC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   !litellm_request_queue_time_metric litellm_guardrail_latency_metricrb   rd   rf   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rt   ru   rv   rw   rx   ry   rz   litellm_total_users_metriclitellm_teams_count_metricr}   r   BATCH_DURATION_BUCKETSr   r   r   r   r   r   r   	Exceptionr   str)selfkwargsr5   r6   r7   _custom_bucketses          r0   __init__PrometheusLogger.__init__N   s   k	CC "&!>!>!@D @BD&%@@O #. o&$  
 7U6VD3 %)$?$?$HD!"&"="=e"DD&*&A&A)&LD#8<8M8M; K55: 9N 9D5 8<7L7L:x559 8M 8D4 9=8O8O6B55: ,, 9P 9D5 372I2I0C556VW,,	 3J 3D/ ?C>U>U<?  55@ ,, ?V ?D;" )-(=(=&-556LM )> )D% *.)>)>-I556ST *? *D& 04/D/D-@556ST 0E 0D, 150E0E.A556TU 1F 1D- 7;6K6K4 I558 7L 7D3 ?C>S>S<q55@ ?T ?D; 6:5J5J3S557 6K 6D2 ;?:O:O8Y55< ;P ;D7 7;6K6K4X558 7L 7D3 9=8K8K6+55: 9L 9D5 372E2E0-556VW 3F 3D/ ?C>Q>Q<<55@ ?R ?D; 8<7J7J5*559 8K 8D4 261D1D/,556UV 2E 2D. >B=P=P;<55? >Q >D: <@;N;N9.55= <O <D8 6:5H5H30557 6I 6D2 BFATAT?@55C BU BD> 9=8K8K6+55: 9L 9D5 372E2E0-556VW 3F 3D/ ?C>Q>Q<=55@ ?R ?D; AE@S@S>^55B AT AD= ?C>Q>Q<\55@ ?R ?D; 6:5H5H3i557 6I 6D2 483F3F1L555 4G 4D0 483J3J1M555 ,, 4K 4D0 6:5L5L4P558 ,, 6M 6D2 594K4K3;R,,	 5L 5D1 372G2G0OH 3H 3D/ 594I4I27D 5J 5D1 =A<O<O:Z*+ =P =D9 -1,?,?*~556PQ -@ -D) 150C0C.6556TU 1D 1D- 150C0C.6556TU 1D 1D- 372G2G0 ]556VW	 3H 3D/ 9=8M8M;o55: 9N 9D5 9=8M8M; @55: 9N 9D5 6:5J5J8x557 6K 6D2 @D?V?VBS55A @W @D< <@;P;P9x**+TU<D8 8<7L7L5t**+PQ8D4 =A<Q<Q?t+, =R =D9 ;?:O:O=U55< ;P ;D7 ,0+@+@. a556OP ,A ,D( .2-B-B0B556QR .C .D* 04/D/D2D556ST 0E 0D, 150E0E3F556TU 1F 1D- DHCXCXF A55E DY DD@ HLG\G\Js55I H] HDD /3.A.A%2 /B /D+ /3.A.A%2 /B /D+ 8<7L7L:G 8M 
8D4 483F3F1mT 4G 4D0 ;?:Q:Q8^#^4.	 ;R ;D7 7;6K6K9E 7L 
7D3 7;6K6K9[$: 7L 7D3 9=8K8K6U 9L 9D5 BFAVAVDc#^4 BW BD> :>9N9N<V(> :O :D6 @D?R?R=C @S @D<  	DSVHMNG	s   e4e7 7
fffc                R   SSK nSSKJn  UR                  nU(       d  0 $ [        R
                  " SU 35        / n[        5       U l        U HY  n[        U[        5      (       a	  U" S0 UD6nOUnUR                  U5        U R                  R                  UR                  5        M[     U R                  U5      nUR                  (       a:  U R                  U5        SSR!                  UR"                  5      -   n[%        U5      eU R'                  U5      n	U R)                  U	5        U	$ )zNParse prometheus metrics configuration for label filtering and enabled metricsr   N)PrometheusMetricsConfigzprometheus config: z!Configuration validation failed:

 )r,   %litellm.types.integrations.prometheusr   prometheus_metrics_configr   debugsetenabled_metricsr.   dictappendupdatemetrics_validate_all_configurations
has_errors_pretty_print_validation_errorsjoinall_error_messages
ValueError_build_label_filters_pretty_print_prometheus_config)
r   r,   r   configparsed_configsgroup_configparsed_configvalidation_resultserror_messager   s
             r0   r   )PrometheusLogger._parse_prometheus_config  s   Q22 I26(;< "u"L,-- 7 G, G ,!!-0  ''(=(=> # ">>~N((001CD@499"55D M ]++ 11.A 	,,];r3   c                :   / n/ nU H  nUR                    Hv  nU R                  U5      nU(       a  UR                  U5        M.  UR                  (       d  MA  U R	                  XTR                  5      nU(       d  Me  UR                  U5        Mx     M     [        X#S9$ )z>Validate all metric configurations and return collected errors)metric_errorslabel_errors)r   _validate_single_metric_namer   include_labels_validate_single_metric_labelsValidationResults)r   r   r   r   r   metric_namemetric_errorlabel_errors           r0   r   -PrometheusLogger._validate_all_configurations  s    $F%~~#@@M!((6 ((("&"E"E#%:%:#K #{$++K8  . %  !}XXr3   c                j    SSK Jn  U[        U" [        5      5      ;  a  [	        UU" [        5      S9$ g)zValidate a single metric namer   )get_args)r   valid_metricsN)typingr   r   DEFINED_PROMETHEUS_METRICSMetricValidationError)r   r   r   s      r0   r   -PrometheusLogger._validate_single_metric_name  s8     	$c(+E"FGG('&'AB  r3   c                    SSK Jn  [        R                  U" [        U5      5      nU Vs/ s H  oUU;  d  M
  UPM     nnU(       a  [        UUUS9$ gs  snf )z#Validate labels for a single metricr   )r   r   invalid_labelsvalid_labelsN)r   r   PrometheusMetricLabels
get_labelsr   LabelValidationError)r   r   labelsr   r   labelr   s          r0   r   /PrometheusLogger._validate_single_metric_labels  sc     	  .88+[9

 .4QVEL7P%VQ''-) 
  Rs
   	AAc                    0 nU HK  nUR                    H8  nUR                  (       d  M  U R                  U5      b  M*  UR                  X$'   M:     MM     U$ )z1Build label filters from validated configurations)r   r   r   )r   r   r   r   r   s        r0   r   %PrometheusLogger._build_label_filters  sR    $F%~~(((88EM5;5J5J2	  . % r3   c                    U R                  X5      nU(       aD  U R                  UR                  UR                  UR                  S9  [        UR                  5      eg)z
Ensure that all the configured labels are valid for the metric

Raises ValueError if the metric labels are invalid and pretty prints the error
r   T)r   "_pretty_print_invalid_labels_errorr   r   r   r   message)r   r   r   r   s       r0   "_validate_configured_metric_labels3PrometheusLogger._validate_configured_metric_labels  s[     99+N33'33*99(55 4 
 [0011r3   c           	         SSK Jn  SSKJn  SSKJn  SSKJn  U" 5       nU" SSS9nUR                  S	5        UR                  U" US
S95        UR                  (       a  UR                   Vs/ s H  oR                  PM     n	nUR                  S   R                  n
U" SSR                  U	5       3SS9nUR                  U" US
S95        U" SSSSSS9nUR                  SSSS9  [        U
5       H  nUR                  U5        M     UR                  U5        UR                   (       a  UR                    H  nU" SUR                   SSR                  UR"                  5       3SS9nUR                  U" US
S95        U" SUR                   S3SSSSS9nUR                  SSSS9  [        UR$                  5       H  nUR                  U5        M     UR                  U5        M     UR                  S	5        gs  snf ! [&         ai    UR                   H#  n[(        R*                  " UR,                  5        M%     UR                    H#  n[(        R*                  " UR,                  5        M%      gf = f)z-Pretty print all validation errors using richr   ConsolePanelTableTextu(   🚨🚨 Configuration Validation Errorsbold redstyler   redborder_stylezInvalid Metric Names: ,    📊 Valid Metric NamesT
bold greenleftgreentitleshow_headerheader_styletitle_justifyr   Available Metricscyanr   no_wrapzInvalid Labels for '': u   🏷️ Valid Labels for ''Valid LabelsN)rich.consoler   
rich.panelr   
rich.tabler   	rich.textr   printr   r   r   r   
add_columnsortedadd_rowr   r   r   ImportErrorr   errorr   )r   r   r   r   r   r   consoler   r   invalid_metricsr   metrics_error_textmetrics_tablemetricr  labels_error_textlabels_tabler   r   r   s                       r0   r   0PrometheusLogger._pretty_print_validation_errors  s{   L	:,((&iG C:VE MM$MM%E:; "//+=+K+K#+KaMM+K   # !3 @ @!-  &*,TYY-G,HI$&" e$6UKL %3 $!-"(!(! (('vt )  %]3F!))&1 4 m, "../<<E(,.u/@/@.ATYYuOcOcEdDef()% MM%(9"NO#( :5;L;L:MQO$(%1&,%,$L !++N&RV+W!'(:(:!;$,,U3 "< MM,/' =* MM$i#l  	: 2 @ @$$\%9%9: !A1>>$$[%8%89  ?		:s&   A,H .G?E;H ?H A0I76I7c                    SSK Jn  SSKJn  SSKJn  SSKJn  U" 5       nU" SU SSR                  U5       S	3S
S9n	U" SSSSSS9n
U
R                  SSSS9  [        U5       H  nU
R                  U5        M     UR                  S5        UR                  U" U	SS95        UR                  U
5        UR                  S5        g! [         a+    [        R                  " SU SU S[        U5       35         gf = f)z8Pretty print error message for invalid labels using richr   r   r   r   r   u%   🚨🚨 Invalid Labels for Metric: 'z'
Invalid labels: r   z'
Please specify only valid labels belowr   r   u$   🏷️ Valid Labels for this MetricTr   r   r   r   r	  r  r  r   r   r   zInvalid labels for metric 'r  z. Valid labels: N)r
  r   r  r   r  r   r  r   r   r  r  r  r  r  r   r  )r   r   r   r   r   r   r   r   r  r   r  r   s               r0   r   3PrometheusLogger._pretty_print_invalid_labels_errorX  s   %	,((&iG 7}DWX\XaXabpXqWr  s[  \ E !< )$$L ##N&$#O-$$U+ . MM$MM%E:;MM,'MM$ 	  -k]#n=MM]^deq^r]st	s   CC 2C;:C;c           	         SSK Jn  SSKJn  SSKJn  SSKJn  U" 5       nU" SU S3SS	9nU" S
SSSSS9n	U	R                  SSSS9  [        U5       H  n
U	R                  U
5        M     UR                  S5        UR                  U" USS95        UR                  U	5        UR                  S5        g! [         a(    [        R                  " SU S[        U5       35         gf = f)z=Pretty print error message for invalid metric name using richr   r   r   r   r   u   🚨🚨 Invalid Metric Name: 'z1'
Please specify one of the allowed metrics belowr   r   r   Tr   r   r   r   r  r  r  r   r   r   zInvalid metric name: z. Valid metrics: N)r
  r   r  r   r  r   r  r   r  r  r  r  r  r   r  )r   invalid_metric_namer   r   r   r   r   r  r   r  r  s              r0   "_pretty_print_invalid_metric_error3PrometheusLogger._pretty_print_invalid_metric_error  s    %	,((&iG 12E1FFxy E "/ )$$M $$%8PT$U /%%f- 0 MM$MM%E:;MM-(MM$ 	  '(;'<<MfUbNcMde	s   B1B4 4/C&%C&c                    U R                  U5      nU(       a9  U R                  UR                  UR                  S9  [	        UR
                  5      eg)zM
Raises ValueError if the metric name is invalid and pretty prints the error
)r  r   N)r   r   r   r   r   r   )r   r   r  s      r0   _valid_metric_name#PrometheusLogger._valid_metric_name  sQ     11+>33$)$5$5UEXEX 4  U]]++	 r3   c                    SSK Jn  SSKJn  SSKJn  SSKJn  U" 5       nU" SSS9nU" S	S
SSS9nUR                  SSS
S9  [        U S5      (       a?  U R                  (       a.  [        U R                  5       H  n	UR                  U	5        M     OUR                  S5        U" SS
SSS9n
U
R                  SSS
S9  U
R                  SSS9  U(       aN  [        UR                  5       5       H0  u  pU(       a  SR                  U5      OSnU
R                  X5        M2     OU
R                  SS5        UR                  S5        UR                  U" USS95        UR                  U5        UR                  U
5        UR                  S5        g!! [          a[    ["        R$                  " S[        U S5      (       a  [        U R                  5      OS 35        ["        R$                  " S U 35         g!f = f)"z>Pretty print the processed prometheus configuration using richr   r   r   r   r   z"Prometheus Configuration Processedz	bold bluer   u   📊 Enabled MetricsTzbold magentar   )r   r   r  r  zMetric Namer  r  r   z8[yellow]All metrics enabled (no filter applied)[/yellow]u   🏷️  Label Filtersr   zAllowed Labelsyellowr   z[dim]No labels specified[/dim]z+[yellow]No label filtering applied[/yellow]z*[dim]All default labels will be used[/dim]r   bluer   zEnabled metrics: zAll metricszLabel filters: N)r
  r   r  r   r  r   r  r   r  hasattrr   r  r  itemsr   r  r  r   info)r   r   r   r   r   r   r  r   r  r  r  r   r   
labels_strs                 r0   r   0PrometheusLogger._pretty_print_prometheus_config  s   @	C,((&iG =[QE ", +$	M $$]&$$Ot.//D4H4H$T%9%9:F!))&1 ; %%N
 !. )$	L ##M#N##$4H#E+1-2E2E2G+H'K " 		&)= 
 !((A ,I $$A@ MM$MM%F;<MM-(MM,'MM$ 	C#GDRcDdDdF4+?+?$@jw#xy /- AB	Cs   FF A"G43G4c                f    [        U S5      (       d  gU R                  (       d  gXR                  ;   $ )z3Check if a metric is enabled based on configurationr   T)r(  r   )r   r   s     r0   _is_metric_enabled#PrometheusLogger._is_metric_enabled  s2     t.// ##2222r3   c                   ^ ^ UU 4S jnU$ )zMCreate a factory function that returns either a real metric or a no-op metricc                    > U (       a  U S   OUR                  SS5      nTR                  U5      (       a  T" U 0 UD6$ [        5       $ )Nr   r9    )getr.  
NoOpMetric)argsr   r   metric_classr   s      r0   factory8PrometheusLogger._create_metric_factory.<locals>.factory  sD    %)$q'vzz&"/EK&&{33#T4V44!|#r3   r   )r   r6  r7  s   `` r0   r   'PrometheusLogger._create_metric_factory  s    	$ r3   c                $   U R                   R                  U5      nUb  U$ [        R                  U5      nXR                  ;  a  X0R                   U'   U$ U R                  U   nU Vs/ s H  oUU;   d  M
  UPM     nnX`R                   U'   U$ s  snf )a  
Get the labels for a metric, filtered if configured.

The result is cached on the instance so the label set used to
construct each Prometheus metric at ``__init__`` time stays in lock
step with the label set passed to ``counter.labels(...)`` at
runtime, even if the underlying module-level toggles consulted by
:meth:`PrometheusMetricLabels.get_labels` (e.g.
``litellm.prometheus_emit_rate_limit_labels``,
``litellm.prometheus_emit_stream_label``) are flipped after the
logger has been created.
)r   r3  r   r   r   )r   r   cacheddefault_labelsconfigured_labelsr   filtered_labelss          r0   r   &PrometheusLogger.get_labels_for_metric  s     ++//<M 0::;G 0006D&&{3!! !..{;  .
-e:K1KE~ 	 
 3B"";/
s   (	B5Bc           	       ^ U R                  U5      n[        R                  R                  U;  a  gTR	                  [        R                  R                  5      c  g[
        R                  n[
        R                  nUc  Uc  gU R                  R                  UU[        U4S jU 5       5      UU[
        R                  S9  g)uZ  
Cap the cardinality of metrics that include the ``end_user`` label.

Called *after* ``metric.labels(...).inc()/observe()`` so the emission is
recorded in prometheus-client's child map before any eviction runs.
Series that get evicted before the next scrape lose updates accrued
since the last scrape — this is inherent to any cardinality cap.
Nc              3  F   >#    U  H  nTR                  U5      v   M     g 7fN)r3  ).0r   r   s     r0   	<genexpr>APrometheusLogger._track_end_user_metric_series.<locals>.<genexpr>_  s     IjUvzz%00js   !)r  r   label_values
max_seriesttl_secondscleanup_interval_seconds)r   UserAPIKeyLabelNamesEND_USERvaluer3  r,   1prometheus_end_user_metrics_max_series_per_metric'prometheus_end_user_metrics_ttl_secondsr   track_seriesr   4prometheus_end_user_metrics_cleanup_interval_seconds)r   r  r   r   r;   rG  rH  s      `   r0   _track_end_user_metric_series.PrometheusLogger._track_end_user_metric_seriesC  s     //<
((..j@::*3399:BNN
EE+"5//<<#IjII!#%,%a%a 	= 	
r3   Nc                    [        U R                  US9UUS9nUR                  " S0 UD6R                  U5        U R	                  XU5        g )Nr   supported_enum_labelsenum_valueslabel_contextr   )prometheus_label_factoryr   r   incrQ  )r   counterr   rW  rX  amount_labelss          r0   _inc_labeled_counter%PrometheusLogger._inc_labeled_countere  sQ     +"&"<"<"<"U#'

 	!!%%f-**7Ir3   c                H  #    [         R                  " S[        U[        5      (       a  [	        UR                  5       5      O[        U5      R                  5        UR                  S5      nUb  [        U[        5      (       d  [        SU 35      eU R                  XS9(       a  g UR                  SS5      nUR                  S0 5      =(       d    0 nUR                  S5      =(       d    0 n[        5       n	U	" US	S
9n
US   S   nUS   S   nUS   S   nUS   S   nUS   S   nUS   R                  S5      nUS   R                  S5      nUS   nUS   nUS   n[        US9nUb  [        U[        5      (       a  US   nO/ n[        SU SU SU SU
 SU 3
5        [        SE0 SU
_SU_SU_SUS    _S US    _S!U_S"U_S#U_S$U_S%U_S&US   S'   _S(S)_SU_S*U_S+U_S,US,   _S-US-   _S.US/   _S0S _S1S _S2[        US39_S4US   R                  S55      _S6US   R                  S75      _S8US   R                  S85      _S9[         R"                  (       a  [%        UR                  S95      5      OS _6nUb9  [        U[$        5      (       a$  UR'                  S:5      (       a  S;S<KJn  U" U5      n[-        U5      nU R/                  U
UUUUUUUUUS=9
  U R1                  UU
UUUUUUUUS>9
  U R3                  UUUUUUUUS?9I S h  vN   U R5                  UUUUUR6                  S@9  U R9                  UUUUUUUUSA9  U R;                  UUUUUUSB9  U R=                  UUUSC9I S h  vN   U R?                  UUUSC9  [@        RC                  U U RD                  SDUUSB9  g  N N97f)FNzFprometheus Logging - Enters success logging function (kwargs keys: %s)standard_logging_objectz)standard_logging_object is required, got=r   standard_logging_payloadr   r2  litellm_paramsmetadata
prometheusservice_typeuser_api_key_user_iduser_api_key_hashuser_api_key_aliasuser_api_key_team_iduser_api_key_team_aliasuser_api_key_org_iduser_api_key_org_aliascompletion_tokenstotal_tokensresponse_costrc  request_tagsz'inside track_prometheus_metrics, model z, response_cost z, tokens_used z, end_user_id z, user_api_key end_userhashed_api_keyr   requested_modelmodel_groupteam
team_aliasorg_id	org_aliasr   r   user_api_key_user_emailstatus_code200litellm_model_nametagsmodel_idapi_baserg   custom_llm_providerexception_statusexception_classcustom_metadata_labelsre  routeuser_api_key_request_route	client_iprequester_ip_address
user_agentstreamzsk-r   )
hash_token)
end_user_iduser_api_keyrk  r   user_api_teamuser_api_team_aliasuser_idrr  rW  rX  )
rc  r  r  rk  r   r  r  r  rW  rX  )r  r  r  rk  rd  rr  r  rn  )r  rk  r   re  r  )r   r   r  rk  r  r  rW  rX  rX  rc  rW  rX  r<   r   )#r   r   r.   r   listkeystype__name__r3  r   $_should_skip_metrics_for_invalid_keyr   ;_get_combined_custom_metadata_from_standard_logging_payloadr   UserAPIKeyLabelValuesget_custom_labels_from_metadatar,   prometheus_emit_stream_labelr   
startswithlitellm.proxy.utilsr  r   ._increment_top_level_request_and_spend_metrics_increment_token_metrics#_increment_remaining_budget_metrics#_set_virtual_key_rate_limit_metricsr  _set_latency_metrics"set_llm_deployment_success_metrics#_async_set_router_remaining_metrics_increment_cache_metricsr)   r^  r<   )r   r   response_obj
start_timeend_timerc  r   rd  	_metadata!get_end_user_id_for_cost_trackingr  r  r  rk  r  r  rn  ro  output_tokenstokens_usedrr  combined_metadata_tagsrW  r  rX  s                             r0   async_log_success_event(PrometheusLogger.async_log_success_eventu  s    T#-fd#;#;DfAVAV	
 FLZZ%F
  $+:$d4
 4
 ;<T;UV  44 5 
 

7B'$4b9?R"&&z28b	,U,W)7
 +:67MN/
;<OP5jABVW0<=ST6zB%
 7zBFF!
 ":*!E!I!I$"
 11DE.~>0AW%=
 $/J$d5
 5
 -^<EE5eW<L]O[ijuiv  wE  FQ  ER  Ra  bn  ao  p	
 , "
 "
'"
 -"
 5]C	"

 1?"
 "
 +"
 '"
 -"
 "
 0
;<UV"
 "
 "
  %"
 "
  .j9!"
" .j9#"
$ 22GH%"
& "'"
( !)"
* $C*$+"
0 +:6::,1"
6 /z:>>?UV7"
8 0
;??M9"
> 77 ,00:;A"
J $<--''..6%l3L5

 	;;#%1' 3'#' 	< 	
 	%% &>#%1' 3#' 	& 	
" 66' 3%1)' 3 7 	
 		
 		
 	00%1 )) 	1 	
 	!!%1' 3 $' 	" 	
 	//' 	0 	
 66%=#' 7 
 	
 	
 	%%%=#' 	& 	
 	--441' 	. 	
E		
d	
s%   LN"NAN"&N '8N" N"c                   [         R                  " S5        Ub  [        U[        5      (       a  US   n[        R                  U U R                  SU	U
[        US   5      S9  [        R                  U U R                  SU	U
[        US   5      S9  [        R                  U U R                  SU	U
[        US	   5      S9  U R                  UU	U
S
9  g )Nz2prometheus Logging - Enters token metrics functionrt  rB   rq  rX  r\  rC   prompt_tokensrD   rp  r  )r   r   r.   r   r)   r^  r   floatrC   rD   _increment_token_detail_metrics)r   rc  r  r  rk  r   r  r  r  rW  rX  r  s               r0   r  )PrometheusLogger._increment_token_metricsN  s     	QR $/J$d5
 5
 -^<E--&&)'1.AB 	. 	
 	--,,)'1/BC 	. 	
 	----*'12EFG 	. 	
 	,,%=#' 	- 	
r3   c                   UR                  S5      =(       d    0 n[        U[        5      (       a  UR                  S5      OSn[        U[        5      (       d  gUR                  S5      =(       d    0 nUR                  S5      =(       d    0 nU R                  S[        U[        5      (       a  UR                  S5      OS4U R                  S[        U[        5      (       a  UR                  S	5      OS4U R
                  S
[        U[        5      (       a  UR                  S5      OS4U R                  S[        U[        5      (       a  UR                  S5      OS4U R                  S[        U[        5      (       a  UR                  S5      OS4/nU HJ  u  pn[        U[        [        45      (       a  US::  a  M)  [        R                  U U	U
UU[        U5      S9  ML     g)a  
Increment per-token-type counters from the Usage object that providers
attach to the request. The Usage dict is plumbed onto
``standard_logging_payload["metadata"]["usage_object"]`` by
``get_standard_logging_object_payload``.

Each counter is only incremented when the underlying value is > 0, so
scrape output stays sparse for providers that don't report these
details (most non-OpenAI/Anthropic models).
re  usage_objectNprompt_tokens_detailscompletion_tokens_detailsrE   cached_tokensrF   cache_creation_tokensrG   audio_tokensrH   reasoning_tokensrI   r   r  )r3  r.   r   rE   rF   rG   rH   rI   intr  r)   r^  )r   rc  rW  rX  re  r  prompt_detailscompletion_detailsdetail_metricsr[  r   rL  s               r0   r  0PrometheusLogger._increment_token_detail_metrics  s     ,//
;Ar,6x,F,FHLL(D 	 ,--%))*ABHb)--.IJPb 774 ".$77 #&&7 ??< ".$77 #&&'>? 663 ".$77 #&&~6 ;;8 ""4d;; '**+=> 774 ""4d;; '**>:K.M
` ,:'G%ec5\22eqj11+U| 2  ,:r3   c           
        UR                  S5      nUc  OUSL ae  [        R                  U U R                  SUUS9  UR                  SS5      nUS:  a+  [        R                  U U R                  SUU[        U5      S	9  O![        R                  U U R                  S
UUS9  SnSnUR                  S0 5      =(       d    0 R                  S5      n[        U[        5      (       Ga  UR                  S5      n	UR                  S5      n
[        U	[        5      (       a  U	n[        U
[        5      (       a  U
nU	cN  UR                  S5      n[        U[        5      (       a(  UR                  S5      n[        U[        5      (       a  UnUS:  a+  [        R                  U U R                  SUU[        U5      S	9  US:  a,  [        R                  U U R                  SUU[        U5      S	9  ggg)z
Increment cache-related Prometheus metrics based on cache hit/miss status.

Args:
    standard_logging_payload: Contains cache_hit field (True/False/None)
    enum_values: Label values for Prometheus metrics
	cache_hitNTrv   r  rq  r   rx   r  rw   re  r  cache_read_input_tokenscache_creation_input_tokensr  r  ry   rz   )r3  r)   r^  rv   rx   r  rw   r.   r   r  ry   rz   )r   rc  rW  rX  r  rq  provider_cache_read_tokensprovider_cache_creation_tokens	usage_obj_read_writer  r  s                r0   r  )PrometheusLogger._increment_cache_metrics  s    -00=	 $11..++ 2  477JLa 55552"/ . 6  1100-+ 2  &'")*&-11*bAGRLL
	 i&&MM";<E]]#@AF%%%-2*&#&&17. }!*/F!Gnd33$2$6$6$GM!-555B2)A- 55HHE"/ !;< 6  .1 55LLI"/ !?@ 6  2; 'r3   c	                v  #    UR                  S5      =(       d    0 n	U	R                  SS 5      n
U	R                  SS 5      nU	R                  SS 5      nU	R                  SS 5      nU	R                  SS 5      nU	R                  SS 5      n[        R                  " U R                  UUUUUS9U R	                  UUU
UUS	9U R                  UUUUS
9U R                  UUS9SS9I S h  vN n[        U5       H>  u  nn[        U[        5      (       d  M  [        R                  " S/ SQU    SU 35        M@     g  NR7f)Nre  user_api_key_team_spenduser_api_key_team_max_budgetuser_api_key_spenduser_api_key_max_budgetuser_api_key_user_spenduser_api_key_user_max_budget)r  rk  rr  key_max_budget	key_spend)r  r  
team_spendteam_max_budgetrr  )r  
user_spenduser_max_budgetrr  r{  rr  T)return_exceptionsz0[Non-Blocking] Prometheus: Budget metric lookup )keyry  r   orgz	 failed: )r3  asynciogather-_set_api_key_budget_metrics_after_api_request*_set_team_budget_metrics_after_api_request*_set_user_budget_metrics_after_api_request)_set_org_budget_metrics_after_api_request	enumerater.   r   r   r   )r   r  r  r  rk  rd  rr  r  rn  r  _team_spend_team_max_budget_api_key_spend_api_key_max_budget_user_spend_user_max_budgetresultsirs                      r0   r  4PrometheusLogger._increment_remaining_budget_metrics:  so     #&&z28b	mm$=tD$==)GN"';TB'mm,EtLmm$=tD$==)GN>>)#5+2( ?  ;;+$7& 0+ <  ;;& 0+	 <  ::*+ ;  #3
 
6 g&DAq!Y''$$FGefgGhFiirstruv '7
s   C"D9$D7%(D9'D9c           
         [         R                  U U R                  SU	U
S9  [         R                  U U R                  SU	U
[	        U5      S9  g )Nru   r  rA   r  )r)   r^  ru   rA   r  )r   r  r  rk  r   r  r  r  rr  rW  rX  s              r0   r  ?PrometheusLogger._increment_top_level_request_and_spend_metricsp  s^     	--((%' 	. 	
 	--%%"'' 	. 	
r3   c                B   SSK Jn  U" U5      nSU 3nSU 3n	UR                  U5      n
U
c  [        R                  n
UR                  U	5      nUc  [        R                  n[        UUUU[        [        UR                  S5      S9S9S9n[        U5      n[        U R                  S	5      UUS
9nU R                  R                  " S0 UD6R                  U
5        [        U R                  S5      UUS
9nU R                  R                  " S0 UD6R                  U5        g )Nr   )#get_model_group_from_litellm_kwargszlitellm-key-remaining-requests-zlitellm-key-remaining-tokens-ra  rs  r  )rv  r   r   r  r  rV   rU  rW   r   ))litellm.proxy.common_utils.callback_utilsr  r3  sysmaxsizer  r  r  r   rY  r   rV   r   r   rW   )r   r  rk  r   re  r  r  rx   remaining_requests_variable_nameremaining_tokens_variable_nameremaining_requestsremaining_tokensrW  rX  requests_labelstokens_labelss                   r0   r  4PrometheusLogger._set_virtual_key_rate_limit_metrics  sG   	
 :&A-k]; 	) ,I)V&%\\*JK%!$#<<(FG#"{{+',#BT-3ZZ8Q-R$

 6kB2"&"<"<># $'
 	99@@S?SWW	
 1"&"<"<<# $'
 	77>>OOSS	
r3   c	                0   UR                  S5      =(       d    [        R                  " 5       n	UR                  S5      n
UR                  SS 5      nUR                  SS 5      nU R                  UUS9nUbv  UR                  SS5      SL aa  [	        U R                  S	S
9UUS9nU R                  R                  " S0 UD6R                  U5        U R                  U R                  S	U5        O[        R                  " S5        U R                  UU	S9nUb`  [	        U R                  SS
9UUS9nU R                  R                  " S0 UD6R                  U5        U R                  U R                  SU5        U R                  U
U	S9nUb`  [	        U R                  SS
9UUS9nU R                  R                  " S0 UD6R                  U5        U R                  U R                  SU5        UR                  S0 5      =(       d    0 nUR                  S5      =(       d    0 R                  S5      nUbh  US:  aa  [	        U R                  SS
9UUS9nU R                  R                  " S0 UD6R                  U5        U R                  U R                  SU5        g g g )Nr  r  api_call_start_timecompletion_start_time)r  r  r  FTr@   rT  rU  zUTime to first token metric not emitted, stream option in model_parameters is not Truer?   r=   rd  re  queue_time_secondsr   r[   r   )r3  r   now_safe_duration_secondsrY  r   r@   r   observerQ  r   r   r?   r=   r   )r   r   r   r  rk  r  r  rW  rX  r  r  r  r  time_to_first_token_seconds_ttft_labelsapi_call_total_time_secondsr]  total_time_seconds_litellm_paramsr  s                       r0   r  %PrometheusLogger._set_latency_metrics  s    $ZZ
3Ex||~)/L)A
$jj)>E &

+BD I&*&A&A** 'B '
#
 (3

8U+t33&*&@&@ L 'A ' (+L ;;BB g12..??<   g '+&A&A* 'B '
# '2.&*&@&@ @ 'A ' (+G //66AAII+ ..330 "88! 9 
 ).&*&@&@ F 'A ' (+G 55<<GwGOO" ..996 !**%5r:@b-11*=CHH 
 ).@A.E.&*&@&@ D 'A ' (+G 2299DGDLL" ..664 /F)r3   c                  #    [         R                  " S[        U[        5      (       a  [	        UR                  5       5      O[        U5      R                  5        UR                  S0 5      nU R                  XS9(       a  g UR                  SS5      nUR                  S0 5      =(       d    0 n[        5       nU" USS9n	US	   S
   n
US	   S   nUS	   S   nUS	   S   nUS	   S   nUS	   R                  S5      n [        U	UUUUUU
UR                  SS5      [        [        US9S9S9	n[        R                  U U R                   SU5        U R#                  U5        U R%                  USS9I S h  vN   g  N! [&         a8  n[         R(                  " SR+                  [-        U5      5      5         S nAg S nAff = f7f)NzFprometheus Logging - Enters failure logging function (kwargs keys: %s)ra  rb  r   r2  rd  rf  rg  re  ri  rj  rk  rl  rm  rn  r  rs  r  )	ru  rv  r   r   ry  rz  r   r  r  rt   r   r  0prometheus Layer Error(): Exception occured - {})r   r   r.   r   r  r  r  r  r3  r  r   r  r  r  r)   r^  rt   "set_llm_deployment_failure_metricsr  r   	exceptionformatr   )r   r   r  r  r  rc  r   rd  r  r  r  r  rk  r  r  rn  rW  r   s                     r0   async_log_failure_event(PrometheusLogger.async_log_failure_event7  s    T#-fd#;#;DfAVAV	

 <B::%r<
  44 5 
 

7B'$4b9?R,U,W)7
 +:67MN/
;<OP5jABVW0<=ST6zB%
 7zBFF!
	/$+0".155j"E'FX1I(K 11;;8	 33F;@@* A    	  	$$BII#a&Q 	sC   C:F?=A5E: 2E83E: 7F?8E: :
F<.F72F?7F<<F?c                B   SnU(       a8  [        US5      (       a'  UR                  (       a   [        UR                  5      nU(       d7  U(       a0  [        USS5      =(       d    [        USS5      nUb   [        U5      nU(       dQ  U(       aJ  UR                  S5      nU(       a2  [        USS5      =(       d    [        USS5      nUb   [        U5      nU$ U$ ! [        [        4 a     Nf = f! [        [        4 a    Sn Nf = f! [        [        4 a    Sn U$ f = f)a  
Extract HTTP status code from various input formats for validation.

This is a centralized helper to extract status code from different
callback function signatures. Handles both ProxyException (uses 'code')
and standard exceptions (uses 'status_code').

Args:
    kwargs: Dictionary potentially containing 'exception' key
    enum_values: Object with 'status_code' attribute
    exception: Exception object to extract status code from directly

Returns:
    Status code as integer if found, None otherwise
Nr~  coder  )r(  r~  r  r   	TypeErrorgetattrr3  )r   r   rW  r  r~  exception_in_kwargss         r0   _extract_status_code%PrometheusLogger._extract_status_code{  s0   *  ]33''!+"9"9: y!)]DA W64FK &'"%k"2K v"(**["9"%' @0&$?  *+&)+&6 {5 	*  #I. '"&K' '	2 +&*+s5   C 5C0 D C-,C-0DDDDc                   ^ US:X  a  gUb8  [        U5      R                  5       m/ SQn[        U4S jU 5       5      (       a  gg)a  
Determine if a request has an invalid API key based on status code and exception.

This method prevents invalid authentication attempts from being recorded in
Prometheus metrics. A 401 status code is the definitive indicator of authentication
failure. Additionally, we check exception messages for authentication error patterns
to catch cases where the exception hasn't been converted to a ProxyException yet.

Args:
    status_code: HTTP status code (401 indicates authentication error)
    exception: Exception object to check for auth-related error messages

Returns:
    True if the request has an invalid API key and metrics should be skipped,
    False otherwise
i  T)zvirtual key expectedzexpected to start with 'sk-'zauthentication errorzinvalid api keyzapi key not validc              3  ,   >#    U  H	  oT;   v   M     g 7frB  r   )rC  patternexception_strs     r0   rD  ?PrometheusLogger._is_invalid_api_key_request.<locals>.<genexpr>  s     O;Nm+;Ns   F)r   lowerany)r   r~  r  auth_error_patternsr   s       @r0   _is_invalid_api_key_request,PrometheusLogger._is_invalid_api_key_request  sL    * #  	N002M# O;NOOOr3   c                    U R                  UUUS9nUc  U(       a  UR                  S5      nU R                  XeS9(       a9  [        R                  " SU SU(       a  [        U5      R                  OS 35        gg)	a  
Determine if Prometheus metrics should be skipped for invalid API key requests.

This is a centralized validation method that extracts status code and exception
information from various callback function signatures and determines if the request
represents an invalid API key attempt that should be filtered from metrics.

Args:
    kwargs: Dictionary potentially containing exception and other data
    user_api_key_dict: User API key authentication object (currently unused)
    enum_values: Object with status_code attribute
    standard_logging_payload: Standard logging payload dictionary
    exception: Exception object to check directly

Returns:
    True if metrics should be skipped (invalid key detected), False otherwise
)r   rW  r  Nr  r  zESkipping Prometheus metrics for invalid API key request: status_code=z, exception=TF)r  r3  r%  r   r   r  r  )r   r   user_api_key_dictrW  rc  r  r~  s          r0   r  5PrometheusLogger._should_skip_metrics_for_invalid_key  s    2 //# 0 
 

;/I++K+M  *m<T]Y8P8Pcg7hj r3   c           
     
  #    SSK Jn  U R                  UUS9(       a  gU R                  US9n UR	                  UUR                  S0 5      S9nUR                  S0 5      =(       d    0 nUR                  S	0 5      R                  S
5      =(       d!    UR                  S	0 5      R                  S
5      n	U R                  U5      u  p[        S'0 SUR                  _SUR                  _SUR                  _SUR                  _SUR                  _SUR                  _SUR                  _SUR                  _SUR                   _SUR                  SS5      _S[#        U5      _S[#        U5      _SU R%                  U5      _SU
_SU_SU_SUR&                  _SUR                  S5      _S UR                  S 5      _S!U	_S"[(        R*                  (       a  [#        UR                  S"5      5      OS_6n[-        U5      n[.        R1                  U U R2                  S#UUS$9  [.        R1                  U U R4                  S%UUS$9  g! [6         a8  n[8        R:                  " S&R=                  [#        U5      5      5         SnAgSnAff = f7f)(z
Track client side failures

Proxy level tracking - failed client side requests

See :attr:`PrometheusMetricLabels.litellm_proxy_failed_requests_metric`
for the authoritative list of labels emitted on this metric.
r   )StandardLoggingPayloadSetup)r)  r  Nr(  proxy_server_request)rd  r-  re  
model_infoidru  r   r   rv  r   ry  rz  r{  r|  rw  r   r2  r~  r  r  rate_limit_categoryrate_limit_typer  r  r  r  r  r  r  r8   r  r<   r  r   )*litellm.litellm_core_utils.litellm_loggingr,  r  r  _get_request_tagsr3  _extract_rate_limit_labelsr  r  r  r   api_key	key_aliasteam_idrz  r{  organization_aliasr   _get_exception_class_namerequest_router,   r  r   r)   r^  r8   r<   r   r   r  r  )r   request_dataoriginal_exceptionr)  traceback_strr,  r~  r  r  r  r0  r1  rW  
_label_ctxr   s                  r0   async_post_call_failure_hook-PrometheusLogger.async_post_call_failure_hook  s    	
 44/( 5 
 //:L/M;	/AA+%1%5%56Lb%Q B E %((R8>BI }}\26::4@ LDTDTbEc$i  483R3R"40 0 *66&.. -77  188	
 099 '.. -77 )// ,>> !- 0 0" =  , "%[!1 !% > >?Q R %8 !0  !" (55#$ $--(>?%& %==6'( "). ;; ((231K6 7{CJ11996( 2  11885( 2   	$$BII#a&Q 		s/   ,JHH> =J>
J .I;6J;J  Jc                   #    g7f)a6  
Proxy level tracking - triggered when the proxy responds with a success response to the client

Note: litellm_proxy_total_requests_metric is NOT incremented here to avoid
double-counting. It is incremented in async_log_success_event which fires
for all successful requests (both streaming and non-streaming).
Nr   )r   datar)  responses       r0   async_post_call_success_hook-PrometheusLogger.async_post_call_success_hook`  s
      	s   c                p    Uc  U$ [        U[        5      (       a  UR                  X#5      $ [        XU5      $ )z&Get value from dict or Pydantic model.)r.   r   r3  r  )r   objr  defaults       r0   	_safe_getPrometheusLogger._safe_getl  s5    ;Nc4  773((s))r3   c           
       ^	^
^ UR                  S0 5      =(       d    0 nUR                  S0 5      =(       d    0 nU R                  US5      =(       d    0 n[        U[        5      (       a  Um
OK[	        USS5      [	        USS5      [	        USS5      [	        USS5      [	        US	S5      [	        US
S5      S.m
UR                  S0 5      =(       d    0 m	T	R                  S5      mSU	U
U4S jjnSU	U
U4S jjnSU	U
U4S jjnSU	U
U4S jjnU" 5       U" 5       U" 5       U" 5       T
R                  S	5      =(       d    T	R                  S	5      T
R                  S
5      =(       d    T	R                  S
5      S.$ )z
Extract label values for deployment failure metrics from all available
sources in request_kwargs. Falls back to litellm_params metadata and
user_api_key_auth when standard_logging_payload has None values.
ra  rd  re  rk  Nrl  rm  rj  r  r  )rk  rl  rm  rj  r  r  user_api_key_authc                 ~   > TR                  S5      n U b  U $ TR                  S5      n U b  U $ Tb  [        TSS 5      $ g )Nrk  r6  r3  r  val_litellm_params_metadatar  rL  s    r0   _get_api_key_aliasUPrometheusLogger._extract_deployment_failure_label_values.<locals>._get_api_key_alias  sP    -- 45C
*../CDC
 ,0+tDDr3   c                 ~   > TR                  S5      n U b  U $ TR                  S5      n U b  U $ Tb  [        TSS 5      $ g )Nrl  r7  rN  rO  s    r0   _get_team_idOPrometheusLogger._extract_deployment_failure_label_values.<locals>._get_team_id  sP    -- 67C
*../EFC
 ,0)TBBr3   c                 ~   > TR                  S5      n U b  U $ TR                  S5      n U b  U $ Tb  [        TSS 5      $ g )Nrm  rz  rN  rO  s    r0   _get_team_aliasRPrometheusLogger._extract_deployment_failure_label_values.<locals>._get_team_alias  sP    -- 9:C
*../HIC
 ,0,EEr3   c                    > TR                  S5      n U b  U $ TR                  S5      n U b  U $ Tb!  [        TSS 5      =(       d    [        TSS 5      $ g )Nrj  r5  api_key_hashrN  rO  s    r0   _get_hashed_api_keyVPrometheusLogger._extract_deployment_failure_label_values.<locals>._get_hashed_api_key  sh    -- 34C
*../BCC
 ,0)TB g%~tG  r3   )r   ry  rz  rv  r  r  )returnOptional[str])r3  rI  r.   r   r  )r   request_kwargsrc  r  _metadata_rawrR  rU  rX  r\  rQ  r  rL  s            @@@r0   (_extract_deployment_failure_label_values9PrometheusLogger._extract_deployment_failure_label_valuest  s    8"=C 	! ),,-=rBHb'?LRPRmT**%I '.!#7' )0!#94) ,3!#<d, &-]<OQU%V(/!#94) &m\4HI  $3#6#6z2#F#L"  5889LM		 				 				 			 	 01 N)+13"'=> D'++,BC#--5 :'++L9	
 		
r3   c                    [         R                  " S5        UR                  S0 5      nUR                  S0 5      =(       d    0 nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  S	S5      nUcs  UR                  S
5      =(       d    0 R                  S5      =(       d2    UR                  S5      =(       d    0 R                  S5      =(       d    0 n	U	R                  S5      nUcY  UR                  S
5      =(       d    0 R                  S5      =(       d)    UR                  S5      =(       d    0 R                  S5      nUR                  SS5      n
U R                  UUS9(       a  gU R	                  U5      nUR                  S0 5      =(       d    0 nUR                  S5      =(       d    UR                  S5      nUR                  S5      =(       d    UR                  S5      nUR                  S5      =(       d    UR                  S5      nUR                  S5      =(       d    UR                  S5      nUR                  S5      =(       d    UR                  S5      nUR                  S5      =(       d    UR                  S5      nSnUb>  [        [        USS5      5      nUS:X  d  U(       d  [        USS5      nUb  [        U5      n[        U5      nU(       a  UnUnUnU
nU=(       d    UnOSnSnSnSnU=(       d    U=(       d    Sn[        UUUUUU(       a  U R                  U5      OSUUUUUUR                  S/ 5      UUS 9n U(       a$  U R                  U=(       d    SUUU
=(       d    SS!9  [        U5      nUb!  [        R                  U U R                  S"UUS#9  [        R                  U U R                  S$UUS#9  g! [          a8  n[         R                  " S%R#                  [        U5      5      5         SnAgSnAff = f)&z
Sets Failure metrics when an LLM API call fails

- mark the deployment as partial outage
- increment deployment failure responses metric
- increment deployment total requests metric

Args:
    request_kwargs: dict

(setting remaining tokens requests metricra  rd  r   Nrx  r  r  r  litellm_metadatar.  re  r/  r  rb  rv  rj  r   rk  ry  rl  rz  rm  r  r  r  r~  Noner  r2  rt  )r  r  r  rg   r  r  rw  rv  r   ry  rz  r  r  r  r  r  r  rg   rm   r  rn   zLPrometheus Error: set_llm_deployment_failure_metrics. Exception occured - {})r   r   r3  r  rb  r   r  boolr  r9  set_deployment_partial_outager   r)   r^  rm   rn   r   r  )r   r`  rc  r  r  rx  r  r  r  _model_infollm_providerfallback_valuesr  rv  r   ry  rz  r  r  r  r  deployment_selectedlabel_litellm_model_namelabel_model_idlabel_api_baselabel_api_providerlabel_requested_modelrW  _deployment_label_ctxr   s                                 r0   r  3PrometheusLogger.set_llm_deployment_failure_metrics  s`   I	  !KL?M?Q?Q)2@$ -001A2FL"O!/!3!3GT!B266}dKK/33JEH/33JEH&**;=I $(();<BGGU '++J7=2BB<P 
 '??40 ".223EFL"QQ! P%))*5;@@O  +../DdKL88%)A 9   #KKO 144ZDJI,001AB imm#GN ,//@ IMM$EM #&&v.W)--@V2WD(,,\: imm)?J (++K8 IMM&=I ),,\: imm?J
  $$#&wy-'N#O #v-5E"9fd;D'+.t9( #'x."+=(!)!)%1"(3(I7I%+-(!#!#%'"(:(Ok(OR%/#;''/!1AJD229=PT 5-+%-11."E#%!K&
 #22'9'?R%%!-!3	 3  %B+$N!$ 55==:"7 6  116633 2   	  ^eeF 	s   FO; I!O; ;
P=.P88P=c           
        UR                  S5      =(       d    UR                  S5      nUR                  S5      =(       d    UR                  S5      nUbL  [        U R                  SS9[        UUUUS9S9n	U R                  R
                  " S	0 U	D6R                  U5        UbM  [        U R                  SS9[        UUUUS9S9n	U R                  R
                  " S	0 U	D6R                  U5        gg)
z/
Set the deployment TPM and RPM limits metrics
tpmrpmNri   rT  rh  rV  rW  rj   r   )r3  rY  r   r  ri   r   r   rj   )
r   r.  rd  r  r  r  rl  rw  rx  r]  s
             r0   %_set_deployment_tpm_rpm_limit_metrics6PrometheusLogger._set_deployment_tpm_rpm_limit_metricsi	  s    nnU#@~'9'9%'@nnU#@~'9'9%'@?.&*&@&@ > 'A ' 2'9%%!-		
G --44?w?CCCH?.&*&@&@ > 'A ' 2'9%%!-		
G --44?w?CCCH r3   c                  #     UR                  S0 5      =(       d    0 R                  S5      =(       d    0 nUR                  S5      SLnUR                  S5      SLnU(       a  U(       a  gUR                  S5      nU(       d  g SSKJn  Uc  g UR	                  U5      I Sh  vN n	U	(       d  gU	R                  S
5      nU	R                  S5      nU(       dF  UbC  [        U R                  SS9UUS9nU R                  R                  " S0 UD6R                  U5        U(       dH  UbD  [        U R                  SS9UUS9nU R                  R                  " S0 UD6R                  U5        ggg! [         a    Sn Nf = f N! [
         a"  n
[        R                  " S	UU
5         Sn
A
gSn
A
ff = f! [
         a8  n
[        R                  " SR                  [        U
5      5      5         Sn
A
gSn
A
ff = f7f)a  
Populate ``litellm_remaining_tokens_metric`` /
``litellm_remaining_requests_metric`` from the router's internal usage
counters when the upstream provider did not return
``x-ratelimit-remaining-*`` response headers.

OpenAI / Anthropic / Azure return remaining tokens/requests in response
headers, but Bedrock and Vertex AI do not. This fallback computes
``configured_limit - current_usage`` via
``Router.get_remaining_model_group_usage`` so the same gauges are
emitted for every provider when tpm/rpm is configured on the
deployment.
hidden_paramsadditional_headersx_ratelimit_remaining_tokensNx_ratelimit_remaining_requestsrx  r   )
llm_routerzIPrometheus: get_remaining_model_group_usage failed for model_group=%s: %szx-ratelimit-remaining-tokenszx-ratelimit-remaining-requestsrY   rT  rU  rX   zMPrometheus Error: _async_set_router_remaining_metrics. Exception occured - {}r   )r3  litellm.proxy.proxy_serverr  r  get_remaining_model_group_usager   r   r  rY  r   rY   r   r   rX   r  r   )r   rc  rW  rX  r~  already_have_tokensalready_have_requestsrx  r  remaining_usager   r  r  r]  s                 r0   r  4PrometheusLogger._async_set_router_remaining_metrics	  s+    &H	(,,_bAGRc&'".+- 
 #&&'EFdR   #&&'GHPTT " #'<266}EK"A !(2(R(R) # #.223QR!0!4!45U!V&+;+G2*.*D*D$E +E + !,"/ 44;;FgFJJ$ )-?-K2*.*D*D$G +E + !,"/ 66==HHLL& .L(K  "!
"#  (()	 L  	$$))/A 	s   G9A'F4 +G9,F4 G9E1 F4 G9F %F&F *F4 1G92B<F4 .G91F =F4 ?F  F4 F 
F1F,'F4 +G9,F11F4 4
G6>.G1,G91G66G9c           	         [         R                  " S5        UR                  S5      nUc  g U R                  UUUS9(       a  g US   nUR                  S0 5      =(       d    0 n	[	        U5      n
UR                  SS 5      nU	R                  SS 5      nU
R                  S5      =(       d    0 nUR                  S	S 5      nU(       d  U	(       a  U R                  UU	UUUUS
9  S nS nUS   S   =n(       a$  UR                  SS 5      nUR                  SS 5      nUS   R                  S5      =n(       aF  [        U R                  SS9UUS9nU R                  R                  " S!0 UD6R                  US-  5        U(       aD   [        U R                  SS9UUS9nU R                  R                  " S!0 UD6R                  U5        U(       aC  [        U R                  SS9UUS9nU R                  R                  " S!0 UD6R                  U5         U R                  U=(       d    SU=(       d    SU=(       d    SU=(       d    SS9  [        R!                  U U R"                  SUUS9  [        R!                  U U R$                  SUUS9  X2-
  nS nUR                  SS 5      b  US   SL a  UR                  SU5      U-
  nU=(       d    UnUR'                  5       nS nUbP  US:  aI  UU-  n[        U R                  SS9UUS9nU R(                  R                  " S!0 UD6R                  U5        g g g ! [*         a8  n[         R,                  " S R/                  [1        U5      5      5         S nAg S nAff = f)"Nre  ra  )r   rW  rc  r  rd  r   r  r.  r/  )r.  rd  r  r  r  rl  r}  r~  r  r  litellm_overhead_time_msrZ   rT  rU  i  rX   rY   r2  rh  rl   r  rn   r  Tr  r   ro   zLPrometheus Error: set_llm_deployment_success_metrics. Exception occured - {}r   )r   r   r3  r  r   rz  rY  r   rZ   r   r  rX   r   rY   set_deployment_healthyr)   r^  rl   rn   total_secondsro   r   r  r  r   )r   r`  r  r  rW  r  rX  rc  r  r  r  r  rl  rk  r  r  r  r~  r  r]  response_ms!time_to_first_token_response_time_latency_latency_secondslatency_per_tokenr   s                             r0   r  3PrometheusLogger.set_llm_deployment_success_metrics	  s,   ^	  !KL""#<= % (/ 88%')A 9 
 /
;H,001A2FL"O8HI!/!3!3GT!B*../DdKL#--5;K"tT2Ho::*#2'9%%!- ;  15.2%=o%N$& !  &8%;%;4d&" $6#9#92D$  ,D,c,-.' . 3*.*D*D$E +E + !,"/ 44;;FgFNN,t3 " 3*.*D*D$G +E + !,"/ 66==HHLL&  2*.*D*D$E +E + !,"/ 44;;FgFJJ$ ''#5#;!R!R)/R	 (  11996+ 2  11663+ 2  &.%:KEI- ""8T2>"8,4 #&&'>IJV 2 #D"R{H'557 !%(]Q->$4}$D!2*.*D*D$Q +E + !,"/ @@GG '+, .?(  	$$^eeF
 	s#   *L. L. K'L. .
M08.M++M0c                    U R                   R                  UUU=(       d    SUS9R                  U5        U R                  R                  UUUS9R	                  5         US:X  a2  U(       a*  U R
                  R                  UUUS9R	                  5         ggg! [         a,  n[        R                  " S[        U5       35         SnAgSnAff = f)a+  
Record guardrail metrics for prometheus.

Args:
    guardrail_name: Name of the guardrail
    latency_seconds: Execution latency in seconds
    status: "success", "error", or "intervened"
    error_type: Type of error if any, None otherwise
    hook_type: "pre_call", "during_call", or "post_call"
noner]   re   r  rc   z#Error recording guardrail metrics: N)
r   r   r  rd   rZ  rb   r   r   r   r   )r   r^   latency_secondsr_   r`   ra   r   s          r0   _record_guardrail_metrics*PrometheusLogger._record_guardrail_metrics
  s    $	Q1188-%/#	 9 
 go& 1188-# 9  ce  Z33::#1)' ;  #% &0   	Q  #Fs1vh!OPP	Qs   BB 
C""C		Cc                     U R                   R                  UUUUUS9R                  5         g ! [         a#  n[        R
                  " SU 35         S nAg S nAff = f)Nr~   z&Error recording batch created metric: )r}   r   rZ  r   r   warningr   r   rg   r   r   r   r   s          r0   record_managed_batch_created-PrometheusLogger.record_managed_batch_created
  sd    		Q44;;)%+ <  ce 	Q""%KA3#OPP	Q   +. 
AAAc                     U R                   R                  UUU=(       d    SU=(       d    SU=(       d    SS9R                  U5        g! [         a#  n[        R
                  " SU 35         SnAgSnAff = f)zXRecord the size of a managed file. Uses a gauge (last-seen value per label combination).r2  r   z"Error recording file size metric: N)r   r   r   r   r   r  )r   
size_bytesr   r   r   rg   r   r   s           r0   record_managed_file_size)PrometheusLogger.record_managed_file_size
  ss    		M0077#kr)/RZR 8  c*o 	M""%Gs#KLL	Ms   AA
 

A7A22A7c                     U R                   R                  U=(       d    SU=(       d    SS9R                  U5        g ! [         a#  n[        R
                  " SU 35         S nAg S nAff = f)Nr2  r   rg   z'Error recording batch duration metric: )r   r   r  r   r   r  )r   duration_secondsr   rg   r   s        r0   record_managed_batch_duration.PrometheusLogger.record_managed_batch_duration
  sh    	R77>>kr)/R ?  g&' 	R""%LQC#PQQ	Rs   ;> 
A+A&&A+c                     U R                   R                  UUUUUS9R                  5         g ! [         a#  n[        R
                  " SU 35         S nAg S nAff = f)Nr~   z%Error recording file created metric: )r   r   rZ  r   r   r  r  s          r0   record_managed_file_created,PrometheusLogger.record_managed_file_created   sd    		P33::)%+ ;  ce 	P""%J1##NOO	Pr  c                     U R                   R                  US9R                  5         g! [         a#  n[        R
                  " SU 35         SnAgSnAff = f)zIRecord a managed file deletion attempt. result is 'success' or 'blocked'.)r   z%Error recording file deleted metric: N)r   r   rZ  r   r   r  )r   r   r   s      r0   record_managed_file_deleted,PrometheusLogger.record_managed_file_deleted  sS    	P33::&:IMMO 	P""%J1##NOO	P   '* 
AAAc                   SSK n U R                  R                  UR                  5       5        U R                  R                  U5        U(       aF  U H?  u  pEU R                  R                  U=(       d    SU=(       d    SS9R                  5         MA     gg! [         a#  n[        R                  " SU 35         SnAgSnAff = f)z
Record CheckBatchCost polling metrics.

Args:
    jobs_polled: Number of unprocessed batches found
    processed_models: List of (model, api_provider) tuples for processed jobs
r   Nr2  r  z*Error recording check batch cost metrics: )
timer   r   r   r   r   rZ  r   r   r  )r   jobs_polledprocessed_modelsr  r   rg   r   s          r0   record_check_batch_cost_run,PrometheusLogger.record_check_batch_cost_run  s     		U<<@@M5599+F+;'EFFMM#kr%1%7R N  ce	 ,<    	U""%OPQs#STT	Us   BB 
C"C  Cc                     U R                   R                  US9R                  5         g ! [         a#  n[        R
                  " SU 35         S nAg S nAff = f)N)r`   z/Error recording check batch cost error metric: )r   r   rZ  r   r   r  )r   r`   r   s      r0   record_check_batch_cost_error.PrometheusLogger.record_check_batch_cost_error5  sW    	66==% > ce 	""A!E 	r  c                   [        U SS 5      n[        U[        5      (       a	  U(       a  U$  SSKJn  Ub  [        X5      (       a  gSn[        U S5      (       a  [        U S5      =(       d    Sn[        U5      S:  a  US   R                  5       USS  -   S-   nX0R                  R                  -  nU$ ! [
         a    S n Nf = f)	Nprometheus_exception_class_namer   )BudgetExceededErrorr  r2  rl     .)r  r.   r   litellm.exceptionsr  r  r(  lenupper	__class__r  )r  legacy_class_namer  exception_class_names       r0   r9  *PrometheusLogger._get_exception_class_name?  s     $I/PRVW'--2C$$	'> *z0
 0
 )!9n--#*9n#E#K  #$)$Q'--/2Fqr2JJSP ! 	 3 3 < <<##)  	'"&	's   B4 4CCc                b    U c  g[        [        U SS5      5      [        [        U SS5      5      4$ )a  
Pull the unified ``category`` / ``rate_limit_type`` fields off any
exception that declares them (``litellm.RateLimitError`` and bare-
Exception subclasses like ``BudgetExceededError``).

Values are validated against the :class:`RateLimitErrorCategory` /
:class:`RateLimitType` enums so unrelated third-party exceptions that
happen to declare ``.category`` / ``.rate_limit_type`` string attributes
can't leak garbage into Prometheus label cardinality.
NNNcategoryr1  )r   r  r   r(  s    r0   r4  +PrometheusLogger._extract_rate_limit_labelsj  s:     (J)MN$WY8I4%PQ
 	
r3   c                  #    SSK JnJn  [        R                  " SUU5        [        U5      nUR                  U5      =(       d    0 nUR                  US9nUR                  S5      n	[        [        [           UR                  S5      =(       d    / 5      n
[        UU	US   US   US	   US
   [        [        USS5      5      U R                  U5      U
S9	n[        R                  U U R                   SU[#        U5      S9  g7f)z6

Logs a successful LLM fallback event on prometheus

r   StandardLoggingMetadatar,  zLPrometheus: log_success_fallback_event, original_model_group: %s, kwargs: %sr  r   r  rj  rk  rl  rm  r~  N	rw  fallback_modelrv  r   ry  rz  r  r  r  rp   r  )r2  r  r,  r   r   r   r3  get_standard_logging_metadatar   r   r   r  r  r9  r)   r^  rp   r   )r   original_model_groupr   r<  r  r,  _metadata_keyr  standard_metadata
_new_modelr  rW  s               r0   log_success_fallback_event+PrometheusLogger.log_success_fallback_event  s    	

 	Z 	

 ?vFJJ}-3	'EE" F  	
 ZZ(
T#Y

6 2 8b9+0%,-@A+,@A"#9:()BC );]D!QR ::;MN

 	--8857D 	. 	
   D Dc                  #    SSK JnJn  [        R                  " SUU5        UR                  S5      n[        U5      nUR                  U5      =(       d    0 n[        [        [           UR                  S5      =(       d    / 5      n	UR                  US9n
[        UUU
S   U
S   U
S	   U
S
   [        [        USS5      5      U R                  U5      U	S9	n[        R                  U U R                   SU[#        U5      S9  g7f)z0
Logs a failed LLM fallback event on prometheus
r   r  zLPrometheus: log_failure_fallback_event, original_model_group: %s, kwargs: %sr   r  r  rj  rk  rl  rm  r~  Nr  rq   r  )r2  r  r,  r   r   r3  r   r   r   r   r  r  r  r9  r)   r^  rq   r   )r   r  r   r<  r  r,  r  r  r  r  r  rW  s               r0   log_failure_fallback_event+PrometheusLogger.log_failure_fallback_event  s    	

 	Z 	

 ZZ(
>vFJJ}-3	T#Y

6 2 8b9'EE" F  	 ,0%,-@A+,@A"#9:()BC );]D!QR ::;MN

 	--4417D 	. 	
r  c           
         [        U R                  SS9[        UUUUS9S9nU R                  R                  " S0 UD6R                  U5        g)z
Set the deployment state.
rh   rT  rh  ry  Nr   )rY  r   r  rh   r   r   )r   stater  r  r  rg   r]  s          r0   set_litellm_deployment_state-PrometheusLogger.set_litellm_deployment_state  s^     +"&"<"<6 #= # .#5!!)		

 	%%,,7w7;;EBr3   c                *    U R                  SXX45        g )Nr   r  r   r  r  r  rg   s        r0   r  'PrometheusLogger.set_deployment_healthy       	))!X	
r3   c                *    U R                  SXX45        g )Nr  r  r  s        r0   rj  .PrometheusLogger.set_deployment_partial_outage   r  r3   c                *    U R                  SXX45        g )N   r  r  s        r0   set_deployment_complete_outage/PrometheusLogger.set_deployment_complete_outage  r  r3   c           	         U R                   R                  [        U5      [        U5      [        U5      [        U5      [        U5      5      R                  5         g)z^
increment metric when litellm.Router / load balancing logic places a deployment in cool down
N)rk   r   r%   rZ  )r   r  r  r  rg   r  s         r0    increment_deployment_cooled_down1PrometheusLogger.increment_deployment_cooled_down  sK     	++22,-?@,X6,X6,\:,-=>	
 #%r3   c                R    U R                   R                  US9R                  5         g)zQ
Increment metric when logging to a callback fails (e.g., s3_v2, langfuse, etc.)
)rs   N)rr   r   rZ  )r   rs   s     r0   "increment_callback_logging_failure3PrometheusLogger.increment_callback_logging_failure)  s'     	55<<' 	= 	

#%r3   c                t    U R                   R                  U5      R                  U R                  UUS95        g)z/
Track provider remaining budget in Prometheus

max_budgetspendN)rf   r   r   _safe_get_remaining_budget)r   providerr  budget_limits       r0   track_provider_remaining_budget0PrometheusLogger.track_provider_remaining_budget4  s<     	55<<XFJJ++' , 	
r3   c                0    Uc  [        S5      $ Uc  U$ X-
  $ )Ninf)r  )r   r  r  s      r0   r  +PrometheusLogger._safe_get_remaining_budgetA  s(     <=!!r3   c           	       #    SSK Jn  Uc  g SnSnU" XeS9I Sh  vN u  pxUc  [        U5      nX-   S-
  U-  n	U" U5      I Sh  vN   [        SU	S-   5       H#  nU" XeS9I Sh  vN u  pzU" U5      I Sh  vN   M%     g Ne N= N N! [         a/  n[
        R                  " SU S	[        U5       35         SnAgSnAff = f7f)
a2  
Generic method to initialize budget metrics for teams or API keys.

Args:
    data_fetch_function: Function to fetch data with pagination.
    set_metrics_function: Function to set metrics for the fetched data.
    data_type: String representing the type of data ("teams" or "keys") for logging purposes.
r   prisma_clientNr  2   )	page_sizepager  zError initializing z budget metrics: )r  r  r  ranger   r   r  r   )r   data_fetch_functionset_metrics_function	data_typer  r  r  rB  total_counttotal_pages_r   s               r0   _initialize_budget_metrics+PrometheusLogger._initialize_budget_metricsL  s      	= 	DI&9#' !D "!$i '2Q69DK 't,,, aq1 3i SS*4000 2! - T0 	$$%i[0A#a&J 	st   CB	 B)B	 BB	 %B&B	 7B8B	  CB	 B	 B	 B	 	
C%B=8C=CCc                   ^^#    SSK Jm  SSKJm  Tc  [        R
                  " S5        g      S	UU4S jjnU R                  UU R                  SS9I Sh  vN   g N7f)
zI
Initialize team budget metrics by reusing the generic pagination logic.
r   )get_paginated_teamsr  NzDPrometheus: skipping team metrics initialization, DB not initializedc                T   >#    T" TXS9I S h  vN u  p#Uc  [        U5      nX#4$  N7f)N)r  r  r  )r  )r  r  teamsr  r  r  s       r0   fetch_teamsEPrometheusLogger._initialize_team_budget_metrics.<locals>.fetch_teams  s>      (;+y( "E "!%j%%"s   (&(r	  r  r  r   )r  r  r  r  r^  z-Tuple[List[LiteLLM_TeamTable], Optional[int]])1litellm.proxy.management_endpoints.team_endpointsr  r  r  r   r   r  _set_team_list_budget_metrics)r   r
  r  r  s     @@r0   _initialize_team_budget_metrics0PrometheusLogger._initialize_team_budget_metricsy  s|     	
 	=   V 	&	&"%	&:	& 	& -- +!%!C!C . 
 	
 	
s   AA AA c                   ^^^#    SSK Jm  SSKJm  SSKJm  Tc  [        R                  " S5        gS
UUU4S jjnU R                  UU R                  SS	9I Sh  vN   g N7f)zL
Initialize API key budget metrics by reusing the generic pagination logic.
r   )UI_SESSION_TOKEN_TEAM_ID)_list_key_helperr  NzCPrometheus: skipping key metrics initialization, DB not initializedc                   >#    T" TUU S S S S TSS S9
I S h  vN nUR                  S/ 5      nUR                  S5      nUc  [        U5      nX44$  N87f)NT)
r  r  sizer  r7  r6  key_hashexclude_team_idreturn_full_objectorganization_idr  r  )r3  r  )r  r  key_list_responser  r  r  r  r  s        r0   
fetch_keysGPrometheusLogger._initialize_api_key_budget_metrics.<locals>.fetch_keys  sw      '7+ 8#' $' ! %((4D+//>K"!$i$$!!s   AA9Ar  r  )r  r  r  r  r^  zXTuple[List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]], Optional[int]])
litellm.constantsr  ;litellm.proxy.management_endpoints.key_management_endpointsr  r  r  r   r   r  _set_key_list_budget_metrics)r   r  r  r  r  s     @@@r0   "_initialize_api_key_budget_metrics3PrometheusLogger._initialize_api_key_budget_metrics  si      	?	
 	=   U 	% 	%, -- *!%!B!B . 
 	
 	
s   AA"A A"c                   ^#    SSK Jm  Tc  [        R                  " S5        g      SU4S jjnU R	                  UU R
                  SS9I Sh  vN   g N7f)	zI
Initialize user budget metrics by reusing the generic pagination logic.
r   r  NzDPrometheus: skipping user metrics initialization, DB not initializedc                   >#    US-
  U -  n[        T5      R                  R                  UU SS0S9I S h  vN n[        T5      R                  R                  5       I S h  vN nX44$  N2 N	7f)Nr  
created_atdesc)skiptakeorder)r"   table	find_manycount)r  r  r&  usersr  r  s        r0   fetch_usersEPrometheusLogger._initialize_user_budget_metrics.<locals>.fetch_users  s|      1H	)D(7==GG#V, H  E
 !/} = C C I I KKK%%
 Ls!   2A,A(*A, A*!A,*A,r,  r  )r  r  r  r  r^  z-Tuple[List[LiteLLM_UserTable], Optional[int]])r  r  r   r   r  _set_user_list_budget_metrics)r   r-  r  s     @r0   _initialize_user_budget_metrics0PrometheusLogger._initialize_user_budget_metrics  sr      	=   V 
	&
	&"%
	&:
	& -- +!%!C!C . 
 	
 	
s   AAAAc                   ^#    SSK Jm  Tc  [        R                  " S5        gSU4S jjnU R	                  UU R
                  SS9I Sh  vN   g N7f)	zH
Initialize org budget metrics by reusing the generic pagination logic.
r   r  NzCPrometheus: skipping org metrics initialization, DB not initializedc                   >#    US-
  U -  n[        T5      R                  R                  UU SS0SS0S9I S h  vN n[        T5      R                  R                  5       I S h  vN nX44$  N2 N	7f)Nr  r$  r%  litellm_budget_tableT)r&  r'  r(  include)r    r)  r*  r+  )r  r  r&  orgsr  r  s        r0   
fetch_orgsCPrometheusLogger._initialize_org_budget_metrics.<locals>.fetch_orgs  s     1H	)D/>DDNN#V,/6	 O  D !7} E K K Q Q SSK$$ Ts!   5A/A+*A/#A-$A/-A/r6  r  )r  r  r  r  r^  zTuple[list, Optional[int]])r  r  r   r   r  _set_org_list_budget_metrics)r   r7  r  s     @r0   _initialize_org_budget_metrics/PrometheusLogger._initialize_org_budget_metrics  sZ      	=   U 		% -- *!%!B!B . 
 	
 	
s   AA
AAc                  #    SSK Jn  SSKJn  UR                  R
                  nU(       a_  UR                  (       aN  UR                  US9I Sh  vN (       a1   U R                  5       I Sh  vN   UR                  US9I Sh  vN   ggU R                  5       I Sh  vN   g NT N8 N#! UR                  US9I Sh  vN    f = f N(7f)aq  
Handler for initializing remaining budget metrics for all teams to avoid metric discrepancies.

Runs when prometheus logger starts up.

- If redis cache is available, we use the pod lock manager to acquire a lock and initialize the metrics.
    - Ensures only one pod emits the metrics at a time.
- If redis cache is not available, we initialize the metrics directly.
r   )'PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME)proxy_logging_obj)
cronjob_idN)
r  r=  r  r>  db_spend_update_writerpod_lock_managerredis_cacheacquire_lock$_initialize_remaining_budget_metricsrelease_lock)r   r=  r>  rA  s       r0   #initialize_remaining_budget_metrics4PrometheusLogger.initialize_remaining_budget_metrics  s      	N@,CCTT  0 < <%22B 3   CCEEE*77#J 8    ;;=== F*77#J 8   
 >sf   AC	B$	C	B* -B&.B* 2C	B(C	CC	&B* (C	*C=C >CC	c                <  #    [         R                  " S5        U R                  5       I Sh  vN   U R                  5       I Sh  vN   U R	                  5       I Sh  vN   U R                  5       I Sh  vN   U R                  5       I Sh  vN   g Ne NO N9 N# N7f)zS
Helper to initialize remaining budget metrics for all teams, API keys, and users.
z0Emitting key, team, user, org budget metrics....N)r   r   r  r   r0  r:  '_initialize_user_and_team_count_metrics)r   s    r0   rD  5PrometheusLogger._initialize_remaining_budget_metrics  s      	OP22444557772244411333::<<<	 	5743<sW   *BBBBBBB4B5BBBBBBBc                D  #    SSK Jn  Uc  [        R                  " S5        g [	        U5      R
                  R                  5       I Sh  vN nU R                  R                  U5        [        R                  " SU 35        [        U5      R
                  R                  5       I Sh  vN nU R                  R                  U5        [        R                  " SU 35        g N N;! [         a,  n[        R                  " S[        U5       35         SnAgSnAff = f7f)z
Initialize user and team count metrics by querying the database.

Updates:
- litellm_total_users: Total count of users in the database
- litellm_teams_count: Total count of teams in the database
r   r  NzOPrometheus: skipping user/team count metrics initialization, DB not initializedz'Prometheus: set litellm_total_users to z'Prometheus: set litellm_teams_count to z,Error initializing user/team count metrics: )r  r  r   r   r"   r)  r+  r   r   r!   r   r   r  r   )r   r  total_userstotal_teamsr   s        r0   rI  8PrometheusLogger._initialize_user_and_team_count_metrics(  s      	=   a 	 .} = C C I I KKK++//<  9+G
 !/} = C C I I KKK++//<  9+G L L
  	$$>s1vhG 	sS   !D &C' 
C#AC' )C%*8C' "D #C' %C' '
D1"DD DD c                n   #    U H+  n[        U[        5      (       d  M  U R                  U5        M-     g7f)z8Helper function to set budget metrics for a list of keysN)r.   r   _set_key_budget_metrics)r   r  r  s      r0   r  -PrometheusLogger._set_key_list_budget_metricsK  s,      C#~..,,S1 s   55c                @   #    U H  nU R                  U5        M     g7f)z9Helper function to set budget metrics for a list of teamsN)_set_team_budget_metrics)r   r	  ry  s      r0   r  .PrometheusLogger._set_team_list_budget_metricsS       D))$/    c                @   #    U H  nU R                  U5        M     g7f)z9Helper function to set budget metrics for a list of usersN)_set_user_budget_metrics)r   r,  r   s      r0   r/  .PrometheusLogger._set_user_list_budget_metricsX  rU  rV  c                  #    U H  n[        USS5      nU R                  UR                  =(       d    SUR                  =(       d    SUR                  =(       d    SU(       a  UR
                  OSU(       a  [        USS5      OSS9  M     g7f)z8Helper function to set budget metrics for a list of orgsr4  Nr2          budget_reset_atr{  r|  r  r  r\  )r  _set_org_budget_metricsr  r8  r  r  )r   r6  r  budget_tables       r0   r9  -PrometheusLogger._set_org_list_budget_metrics]  s}     C"3(>EL((**0b006Bii&36B<22 $ L*;TB ) 
 s   BBc                   #    U(       a6  U R                  UU=(       d    SUUUS9I Sh  vN nU R                  U5        gg N7f)z
Set team budget metrics after an LLM API request

- Assemble a LiteLLM_TeamTable object
    - looks up team info from db if not available in metadata
- Set team budget metrics
r2  )r7  rz  r  r  rr  N)_assemble_team_objectrS  )r   r  r  r  r  rr  team_objects          r0   r  ;PrometheusLogger._set_team_budget_metrics_after_api_requestm  sR       $ : :%.4" *+ !; ! K ))+6    'AAAc                  #    SSK Jn  SSKJnJn  U=(       d    SU-   n	[        UUU	US9n
 U" UUUS9I Sh  vN nU(       a<  UR                  U
l
        U
R                  c  UR                  b  UR                  U
l        U
$  NI! [         a.  n[        R                  " S[        U5       35        U
s SnA$ SnAff = f7f)z
Assemble a LiteLLM_TeamTable object

for fields not available in metadata, we fetch from db
Fields not available in metadata:
- `budget_reset_at`
r   )get_team_objectr  user_api_key_cache)r7  rz  r  r  )r7  r  ri  Nz4[Non-Blocking] Prometheus: Error getting team info: )litellm.proxy.auth.auth_checksrg  r  r  ri  r   r   r   r   r   r\  r  )r   r7  rz  r  r  rr  rg  r  ri  _total_team_spendrc  	team_infor   s                r0   rb  &PrometheusLogger._assemble_team_object  s      	CP"Za=8'!#!	

	-+#5 I *3*C*CK'%%-)2F2F2R)2)=)=&!
  	  Fs1vhO 		sD   )B>B BB AB>B 
B;#B60B;1B>6B;;B>c                   [        UR                  UR                  =(       d    SS9n[        U R	                  SS9US9nU R
                  R                  " S0 UD6R                  U R                  UR                  UR                  S95        UR                  bL  [        U R	                  SS9US9nU R                  R                  " S0 UD6R                  UR                  5        UR                  bZ  [        U R	                  S	S9US9nU R                  R                  " S0 UD6R                  U R                  UR                  S
95        gg)z^
Set team budget metrics for a single team

- Remaining Budget
- Max Budget
- Budget Reset At
r2  )ry  rz  rJ   rT  ry  r  NrK   rL   r\  r   )r  r7  rz  rY  r   rJ   r   r   r  r  r  rK   r\  rL   %_get_remaining_hours_for_budget_reset)r   ry  rW  r]  s       r0   rS  )PrometheusLogger._set_team_budget_metrics  s^    ,,"

 +"&"<"<B #= # $	
 	1188C7CGG++??jj , 	
 ??&.&*&@&@ @ 'A ' (	G //66AAEEdooV+.&*&@&@ L 'A ' (	G ;;BBMWMQQ::$($8$8 ;  ,r3   c                  #    U(       d  gSSK Jn  SSKJnJn  Uc  g U" UUUSS9I Sh  vN nUc  gUR                  =(       d    SnUR                  =(       d    S	U-   n	UR                  n
U
(       a  U
R                  OSnU
(       a  [        U
S
S5      OSnU R                  UUU	UUS9  g N! [
         a,  n[        R                  " S[        U5       35         SnAgSnAff = f7f)zz
Set org budget metrics after an LLM API request

- Fetches org info via cache (get_org_object)
- Sets org budget metrics
Nr   )get_org_objectrh  T)r{  r  ri  include_budget_tablez3[Non-Blocking] Prometheus: Error getting org info: r2  r[  r\  r]  )rj  rs  r  r  ri  r   r   r   r   r8  r  r4  r  r  r^  )r   r{  rr  rs  r  ri  org_infor   r|  _total_org_spendr_  r  r\  s                r0   r  :PrometheusLogger._set_org_budget_metrics_after_api_request  s      AP 	++#5%)	 H //52	$NN1c]B440<\,,$
>JGL"3T:PT 	 	$$"!+ 	% 	
/  	  Ec!fXN 		s>   C&B- B+B- A<C&+B- -
C#7"CC&C##C&c                   [        UUS9n[        U R                  SS9US9nU R                  R                  " S
0 UD6R                  U R                  UUS95        UbB  [        U R                  SS9US9nU R                  R                  " S
0 UD6R                  U5        UbP  [        U R                  SS9US9nU R                  R                  " S
0 UD6R                  U R                  US	95        gg)z\
Set org budget metrics for a single org

- Remaining Budget
- Max Budget
- Budget Reset At
)r{  r|  rM   rT  ry  r  NrN   rO   ro  r   )
r  rY  r   rM   r   r   r  rN   rO   rp  )r   r{  r|  r  r  r\  rW  r]  s           r0   r^  (PrometheusLogger._set_org_budget_metrics  s5    ,

 +"&"<"<A #= # $	
 	0077B'BFF++% , 	
 !.&*&@&@ ? 'A ' (	G ..55@@DDZP&.&*&@&@ K 'A ' (	G ::AALGLPP::$3 ;  'r3   c                l   [        UR                  UR                  =(       d    SS9n[        U R	                  SS9US9nU R
                  R                  " S
0 UD6R                  U R                  UR                  UR                  S95        UR                  bL  [        U R	                  SS9US9nU R                  R                  " S
0 UD6R                  UR                  5        UR                  bC  U R                  R                  " S
0 UD6R                  U R                  UR                  S	95        gg)zS
Set virtual key budget metrics

- Remaining Budget
- Max Budget
- Budget Reset At
r2  )rv  r   rP   rT  ry  r  NrQ   ro  r   )r  tokenr6  rY  r   rP   r   r   r  r  r  rQ   r\  rR   rp  )r   r)  rW  r]  s       r0   rP  (PrometheusLogger._set_key_budget_metricsQ  sG    ,,22+55;
 +"&"<"<E #= # $	
 	44;;FgFJJ++,77'-- , 	
 ''3.&*&@&@ C 'A ' (	G 2299DGDHH!,, ,,8>>EEPPTT::$5$E$E ;  9r3   c                   #    U(       a6  U R                  UU=(       d    SUUUS9I S h  vN nU R                  U5        g g  N7f)Nr2  )r  rk  r  r  rr  )_assemble_key_objectrP  )r   r  rk  rr  r  r  r)  s          r0   r  >PrometheusLogger._set_api_key_budget_metrics_after_api_request|  sU      &*&?&?)#5#;-#+ '@ ' ! (():; !re  c                ^  #    SSK Jn  SSKJnJn  U=(       d    SU-   n	[        UUUU	S9n
 U
R                  (       a2  U" U
R                  UUS9I Sh  vN nU(       a  UR                  U
l        U
$  N! [         a-  n[        R                  " S[        U5       35         SnAU
$ SnAff = f7f)z"
Assemble a UserAPIKeyAuth object
r   )get_key_objectrh  )r{  r6  r  r  )hashed_tokenr  ri  Nz3[Non-Blocking] Prometheus: Error getting key info: )rj  r  r  r  ri  r   r{  r\  r   r   r   r   )r   r  rk  r  r  rr  r  r  ri  _total_key_spendr)  
key_objectr   s                r0   r~  %PrometheusLogger._assemble_key_object  s      	BP%Nm;*(%"	
	 &&#1!2!8!8"/'9$ 

 8B8R8R%5 !   	  Ec!fXN  ! 	s@   )B-&A3 A1A3 /B-1A3 3
B*="B%B-%B**B-c                v   #    U(       a,  U R                  UUUUS9I Sh  vN nU R                  U5        gg N7f)z
Set user budget metrics after an LLM API request

- Assemble a LiteLLM_UserTable object
    - looks up user info from db if not available in metadata
- Set user budget metrics
)r  r  r  rr  N)_assemble_user_objectrX  )r   r  r  r  rr  user_objects         r0   r  ;PrometheusLogger._set_user_budget_metrics_after_api_request  sJ       $ : : *+	 !; ! K ))+6 s   979c                  #    SSK Jn  SSKJnJn  U=(       d    SU-   n[        UUUS9n	 U" UUUSSS9I Sh  vN n
U
(       ax  U
R                  U	l
        U	R                  c  U
R                  b  U
R                  U	l        U
R                  b  U
R                  U	l        U
R                  b  U
R                  U	l        U	$  N! [         a.  n[        R                  " S[        U5       35        U	s SnA$ SnAff = f7f)	z
Assemble a LiteLLM_UserTable object

for fields not available in metadata, we fetch from db
Fields not available in metadata:
- `budget_reset_at`
r   )get_user_objectrh  )r  r  r  F)r  r  ri  user_id_upsertcheck_db_onlyNz4[Non-Blocking] Prometheus: Error getting user info: )rj  r  r  r  ri  r   r   r   r   r   r\  r  r   
user_alias)r   r  r  r  rr  r  r  ri  _total_user_spendr  	user_infor   s               r0   r  &PrometheusLogger._assemble_user_object  s     	CP"Za=8'#!

	 .+#5$# I *3*C*CK'%%-)2F2F2R)2)=)=&##/)2)=)=&##/)2)=)=&-  	  Fs1vhO 		sD   (C;C  B>C  BC;>C   
C8
#C3-C8.C;3C88C;c                   [        UR                  UR                  =(       d    SUR                  =(       d    SS9n[	        U R                  SS9US9nU R                  R                  " S0 UD6R                  U R                  UR                  UR                  S95        UR                  bL  [	        U R                  SS9US9nU R                  R                  " S0 UD6R                  UR                  5        UR                  bZ  [	        U R                  S	S9US9nU R                  R                  " S0 UD6R                  U R                  UR                  S
95        gg)z^
Set user budget metrics for a single user

- Remaining Budget
- Max Budget
- Budget Reset At
r2  )r   r   r  rS   rT  ry  r  NrT   rU   ro  r   )r  r  r   r  rY  r   rS   r   r   r  r  r  rT   r\  rU   rp  )r   r   rW  r]  s       r0   rX  )PrometheusLogger._set_user_budget_metrics  sj    ,,","
 +"&"<"<B #= # $	
 	1188C7CGG++??jj , 	
 ??&.&*&@&@ @ 'A ' (	G //66AAEEdooV+.&*&@&@ L 'A ' (	G ;;BBMWMQQ::$($8$8 ;  ,r3   c                j    U[         R                  " UR                  5      -
  R                  5       S-  $ )z&
Get remaining hours for budget reset
i  )r   r  tzinfor  )r   r\  s     r0   rp  6PrometheusLogger._get_remaining_hours_for_budget_reset1  s.    
 hll?+A+ABB
-/D! 	!r3   c                |    [        U[        5      (       a'  [        U[        5      (       a  X!-
  R                  5       $ g)z
Compute the duration in seconds between two objects.

Returns the duration as a float if both start and end are instances of datetime,
otherwise returns None.
N)r.   r   r  )r   r  r  s      r0   r  'PrometheusLogger._safe_duration_seconds9  s2     j(++
8X0N0N)88::r3   c                D   SSK Jn  [        R                  R	                  [
        S9n[        R                  " S[        U5      5        [        U5      S:  aJ  [        [
        US   5      n[        R                  " SU-  5        U R                  UR                  SUSSS	9  g
g
)z
Initialize budget metrics as a cron job. This job runs every `PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES` minutes.

It emits the current remaining budget metrics for all Keys and Teams.
r   )2PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES)callback_typezfound %s prometheus loggerszNInitializing remaining budget metrics as a cron job executing every %s minutesintervalprometheus_budget_metrics_jobT)minutesr/  replace_existingN)r  r  r,   logging_callback_managerget_custom_loggers_for_typer)   r   r   r  r   add_jobrF  )	schedulerr  prometheus_loggersprometheus_loggers       r0   "initialize_budget_metrics_cron_job3PrometheusLogger.initialize_budget_metrics_cron_jobH  s     	Y ,,HH. I  	 	:C@R<ST!"Q& $%57I!7L M  `DE !EEJ2!%   'r3   c                     SSK Jn   SSKJn  SSKJn  S[        R                  ;   a)  SSK JnJ	n  U" 5       nUR                  U5        U " U5      nOU " 5       nUR                  SU5        UR                  S5        g	)
z
Mount the Prometheus metrics endpoint with optional authentication.

Args:
    require_auth (bool, optional): Whether to require authentication for the metrics endpoint.
                                Defaults to False.
r   )make_asgi_app)verbose_proxy_logger)appPROMETHEUS_MULTIPROC_DIR)CollectorRegistrymultiprocessz/metricsz;Starting Prometheus Metrics on /metrics (no authentication)N)r   r  litellm._loggingr  r  r  osenvironr  r  MultiProcessCollectormountr   )r  r  r  r  r  registrymetrics_apps          r0   _mount_metrics_endpoint(PrometheusLogger._mount_metrics_endpointg  sf     	492 &3I(*H..x8'1K'/K 			*k*""I	
r3   )Ir   r   r   r   r   r   r   r   rR   rQ   rv   rw   rx   rr   r   r   r   r   rk   rq   rm   ro   rj   rh   rl   rp   rn   ri   rb   r   rd   rG   rF   rE   rC   rt   r?   r@   r}   r   r   r   r   rO   rN   rI   rH   rD   rZ   rz   ry   rf   r8   r<   rP   rV   rW   rM   rX   rJ   rY   rS   r   r=   ru   rA   rL   rK   r   r   r   rU   rT   )r^  zOptional['PrometheusLogger'])r^  Dict[str, List[str]])r   r   r^  r   )r   r   r^  zOptional[MetricValidationError])r   r   r   	List[str]r^  zOptional[LabelValidationError])r   r   r^  r  )r   r   r   r  )r   r   r^  rg  )r   r   r   r  r   r  r^  rg  )r  r   r   r   r^  rg  )r   r   )r   r  r^  rg  )r   r   r^  ri  )r   r   r^  r  )r  r   r   r   r   Dict[str, Optional[str]]r^  rg  )N      ?)r[  r   r   r   rW  r  rX  'Optional[PrometheusLabelFactoryContext]r\  r  r^  rg  rB  )rc  r&   r  r_  r  r_  rk  r_  r   r_  r  r_  r  r_  r  r_  rW  r  rX  r  )rc  r&   rW  r  rX  r  r^  rg  )rc  r&   rW  r  rX  r  r  )r  r_  r  r_  r  r_  rk  r_  rd  r   rr  r  r  r_  rn  r_  )r  r_  r  r_  rk  r_  r   r_  r  r_  r  r_  r  r_  rr  r  rW  r  rX  r  )
r  r_  rk  r_  r   r   re  r   r  r_  )r   r   r   r_  r  r_  rk  r_  r  r_  r  r_  rW  r  rX  r  )NNN)r   Optional[dict]rW  Optional[Any]r  Optional[Exception]r^  Optional[int])r~  r  r  r  r^  ri  )NNNNN)r   r  r)  r  rW  r  rc  z-Optional[Union[dict, StandardLoggingPayload]]r  r  r^  ri  )r;  r   r<  r   r)  r   r=  r_  )rB  r   r)  r   )rG  r   r  r   rH  r   r^  r   )r`  r   r^  r  )r`  r   )r.  r   rd  r   r  r_  r  r_  r  r_  rl  r_  )r  N)r`  r   rW  r  r  r  rX  r  )
r^   r   r  r  r_   r   r`   r_  ra   r   )
r   r_  rg   r_  r   r_  r   r_  r   r_  )r  r  r   r   r   r   r   r_  rg   r_  r   r_  )r  r  r   r_  rg   r_  )r   r   )r  r  r  z3Optional[List[Tuple[Optional[str], Optional[str]]]])r`   r   )r  r   r^  r   )r  r  r^  z#Tuple[Optional[str], Optional[str]])r  r   r   r   r<  r   )
r  r  r  r   r  r_  r  r_  rg   r   )r  r   r  r   r  r   rg   r   )r  r   r  r_  r  r_  rg   r   )
r  r   r  r   r  r   rg   r   r  r   )rs   r   )r  r   r  r  r  r  )r  Optional[float]r  r  r^  r  )r  z9Callable[..., Awaitable[Tuple[List[Any], Optional[int]]]]r  z&Callable[[List[Any]], Awaitable[None]]r   z)Literal['teams', 'keys', 'users', 'orgs'])r  z List[Union[str, UserAPIKeyAuth]])r	  zList[LiteLLM_TeamTable])r,  zList[LiteLLM_UserTable])r6  r  )
r  r_  r  r_  r  r  r  r  rr  r  )r7  r   rz  r   r  r  r  r  rr  r  r^  r   )ry  r   )r{  r_  rr  r  )
r{  r   r|  r   r  r  r  r  r\  zOptional[datetime])r)  r   )
r  r_  rk  r_  rr  r  r  r  r  r  )r  r   rk  r   r  r  r  r  rr  r  r^  r   )r  r_  r  r  r  r  rr  r  )
r  r   r  r  r  r  rr  r  r^  r   )r   r   )r\  r   r^  r  )r  r   r  r   r^  r  )r  r'   )[r  
__module____qualname____firstlineno__staticmethodr1   r   r   r   r   r   r   r   r   r   r   r#  r   r.  r   r   rQ  r^  r  r  r  r  r  r  r  r  r  r  r%  r  r?  rD  rI  rb  r  rz  r  r  r  r  r  r  r  r  r  r  r9  r4  r  r  r  r  rj  r  r  r  r  r  r  r  r   r0  r:  rF  rD  rI  r  r  r/  r9  r  rb  rS  r  r^  rP  r  r~  r  r  rX  rp  r  r  r  __static_attributes__r   r3   r0   r)   r)   A   s     ob)VY.	((1	',*P:"3P:	P:d))09)IR)	)V)#&)7<)	)^	,DC1DC	DCL
3$5$	$L 
 
 0 
 )	 

 
 
N BFJJ 0J +	J
 ?J J 
J W
H BF5
"85
 #5
 $	5

 *5
 5
 %5
 +5
 5
 +5
 ?5
v BF	T"8T +T ?	T
 
Tt BF	]"8] +] ?	]N "&-14$4 +4 $	4
 *4 4 4 4 +4B BF
"
 $
 *	

 
 %
 +
 
 
 +
 ?
F #';
#;
 *;
 	;

 ;
  ;
L BFkk k $	k
 *k %k +k +k ?kZBL "&%))-	99 #9 '	9
 
9| *.%"% '% 
	%R "&+/%)RV)-)) )) #	)
 #P) ') 
)` (,VV &V *	V
 %Vp

-;
*\
"\
	!\
|Un)I)I )I *	)I
  )I  )I $)I^ BF	["8[ +[ ?	[
 
[F  #AEgg
 +g g ?gR*Q*Q *Q 	*Q
 "*Q *Q`QQ $Q 	Q
 "Q %Q0  $&*"MM M 	M
 M $M M0  $&*	RR R $	RPP $P 	P
 "P %P&P QUUU NU6 ($ ($T 
&
	,
 
(-
$'-
15-
KT-
^,
$',
15,
KT,
\CC  C  	C
  C C2	
	
 	
 		

 	
	
	
  	
  		

 	
	
	
  	
  		

 	
  	
  &		

$)
9>
	")	"2A	"		"+V+ E+ =	+Z
>*
X
<
:><	=!F2420
0
 7$7 +7 $	7
 )7 74** * 	*
 $* * 
*X11f2
2
 2
h55 5 	5
 $5 ,5n)V<#< *< 	<
 (< #<$#!#!  #! (	#!
 ##! #! 
#!J77 $7 )	7
 7000 0 $	0
 0 
0d22h!  
	  < 
 
r3   r)   c                   U  Vs0 s H#  nX!R                   ;   d  M  X!R                   U   _M%     nn[        R                  R                  U;   a+  UR	                  5       U[        R                  R                  '   UR
                  R                  5        H  u  pEX@;   d  M  XSU'   M     UR                  R                  5        H  u  pgX`;   d  M  XsU'   M     U  H  nX#;  d  M
  S X2'   M     U$ s  snf rB  )_sanitized_enumrJ  rK  rL  get_resolved_end_user_custom_by_sanitized_keyr)  _tag_labels)rV  ctxr   r>  skrP  kvs           r0   _prometheus_labels_from_contextr    s     +1*E''' 	*""5))*  1 $$**o=%%' 	,55;;< //557&"%B 8 %%'%!"A ( ''%)O" ' /1s
   C,C,r  c                  Ub%  UR                   ULa  [        S5      e[        X5      $ UR                  5       nUR	                  5        VVs0 s H  u  pVXP;   d  M  U[        U5      _M     nnn[        R                  R                  U;   a   [        5       nU" SUR                  0SS9US'   UR                  bB  UR                  R	                  5        H$  u  p[        U	5      n
X;   d  M  [        U5      Xz'   M&     UR                  bB  [        UR                  5      nUR	                  5        H  u  pX;   d  M  [        U5      Xy'   M     U  H  nXW;  d  M
  SXu'   M     U$ s  snnf )a  
Returns a dictionary of label + values for prometheus.

Ensures end_user param is not sent to prometheus if it is not supported.

When ``label_context`` is provided, it must have been built from the same
``enum_values`` object; work is amortized (single model_dump, tag map, etc.).
Nz@label_context.enum_values must be the same object as enum_valuesuser_api_key_end_user_idrf  )rd  rh  ru  )rW  r   r  
model_dumpr)  r%   rJ  rK  rL  r   ru  r  r$   r  get_custom_labels_from_tags)rV  rW  tagrX  	enum_dictr   rL  r>  r  r  sanitized_keycustom_tag_labelss               r0   rY  rY    sz     $$K7R  //DTT &&(I &OO--LE) 	7/66-   $$**o=,U,W)&G68L8LM%'

#
 ))5%<<BBDJC;C@M51QRW1X.	 E #78H8HI+113JC+'G'N$ 4 ''%)O" ' As   E E c                   [         R                  nUb  [        U5      S:X  a  0 $ 0 nU H  nUnUR                  S5      (       a  UR	                  SSS5      OUnUR                  S5      nU nU H2  n[        U[        5      (       a  UR                  US5      nOSnUb  M2    O   Uc  M  [        U[        5      (       d  M  XbUR	                  SS5      '   M     U$ )z!
Get custom labels from metadata
Nr   z	metadata.r2  r  r  r  )
r,   !custom_prometheus_metadata_labelsr  r  replacesplitr.   r   r3  r   )re  r  r   r  original_key
keys_partsrL  parts           r0   r  r    s     44D|s4yA~	F141L1Lckk+r1-RUYYs^
D%&&		$-}  E3!7!75:<''S12# & Mr3   c                   [        U [        5      (       d  0 $ U R                  S5      =(       d    0 n[        U[        5      (       d  0 $ UR                  S5      nUR                  S5      nUR                  S5      n0 [        U[        5      (       a  UO0 E[        U[        5      (       a  UO0 E[        U[        5      (       a  UE$ 0 E$ )zH
Combine the metadata sources that can supply custom Prometheus labels.
re  requester_metadatauser_api_key_auth_metadataspend_logs_metadata)r.   r   r3  )rc  standard_logging_metadatar  r  r  s        r0   r  r    s     .55	 8 < <Z H NB/66	2667KL!:!>!>$" 4778MN!+,>!E!E2 4d;; ' #--@$"G"G  NP r3   c                v   ^^ SSK mSSKJn  U" 5       nUR                  U5      m[	        UU4S jU  5       5      $ )a  
Check if any of the request tags matches a wildcard configured pattern

Args:
    tags: List[str] - The request tags
    configured_tag: str - The configured tag

Returns:
    bool - True if any of the request tags matches the configured tag, False otherwise

e.g.
tags = ["User-Agent: curl/7.68.0", "User-Agent: python-requests/2.28.1", "prod"]
configured_tag = "User-Agent: curl/*"
_tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True

configured_tag = "User-Agent: python-requests/*"
_tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True

configured_tag = "gm"
_tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # False
r   N)PatternMatchRouterc              3  D   >#    U  H  nTR                  TUS 9v   M     g7f))r  stringN)match)rC  r  reregex_patterns     r0   rD  ;_tag_matches_wildcard_configured_pattern.<locals>.<genexpr>>  s     Kdsrxxcx:ds    )r  .litellm.router_utils.pattern_match_deploymentsr  _pattern_to_regexr#  )r  configured_tagr  pattern_routerr  r  s       @@r0   (_tag_matches_wildcard_configured_patternr     s4    0 Q')N"44^DMKdKKKr3   c                    SSK Jn  [        R                  nUb  [	        U5      S:X  a  0 $ 0 nU H7  nU" SU 35      nX@;   a  SX5'   M  SU;   a  [        XS9(       a  SX5'   M3  SX5'   M9     U$ )an  
Get custom labels from tags based on admin configuration.

Supports both exact matches and wildcard patterns:
- Exact match: "prod" matches "prod" exactly
- Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0"

Reuses PatternMatchRouter for wildcard pattern matching.

Returns dict of label_name: "true" if the tag matches the configured tag, "false" otherwise

{
    "tag_User-Agent_curl": "true",
    "tag_User-Agent_python_requests": "false",
    "tag_Environment_prod": "true",
    "tag_Environment_dev": "false",
    "tag_Service_api_gateway_v2": "true",
    "tag_Service_web_app_v1": "false",
}
r   )r$   tag_truer#   )r  r  false)r   r$   r,   custom_prometheus_tagsr  r  )r  r$   configured_tagsr   r  
label_names         r0   r  r  A  s    , V44O#o"6!";	F)4tN;K5LM
 !!'F . %M&
 "(F %! *$ Mr3   )rV  r  r  r   r^  r  rB  )
rV  r  rW  r  r  r_  rX  r  r^  r   )re  r   r^  Dict[str, str])rc  r  r^  zDict[str, Any])r  Sequence[str]r  r   r^  ri  )r  r  r^  r  )>
__future__r   r  r  r  r   r   r   r   r   r   r	   r
   r   r   r   r   r   r   r   r,   r  r   r   r  r   r   "litellm.integrations.custom_loggerr   'litellm.integrations.prometheus_helpersr   r   Ilitellm.integrations.prometheus_helpers.bounded_prometheus_series_trackerr   'litellm.litellm_core_utils.core_helpersr   r   litellm.proxy._typesr   r   r   r   ,litellm.repositories.organization_repositoryr    $litellm.repositories.team_repositoryr!   $litellm.repositories.user_repositoryr"   r   r$   r%   litellm.types.utilsr&   apscheduler.schedulers.asyncior'   r)   r  rY  r  r  r  r  r   r3   r0   <module>r     s0   #  	 
 (     : <  P ? ? 3 7?B=
| B=
Jz$	& B ;
 >B;$;&; 
;
 ;; 
;|@,<L
L),L	LB0r3   