
    CjD                       S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKJr  S SKJr  S SKJrJrJrJrJrJrJrJrJrJrJrJrJr  S SKrS SKrS SKrS SKJ r   S SK!J"r"  S SK#J$r$  S SK%r%S SK&r%S SK'r%S SK%J(r(  S S	K)J*r*  S S
K+J,r,  S SK-J.r.J/r/J0r0J1r1  S SK2J3r3J4r4J5r5  S SK6J7r7  S SK8J9r9  S SK:J;r;J<r<  S SK=J>r>  S SK?J@r@  S SKAJBrB  S SKCJDrE  S SKFJGrG  S SKHJIrI  S SKJJKrK  S SKLJMrM  S SKNJOrO  S SKPJQrQ  S SKRJSrS  S SKTJUrU  S SKVJWrW  S SKXJYrY  S SKZJ[r[J\r\  S SK]J^r^J_r_J`r`  S S KaJbrb  S S!KcJdrdJere  S S"KfJgrgJhrh  S S#KiJjrj  S S$KkJlrlJmrmJnrnJoroJprp  S S%KqJrrrJsrsJtrt  S S&KuJvrw  S S'KxJyryJzrz  S S(K{J|r|  S S)K}J~r~  S S*KJr  S S+KJr  S S,KJrJr  S S-KJrJr  S S.KJrJrJrJr  S S/KJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJr  S S0KJr  S S1KJrJrJr  S S2KJr  S S2KJr  S S3KJrJrJr  S S4KJrJrJrJrJrJrJrJrJrJr  S5S6KJr  \(       a>  S S7KJr  S S8KJrJr  S S9KJr  S S:KJr  S S;KJr  S S<KJr  S S=KJr  S S>KJrJrJr  \\\4   rO\r\r\r\r\r\r " S? S@\GR                  5      r " SA SB5      rg)C    Ndefaultdict)	lru_cache)TYPE_CHECKINGAnyAsyncGeneratorCallableDict	GeneratorListLiteralOptionalSetTupleUnioncast)AsyncOpenAI)	BaseModel)overload)get_secret_str)verbose_router_logger)uuid)	DualCacheInMemoryCache
RedisCacheRedisClusterCache)DEFAULT_HEALTH_CHECK_INTERVAL)DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIERDEFAULT_MAX_LRU_CACHE_SIZE)CustomLogger)run_async_function)!_get_parent_otel_span_from_kwargs&get_metadata_variable_name_from_kwargs)coroutine_checker)CredentialAccessor)tracer)Logging)SensitiveDataMasker)JSONProviderRegistry)RouterBudgetLimiting)LeastBusyLoggingHandler)LowestCostLoggingHandler)LowestLatencyLoggingHandler)LowestTPMLoggingHandler)LowestTPMLoggingHandler_v2simple_shuffle)get_deployments_for_tag) add_fallback_headers_to_responseadd_retry_headers_to_response)"_get_router_metadata_variable_namereplace_model_in_jsonlshould_replace_model_in_jsonl)InitalizeCachedClient)get_dynamic_litellm_paramsis_clientside_credential)filter_team_based_modelsfilter_web_search_deployments)CooldownCache)DEFAULT_COOLDOWN_TIME_SECONDS_async_get_cooldown_deployments/_async_get_cooldown_deployments_with_debug_info_get_cooldown_deployments_set_cooldown_deployments)#_check_non_standard_fallback_formatget_fallback_model_grouprun_async_fallback)!get_num_retries_from_retry_policy)#async_raise_no_deployment_exceptionsend_llm_exception_alert)DeploymentHealthCache)DeploymentAffinityCheck)ModelRateLimitingCheck)PromptCachingDeploymentCheck)0increment_deployment_failures_for_current_minute1increment_deployment_successes_for_current_minute)FlowItem	Scheduler)AllMessageValues	FileTypesOpenAIFileObjectOpenAIFilesPurpose)#CONFIGURABLE_CLIENTSIDE_AUTH_PARAMSVALID_LITELLM_ENVIRONMENTSAlertingConfigAllowedFailsPolicyAssistantsTypedDictCredentialLiteLLMParamsCustomRoutingStrategyBase
DeploymentDeploymentTypedDictGuardrailTypedDictLiteLLM_ParamsMockRouterTestingParamsModelGroupInfoOptionalPreCallChecksRetryPolicyRouterCacheEnumRouterGeneralSettingsRouterModelGroupAliasItemRouterRateLimitErrorRouterRateLimitErrorBasicRoutingGroupRoutingStrategySearchToolTypedDict)ServiceTypes)CustomPricingLiteLLMParamsGenericBudgetConfigTypeLiteLLMBatch)	ModelInfo)ModelResponseStreamStandardLoggingPayloadUsage)
CustomStreamWrapperEmbeddingResponseModelResponseRulesfunction_setupget_llm_provider!get_non_default_completion_params
get_secretget_utc_datetimeis_region_allowed   )PatternMatchRouter)Span)
AutoRouterPreRoutingHookResponseComplexityRouterAdaptiveRouterQualityRouter!BaseResponsesAPIStreamingIterator)BaseLiteLLMOpenAIResponseObject)ResponseAPIUsageResponseInputParamResponsesAPIResponsec                       \ rS rSrSrSrg)RoutingArgs   <    N)__name__
__module____qualname____firstlineno__ttl__static_attributes__r       `/home/rurouni/.local/share/pipx/venvs/strix-agent/lib/python3.13/site-packages/litellm/router.pyr   r      s    
Cr   r   c            k       <"   \ rS rSr% \" 5       r\\S'   Sr\\	   \S'   Sr
\\S'   SrSr\\   \S'   Sr\\   \S	'   Sr\\\\\\4         \S
'   SSSSSSSSSS0 SSSSSSSSSSSSS/ / / 0 SSSSS0 SSSSSS0 SSS\" 5       SSSSSSS44S\\\\   \\\\4      4      S\\   S\\\      S\\\      S\\   S\\   S\\   S\\   S\\   S\\	   S\S\\\      S\S\\    S\\   S\\   S\\   S\\    S \\    S!\\   S"\\   S#\	S$\!S%   S&\\\      S'\S(\S)\S*\\\\\\"4   4      S+\	S,\	S-\	S.\S/\\\#\4      S0\\\#4   S1\\   S2\\$   S3\\    S4\\	   S5\!S6   S7\\%   S8\S9\\\\&\4         S:\\'   S;\\(   S<\\   S=\S>\\\\\   4      S?\	S@\	SA\\   SB\	SC\	SDS4jSE jjr)\*SD\\   4SF j5       r+SG r,SH r-\*SI\\\4   SD\\.\/4   4SJ j5       r0SK\.4SL jr1SS	SMSNSOSP.r2\\\4   \SQ'   \*SR\\3\S4   SD\\   4SS j5       r4S5\\3\S4   SDS4ST jr5 GSSR\\3\4   S8\SU\	SD\\   4SV jjr6SW\\   SDS4SX jr7S5\\3\4   S8\4SY jr8SZ\\\\&\4         SDS4S[ jr9S\\SD\:\\   \\   4   4S] jr;SR\\   S^\\   S\\S_\<S`\\\\\4         Sa\\\\4      Sb\\   SD\\   4Sc jr=SR\\   S^\\   S\\S_\<S`\\\\\4         Sa\\\\4      Sb\\   SD\\   4Sd jr>Se r?Sf r@Sg rASh rBSi rCSj rDSk rESl rFSm rGSn rHSo rISp rJSq rKSr\\   4Ss jrLSt\	SDS4Su jrMS7\\%   4Sv jrNSw\4Sx jrOS\\S`\\\\4      SD\\P\Q4   4Sy jrRS\\S`\\\\4      SD\\P\Q4   4Sz jrSSD\4S{ jrTS|\S`\\   4S} jrU\VS\\S`\\W   S~\!S   SD\Q4S j5       rX\V GSS\\S`\\W   S~\!S   SD\P4S jj5       rX\V GSS\\S`\\W   S~\\!S   \!S   4   SD\\Q\P4   4S jj5       rX GSS\\S`\\W   S~\	4S jjrX\*S\YS\\Z   SDS4S j5       r[S\QS`\\\\4      S\SD\Q4S jr\\*SSSD\S   4S j5       r]\*      GSS j5       r^\*S\\\S4      S\SDS4S j5       r_SSS\\\4   SDS4S jr`S\QS`\\\\4      S\SD\Q4S jraS|\S`\\   4S jrbS\\S`\\\\4      SD\\P\Q4   4S jrc GSS\\S\S\\   SDS4S jjrdS\eSw\SDS4S jrfS\eSw\SDS4S jrg GSS\S\\   SDS4S jjrh GSSw\S\S\\   SD\i4S jjrj\*Sw\S\SDS4S j5       rk GSSw\S\S\\   SDS4S jjrlSw\S\4S jrmS\S\SD\\\ \4      4S jrnS\S\SD\\\ \4      4S jroS\S\SD\\\ \4      4S jrpS\\   S`\\\\\4      \\\\\4         4   4S jrqS\\S`\\\W      4S jrr\VS\\S`\\\\4      S~\!S   SD\Q4S j5       rs\V GSS\\S`\\\\4      S~\!S   SD\P4S jj5       rs GSS\\S`\\\\4      S~\	4S jjrs\V GSS\\S`\\W   S\S~\!S   SD\P4
S jj5       rt\VS\\S`\\W   S\S~\!S   SD\Q4
S j5       rt GSS\\S`\\W   S\4S jjrtS\\S\S\S\:\S4   S\\\4   4
S jruS\\SD\	4S jrvS\\S`\\W   S\\\4   4S jrwS\S\\4S jrxS\S\\4S jryS\S\\4S jrzS\S\\4S jr{S\|S\\4S jr}S\|S\\4S jr~S\\Sa\S\4S jrS\\Sa\S\4S jrS\\4S jrS\\4S jr   GSS\\S\S\\	   S\\	   S\\	   4
S jjr   GSS\\S\S\\	   S\\	   S\\	   4
S jjrS\\S\4S jr   GSS\S\\S\\	   S\\	   S\\	   4
S jjrS\S\\4S jrS\4S jrS\\S\4S jrS\S\4S jrS\\S\4S jrS\SDS4S jrS\\S\4S jrS\\\4   S\\S\SD\\\4   4S jrS\\S\4S jrS\S\SD\S   4S jrS\\S\4S jr GSS\\Sa\\\4   S\\	   SD\4S jjrSa\\\4   S\\4S jr GSS\\Sa\\\4   S\\	   SD\4S jjrSa\\\4   S\\4S jrS\\SD\4S jrS\\SD\4S jrS\\\S4   4S jrS rS\\SD\4S jrS\\SD\4S jr GSS\\\   SD\4S jjrS\\SD\4S jrS\\SD\4S jrS\\4S jr GSS\S\\   4S jjr GSS\S\!S   4S jjr GSS\S\\   4S jjr GSS\S\\   4S jjrS\4S jr GSS\S\\   4S jjr GSS\S\\   4S jjr  GSS\S\\   S\\   4S jjrS\eS\S\\   S\S\S\SD\\   4S jrS\eS\\	   S'\\   S(\\   S)\\   S\\   S\S\4S jr\GR^                  " 5       S 5       r    GSS\S\\   S'\\   S(\\   S)\\   4
S jjr\GR^                  " 5       GS  5       rS\4GS jr GSS\S\\   4GS jjr     GSGS\eS_\\   S\\   S(\\   S)\\   GS\\   4GS jjrGS r GSS'\\\\\   4      S\\   SD\\\      4GS jjrSD\\   4GS jr  GSS\eGS	\S\S_\\   S\\   SD\\\ 4   4GS
 jjrGS rSD\\   4GS jrSD\	4GS jrGS\\   4GS jrS\SD\!GS   4GS jrS\S\eSD\4GS jrGS\GS\\   SD\4GS jrSD\	4GS jrS\\S\PS\SD\	4GS jrS\\GS\\   4GS jrS\\GS\\   SD\:\\   \\   4   4GS jrSw\4GS jr GSSw\GS\\   GS\\   4GS jjr  GSS\\S_\\   S`\\\W      GS\\   Sb\\   GS\\   4GS jjrS\GS\4GS jr\*GS \GS!\S\\   SDS4GS" j5       rGS#\GS$\GS%\GS&\SD\\i   4
GS' jrGS\SD\	4GS( jrSw\i4GS) jrGS\SD\	4GS* jrSw\i4GS+ jrGS\SD\	4GS, jrGSGS- jrSw\iSDS4GS. jrGS\SD\	4GS/ jrSw\i4GS0 jrSw\iSD\	4GS1 jrS\<4GS2 jrSw\iSD\i4GS3 jrSw\iS\S\\4GS4 jrSw\iSD\\i   4GS5 jrGS6\GS7\SDS4GS8 jrS\\GS9\SDS4GS: jr GSS\\GS6\\   SDS4GS; jjrSw\iSD\\i   4GS< jrGS=\SD\\i   4GS> jrSD\\   4GS? jrSw\iSD\	4GS@ jrSw\iSDS4GSA jrGS6\SD\\i   4GSB jrGS6\SD\\   4GSC jrGSD\SD\\i   4GSE jrGS6\SD\\\\4      4GSF jr\V GSSw\\GSG4   GSH\GS=SSD\4GSI jj5       r\VSwSGSH\GS=\SD\4GSJ j5       r GSSw\\\GSG4      GSH\GS=\\   SD\4GSK jjrGS=\SD\\   4GSL jrGS=\SD\\   4GSM jrGS6\S\SD\\   4GSN jrS\GSO\SD\\   4GSP jrS\SD\\   4GSQ jrS\SD\:\\   \\   4   4GSR jr\" \GSS9S\SD\\   4GST j5       rS\SD\\\4   4GSU jr  GSS\S\\   Sb\\   SD\4GSV jjrS\<SDS4GSW jrS\<4GSX jr GSS\\   GSY\	SD\\   4GSZ jjrGS[\SD\	4GS\ jrGS6\\   SD\\   4GS] jrGS^\\   GS_\SD\\   4GS` jr GSS\S\\GS_\\   SD\	4GSa jjr  GSS\GSb\\   GS_\\   SD\\   4GSc jjrGSGS_\\   SD\\   4GSd jjGr SDG\\   4GSe jGr\*GSf\\   SD\	4GSg j5       GrS\\SD\	4GSh jGrSDG\\   4GSi jGr GSSw\GS_\\   SD\\   4GSj jjGrGS \\   SD\	4GSk jGr GSS\\   SD\\   4GSl jjGr GSS\\   GS_\\   SD\\\      4GSm jjGr	GSGSn jGr
GSGSo jGr   GSS\\   GSp\\   GS_\\   SD\\\\   4   4GSq jjGrGSp\SD\	4GSr jGrGSs GrGSt GrGSGSu jGr GSS\\S_\S`\\\\4      Sb\\   4GSv jjGrS\\SD\\   4GSw jGrS\\SD\4GSx jGrS\\GSy\\   SD\\:\\\\4   4      4GSz jGr    GSS\\S`\\\\\4         Sa\\\\4      GS{\\	   Sb\\   SD\:\\\\4   4   4GS| jjGrS\\S_\Sb\\   GSy\\   SD\4
GS} jGr    GSS\\Sb\S`\\\\\4         Sa\\\\4      GS{\\	   GS\\   SD\\\   \4   4GS~ jjGr   GSS\\Sb\S`\\\\\4         Sa\\\\4      GS{\\	   4
GS jjGr   GSS\\Sb\S`\\\\\4         Sa\\\\4      GS{\\	   4
GS jjGr   GSS\\Sb\S`\\\\\4         Sa\\\\4      GS{\\	   SD\G\   4GS jjGr    GSS\\S`\\\\\4         Sa\\\\4      GS{\\	   Sb\\   4
GS jjGr    GSS\\S`\\\\\4         Sa\\\\4      GS{\\	   Sb\\   4
GS jjGrS_\\   GS\\   SD\\   4GS jGrS_\\   SD\\   4GS jGr\*SwGSGSD\	4GS j5       Gr  GSS_\\   GS\\   SD\\   4GS jjGr! GSS_\\   GS\\   SD\\   4GS jjGr"S_\\   SD\\   4GS jGr# GSGS\\   4GS jjGr$ GSS\eS\\   4GS jjGr%S\e4GS jGr&GS Gr'GSG\(4GS jGr)GS Gr*GS Gr+GSGr,g(  Router   model_namesFcache_responsesi  default_cache_time_secondsNleastbusy_loggerlowesttpm_loggeroptional_callbacksINFOTr   simple-shuffle
model_listassistants_configsearch_toolsguardrail_list	redis_url
redis_host
redis_portredis_passwordredis_dbcache_kwargscaching_groups
client_ttlpolling_intervaldefault_prioritynum_retriesmax_fallbackstimeoutstream_timeoutdefault_litellm_paramsdefault_max_parallel_requestsset_verbosedebug_level)DEBUGr   default_fallbacks	fallbackscontext_window_fallbackscontent_policy_fallbacksmodel_group_aliasenable_pre_call_checksenable_tag_filteringtag_filtering_match_anyretry_afterretry_policymodel_group_retry_policyallowed_failsallowed_fails_policycooldown_timedisable_cooldownsrouting_strategy)r   
least-busyusage-based-routinglatency-based-routingcost-based-routingusage-based-routing-v2optional_pre_call_checksrouting_strategy_argsrouting_groupsprovider_budget_configalerting_configrouter_general_settingsdeployment_affinity_ttl_secondsmodel_group_affinity_configignore_invalid_deploymentsenable_health_check_routing health_check_staleness_threshold$health_check_ignore_transient_errorsenable_weighted_failoverreturnc5           	      D   UU l         U0U l        UU l        UU l        UU l        UU l        SSKJn5  U5" 5       U l        S[        l
        U R                   SL aU  US:X  a%  [        R                  " [        R                  5        O*US:X  a$  [        R                  " [        R                  5        U-=(       d
    [!        5       U l        X l        U=(       d    / U l        U=(       d    / U l        / U l        0 U l        Sn6Sn70 n8Xl        Uc  Ubw  Ubt  Sn6Ub  UU8S	'   Ub  UU8S
'   Ub  [1        U5      U8S'   Ub  UU8S'   U	b$  [        R2                  " S5        [1        U	5      U8S'   U8R5                  U5        U R7                  U85      n7U
(       a8  [        R8                  c!  [        R:                  " S&SU60U8D6[        l        Xl        [?        U7[A        5       S9U l        [C        UU7S9U l"        Xl#        SU l$        UU l%        / U l&        [O        5       U l(        0 U l)        0 U l*        0 U l+        0 U l,        0 U l-        U=(       d    0 U l.        0 U l/        0 U l0        0 U l1        SU l2        0 U l3        UbL  U Ri                  U5        U Rj                  U l6        U H#  n9SU9S   ;   d  M  SU R,                  U9S   S   '   M%     O/ U l5        U#b  U#U l7        O[        Rn                  U l7        U%=(       d    [p        U l9        [u        U R8                  U Rr                  S9U l;        U&U l<        U1U l=        U4U l>        U3U l?        U2=(       d    [        [        -  n:[        U R8                  [        U:5      S9U lD        [A        5       U lE        Ub  UU lF        O<[        R                  b  [        R                  U lF        O[        R                  U lF        Ub  UU lI        O<[        R                  b  [        R                  U lI        O[        R                  U lI        UU lK        U=(       d    [        R                  U lM        UU lN        U U lO        U R                  U'5      U lQ        U*U lR        U=(       d    [        R                  n;U R                  U;S9  U;U lS        Uc  [        R                  bN  U=(       d    [        R                  n;U R                  b  U R                  R                  SU;05        O
SU;0/U lS        U=(       d    [        R                  U lW        U=(       d    [        R                  n<U R                  U<S9  U<U lX        [        [        5      U l[        [        [        5      U l\        [        [        5      U l]        / U l^        U=(       d    0 n[        R                  " UU S9U l`        UU la        U R                  R                  SU5        U R                  R                  SS5        U R                  R                  S0 5      R5                  SU05        0 U lc         U R                  U'U)S9  U R                  U R                  5        SU lf        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        O)[        R                  R                  U R                  5        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        OU R                  /[        lm        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        OU R                  /[        lp        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        OU R                  /[        ls        U)U lv        U+U lw        U.U lx        SU ly        [        R                  " XR                  S9(       a  U(b  U(R                  S5        OS/n(SU l|        U!b  [        U![        5      (       a  [        S&0 U!D6U l|        O[        U![        5      (       a  U!U l|        U R                  b>  [        R                  " S GR                  U R                  GR                  SS!95      5        U"U l        U/U l        SU l        U$b  [        U$[        5      (       a  G[        S&0 U$D6U l        O[        U$G[
        5      (       a  U$U l        U GR                  b?  [        R                  " S"GR                  U GR                  GR                  SS!95      5        U,U l        U(b  U GR                  U(5        U GR                  (       a  G[        S# U GR                  =(       d    /  5       5      (       d  U GR                  c  / U l        G[        U R8                  U R                  S$S$S$U GR                  S%9n=U GR                  R                  U=5        [        R                  GR                  U=5        U GR                  b  U GR                  5         U GR                  5         U GR                  5         U GR                  5         g)'ae  
Initialize the Router class with the given parameters for caching, reliability, and routing strategy.

Args:
    model_list (Optional[list]): List of models to be used. Defaults to None.
    redis_url (Optional[str]): URL of the Redis server. Defaults to None.
    redis_host (Optional[str]): Hostname of the Redis server. Defaults to None.
    redis_port (Optional[int]): Port of the Redis server. Defaults to None.
    redis_password (Optional[str]): Password of the Redis server. Defaults to None.
    cache_responses (Optional[bool]): Flag to enable caching of responses. Defaults to False.
    cache_kwargs (dict): Additional kwargs to pass to RedisCache. Defaults to {}.
    caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None.
    client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600.
    polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms.
    default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None.
    num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2.
    timeout (Optional[float]): Timeout for requests. Defaults to None.
    default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}.
    set_verbose (bool): Flag to set verbose mode. Defaults to False.
    debug_level (Literal["DEBUG", "INFO"]): Debug level for logging. Defaults to "INFO".
    fallbacks (List): List of fallback options. Defaults to [].
    context_window_fallbacks (List): List of context window fallback options. Defaults to [].
    enable_pre_call_checks (boolean): Filter out deployments which are outside context window limits for a given prompt
    model_group_alias (Optional[dict]): Alias for model groups. Defaults to {}.
    retry_after (int): Minimum time to wait before retrying a failed request. Defaults to 0.
    allowed_fails (Optional[int]): Number of allowed fails before adding to cooldown. Defaults to None.
    cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1.
    routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy used for the implicit "default" group (any model not claimed by an entry in `routing_groups`). Defaults to "simple-shuffle".
    routing_strategy_args (dict): Additional args for the default group's routing strategy (e.g. latency window). Defaults to {}.
    routing_groups (Optional[List[RoutingGroup]]): Named subsets of `model_name`s that use a per-group routing strategy and args. Each model belongs to at most one explicit group; everything else lands in the implicit "default" group driven by `routing_strategy` / `routing_strategy_args`. Defaults to None.
    alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None.
    provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None.
    deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600.
    ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
    enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False.
Returns:
    Router: An instance of the litellm.Router class.

Example Usage:
```python
from litellm import Router
model_list = [
{
    "model_name": "azure-gpt-3.5-turbo", # model alias
    "litellm_params": { # params for litellm completion/embedding call
        "model": "azure/<your-deployment-name-1>",
        "api_key": <your-api-key>,
        "api_version": <your-api-version>,
        "api_base": <your-api-base>
    },
},
{
    "model_name": "azure-gpt-3.5-turbo", # model alias
    "litellm_params": { # params for litellm completion/embedding call
        "model": "azure/<your-deployment-name-2>",
        "api_key": <your-api-key>,
        "api_version": <your-api-version>,
        "api_base": <your-api-base>
    },
},
{
    "model_name": "openai-gpt-3.5-turbo", # model alias
    "litellm_params": { # params for litellm completion/embedding call
        "model": "gpt-3.5-turbo",
        "api_key": <your-api-key>,
    },
]

router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
```
r   )ServiceLoggingTr   r   localNredisurlhostportpasswordzwDeprecated 'redis_db' argument used. Please remove 'redis_db' from your config/database and use 'cache_kwargs' instead.dbtype)redis_cachein_memory_cache)r   r   modellitellm_params)cachedefault_cooldown_time)r   staleness_threshold)fallback_param*)params
router_objr   max_retriesmetadatar   r   r   )r   r   router_budget_limitingz+[32mRouter Custom Retry Policy Set:
{}[0mexclude_nonez3[32mRouter Custom Allowed Fails Policy Set:
{}[0mc              3   B   #    U  H  n[        U[        5      v   M     g 7fN
isinstancerJ   ).0cbs     r   	<genexpr>"Router.__init__.<locals>.<genexpr>  s"      8
4 r2334   Fr   ttl_secondsenable_user_key_affinityenable_responses_api_affinityenable_session_id_affinityr   r   )r   r   r   r   r   r   litellm._service_loggerr   service_logger_objlitellmsuppress_debug_infor   setLevelloggingr   r   re   r   r   r   r   deployment_namesdeployment_latency_mapr   strwarningupdate_create_redis_cacher   Cacher   r   r   rP   	schedulerr   default_deploymentr   provider_default_deployment_idsr   pattern_routerteam_pattern_routersauto_routerscomplexity_routersadaptive_routersquality_routersr    model_id_to_deployment_index_map model_name_to_deployment_indices team_model_to_deployment_indices_access_groups_cache_zero_cost_cacheset_model_listr   healthy_deploymentsr   r>   r   r=   cooldown_cacher   r   r   r   r   r   rI   floathealth_state_cachefailed_callsr   openaiDEFAULT_MAX_RETRIESr   ROUTER_MAX_FALLBACKS_explicit_timeoutrequest_timeoutr   r   r   _normalize_strategyr   _routing_groups_inputr   validate_fallbacksr   appendr   r   r   inttotal_calls
fail_callssuccess_callsprevious_modelsChatchatr   
setdefaultdeployment_statsrouting_strategy_init_init_routing_groupsaccess_groupsr   _async_success_callbacklistlogging_callback_manager"add_litellm_async_success_callbackdeployment_callback_on_successsuccess_callbackadd_litellm_success_callback#sync_deployment_callback_on_success_async_failure_callback"add_litellm_async_failure_callback$async_deployment_callback_on_failurefailure_callbackadd_litellm_failure_callbackdeployment_callback_on_failurer   r   r   router_budget_loggerr*   !should_init_router_budget_limiterr   dictrc   infoformat
model_dumpr   r   r   rX   r   add_optional_pre_call_checksanyr   rJ   add_litellm_callback_initialize_alertinginitialize_assistants_endpointinitialize_router_endpointsapply_default_settings)>selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
cache_typer   cache_configm
_staleness
_fallbacks_content_policy_fallbacksaffinity_callbacks>                                                                 r   __init__Router.__init__   s	   H '*D'&&<#$8!'>$:2@2B&*#t#f$%..w||<'%..w}}=#>'<'> 	$ "3(.B,2 	 ')#  	 ')$ Z%;
@V J$&/U#%'1V$%'*:V$)+9Z(#%-- N &)]T" -22<@K}}$ ' N: N N#2 #]_


 #-;
 !1"&-J*:<,02 	! 68AC=?;= # 	
 AC- GI-RT-
 EI! 24!
+-1__D$a 011PQD//2B0CG0LM  
  O $!.D!(!6!6D*K.K+**D4F4F
 "3+F((@%4X15 
),UU 	 #8**%
2C#
 O 	 "*D  ,&22D%99D$!.D"".!(!6!6D!(!=!=D!(9'"9"9,& $ 8 89I J 	" 3'"3"3
z:#(G,E,E,Q*Gg.G.GJ~~)%%sJ&78#&
"3!4 %H(H(H 	%
 %H(H(H 	" 	/HI(A%(3)
 (3(
 +6+
  	
 "8!=2LL(>4P	 '=###..y'B##..}a@##..z2>EE~.	
 ')	 	""-"7 	# 	
 	!!$"<"<=!g55t<<,,OO33 ,,OO33 g..55,,II88 )-(P(P'QG$g55t<<,,OO99
 99/G+ g..55,,II33 )-(K(K'LG$%:"&<#/N,DH!AA!:U:U
 (3(//0HI,D+E(37#,--$/$?,$?!L+66$0!  ,%**HOO))44$4G % 	% ( 	( CG!+.55,>,VAU,V)02DEE,@)((4%**PWW11<<$<O :I#/--.FG
 +++C 8
..4"48
 5
 5
 &&.*,' 7jj @@)..3+0,0,L,L! ##**+<=,,AABST+%%'++-((*##%r   c                      [         R                  " [        R                  5      n U R                  U R
                  -   nSU;   a  UR                  S5        U$ )zC
Returns a list of valid arguments for the Router.__init__ method.
r[  )inspectgetfullargspecr   rc  args
kwonlyargsremove)arg_spec
valid_argss     r   get_valid_argsRouter.get_valid_args  sH    
 ))&//:]]X%8%88
Zf%r   c                 *    / nU R                  U5        g)z+
Apply the default settings to the router.
N)rT  )r[  default_pre_call_checkss     r   rZ  Router.apply_default_settings  s    
 :<))*ABr   c                 $   [         R                  R                  [         R                  U 5        [         R                  R                  [         R                  U 5        [         R                  R                  [         R
                  U 5        [         R                  R                  [         R                  U 5        [         R                  R                  [         R                  U 5        [         R                  R                  [         R                  U 5        [         R                  R                  [         R                  U 5        U R                  bA  U R                   H0  n[         R                  R                  [         R                  USS9  M2     gg)z
Pseudo-destructor to be invoked to clean up global data structures when router is no longer used.
For now, unhook router's callbacks from all lists
NF)require_self)r  rB  #remove_callback_from_list_by_objectr@  rE  rH  rK  input_callbackservice_callback	callbacksr   )r[  callbacks     r   discardRouter.discard  s3   
 	((LL++T	
 	((LL$$d	
 	((LL++T	
 	((LL$$d	
 	((LL""D	
 	((LL$$d	
 	((LLt	

 "". 3300TT%%xe U  4 /r   r]  c                     U R                  S5      nU(       d9  [        S5      nUb+  [        U[        5      (       a  [        R
                  " U5      nU(       a  [        S0 0 U ESU0ED6$ [        S0 U D6$ )zQ
Initializes either a RedisCache or RedisClusterCache based on the cache_config.
startup_nodesREDIS_CLUSTER_NODESr   )getr{   r   r  jsonloadsr   r   )r]  r|  _env_cluster_nodess      r   r  Router._create_redis_cache'  sr     %((9!+,A!B!-*=OQT2U2U $

+= >$X'W,'W'WXX---r   r   c                 T    U R                   R                  c  XR                   l        gg)z
Update the redis cache for the router, if none set.

Allows proxy user to just do
```yaml
litellm_settings:
    cache: true
```
and caching to just work.
N)r   r   )r[  r   s     r   _update_redis_cacheRouter._update_redis_cache9  s#     ::!!)%*JJ" *r   lowesttpm_logger_v2lowestlatency_loggerlowestcost_logger)r   r   r   r   r   "_DEFAULT_SELECTOR_ATTR_BY_STRATEGYstrategyc                 P    U c  g [        U [        5      (       a  U R                  $ U $ r   )r   rj   value)r  s    r   r0  Router._normalize_strategyR  s)     h00>>!r   c                     S/[          Vs/ s H  o"R                  PM     sn-   nUc  g [        U[        5      =(       a    X;   n[        U[         5      nU(       d  U(       d  [	        SU SU S35      eg g s  snf )Nr   zInvalid routing_strategy: 'z'. Valid options: z. Check 'router_settings.routing_strategy' in your config.yaml or the 'routing_strategy' parameter if using the Router SDK directly.)rj   r  r   r  
ValueError)r[  r   svalid_strategy_stringsis_valid_stringis_valid_enums         r   _validate_routing_strategy!Router._validate_routing_strategy\  s     #3!36X1ww6X!X#'- ; : 	 ##3_E}-.>-? @""8!9 :XY  (5 7Ys   A6register_callbacksc                 p   SnU R                  U5      =[        R                  R                  :X  al    [	        U R
                  S9nU(       Ga  [        [        R                  [        5      (       a   [        R                  R                  U5        OU/[        l        O=[        R                  R                  :X  a    [        U R
                  US9nO=[        R                  R                  :X  a    [        U R
                  US9nOe=[        R                  R                  :X  a    [!        U R
                  US9nO1[        R"                  R                  :X  a  [%        U R
                  0 S9nUbI  U(       aB  [        [        R&                  [        5      (       a  [        R(                  R+                  U5        U$ )z
Constructs a strategy selector for a given strategy.
Returns None for `simple-shuffle` (no selector needed) and unknown
strategies.
N)router_cache)r  routing_args)r0  rj   
LEAST_BUSYr  r+   r   r   r  ru  rA  r3  USAGE_BASED_ROUTINGr.   USAGE_BASED_ROUTING_V2r/   LATENCY_BASEDr-   
COST_BASEDr,   rw  rB  rV  )r[  r  r   r  selectors        r   _build_strategy_selectorRouter._build_strategy_selectorp  s9    #'&&x01++112

K%!'"8"8$??..55h?2:.:44::2!%!6 >77==5!%!6 5..446!%!6 !++113!%!#  "7,,d33,,AA(Kr   	selectorsc                    U Vs1 s H  o"c  M  [        U5      iM     nnU(       d  g[        [        R                  [        5      (       a;  [        R                   Vs/ s H  n[        U5      U;  d  M  UPM     sn[        l        [        [        R
                  [        5      (       a<  [        R
                   Vs/ s H  n[        U5      U;  d  M  UPM     sn[        l        ggs  snf s  snf s  snf )a  
Drop router-owned strategy selectors from litellm's global callback
lists by identity. Used before re-init (`routing_strategy_init` /
`_init_routing_groups`) so repeated `update_settings` calls don't
accumulate dead selectors that keep receiving callback events.
N)idr   r  rw  rA  ru  )r[  r  r  selector_idscs        r   _unregister_router_selectors#Router._unregister_router_selectors  s     (1By!1yBg''..",,!,a1\0I,!G g,,d33"11&1aRU,5N1&G" 4 C!&s"   C&C&C+4C+;C0C0c           
         [         R                  " SU 35        U R                  U5        U R                  U R                  R                  5        Vs/ s H  n[        XS 5      PM     sn5        S U l        S U l        S U l	        S U l
        S U l        U R                  UUS9nU R                  R                  U R                  U5      =(       d    S5      nUb  [        XU5        g g s  snf )NzRouting strategy: r  r    )r   rQ  r  r  r  valuesgetattrr   r   r  r  r  r  r~  r0  setattr)r[  r   r   attrr  s        r   r=  Router.routing_strategy_init  s     	""%78H7I#JK''(89)) !CCJJLLD D)L	
 DHCGIM KO!EI00%"7 1 
 66::$$%56<"
 D) /s   C&groups_inputc                 h   U R                  [        U S0 5      R                  5        VVs/ s H  nUR                  5         H  nUPM     M     snn5        0 U l        0 U l        0 U l        U(       d  gU R                  =(       d    /  Vs1 s H+  oDR                  S5      (       d  M  UR                  S5      iM-     nn[        5       nU GH  n[        U[        5      (       a  UO
[        S0 UD6nUR                  (       d  [        S5      eUR                  S:X  a  [        S5      eUR                  U;   a  [        SUR                   S35      eUR                  UR                  5        U R                  UR                  5        UR                    H  n	XR                  ;   a,  [        S	U	 S
U R                  U	    SUR                   S35      eU(       a'  X;  a"  ["        R$                  " SU	UR                  5        UR                  U R                  U	'   M     XR                  UR                  '   U R'                  UR                  5      =(       d    Sn
U R)                  UR                  UR*                  =(       d    0 S9nUb  X0O0 U R
                  UR                  '   GM     gs  snnf s  snf )a  
Validates and indexes `routing_groups`. Each `model_name` may belong to
at most one explicit group. Constructs per-group strategy selectors so
groups with different `routing_strategy_args` track independent state.

Models not claimed by any explicit group are served by the implicit
`"default"` group, whose selectors are the `self.<strategy>_logger`
attributes set up in `routing_strategy_init`.
_group_selectorsN
model_namez-routing_groups: group_name must be non-empty.defaultzFrouting_groups: 'default' is reserved for the implicit fallback group.zBrouting_groups: group names must be unique, duplicate group_name ''.zrouting_groups: model_name 'z' appears in both 'z' and 'z.'. Each model may belong to at most one group.zrouting_groups: model_name '%s' (group '%s') is not in model_list; the group entry will only take effect once a deployment with that model_name is added.r  r  r   )r  r  r  _routing_groups_model_to_groupr  r   r~  setr   ri   
group_namer  addr  r   modelsr   r  r0  r  r   )r[  r  r  selr^  known_model_namesseen_group_namesrawgroupr  strategy_valuegroup_selectors               r   r>  Router._init_routing_groups  s    	)) ")/A2!F!M!M!O!OI$++-C - !O	
 9;/1;= +///*?R*?
*?AEE,DWAEE,*? 	 
 !$C%c<88Cl>QS>QE## !PQQ9, \  #33 XY^YiYiXjjlm    !1!12++E,B,BC#ll
!5!55$6zl C!!%!5!5j!A B C!,,--[] 
 %)L)11/ #(( 493C3C$$Z0 +" 6;  !1!12!55e6L6LMSQSN!:://&+&A&A&GR ; N
 5C4N0TV !!%"2"23S  '
s   $J)

J/'J/r   c                    U R                   R                  U5      nUcj  U R                  U R                  5      nU R                  R                  U=(       d    S5      nUb  [        XS5      OSn[        R                  " SX5        X54$ U R                  U   nU R                  UR                  5      nU R                  R                  U0 5      R                  U=(       d    S5      n[        R                  " SX!U5        X54$ )a  
Resolves the routing strategy and selector to use for the given model.

Every model belongs to exactly one group: an explicit entry from
`routing_groups`, or the implicit `"default"` group driven by the
router's top-level `routing_strategy` / `routing_strategy_args`.

`self.routing_strategy` may be either a string or a `RoutingStrategy`
enum member (the constructor accepts both), so it is normalized to a
string here. Downstream call sites and `_select_deployment_*` arms
compare against string literals.
Nr  z*routing_group=default model=%s strategy=%sz%routing_group=%s model=%s strategy=%s)
r  r~  r0  r   r  r  r   debugr  r  )r[  r   r  r  r  r  r  s          r   _get_routing_contextRouter._get_routing_context$  s     ))--e4
//0E0EFH::>>x~2ND484Dwt40$H!''<e %%$$Z0++E,B,BC((,,Z<@@RP##3Z	
 !!r   r  r&  messagesinputrequest_kwargsc                .  #    Uc  gU=S:X  a    UR                  UUS9I Sh  vN $ =S:X  a    UR                  UUUUS9$ ==S:X  a  O	=S:X  a  O  O    UR                  UUUUS9I Sh  vN $ S:X  a  UR                  UUUUUS	9I Sh  vN $  g Nl N( N
7f)
z
Asks the strategy selector for a deployment. Caller handles
`simple-shuffle` separately (it does not flow through a selector).
Returns None for unknown strategies or when the selector is missing.
Nr   model_groupr&  r   r  r&  r  r  r   r   r   r  r&  r  r  r  )async_get_available_deploymentsget_available_deploymentsr[  r  r  r   r&  r  r  r  s           r   _select_deployment_asyncRouter._select_deployment_asyncC  s       %EE %(; F    '
  99 %(;%	 :   A),@@%EE %(;%	 F    )%EE %(;%#1 F    ? s4    BBAB(B)BB	BBBc                    Uc  gU=S:X  a    UR                  UUS9$ ==S:X  a  O	=S:X  a  O  O    UR                  UUUUS9$ S:X  a  UR                  UUUUUS9$  g)	zY
Sync sibling of `_select_deployment_async`. Caller handles
`simple-shuffle` separately.
Nr   r  r   r   r  r   r  )r  r  s           r   _select_deployment_syncRouter._select_deployment_syncx  s     
 99 %(; :   B&)AA99 %(;%	 :   )99 %(;%#1 :   r   c                 D   U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R
                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l	        g r   )
factory_functionr  acreate_assistantsadelete_assistantaget_assistantsacreate_threadaget_threada_add_messageaget_messagesarun_threadr[  s    r   rX  %Router.initialize_assistants_endpoint  s    "&"7"78R8R"S!%!6!6w7P7P!Q#44W5L5LM"33G4J4JK001D1DE!2273H3HI!2273H3HI001D1DEr   c                 N   U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R
                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l	        U R                  [        R                  S	S9U l
        U R                  [        R                  S
S9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                   SS9U l        U R                  [        R"                  SS9U l        U R                  [        R$                  SS9U l        U R                  [        R&                  SS9U l        U R                  [        R(                  SS9U l        U R                  [        R*                  SS9U l        U R                  [        R,                  SS9U l        U R                  [        R.                  SS9U l        U R                  [        R0                  SS9U l        g)z+Helper to initialize core router endpoints.
moderation	call_typeanthropic_messagesagenerate_contentaadapter_generate_content
aresponsesafile_deleteafile_content	responsesaget_responsesacancel_responsesacompact_responsesadelete_responsesalist_input_items
_arealtime_aresponses_websocketacreate_fine_tuning_jobacancel_fine_tuning_jobalist_fine_tuning_jobsaretrieve_fine_tuning_jobalist_filesaimage_editallm_passthrough_routeN)r  r  amoderationr  aanthropic_messagesr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  
afile_listr  r  r  s    r   _initialize_core_endpoints!Router._initialize_core_endpoints  sW   00< 1 
 $(#8#8&&2F $9 $
  #'"7"7&&2F #8 #
 "&!6!6%%1D "7 "
 *.)>)>--9T *? *
& //, 0 
 !11  N 2 
 "22!!_ 3 
 ..w/@/@K.X"33"".> 4 
 "&!6!6%%1D "7 "
 #'"7"7&&2F #8 #
 "&!6!6%%1D "7 "
 "&!6!6%%1D "7 "
 //, 0 
 &*%:%:))5L &; &
" (,'<'<++7P (= (
$ (,'<'<++7P (= (
$ '+&;&;**6N '< '
# *.)>)>--9T *? *
& //- 0 
  00= 1 
 '+&;&;**6N '< '
#r   c                    SSK JnJnJnJnJnJnJnJnJ	n	J
n
Jn  U R                  USS9U l        U R                  U
SS9U l        U R                  USS9U l        U R                  USS9U l        U R                  U	SS9U l        U R                  US	S9U l        U R                  US
S9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        g)z"Initialize vector store endpoints.r   )adeletealist	aretrieveasearchaupdatecreatedeleterA  retrievesearchr  avector_store_searchr  vector_store_searchvector_store_createavector_store_retrievevector_store_retrieveavector_store_listvector_store_listavector_store_updatevector_store_updateavector_store_deletevector_store_deleteN)litellm.vector_stores.mainr  r  r  r  r  r	  r
  rA  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )r[  r  r  r  r  r  r	  r
  rA  r  r  r  s               r   "_initialize_vector_store_endpoints)Router._initialize_vector_store_endpoints  sv   	
 	
 	
 	
 %)$9$95 %: %
! $(#8#83 $9 $
  $(#8#83 $9 $
  '+&;&;!9 '< '
# &*%:%: 7 &; &
" #'"7"71 #8 #
 "&!6!6/ "7 "
 %)$9$95 %: %
! $(#8#83 $9 $
  %)$9$95 %: %
! $(#8#83 $9 $
 r   c                    SSK Jn  SSK Jn  SSK Jn  SSK Jn  SSK Jn  SSK Jn  SSK Jn  SS	K Jn  SS
K J	n	  SSK J
n
  SSK Jn  SSK Jn  U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  U	SS9U l        U R                  USS9U l        U R                  U
SS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        g)z'Initialize vector store file endpoints.r   acreater  r  )r  )aretrieve_content)r  r	  r
  rA  )r  )retrieve_content)r  avector_store_file_creater  vector_store_file_createavector_store_file_listvector_store_file_listavector_store_file_retrievevector_store_file_retrieveavector_store_file_contentvector_store_file_contentavector_store_file_updatevector_store_file_updateavector_store_file_deletevector_store_file_deleteN)litellm.vector_store_files.mainr  r  r  r  r   r  r	  r
  rA  r  r$  r  r  r%  r&  r'  r(  r)  r*  r+  r,  r-  r.  r/  r0  )r[  avector_store_file_create_fnavector_store_file_delete_fnavector_store_file_list_fnavector_store_file_retrieve_fnavector_store_file_content_fnavector_store_file_update_fnvector_store_file_create_fnvector_store_file_delete_fnvector_store_file_list_fnvector_store_file_retrieve_fnvector_store_file_content_fnvector_store_file_update_fns                r   '_initialize_vector_store_file_endpoints.Router._initialize_vector_store_file_endpoints*  s   	
	
 	X	
	
	
	
	
 	V	
	
	
 *.)>)>(4O *? *
& )-(=(='3M )> )
% (,'<'<&2K (= (
$ '+&;&;%1I '< '
# ,0+@+@*6S ,A ,
( +/*?*?)5Q +@ +
' +/*?*?)5Q +@ +
' *.)>)>(4O *? *
& *.)>)>(4O *? *
& )-(=(='3M )> )
% *.)>)>(4O *? *
& )-(=(='3M )> )
%r   c                     SSK JnJnJnJn  U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        g)	z"Initialize Google GenAI endpoints.r   )r  agenerate_content_streamgenerate_contentgenerate_content_streamr  r  rB  rA  rC  N)litellm.google_genair  rA  rB  rC  r  )r[  r  rA  rB  rC  s        r   "_initialize_google_genai_endpoints)Router._initialize_google_genai_endpointsr  s    	
 	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 )-(=(=$0J )> )
% (,'<'<#/H (= (
$r   c                     SSK JnJn  U R                  USS9U l        U R                  USS9U l        SSKJnJn  U R                  USS9U l        U R                  USS9U l        g	)
z$Initialize OCR and search endpoints.r   )aocrocrrH  r  rI  r  r  r  r  N)litellm.ocrrH  rI  r  litellm.searchr  r  )r[  rH  rI  r  r  s        r    _initialize_ocr_search_endpoints'Router._initialize_ocr_search_endpoints  sd    )))$&)A	(((>2,,W	,J++Fh+Gr   c                 H   SSK JnJnJnJnJnJnJnJnJ	n	J
n
JnJnJnJnJnJnJnJn  U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        U R'                  U	SS9U l	        U R'                  US	S9U l        U R'                  US
S9U l        U R'                  U
SS9U l
        U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        U R'                  USS9U l        g)zInitialize video endpoints.r   )avideo_contentavideo_create_characteravideo_editavideo_extensionavideo_generationavideo_get_characteravideo_listavideo_remixavideo_statusvideo_contentvideo_create_character
video_editvideo_extensionvideo_generationvideo_get_character
video_listvideo_remixvideo_statusrT  r  r]  rV  r_  rX  ra  rP  rY  rW  r`  rQ  rZ  rU  r^  rR  r[  rS  r\  N)litellm.videosrP  rQ  rR  rS  rT  rU  rV  rW  rX  rY  rZ  r[  r\  r]  r^  r_  r`  ra  r  )r[  rP  rQ  rR  rS  rT  rU  rV  rW  rX  rY  rZ  r[  r\  r]  r^  r_  r`  ra  s                      r   _initialize_video_endpoints"Router._initialize_video_endpoints  s-   	
 	
 	
 	
 	
* "&!6!6)< "7 "
 !% 5 5(: !6 !
  000V//
l/S!22_ 3 
 !11N 2 
 #33&6 4 
 "22_ 3 
 !11N 2 
  000V'+'<'<#/H (= (
$ '+&;&;".F '< '
# %)$9$9 ,B %: %
! $(#8#8+@ $9 $
   000V//
l/S $ 5 5(: !6 !
  $44'8  5  
r   c           
         SSK JnJnJnJnJnJnJnJn  SSK	J
n	  U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  US	S9U l        U R                  US
S9U l        U R                  USS9U l        U R                  USS9U l        U	R                  5        H  u  p[        X
U R                  XS95        M      g)zInitialize container endpoints.r   )acreate_containeradelete_containeralist_containersaretrieve_containercreate_containerdelete_containerlist_containersretrieve_container)_generated_endpointsrf  r  rj  rh  rl  ri  rm  rg  rk  N)litellm.containersrf  rg  rh  ri  rj  rk  rl  rm  #litellm.containers.endpoint_factoryrn  r  itemsr  )r[  rf  rg  rh  ri  rj  rk  rl  rm  container_file_endpointsnamefuncs               r   _initialize_container_endpoints&Router._initialize_container_endpoints  sE   		
 		
 		
	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 !% 5 5(: !6 !
  $44'8  5  
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
 "&!6!6)< "7 "
 !% 5 5(: !6 !

 388:JDD 5 5d 5 KL ;r   c                    U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R
                  SS9U l        g)z*Initialize Anthropic Skills API endpoints.acreate_skillr  alist_skills
aget_skilladelete_skillN)r  r  rx  ry  rz  r{  r  s    r   _initialize_skills_endpoints#Router._initialize_skills_endpoints  s    !22!!_ 3 
 !11  N 2 
 //, 0 
 "22!!_ 3 
r   c                    SSK Jn  SSK Jn  SSK Jn  SSK Jn  SSK Jn  SSK Jn  SSK Jn  SS	K Jn  U R                  US
S9U l
        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        g)z-Initialize Google Interactions API endpoints.r   )acancelr  r  aget)cancelr!  r"  r~  acreate_interactionr  create_interactionaget_interactionget_interactionadelete_interactiondelete_interactionacancel_interactioncancel_interactionN)litellm.interactionsr  r  r  r  r  r	  r
  r~  r  r  r  r  r  r  r  r  r  )	r[  r  r  r  r  r  r  r  r  s	            r   "_initialize_interactions_endpoints)Router._initialize_interactions_endpoints  s   GGGAEEE?#'#8#8+@ $9 $
  #'"7"7*> #8 #
 !% 5 5(: !6 !
  $44'8  5  
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
r   c                     SSK Jn  SSK Jn  SSK Jn  SSK Jn  SSK Jn  SSK Jn  SSK Jn  SS	K Jn  SS
K J	n	  SSK J
n
  U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  U	SS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  U
SS9U l        g)z?Initialize Google Managed Agents API endpoints (v1beta/agents).r   r  r  r  r  )alist_versionsr!  r"  r  r#  )list_versionsacreate_agentr  create_agentalist_agentslist_agents
aget_agent	get_agentadelete_agentdelete_agentalist_agent_versionslist_agent_versionsN)litellm.interactions.agentsr  r  r  r  r  r	  r
  r~  rA  r  r  r  r  r  r  r  r  r  r  r  r  )r[  r  r  r  r  r  r  r  r  r  r  s              r   $_initialize_managed_agents_endpoints+Router._initialize_managed_agents_endpoints8  s?   HHBEVFF@CT!22_ 3 
 !11N 2 
 !11N 2 
  000V//
l/S..yK.P!22_ 3 
 !11N 2 
 %)$9$9 ,B %: %
! $(#8#8+@ $9 $
 r   c                 D   U R                  5         U R                  5         U R                  5         U R                  5         U R	                  5         U R                  5         U R                  5         U R                  5         U R                  5         U R                  5         g)zvHelper to initialize specialized router endpoints (vector store, OCR, search, video, container, skills, interactions).N)
r  r>  rE  rM  )_override_vector_store_methods_for_routerrc  ru  r|  r  r  r  s    r   !_initialize_specialized_endpoints(Router._initialize_specialized_endpoints^  sz    //1446//1--/668((*,,.))+//1113r   c                 D    U R                  5         U R                  5         g r   )r  r  r  s    r   rY  "Router.initialize_router_endpointsl  s    '')..0r   r   c           	          Uc  gU HQ  n[        U[        5      (       d  [        SU S35      e[        U5      S:w  d  M8  [        SU S[        U5       S35      e   g)z#
Validate the fallbacks parameter.
NzItem 'z' is not a dictionary.r~   zDictionary 'z%' must have exactly one key, but has z keys.)r   rP  r  len)r[  r   fallback_dicts      r   r2  Router.validate_fallbacksp  so     !+MmT22 6-8N!OPP=!Q& "=/1VWZ[hWiVjjpq 	 ,r   enable_global_affinityc                 r   SSK Jn  S[        [           SUSS 4S jnU(       d   UR	                  U R
                  5      (       a  U R                  c  / U l        S nU R                   H  n[        XR5      (       d  M  Un  O   Ub=  Xl        UR                  =(       d    UUl	        U R
                  =(       d    0 Ul        UnOLU" U UU R
                  S9nU R                  R                  U5        [        R                  R                  U5        U" U R                  U5        U" [        R                  U5        g g )Nr   )EncryptedContentAffinityCheckcallback_listcallback_to_mover   c                     X;  a  g U R                  U5        [        S [        U 5       5       [        U 5      5      nU R	                  X!5        g )Nc              3   X   #    U  H   u  p[        U[        5      (       d  M  Uv   M"     g 7fr   r   )r   idxrx  s      r   r  iRouter._add_encrypted_content_affinity_check.<locals>._move_before_deployment_affinity.<locals>.<genexpr>  s&      )A!(,CD C)As   *	*)rj  next	enumerater  insert)r  r  insert_indexs      r    _move_before_deployment_affinityVRouter._add_encrypted_content_affinity_check.<locals>._move_before_deployment_affinity  sR      4  !12)2=)A
 M"L   @r   )routerr  r   )Elitellm.router_utils.pre_call_checks.encrypted_content_affinity_checkr  r   r    has_model_group_affinity_enabledr   r   r   r  r  r3  r  rB  rV  rw  )r[  r  r  r  existing_ec_callbackr  ec_callbacks          r   %_add_encrypted_content_affinity_check,Router._add_encrypted_content_affinity_check~  s>   	
	A9	A;	A 	A$ #,MM00  &&.*,'LP --b@@+-( .
 $/.2+(?? .- %;
 44: %@ 3;+A040P0P
 ''..{;00EEkR,T-D-DkR,W->->LCr   c           	         Uc  g SU;   nSU;   nSU;   nU(       d  U(       d  U(       Ga-  U R                   c  / U l         S nU R                    H  n[        U[        5      (       d  M  Un  O   Ub  UR                  =(       d    UUl        UR                  =(       d    UUl        UR
                  =(       d    UUl        U R                  Ul        U R                  (       a  U R                  Ul        Of[        U R                  U R                  UUUU R                  S9nU R                   R                  U5        [        R                  R                  U5        U R                  SU;   S9  U H  nS n	US;   a  M  US:X  a  [        U R                  S	9n	ObUS
:X  aC  U R!                  5       b  M@  [#        U R                  U R$                  U R&                  S9n	Xl        OUS:X  a  [+        U R                  S9n	U	c  M  U R                   c  / U l         U R                   R                  U	5        [        R                  R                  U	5        M     g )Ndeployment_affinityresponses_api_deployment_checksession_affinityr  encrypted_content_affinity)r  )r  r  r  r  prompt_caching)r   r   )
dual_cacher   r   enforce_model_rate_limits)r  )r   r   rJ   r  r  r	  r   r  r   r   r3  r  rB  rV  r  rL   %_get_router_deployment_budget_limiterr*   r   r   rN  rK   )
r[  r   r  r  r	  existing_affinity_callbackr  rb  pre_call_check	_callbacks
             r   rT  #Router.add_optional_pre_call_checks  s[    $+
 $9<T#T ,0HH 	& &8;S%S"$,)&&.*,'LP&--b"9::13. .
 *5.GG 0/ +C
 /LL 54 +H
 /II 21 +E
 88 +6 3388 /J %<** $ D D-E2O/I040P0P%! ''../@A00EEFWX
 	22,0HH 	3 	
 7N04I "  !118tzzJ	#;;==?K0#zz+/+F+F#	
 -6)#>>2djjI	 &&.*,'##**95,,AA)L; 7r   
deploymentc                 <    [         R                  " U5      nUS   n[        R                  (       a   [	        SSS9nUR                  U5      US'   U$ SU;   a  US   SS S-   US'   U$ ! [         a)  n[        R                  " S[        U5       35        UeSnAff = f)	z
returns a copy of the deployment with the api key masked

Only returns 2 characters of the api key and masks the rest with * (10 *).
r      r   )visible_prefixvisible_suffixapi_keyNz
**********z+Error occurred while printing deployment - )
copydeepcopyr  redact_user_api_key_infor(   	mask_dict	Exceptionr   r  r  )r[  r  _deployment_copyr   maskeres         r   print_deploymentRouter.print_deployment"  s    	#}}Z8#34D#EN//,AaP5;5E5En5U !12 $# n,,:9,Ebq,IH,Ty)## 	!''=c!fXF G		s   AA( A( (
B2$BBc                      [         R                  " SU S35        XS'   X#S'   U R                  US'   U R                  XS9  U R                  " S0 UD6nU$ ! [
         a  nUeSnAff = f)	z
Example usage:
response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}]
zrouter.completion(model=z,..)r   r  original_functionr   kwargsNr   )r   r  _completion_update_kwargs_before_fallbacksfunction_with_fallbacksr  )r[  r   r  r  responser  s         r   
completionRouter.completion;  s}    
	!''*B5'(NO#7O!):*.*:*:F&'00u0L33=f=HO 	G	s   AA 
A' A""A'c           	      Z   S nS n UR                  5       nXS'   U R                  UUUR                  SS 5      US9nUS   R                  5       nUR                  SS 5      nUb2  [        R                  " U R
                  X4USS9n	U	R                  5         U R                  XSS9  UR                  SS 5        US   nU R                  XSS9n
UR                  S	S 5      nUb  U
b  XR                  :w  a  S nOU
nU R                  U5      (       d  U R                  US
9  0 UEUU R                  US.EUEn[        R                  " S0 UD6n[         R"                  " SU S35        [%        U[&        5      (       a-  U R)                  XUS9nU(       a  [        R*                  " SUSS9e[%        U[,        5      (       a  U R/                  UUUS9$ U$ ! [0         aT  n[         R"                  " SU S[3        U5       S35        Ub$  U R5                  UU5        U R7                  UU5        UeS nAff = f)Nr   specific_deploymentr   r  r  r  r   silent_modelT)targetrh  r  daemonr  r  r  r  r  cachingclientzlitellm.completion(model=)[32m 200 OK[0mr   r  r  Response output was blocked.r  messager   llm_providermodel_responser  initial_kwargs)[31m Exception [0mr   )r  get_available_deploymentpop	threadingThread_silent_experiment_completionstart_update_kwargs_with_deployment_get_clientr~  r  has_model_id routing_strategy_pre_call_checksr   r  r  r   rQ  r   rv   "_should_raise_content_policy_errorContentPolicyViolationErrorrt   _completion_streaming_iteratorr  r  (_set_deployment_num_retries_on_exception&_set_failed_deployment_id_on_exception)r[  r   r  r  r  r  #input_kwargs_for_streaming_fallbackr   r  threadpotential_model_clientdynamic_api_keymodel_clientinput_kwargsr  _should_raiser  s                    r   r  Router._completionN  s    

_	 39++-/;@8 66!$*JJ/Dd$K%	 7 J ((89>>@N)--ndCL'
 #))==&1!	 //:/UJJ~t,'0J%)%5%5% &6 &" %jjD9O+*6#'E'EE#5 $$U++555L $//&	
 L ))9L9H!&&+J<7NO
 (M22 $ G G6 !H ! !!== >#%'  ($788::#+%#F ;   O 	!&&+J<7KCPQF8SZ[ %==aL;;AzJG	s   GG 
G 
H*AH%%H*c                 T   SSK Jn  U" U5      nUR                  S5      nUb"  UR                  S5      UL a  [        U5      US'   SU;  a  0 US'   US   R	                  SS5        SUS   S'   SUS	'   UR	                  S
S5        UR	                  SS5        UR	                  SS5        U$ )u  
Prepare kwargs for a silent experiment by ensuring isolation from the primary call.

Guarantee metadata isolation: safe_deep_copy falls back to the original
reference when deepcopy fails (e.g. metadata contains UserAPIKeyAuth with
parent_otel_span — an OTel Span that is not deepcopy-able). Force a shallow
copy of the metadata dict so mutations (model_group, is_silent_experiment)
never corrupt the main call's metadata.
r   safe_deep_copyr   Nlitellm_parent_otel_spanTis_silent_experimentFstreamlitellm_call_idlitellm_logging_objstandard_logging_object)'litellm.litellm_core_utils.core_helpersr  r~  rP  r  )r[  r  r  silent_kwargsoriginal_metadatas        r   _get_silent_experiment_kwargs$Router._get_silent_experiment_kwargs  s     	K&v. #JJz2)!!*-1BB(,->(?M*%]*(*M*%
 	j!%%&@$G<@j!"89 #(h 	+T2/63T: r   r  c           	      H  ^ ^^^  UR                  S0 5      R                  SS5      (       a  g[        R                  " T5      m[        R                  " ST 35        T R
                  " S
0 UD6mTTS   S'   [        R                  " 5       n[        R                  " U5         UU UU4S jnUR                  U" 5       5        UR                  5         g! UR                  5         f = f! [         a/  n[        R                  " ST S	[        U5       35         SnAgSnAff = f)z5
Run a silent experiment in the background (thread).
r   r  FN%Starting silent experiment for model r  c                  B  >#    TR                   " ST[        [        [           T5      S.TD6I S h  vN   [        R
                  " 5       n [        R                  " 5       nUb  U R                  U5        U (       a  [        R                  " U SS06I S h  vN   g g  Ni N7f)Nr   r  return_exceptionsTr   )	acompletionr   r   rQ   asyncio	all_taskscurrent_taskry  gather)pendingcurrentr  r[  r  r  s     r   _run_silent_completionDRouter._silent_experiment_completion.<locals>._run_silent_completion   s     ** *!%d+;&<h!G (   &//1G%224G*0%nngNNNN  Os"   .BBA"BBBB#Silent experiment failed for model : r   )r~  r  r  r   rQ  r  r'  new_event_loopset_event_looprun_until_completecloser  errorr  )r[  r  r  r  loopr-  r  r  s   ```    @r   r   $Router._silent_experiment_completion  s   +	zz*b)--.DeLL}}X.H!&&7~F !>>HHM 8DM*%m4 ))+D""4(O O ''(>(@A



 	!''5l^2c!fXN 	s5   'C( A4C( #C C( C%%C( (
D!2%DD!r  c                    #    g 7fr   r   r[  r   r  r  r  s        r   r&  Router.acompletion  
      	   c                    #    g 7fr   r   r9  s        r   r&  r:    r;  r<  c                    #    g 7fr   r   r9  s        r   r&  r:  %  r;  r<  c                 0  #     XS'   X$S'   X4S'   U R                   US'   U R                  XS9  UR                  S5      =(       d    U R                  n[        R                  " 5       nU R                  U5      nU(       a  U R                  UUUS9I S h  vN $ Ub0  [        U[        5      (       a  U R                  " S0 UD6I S h  vN nOU R                  " S0 UD6I S h  vN n[        R                  " 5       n	X-
  n
[        R                  " U R                  R                  [        R                   U
SUU	[#        U5      S	95        U$  N N Nl! [$         a:  n[        R                  " ['        U U[(        R*                  " 5       US
95        UeS nAff = f7f)Nr   r  r  r  r  priority)r   r  r  r&  servicedurationr  
start_timeend_timeparent_otel_spanlitellm_router_instancer  error_traceback_stroriginal_exceptionr   )_acompletionr  r~  r   time_is_prompt_management_model_prompt_management_factoryr   r4  schedule_acompletionasync_function_with_fallbacksr'  create_taskr  async_service_success_hookrl   ROUTERr"   r  rH   	traceback
format_exc)r[  r   r  r  r  request_priorityrD  rM  r  rE  	_durationr  s               r   r&  r:  .  s    ,	#7O!):%8*.*;*;F&'00u0L%zz*5N9N9NJ*.*J*J5*Q'*!<<%! =   
  +
;KS0Q0Q!%!:!:!DV!DD!%!C!C!Mf!MMyy{H -I''BB(//&+)%%Fv%N C 	 O- EM  		(,0#)(1(<(<(>'(	 G		sl   FBE E	E F-E EE  E!A'E F	E E E 
F5FFFfallback_itemcomplete_response_object_usagec                     SSK Jn  [        [        [           [        U SS5      5      nUb  U/O/ nUb/  [        US5      (       a  UR                  b  UR                  U5        UR                  US9n[        U SU5        g)zCMerge partial-stream usage with fallback-stream usage on the chunk.r   )BaseTokenUsageProcessorusageN)usage_objects)litellm.cost_calculatorr[  r   r   rs   r  hasattrr\  r3  combine_usage_objectsr  )rX  rY  r[  r\  r]  combined_usages         r   _combine_fallback_usageRouter._combine_fallback_usagec  s     	DXe_gmWd&KL#(#4"*66@@.44@  !?@0FF' G 
 	w7r   r  r  c                 r   ^ ^^^^#    SSK Jm   " U4S jS[        5      nUUUUU 4S jnU" U" 5       5      $ 7f)zo
Helper to iterate over a streaming response.

Catches errors for fallbacks using the router's fallback system
r   MidStreamFallbackErrorc                   @   >^  \ rS rSrS\4U U4S jjrS rS rSrU =r	$ )ERouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapperi  async_generatorc                    > [         TU ]  UTR                  TR                  TR                  S9  Xl        [        TS5      (       a   TR                  R                  5       U l        g g N)completion_streamr   custom_llm_providerlogging_obj_hidden_params)	superrc  r   rm  rn  _async_generatorr_  ro  r  )r[  ri  	__class__r  s     r   rc  NRouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__init__  se     &5(..(6(J(J . : :	 !  )8%>+;<<*8*G*G*L*L*ND' =r   c                     U $ r   r   r  s    r   	__aiter__ORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__aiter__      r   c                 R   #    U R                   R                  5       I S h  vN $  N7fr   )rq  	__anext__r  s    r   ry  ORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__anext__  s      !22<<>>>>s   '%')rq  ro  )
r   r   r   r   r   rc  ru  ry  r   __classcell__rr  r  s   @r   FallbackStreamWrapperrh    s&    O O O? ?r   r}  c                   >#    S n  T  S h  vN nU7v   M   N
 GO! T Ga  nSSK Jn  U" TR                  S9n[        [        [
           [        USS 5      5      n [        [        TR                  S5      5      nTR                  STR                  5      nTR                  STR                  5      nTR                  STR                  5      n	TR                  TS	'   UR                  (       d  UR                  (       d  TTS
'   OTSSS.SUR                  SS./-   TS
'   TR                  UTS9  TR!                  USUUU	USTS9I S h  vN  n [#        U S5      (       aW  U   S h  vN  n
U
(       a7  [%        U
[&        5      (       a"  [#        U
S5      (       a  TR)                  X5        U
7v   MO  
  S nAO?S 7v    S nAO5! [*         a   n[,        R.                  " SU 35        UeS nAff = fS nAff = f[0        R2                  " SS9   [#        TS5      (       aI   TR5                  5       I S h  vN    O.! [6         a!  n[,        R8                  " SU5         S nAOS nAff = fU bZ  [#        U S5      (       aI   U R5                  5       I S h  vN    O.! [6         a!  n[,        R8                  " SU5         S nAOS nAff = fS S S 5        g ! , (       d  f       g = f! [0        R2                  " SS9   [#        TS5      (       aI   TR5                  5       I S h  vN    O.! [6         a!  n[,        R8                  " SU5         S nAOS nAff = fU bZ  [#        U S5      (       aI   U R5                  5       I S h  vN    O.! [6         a!  n[,        R8                  " SU5         S nAOS nAff = fS S S 5        f ! , (       d  f       f = f= f7f)Nr   stream_chunk_builderchunksr\  r   r   r   r   r  r  system  You are a helpful assistant. You are given a message and you need to respond to it. You are also given a generated content. You need to respond to the message in continuation of the generated content. Do not repeat the same content. Your response should be in continuation of this text: rolecontent	assistantTr  r  prefixr  Fr   r  disable_fallbacksr   r   r   r  rh  r  ru  Fallback also failed: shieldaclose7stream_with_fallbacks: error closing model_response: %s:stream_with_fallbacks: error closing fallback_response: %s)litellm.mainr  r  r   r   rs   r  r  r~  r   r   r   rK  is_pre_first_chunkgenerated_contentr  *async_function_with_fallbacks_common_utilsr_  r   rq   rb  r  r   r5  anyioCancelScoper  BaseExceptionr  )fallback_responseitemr  r  complete_response_objectrY  r  r   r   r   rX  fallback_errorrf  r  r  r  r[  s               r   stream_with_fallbacksERouter._acompletion_streaming_iterator.<locals>.stream_with_fallbacks  s     $i"0 $J.) N)=+?)00,( 26UO4gtD2.D)"&sN,>,>w,G"HK0>0B0B#T^^1I @N?Q?Q2D4Q4Q@, @N?Q?Q2D4Q4Q@, ;?:K:KN#67++13F3F
 6>z25=(0 ,M
 )4+,+>+>*.
A 
6z2 88). 9  #MM.3&/5M5M(3!##1 N 	 	 	 & 0+>>3D 	0 	0- -$.}>Q$R$R$+M7$C$C $ < <$1!" #0/ 4E #

  ))//00@A )()SN)h &&d3~x88"0"7"7"999, 177 Y ! 
 )4)8: :"3":":"<<<, 177 \ !  433U&&d3~x88"0"7"7"999, 177 Y ! 
 )4)8: :"3":":"<<<, 177 \ !  433s  O	   K G68G1C.GEG%F4)E,*F4.AG5K :G?K 
G.G))G..G11G66K 9O	K H:3H64H:9K:
I%I K I%%K=JJJK
K!J=8K=KK	O	
KO	O4N5L!L
L! N5!
M	+M	N5M	N5$M>7M:
8M>=N5>
N)	N$	N5$N)	)N5,	O5
O?OO	litellm.exceptionsrf  rt   )r[  r  r  r  r}  r  rf  s   ````  @r   _acompletion_streaming_iterator&Router._acompletion_streaming_iteratorx  s6      	>	?$7 	?(k	 k	Z %%:%<==s   07source_iteratorr   r   c                    SSK Jn  SSKJnJnJnJn  [        X5      (       a  U R                  nU(       a   SSK	J
n  U" US9nUb  [        USS5      OSn	U	bi  [        [        U	SS5      =(       d    S5      n
[        [        U	S	S5      =(       d    S5      n[        [        U	S
X-   5      =(       d    X-   5      nU" U
UUS9$  U R                  n[        UX4U45      (       a  UR                  R                   $ g! [         a     NBf = f)u$  
Best-effort: pull partial token usage from a Responses-API streaming
iterator that errored mid-stream, normalized to ResponseAPIUsage so
the caller can combine without crossing token-naming conventions.

Two sources, in priority order:
  1. The bridge path (LiteLLMCompletionStreamingIterator) accumulates
     chat-completion chunks while streaming — feed them through
     stream_chunk_builder to recover chat Usage, then translate
     (prompt_tokens → input_tokens, completion_tokens → output_tokens).
  2. The native path (ResponsesAPIStreamingIterator) only has a
     completed_response object if the stream reached
     RESPONSE_COMPLETED before erroring — uncommon mid-stream but
     worth checking. Already ResponseAPIUsage-shaped.

Returns None when no partial usage is recoverable.
r   )"LiteLLMCompletionStreamingIteratorr   ResponseCompletedEventResponseFailedEventResponseIncompleteEventr  r  Nr\  prompt_tokenscompletion_tokenstotal_tokensinput_tokensoutput_tokensr  )Flitellm.responses.litellm_completion_transformation.streaming_iteratorr  litellm.types.llms.openair   r  r  r  r    collected_chat_completion_chunksr  r  r  r4  r  completed_responser  r\  )r  r  r   r  r  r  r  r  builtr:  promptr  total	completeds                 r    _extract_partial_responses_usage'Router._extract_partial_responses_usage	  s#   *	
	
 	
 oJJ$EEFA0?E =B<M75'48SWD' "%WT?A%F%K!!L%(7JA)N)SRS%T
 ##D.&:MN 5 & 3!  0)/*4).   (, $66	#:QR
 
 %%+++ ! s   B	C8 8
DDc                    SSK JnJnJnJn  [        U X4U45      (       d  gU R                  nUR                  c  gUR                  nU" UR                  =(       d    SUR                  =(       d    S-   UR                  =(       d    SUR                  =(       d    S-   UR                  =(       d    SUR                  =(       d    S-   S9Ul        g)u  
Merge partial-stream usage with fallback-stream usage on a
Responses-API streaming event.

Only mutates events that carry a `response` with a `usage` field
(response.completed / response.failed / response.incomplete). Other
events pass through unchanged.

Both inputs are ResponseAPIUsage-shaped (see
_extract_partial_responses_usage which normalizes the bridge path),
so we can sum input_tokens / output_tokens / total_tokens directly
and produce a clean ResponseAPIUsage — no token-naming split, no
setattr bypass.
r   r  Nr  )r  r   r  r  r  r   r  r\  r  r  r  )rX  partial_usager   r  r  r  r  fbs           r   !_combine_responses_fallback_usage(Router._combine_responses_fallback_usageU	  s    &	
 	
 #:QR
 
  ))>>!^^)'449boo>RQRS(66;!@P@P@UTUV'449boo>RQRS
r   	input_valr   r  c                     [        U [        5      (       a  SSSU S./S./nO#[        U [        5      (       a  [        U 5      nO/ nSSSSS./S.SSS	US./S./n[        S
X#-   5      $ )u  
Convert Responses-API input + partial assistant output into a
continuation input that asks the fallback model to pick up where the
prior assistant message stopped.

Best effort across providers. The chat-completions path uses
Anthropic's `prefix: True` prefill trick on the assistant message;
the Responses-API input schema has no direct equivalent, so we
append an instruction (developer role) plus a prior assistant
message containing the partial output. Providers without prefill
semantics (OpenAI, Vertex) treat this as conversational context
and may regenerate — same trade-off as the chat-completions path
for non-Anthropic fallbacks.
r  user
input_text)r   text)r   r  r  	developeru   The previous assistant response was interrupted mid-stream. Continue exactly where it stopped — do not repeat any of its content. Your response must read as a seamless continuation.r  output_textr   )r   r  rA  r   )r  r  basecontinuations       r   #_build_responses_continuation_input*Router._build_responses_continuation_input	  s    2 i%% &")5y IJD 	4((	?DD "# !-D
  "#%2<MNO!#
, ($*=>>r   r  c                   ^ ^^^	^
#    SSK Jm  SSKJnJn  Um
U" 5       nUR
                  R                  UR
                  R                  UR
                  R                  4m	 " U	U
4S jSU5      nUUU U
4S jnU" U" 5       5      $ 7f)a  
Wrap a Responses-API streaming iterator so MidStreamFallbackError
triggers the Router's fallback chain (parity with
_acompletion_streaming_iterator for the chat-completions path).

The Responses-API streaming path goes through
_ageneric_api_call_with_fallbacks rather than _acompletion, so the
returned iterator is never wrapped by the chat completions
fallback handler. Without this wrapper, MidStreamFallbackError
raised mid-stream from the underlying CustomStreamWrapper (used by
LiteLLMCompletionStreamingIterator when the Responses API is
served via the completion bridge) propagates unhandled and the
configured cross-provider fallback never fires.

Full parity with the chat-completions path:
  - Pre-first-chunk: retry with the original input unchanged.
  - Partial content: inject a developer instruction + prior
    assistant message carrying the generated text so the fallback
    model continues rather than restarts.
  - Usage combining: merge partial-stream usage onto the fallback's
    response.completed event so accounting reflects both attempts.
  - Stream cleanup: shielded aclose() on both source and fallback
    iterators on terminate.
r   re  )r   _get_openai_response_typesc                   F   > \ rS rSrSrS\4U4S jjrS rU 4S jrS r	Sr
g	)
MRouter._aresponses_streaming_iterator.<locals>.FallbackResponsesStreamWrapperi	  z
Subclasses BaseResponsesAPIStreamingIterator only for isinstance
compatibility (proxy + interactions code paths check the type).
Bypasses the parent constructor and delegates iteration to an
async generator.
ri  c           	        > SS K nSSKJn  Xl        [        TSS 5      U l        [        TSS 5      U l        [        TS[        TSS 5      5      U l        SU l        [        TSS 5      U l        S U l	        [        TS	UR                  5       5      U l        SU l        SU l        SU l        S U l        S
U l        UR                   " 5       U l        [        TSS 5      U l        [        TSS 5      U l        [        TS0 5      =(       d    0 U l        [        TSS 5      U l        [-        [        TSS 5      =(       d    0 5      U l        g )Nr   )datetimer  r   rn  r  Fresponses_api_provider_configrD  Tlitellm_metadatarm  request_datar  ro  )rL  r  rq  r  r  r   rn  finishedr  r  nowrD  _failure_handled_completed_response_cached_completed_response_logged_completed_response_cache_hit*_persist_completed_response_before_logging_stream_created_timer  rm  r  r  rP  ro  )r[  ri  rL  r  r  s       r   rc  VRouter._aresponses_streaming_iterator.<locals>.FallbackResponsesStreamWrapper.__init__	  s;   -(7%( !(T J$_gtD
#*#!O-BDI$ 
 !&5<#%Dd62 +/'")/<"X(-%27/27/592BF?,0IIK)(/#%7)% ,3#%:D,( %,O^R$P$VTV!!(+t!L '+O-=tDJ'#r   c                     U $ r   r   r  s    r   ru  WRouter._aresponses_streaming_iterator.<locals>.FallbackResponsesStreamWrapper.__aiter__-
  rw  r   c                    >#    U R                   R                  5       I S h  vN nU R                  c  [        USS 5      T;   a  Xl        U$  N*7f)Nr   )rq  ry  r  r  )r[  chunk_RESPONSES_TERMINAL_EVENT_TYPESs     r   ry  WRouter._aresponses_streaming_iterator.<locals>.FallbackResponsesStreamWrapper.__anext__0
  sN     "33==?? ++3vt48WW.3+! @s   AA+Ac                 T   #    U R                   R                  5       I S h  vN   g  N7fr   )rq  r  r  s    r   r  TRouter._aresponses_streaming_iterator.<locals>.FallbackResponsesStreamWrapper.acloseC
  s     ++22444s   (&()rq  r  r  r  r  ro  r  r  r  r  rm  r  r  rn  r   r  r  r  rD  N)r   r   r   r   __doc__r   rc  ru  ry  r  r   )r  r  s   r   FallbackResponsesStreamWrapperr  	  s$    7 7r&5r   r  c                 *  >#    S n  T  S h  vN nU7v   M   N
 GO! T Ga  n[         R                  T5      n [        [        TR	                  S5      5      nTR	                  STR
                  5      nTR	                  STR                  5      nTR	                  STR                  5      nTR                  TS'   UR                  (       d  UR                  (       d  O2[         R                  TR	                  S5      UR                  5      TS'   TR                  UTSS9  TR                  US	UUUUS
TS9I S h  vN  n [        U S5      (       a1  U   S h  vN  nUb  [         R                  X5        U7v   M)  
  S nAO?U 7v    S nAO5! [          a   n	["        R$                  " SU	 35        U	eS n	A	ff = fS nAff = f[&        R(                  " SS9   [        TS5      (       aI   TR+                  5       I S h  vN    O.! [,         a!  n
["        R.                  " SU
5         S n
A
OS n
A
ff = fU bZ  [        U S5      (       aI   U R+                  5       I S h  vN    O.! [,         a!  n
["        R.                  " SU
5         S n
A
OS n
A
ff = fS S S 5        g ! , (       d  f       g = f! [&        R(                  " SS9   [        TS5      (       aI   TR+                  5       I S h  vN    O.! [,         a!  n
["        R.                  " SU
5         S n
A
OS n
A
ff = fU bZ  [        U S5      (       aI   U R+                  5       I S h  vN    O.! [,         a!  n
["        R.                  " SU
5         S n
A
OS n
A
ff = fS S S 5        f ! , (       d  f       f = f= f7f)Nr   r   r   r   r  r  r  r   r  metadata_variable_nameFr   r  ru  z*Responses streaming fallback also failed: Tr  r  z;stream_with_fallbacks(aresponses): error closing source: %sz=stream_with_fallbacks(aresponses): error closing fallback: %s)r   r  r   r  r~  r   r   r   (_ageneric_api_call_with_fallbacks_helperr  r  r  r  r  r_  r  r  r   r5  r  r  r  r  r  )r  r  r  r  r  r   r   r   rX  r  excrf  r  r[  r  s              r   r  DRouter._aresponses_streaming_iterator.<locals>.stream_with_fallbacksG
  s}     $\"1 $J/) E) & G G XC)"&sN,>,>w,G"HK0>0B0B#T^^1I @N?Q?Q2D4Q4Q@, @N?Q?Q2D4Q4Q@, EE ##67 ++13F3F  #FF . 2 27 ; ! 3 3 'w/ 88)-/A 9  #MM.3&/5M5M(3!##1 N 	 	 	 & 0+>>3D 0 0-,8 & H H$1!" #0/ 4E 0//  ))//D^DTU )(	)CE)N &&d399"1"8"8":::, 177 ] # 
 )4)8: :"3":":"<<<, 177 _ #  433U&&d399"1"8"8":::, 177 ] # 
 )4)8: :"3":":"<<<, 177 _ #  433s  N   J) G F;DF<D?=FE>EE>!F?J) F	J) 
F8F33F88F;;G  J) NJ*H=H >HJ
H/H*%J*H//JI!II! J!
J+JJJJ	N
J&"N)N>M?K+$K'
%K+*M?+
L	5L	M?L	M?.MM
MM?
M3	M.	)M?.M3	3M?6	N?
N	NN)	r  rf  $litellm.responses.streaming_iteratorr   r  ResponsesAPIStreamEventsRESPONSE_COMPLETEDRESPONSE_INCOMPLETERESPONSE_FAILED)r[  r  r  r   r  _openai_typesr  r  rf  r  r  s   ` `     @@@r   _aresponses_streaming_iterator%Router._aresponses_streaming_iterator	  s     : 	>	

 # 3422EE22FF22BB+
'Y	5 Y	5-N Y	5v^	 ^	@ ..C.EFFs   A>Bc                 n   ^^^^^ SSK Jm   " U4S jS[        5      nU mUUUUU4S jnU" U" 5       5      $ )z
Sync equivalent of _acompletion_streaming_iterator.

Wraps a sync streaming response so that MidStreamFallbackError
(raised by CustomStreamWrapper.__next__) triggers the Router's
fallback chain instead of surfacing directly to the caller.
r   re  c                   @   >^  \ rS rSrS\4U U4S jjrS rS rSrU =r	$ )HRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapperi
  sync_generatorc                    > [         TU ]  UTR                  TR                  TR                  S9  Xl        [        TS5      (       a   TR                  R                  5       U l        g g rk  )	rp  rc  r   rm  rn  _sync_generatorr_  ro  r  )r[  r  rr  r  s     r   rc  QRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapper.__init__
  se     &4(..(6(J(J . : :	 !  (6$>+;<<*8*G*G*L*L*ND' =r   c                     U $ r   r   r  s    r   __iter__QRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapper.__iter__
  rw  r   c                 ,    [        U R                  5      $ r   )r  r  r  s    r   __next__QRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapper.__next__
  s    D0011r   )ro  r  )
r   r   r   r   r   rc  r  r   r   r{  r|  s   @r   SyncFallbackStreamWrapperr  
  s&    	Oy 	O 	O2 2r   r  c            	   3   l  >#    S n  T H  nUv   M	     [#        TS5      (       a   TR1                  5         U b$  [#        U S5      (       a   U R1                  5         g g g ! T Ga  nSSK Jn  U" TR                  S9n[        [        [
           [        USS 5      5      n [        [        TR                  S5      5      nTR                  STR                  5      nTR                  STR                  5      nTR                  STR                  5      n	TR                  TS	'   UR                  (       d  UR                  (       d  TTS
'   OTSSS.SUR                  SS./-   TS
'   TR                  UTS9  TR                   " S0 TDUUU	S.D6n [#        U S5      (       aQ  U  HE  n
U
(       a7  [%        U
[&        5      (       a"  [#        U
S5      (       a  TR)                  X5        U
v   MG      S nAGNS v    S nAGN! [*         a   n[,        R.                  " SU 35        UeS nAff = fS nAff = f! [2         a"  n[,        R4                  " SU5         S nAGNS nAff = f! [2         a!  n[,        R4                  " SU5         S nAg S nAff = f! [#        TS5      (       a@   TR1                  5         O.! [2         a!  n[,        R4                  " SU5         S nAOS nAff = fU bR  [#        U S5      (       a@   U R1                  5         f ! [2         a!  n[,        R4                  " SU5         S nAf S nAff = ff f = f7f)Nr   r  r  r\  r   r   r   r   r  r  r  r  r  r  Tr  r  )r   r   r   r  r  r4  r  r  r   )r  r  r  r   r   rs   r  r  r~  r   r   r   r  r  r  r  r  r_  r   rq   rb  r  r   r5  r4  r  r  )r  r  r  r  r  rY  r  r   r   r   rX  r  	close_errrf  r  r  r  router_selfs                r   r  DRouter._completion_streaming_iterator.<locals>.stream_with_fallbacks
  sl     $X*DJ +L >733&,,. %0W%w6 6)//1	60Y * B)=+?)00,( 26UO4gtD2.8)"&sN,>,>w,G"HK0>0B0B#[%:%:1I @N?Q?Q2#<<@, @N?Q?Q2#<<@, ;F:Q:QN#67++13F3F5=z25=(0 ,M
 )4+,+>+>*.
A 
6z2  ??). @  )4(K(K )()"+1I1I	)% 0*==->M -$.}>Q$R$R$+M7$C$C + C C$1!" #0/ .? #

  ))//00@A )(	)}B)N ) -33U%  ) -33X%  >733&,,.( -33U% 
 %0W%w6 6)//1( -33X% 60s
  L4A L4H, L4I L4H)%8H$E	G7'J	 -G71J	 7
H!HH!!H$$H))J	 ,
I6IL4IL4
J%J<L4JL4	L1J-,L1-
K7KL1KL10L L1
L,L'"L1'L,,L11L4r  )r[  r  r  r  r  r  rf  r  s    ```  @@r   r  %Router._completion_streaming_iterator
  s:     	>	2(; 	2$ Z	 Z	x ))>)@AAr   c           	        #     UR                  S0 5      R                  SS5      (       a  g[        R                  " U5      n[        R                  " SU 35        U R
                  " S
0 UD6nXS   S'   U R                  " S
U[        [        [           U5      S.UD6I Sh  vN   g N! [         a/  n[        R                  " SU S	[        U5       35         SnAgSnAff = f7f)z,
Run a silent experiment in the background.
r   r  FNr"  r  r$  r/  r0  r   )r~  r  r  r   rQ  r  r&  r   r   rQ   r  r5  r  )r[  r  r  r  r  r  s         r   _silent_experiment_acompletion%Router._silent_experiment_acompletion*  s     	zz*b)--.DeLL}}X.H!&&7~F !>>HHM7C*%m4 "" "d#34h?    
  	!''5l^2c!fXN 	sK   C%'B) C%A5B) !B'"B) &C%'B) )
C"3%CC%C""C%c                   #    SnSn0 n UR                  5       nXS'   [        U5      n[        R                  " 5       n	U R                  UUUR	                  SS5      US9I Sh  vN nUn[        R                  " 5       n
X-
  n[
        R                  " U R                  R                  [        R                  USU	U
[        U5      S95        U R                  XXS9  US   R                  5       nUR	                  S	S5      nUb)  [
        R                  " U R                  " S UUS
.UD65        U R                  XSS9  UR	                  S	S5        US   nU R                  UUS9nU R                  U==   S-  ss'   0 UEUU R                   US.EUEnUR	                  S	S5        ["        R$                  " S 0 UD6nUR'                  SS5      nU R)                  UUSS9nUb_  [+        U[
        R,                  5      (       a@  U ISh  vN    U R/                  UUUS9I Sh  vN   UI Sh  vN nSSS5      ISh  vN   O#U R/                  UUUS9I Sh  vN   UI Sh  vN n[+        W[0        5      (       a.  U R3                  UUUS9nU(       a  ["        R4                  " SUSS9eU R6                  U==   S-  ss'   [8        R:                  " SU S35        U R                  UUUS9  [+        U[<        5      (       a  U R?                  UUUS9I Sh  vN $ U$  GN GN N N N! , ISh  vN  (       d  f       N= f N N N0! ["        R@                   a  nUR'                  S0 5      R'                  SS5      nUR'                  S0 5      R'                  SS5      n["        RB                  (       a  U=RD                  SU SU 3-  sl"        Ub$  U RG                  UU5        U RI                  UU5        UeSnAf[J         an  n[8        R:                  " SU S[M        U5       S35        Ub  U RN                  U==   S-  ss'   Ub$  U RG                  UU5        U RI                  UU5        UeSnAff = f7f)!z
- Get an available deployment
- call it with a semaphore over the call
- semaphore specific to it's rpm
- in the semaphore,  make a check against it's local rpm before running
Nr   r  r  async_get_available_deploymentrA  r  rF  r   r  )r  r  r  r~   r  r  max_parallel_requestsr  r  client_type)r  rn  rF  r  r  r  r  zlitellm.acompletion(model=r  )r  r  rF  r  r/  r   z$

Deployment Info: request_timeout: z

timeout: r  r  r   )(r  r"   rL  r  r  r'  rQ  r  rR  rl   rS  _track_deployment_metricsr	  r  _get_async_openai_model_clientr5  r   r  r&  r~  r  r   	Semaphore&async_routing_strategy_pre_call_checksrv   r  r  r7  r   rQ  rt   r  Timeoutexpose_router_debug_in_errorsr  r	  r
  r  r  r6  )r[  r   r  r  r  r  _timeout_debug_deployment_dictr  rF  rD  rE  rW  r   r  r  r  	_responsern  rpm_semaphorer  r  r   deployment_request_timeout_paramdeployment_timeout_params                           r   rK  Router._acompletionJ  s     

 	'^	28++-/;@8@HJ#BB!$*JJ/Dd$K%	  C   J .8*yy{H -I''BB(//&>)%%Fv%N C 	 **% +  ((89>>@N)--ndCL' ##77 %1!) ! //:/UJJ~t,'0J>>% ? L Z(A-( $//&	
 L ^T2++;l;I4:JJ%t5K !,,%3 - M
 (Zw00. . )= EE#-$/)9 F   
 &/H )== AA) +%5 B    "+? (M22 $ G G(6 !H ! !!== >#%'  z*a/*!&&,ZL8OP **%!!1 +  ($788!AA#+%#F B    OZ )

  / )=== +4  	/M/Q/Q "0c#T* - (F'I'I "(c)T" % 44		EFfEggr  tL  sM  N  N	%==aL;;AzJG 
	!&&,ZL8LSQRVHT[\ %
+q0+%==aL;;AzJG
	s   Q/AM  #L$FM  LM  L L	L &L'L +M  6L7M  L:	M  L<B2M  L>M  Q/M  Q/M  M  L L M   L7&L)'L73M  <M  >M   Q,BO11Q,>A)Q''Q,,Q/r  r  c                    UR                  SU R                  5      US'   UR                  S[        [        R
                  " 5       5      5        SnU R                  US9(       a  UnUR                  U0 5      R                  XS.5        g)zE
Adds/updates to kwargs:
- num_retries
- litellm_trace_id
- metadata
r   litellm_trace_idNr   )r  r   )r~  r   r;  r  r   uuid4_get_model_from_aliasr  )r[  r   r  r  r   s        r   r  &Router._update_kwargs_before_fallbacks  sz     !'

=$:J:J K},c$**,.?@+/%%E%2 %0"5<<!J	
r   	exceptionc                     [        US5      (       a  UR                  b  gUR                  S0 5      nUR                  S5      nUb   [        U5      Ul        gg! [        [
        4 a     gf = f)z
Set num_retries from deployment litellm_params on the exception.

This allows the retry logic in async_function_with_retries to use
per-deployment retry settings instead of the global setting.
r   Nr   )r_  r   r~  r4  r  	TypeError)r[  r#  r  r   dep_num_retriess        r   r	  /Router._set_deployment_num_retries_on_exception  sv     9m,,1F1F1R#(8"=(,,];&(+O(<	% ' 	* s   A A,+A,c                     [        USS5      (       a  gUR                  S5      =(       d    0 R                  S5      nU(       a   X1l        gg! [         a     gf = f)aQ  
Stamp the failed deployment's `model_info.id` on the exception so the
fallback layer can exclude it from subsequent re-picks within the same
request (used by weighted-routing failover).

Idempotent: never overwrites an existing value, so the id of the
deployment that *first* failed in a chain is preserved if multiple
layers re-raise.
failed_deployment_idN
model_infor  )r  r~  r)  r  )r[  r#  r  deployment_ids       r   r
  -Router._set_failed_deployment_id_on_exception%  s\     94d;;#5;@@F1>.   s   A 
AAc                    U R                   R                  5       nUR                  S0 5      =(       d    0 nUR                  5        H  u  pVUc  M
  UR	                  XV5        M     UR	                  U0 5      R                  U5        g)z
Adds default litellm params to kwargs, if set.

Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name
r   N)r   r  r  rq  r;  r  )r[  r  r  defaultsmetadata_defaultskeyr  s          r   *_update_kwargs_with_default_litellm_params1Router._update_kwargs_with_default_litellm_params:  sx     ..335$LLR8>B #..*JC}c) + 	0"5<<=NOr   function_namec                 h   UR                  S0 5      R                  5       nUS   R                  5       n[        XRS9n[        US9nUR                  U0 5      R                  S5      nU R	                  XS9n	UR                  S5      n
XS'   XS'   [        U[        S0 UD6US	9nU R                  US
9  U$ )z
Handle clientside credential
r*  r   )r   r  r3  r  )r  r   r  original_model_idr  r   r*  r  r   )r~  r  r9   r5   _generate_model_idr\   r_   upsert_deployment)r[  r  r  r3  r*  r   dynamic_litellm_paramsr  r  	_model_idr6  deployment_pydantic_objs               r   _handle_clientside_credential$Router._handle_clientside_credentialO  s      ^^L"5::<
#$45::<!;)"
 "D'"
 jj!7<@@O++# , 
	 'NN40$4*;&'",")C,BC!#

 	. 	 	
 '&r   c                    U R                  S0 5      =(       d    0 n[        U[        5      (       a  UnOUR                  SS9nUR                  S5      =(       d    / nUR                  S5      =(       d    / nU(       d  U(       a  [	        U5      [	        U5      -   nXaS'   SU;  a  UR                  S5      b	  US   US'   ggg)z
Merge tools from deployment litellm_params with request kwargs.
When both have tools, concatenate them (deployment tools first, then request tools).
tool_choice: use request value if provided, else deployment's.
r   Tr   toolstool_choiceN)r~  r   rP  rS  rA  )r  r  dep_params_raw
dep_params	dep_tools	req_toolsmergeds          r   _merge_tools_from_deployment#Router._merge_tools_from_deploymento  s     $(8"=Cnd++'J'222EJNN7+1r	JJw'-2		)_tI6F$7O&:>>-+H+T$.}$=F=! ,U&r   c                    U R                  XS9  UR                  S0 5      R                  5       nUS   S   nUS   R                  S5      nUS   n[        US9(       aV  U R	                  XUS9nUR
                  R                  5       nUR                  R                  nUR                  R                  n[        US	9n	UR                  U	0 5      R                  UUUUS
.5        UR                  S0 5      R                  S5      n
U
(       aO  X)   R                  S5      =(       d    / n[        U5      nU
 H  nX;  d  M
  UR                  U5        M     XU	   S'   UR                  S0 5      R                  S5      nU(       a>  SU 3nX)   R                  S5      =(       d    / nX;  a  UR                  U5        XU	   S'   XBS'   US:X  aQ  SSKJn  [#        U R$                  [&        [(        45      (       a  [)        U R$                  5      OSnU" UUS   US9US'   OU R+                  X!S   S9US'   U R-                  X)S9  g)z
3 jobs:
- Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
- Adds default litellm params to kwargs, if set.
- Merges tools from deployment with request (proxy-configured tools + request tools).
r  r*  r   r   api_baser  )r  r  r  r3  r5  )r  r*  rJ  deployment_model_nametagslitellm_credential_namezCredential: !_ageneric_api_call_with_fallbacksr   )resolve_llm_passthrough_timeoutN)r  r   router_timeoutr   r  data)r  r  )rG  r~  r  r:   r=  r*  rS  r   r   rJ  r5   r;  r  rA  r3  !litellm.passthrough.timeout_utilsrP  r   r.  r4  r(  _get_timeoutr1  )r[  r  r  r3  r*  deployment_litellm_model_namedeployment_api_baserL  r<  r  deployment_tagsexisting_tagsmerged_tagstagcredential_namecredential_tagrP  _router_timeouts                     r   r  %Router._update_kwargs_with_deployment  s    	))Z)O^^L"5::<
(23C(DW(M%()9:>>zJ *< 8#6:&*&H&H%M 'I '# 1;;FFHJ,C,R,R,X,X)"9"H"H"Q"Q!C'"
 	0"5<<;(/)>		
 %..)92>BB6J":>>vFL"M}-K&)&&s+ ' 6A)*62 %..)92>BB%
 +O+<=N":>>vFL"M2$$^45B)*62)|?? d44sElCC d,,- 
 !@)*:;.!F9 !% 1 1/?$@ !2 !F9 	77 	8 	
r   c                     U R                  XSS9nUR                  SS5      nUb  Ub  XCR                  :w  a  SnU$ UnU$ )aY  
Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment

The same OpenAI client is re-used to optimize latency / performance in production

If dynamic api key is provided:
    Do not re-use the client. Pass model_client=None. The OpenAI/ AzureOpenAI client will be recreated in the handler for the llm provider
asyncr  r  N)r  r~  r  )r[  r  r  r  r  r  s         r   r  %Router._get_async_openai_model_client  sc     "&!1!1!g "2 "

 !**Y5'&2#A#AAL  2Lr   rS  c                     UR                  SS5      =(       dH    UR                  SS5      =(       d/    U R                  =(       d    U R                  R                  SS5      $ )z=Helper to get stream timeout from kwargs or deployment paramsr   N)r~  r   r   )r[  r  rS  s      r   _get_stream_timeoutRouter._get_stream_timeout  s_    
 JJ'. Gxx $G ""	G
 **../?F	
r   c                 ,   UR                  SS5      =(       dz    UR                  SS5      =(       da    UR                  SS5      =(       dH    UR                  SS5      =(       d/    U R                  =(       d    U R                  R                  SS5      nU$ )zAHelper to get non-stream timeout from kwargs or deployment paramsr   Nr/  )r~  r   r   r[  r  rS  r   s       r   _get_non_stream_timeoutRouter._get_non_stream_timeout   s    
 JJy$' 	@zz+T2	@xx4	@
 xx!4	@ ||	@ **..y$? 	 r   c                 z    SnUR                  SS5      (       a  U R                  XS9nUc  U R                  XS9nU$ )z6Helper to get timeout from kwargs or deployment paramsNr  FrR  )r~  rd  rh  rg  s       r   rU  Router._get_timeout  sO    /3::h&&..f.HG?22 3 G r   r  c                 \  ^ #    S[         S[        [           4U 4S jjnS[         S[        [           S[        4U 4S jjn[	        U[
        5      (       aX  [        S U 5       5      (       aA  / nU H  nUR                  U" SXrS.UD65        M     [        R                  " U6 I Sh  vN nU$ [	        U[
        5      (       a  [        S	 U 5       5      (       a  / n[        U5       H)  u  pU H  nUR                  U" SUU	U
S
.UD65        M      M+     [        R                  " U6 I Sh  vN n[        [        U5      5       Vs/ s H  n/ PM     nnU HG  n[	        U[        5      (       a  XS      R                  US   5        M3  US   R                  U5        MI     U$ gg N N{s  snf 7f)av  
Async Batch Completion. Used for 2 scenarios:
1. Batch Process 1 request to N models on litellm.Router. Pass messages as List[Dict[str, str]] to use this
2. Batch Process N requests to M models on litellm.Router. Pass messages as List[List[Dict[str, str]]] to use this

Example Request for 1 request to N models:
```
    response = await router.abatch_completion(
        models=["gpt-3.5-turbo", "groq-llama"],
        messages=[
            {"role": "user", "content": "is litellm becoming a better product ?"}
        ],
        max_tokens=15,
    )
```


Example Request for N requests to M models:
```
    response = await router.abatch_completion(
        models=["gpt-3.5-turbo", "groq-llama"],
        messages=[
            [{"role": "user", "content": "is litellm becoming a better product ?"}],
            [{"role": "user", "content": "who is this"}],
        ],
    )
```
r   r  c                 |   >#     TR                   " SXS.UD6I Sh  vN $  N! [         a  nUs SnA$ SnAff = f7fz[
Wrapper around self.async_completion that catches exceptions and returns them as a result
r$  Nr   r&  r  r   r  r  r  r[  s       r   _async_completion_no_exceptionsARouter.abatch_completion.<locals>._async_completion_no_exceptionsA  ?     !--WEWPVWWWW 0   <# !# <# 
949<9<r  c                    >#     TR                   " SXS.UD6I Sh  vN U4$  N! [         a  nXB4s SnA$ SnAff = f7frn  ro  )r   r  r  r  r  r[  s        r   *_async_completion_no_exceptions_return_idxLRouter.abatch_completion.<locals>._async_completion_no_exceptions_return_idxL  sL     **TTVTT T  vs0   ?% #% ?% 
<7<?<?c              3   B   #    U  H  n[        U[        5      v   M     g 7fr   )r   rP  r   r^  s     r   r  +Router.abatch_completion.<locals>.<genexpr>_  s     -T8ajD.A.A8r  r$  Nc              3   B   #    U  H  n[        U[        5      v   M     g 7fr   )r   rA  ry  s     r   r  rz  f  s     /VX
1d0C0CXr  )r   r  r  r~   r   r   )r  r   rQ   r4  r   rA  allr3  r'  r*  r  ranger  tuple)r[  r  r  r  rq  rv  _tasksr   r  r  r  r  _final_responsess   `             r   abatch_completionRouter.abatch_completion  s    H				"&'7"8				+,	 	& h%%#-T8-T*T*TF=gEg`fgh   %^^V44HO$''C/VX/V,V,VF )( 3#EMMB "' #%, %	 $ !4 &nnf55I<A#h-<P/Q<Pq<PO/Q%h..#QK077D#A&--h7	 &
 #") -W' 5 6/Qs8   B#F,&F#'BF,)F%*F,F'AF,%F,'F,c           	         ^ #    S[         S[        [           4U 4S jjn/ nU H  nUR                  U" SXS.UD65        M     [        R
                  " U6 I Sh  vN nU$  N7f)a  
Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router

Use this for sending multiple requests to 1 model

Args:
    model (List[str]): model group
    messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
    **kwargs: additional kwargs
Usage:
    response = await self.abatch_completion_one_model_multiple_requests(
        model="gpt-3.5-turbo",
        messages=[
            [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
            [{"role": "user", "content": "hello good mornign"}],
        ]
    )
r   r  c                 |   >#     TR                   " SXS.UD6I Sh  vN $  N! [         a  nUs SnA$ SnAff = f7frn  ro  rp  s       r   rq  ]Router.abatch_completion_one_model_multiple_requests.<locals>._async_completion_no_exceptions  rs  rt  r$  Nr   )r  r   rQ   r3  r'  r*  )r[  r   r  r  rq  r  message_requestr  s   `       r   -abatch_completion_one_model_multiple_requests4Router.abatch_completion_one_model_multiple_requests|  ss     ,				"&'7"8		 'OMM/ =C  ( !00 1s   AA"A A"c                    #    g 7fr   r   r9  s        r   "abatch_completion_fastest_response)Router.abatch_completion_fastest_response  r;  r<  c                    #    g 7fr   r   r9  s        r   r  r    r;  r<  c                   ^ ^#    UR                  S5       Vs/ s H  oUR                  5       PM     nnS[        S[        [        [        [        4      S[
        S[        S[        [        [        [        4   4
U 4S jjn/ mS[        R                  4U4S	 jjnU H3  n[        R                  " U" SXUS
.UD65      n	TR                  U	5        M5     T(       af  [        R                  " T[        R                   S9I Sh  vN u  n
mU
 H)  nU" U5      I Sh  vN nUc  M  SUR"                  S'   Us  $    T(       a  Mf  [        S5      es  snf  NO N97f)z
model - List of comma-separated model names. E.g. model="gpt-4, gpt-3.5-turbo"

Returns fastest response from list of model names. OpenAI-compatible endpoint.
,r   r  r  r  r   c                    >#     TR                   " SXUS.UD6I Sh  vN nU$  N! [        R                   a'    [        R                  " SR                  U 5      5        e [         a  nUs SnA$ SnAff = f7f)zV
Wrapper around self.acompletion that catches exceptions and returns them as a result
r   r  r  Nz4Received 'task.cancel'. Cancelling call w/ model={}.r   )r&  r'  CancelledErrorr   r  rR  r  )r   r  r  r  resultr  r[  s         r   rq  RRouter.abatch_completion_fastest_response.<locals>._async_completion_no_exceptions  sy     	#//heW]haghh i)) %++JQQRWX  s>   A8& $& A8& AA5)A0*A5+A80A55A8taskc                   >#     U I S h  vN n[        U[        [        45      (       aC  [        R                  " S5        T H  nUR                  5         M     U TR                  U 5        $   TR                  U 5        g  Nv! [         a     $ f = f! [         a     N1f = f! [         a     g f = f!  TR                  U 5        f ! [         a     f f = f= f7f)Nz=Received successful response. Cancelling other LLM API calls.)	r   rv   rt   r   r  r  rj  KeyErrorr  )r  r  tpending_taskss      r   check_responseARouter.abatch_completion_fastest_response.<locals>.check_response  s     #f}6I&JKK)//W +
 +!!((. L!((. $         !((. s   CB BAB B,C-B3 /B#  CB 
BCBC
B B3 B  B3 #
B0-C/B00C3C5CC
CCCCCr  )return_whenNT!fastest_response_batch_completionzAll tasks failedr   )splitstripr  r   r
   boolr   r   rv   rt   r  r'  TaskrQ  r3  waitFIRST_COMPLETEDro  )r[  r   r  r  r  r^  r  rq  r  r  donecompleted_taskr  r  s   `            @r   r  r    sR     &+[[%56%5'')%56		"&tCH~"6	@D	PS	="5y@A	" 	w|| 	, E&&/ 6EKD
   &  (/7+B+B) #D- #'-n==%QUF))*MN!M #'	 m *++C 7j# >s:   EE
CE?E EEE!E?EEr@  c                    #    g 7fr   r   r[  r   r  r@  r  r  s         r   rO  Router.schedule_acompletion  r;  r<  c                    #    g 7fr   r   r  s         r   rO  r    r;  r<  c                 <  #    [        U5      n[        [        R                  " 5       5      n[	        UUUS9nU R
                  R                  US9I S h  vN   [        R                  " 5       U R                  -   n	[        R                  " 5       n
U R
                  R                  nSnX:  a  U R                  XS9I S h  vN u  pU R
                  R                  UR                  UR                  US9I S h  vN nU(       a  O:[        R                   " U5      I S h  vN   [        R                  " 5       n
X:  a  M  U(       a\   U R"                  " SXUS.UD6I S h  vN nUR$                  R'                  S0 5        UR$                  S   R)                  SS	05        U$ U R
                  R/                  UR                  UR                  S9I S h  vN   [0        R2                  " SUSS9e GN GN2 N N N! [*         a  n[-        US
U5        UeS nAff = f NG7f)Nr@  
request_idr  requestFr   rF  r  r  health_deploymentsr  additional_headers%x-litellm-request-prioritization-usedTr@  r  r  %Request timed out while polling queuer+  r  r   )r"   r  r   r   rO   r  add_requestrL  	monotonicr   r   _async_get_healthy_deploymentspollr  r  r'  sleepr&  ro  r;  r  r  r  remove_requestr  r  )r[  r   r  r@  r  r  rF  _request_idr  rE  	curr_timepoll_intervalmake_request_healthy_deploymentsr  r  r  s                    r   rO  r    s     =VD$**,'"
 nn(((666 >>#dll2NN$	77",0,O,O -P - '#  "&!4!4????#7 "5 " L
 mmM222 NN,	 " "&"2"2 #6#EK# 	 ((334H"M(()=>EE<dC !  ..//??t 0    //?% S 	7' 3
  :x0
s   AHG-A*H>G0?8H7G38%HG5H>HG9 G7AG9  2HHH0H3H5H7G9 9
HHHHr  rh  .c                 b  #    [        U5      n[        [        R                  " 5       5      n[	        UUUS9nU R
                  R                  US9I S h  vN   [        R                  " 5       U R                  -   n	[        R                  " 5       n
U R
                  R                  nSnX:  a  U R                  XS9I S h  vN u  pU R
                  R                  UR                  UR                  US9I S h  vN nU(       a  O:[        R                   " U5      I S h  vN   [        R                  " 5       n
X:  a  M  U(       an   U" U0 UD6I S h  vN n[#        UR$                  [&        5      (       a<  UR$                  R)                  S0 5        UR$                  S   R+                  SS05        U$ U R
                  R1                  UR                  UR                  S
9I S h  vN   [2        R4                  " SUSS9e GN GND GN N N! [,         a  n[/        US	U5        UeS nAff = f NH7f)Nr  r  Fr  r  r  r  Tr@  r  r  r+  r  )r"   r  r   r   rO   r  r  rL  r  r   r   r  r  r  r  r'  r  r   ro  rP  r;  r  r  r  r  r  r  )r[  r   r@  r  rh  r  rF  r  r  rE  r  r  r  r  r  r  r  s                    r   _schedule_factoryRouter._schedule_factory^  s     =VD$**,'"
 nn(((666 >>#dll2NN$	77",0,O,O -P - '#  "&!4!4????#7 "5 " L
 mmM222 NN,	 " 
"3T"DV"DD	i66==,,778LbQ,,-ABII@$G !  ..//??t 0    //?% Q 	7' 3
 E  :x0
s   AH/G?A*H/>H?8H/7H8%H/HH/>H/H H
A H 22H/$H-%H/H/H/H/
H 
H*H%%H**H/c                     U R                  US9nUb  [        U5      S:w  a  gUS   S   R                  SS 5      nUb  SU;  a  gUR                  S5      S   nU[        R
                  ;   $ )Nr  r~   Fr   r   r   /)get_model_listr  r~  r  r  )_known_custom_logger_compatible_callbacks)r[  r   r   litellm_modelsplit_litellm_models        r   rM  "Router._is_prompt_management_model  s|    ((E(:
ZA!5"1&67;;GTJ C}$<+11#6q9"g&W&WWWr   c           	      F  #    UR                  SS 5      nUc#  [        S0 S[        5       [        5       S.UED6u  pC[	        [
        U5      nU R                  USSS./UR                  SS 5      S9nU R                  XSS	9  US
   R                  5       nUR                  SS 5      n[        U[        5      =(       a    UR                  S5      nUR                  S5      =(       d    US
   R                  SS 5      n	UR                  S5      =(       d    US
   R                  SS 5      n
UR                  SS 5      =(       d    US
   R                  SS 5      nU(       d2  U	b  [        U	[        5      (       d  [        SU	 S[        U	5       35      eU
b/  [        U
[        5      (       d  [        SU
 S[        U
5       35      eUR!                  UU[#        US9U	U
US9u  nnn1 SknUR%                  5        VVs0 s H  u  pX;  d  M  X_M     nnn0 UEUEUEnXS'   X#S'   XCS'   XS'   XS'   XS'   U R'                  US9nUb  [)        U5      S:X  a/  UR                  S5        [*        R,                  " S0 UD6I S h  vN $ U R.                  " S0 UD6I S h  vN $ s  snnf  N$ N7f)Nr  r&  )r  	rules_objrD  r  r  r  r  r   r  r  r  r   r   zlitellm_agent/	prompt_idprompt_variablesprompt_labelz*Prompt ID is not set or not a string. Got=z, type=z2Prompt variables is set but not a dictionary. Got=)r  )r   r  non_default_paramsr  r  r  >   r  r  prompt_versionbitbucket_configdotprompt_configr  r  r  r   r  r   )r~  rx   rw   r|   r   LiteLLMLoggingr  r  r  r  r   r  
startswithr  r   rP  get_chat_completion_promptrz   rq  r  r  r  r&  rP  )r[  r   r  r  litellm_logging_objectprompt_management_deploymentrS  r  is_litellm_agent_modelr  r  r  optional_paramsprompt_management_paramskvfiltered_data_model_lists                     r   rN  !Router._prompt_management_factory  s*     "(,A4!H!)-; .)6!&"2"4 	.*" "&n6L!M'+'D'D%(;< &

+@$ G (E (
$ 	++3 	, 	
 ,,<=BBD$/ ",3"
 "9&&'78 	 JJ{+ !/K0

#k4
  	 "::
 
)*:;??
 	
 zz.$7 $;W<

#nd
# 	 &Z	3%?%?<YKwtT]N_`  '
;KT0R0RDEUDVV]^bcs^t]uv  #==@O-% > 
		
$
  "ZZ\
)TQQ-NDAD\ 	 
 @M?V??w%z(>$%'{%5!"!-~))U);#k"2a"7JJ*+ ,,6v66677A&AAA#
 7As7   G3J!5JJ
A.J!8J9J!JJ!J!r  c                     X#S'   XS'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6nU$ ! [         a  nUeS nAff = f)Nr   r  r  r   r   r  r   )_image_generationr~  r   r;  r  r  r  r[  r  r   r  r  r  s         r   image_generationRouter.image_generation  s    
	#7O%8*.*@*@F&'$*JJ}d>N>N$OF=!j"-44mU5KL33=f=HO 	G	s   A,A/ /
B 9A;;B c           	         Sn [         R                  " SU SU 35        U R                  USSS./UR                  SS 5      S9nU R	                  XSS	9  US
   R                  5       nU R                  UUS	9nU R                  U==   S-  ss'   U R                  US9  [        R                  " S0 0 UEUU R                  US.EUED6nU R                  U==   S-  ss'   [         R                  " SU S35        U$ ! [         aG  n	[         R                  " SU S[        U	5       S35        Ub  U R                   U==   S-  ss'   U	eS n	A	ff = f)Nr  #Inside _image_generation()- model: 
; kwargs: r  r  r  r  r  r  r   r~   r  r  r  r  zlitellm.image_generation(model=r  r  r  r   )r   r  r  r  r  r  r  r5  r  r  r  r   r7  rQ  r  r  r6  )
r[  r  r   r  r  r  rS  r  r  r  s
             r   r  Router._image_generation  s   
*	!''5eWJvhO 66#)h?@$*JJ/Dd$K 7 J
 //:/U./446D>>% ? L
 Z(A-( 11Z1H// $#33*	
 H z*a/*!&&1*=TU O 	!&&1*=QRUVWRXQYY`a %
+q0+G	s   C5C: :
EAEEc           
      `  #     X#S'   XS'   U R                   US'   UR                  SU R                  5      US'   U R                  X#S9  U R                  " S0 UD6I S h  vN nU$  N! [
         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)Nr   r  r  r   r  rG  r   )_aimage_generationr~  r   r  rP  r  r'  rQ  rH   rT  rU  r  s         r   aimage_generationRouter.aimage_generationI  s     	#7O%8*.*A*AF&'$*JJ}d>N>N$OF=!00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B.AA' A%A' $B.%A' '
B+15B&&B++B.c           	      .  #    Un [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nU R                  XcS9  US	   R                  5       nUS
   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XeS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XeS9I S h  vN   U	I S h  vN n	U R"                  U==   S-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S-  ss'   UeS nAff = f7f)Nr  r  r  r  r  r  r  r  r   r   r~   r  r  r  r  z litellm.aimage_generation(model=r  r  r  r   )r   r  r"   r  r  r  r  r  r5  r  r  r   r  r   r'  r  r  r7  rQ  r  r  r6  )r[  r  r   r  r  rF  r  rS  r  r  r  r  s               r   r  Router._aimage_generation^  s    
D	!''5eWJvhO  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:/U./446DgJ>>% ? L
 Z(A-(00 $#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&2:,>UV OqJ )
  . )=== *  	!&&2:,>RSVWXSYRZZab %
+q0+G	s   HAG FB5G FG F#$F%	F#.F/F#3G >F!?G F=	G  F?!6G HG G F#F#!G #F:)F,*F:6G ?G 
HAHHHfilec           
      "  #     X#S'   XS'   U R                   US'   U R                  X#S9  U R                  " S0 UD6I Sh  vN nU$  N! [         a:  n[        R
                  " [        U U[        R                  " 5       US95        UeSnAff = f7f)aB  
Example Usage:

```
from litellm import Router
client = Router(model_list = [
    {
        "model_name": "whisper",
        "litellm_params": {
            "model": "whisper-1",
        },
    },
])

audio_file = open("speech.mp3", "rb")
transcript = await client.atranscription(
model="whisper",
file=audio_file
)

```
r   r  r  r  NrG  r   )	_atranscriptionr  rP  r  r'  rQ  rH   rT  rU  )r[  r  r   r  r  r  s         r   atranscriptionRouter.atranscription  s     .	#7O!6N*.*>*>F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		9   B;A A A BA 
B5BBBc           	      $  #    Un [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nU R                  XcS9  US	   R                  5       nU R                  UUS9nU R                  U==   S
-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XeS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XeS9I S h  vN   U	I S h  vN n	U R"                  U==   S
-  ss'   [         R$                  " SU S35        U	$  GN= N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S
-  ss'   UeS nAff = f7f)Nz!Inside _atranscription()- model: r  r  r  r  r  r  r  r   r~   )r  r  r  r  r  r  zlitellm.atranscription(model=r  r  r  r   )r   r  r"   r  r  r  r  r  r5  r  r  r   r  r   r'  r  r  r7  rQ  r  r  r6  )r[  r  r   r  r  rF  r  rS  r  r  r  r  s               r   r  Router._atranscription  s|    
B	!''3E7*VHM  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:/U./446D>>% ? L
 Z(A-(--  #33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&/
|;RS OmF )
  . )=== *  	!&&/
|;OPSTUPVxW^_ %
+q0+G	s   HAF< FB0F< FF< FF 	F)F*F.F< 9F:F< F8	F< F:6F< HF< F< FFF< F5$F'%F51F< :F< <
HAHHHvoicec           
      *  #     XS'   X$S'   X4S'   U R                   US'   U R                  XS9  U R                  " S0 UD6I Sh  vN nU$  N! [         a:  n[        R
                  " [        U U[        R                  " 5       US95        UeSnAff = f7f)	a  
Example Usage:

```
from litellm import Router
client = Router(model_list = [
    {
        "model_name": "tts",
        "litellm_params": {
            "model": "tts-1",
        },
    },
])

async with client.aspeech(
    model="tts",
    voice="alloy",
    input="the quick brown fox jumped over the lazy dogs",
    api_base=None,
    api_key=None,
    organization=None,
    project=None,
    max_retries=1,
    timeout=600,
    client=None,
    optional_params={},
) as response:
    response.stream_to_file(speech_file_path)

```
r   r  r  r  r  NrG  r   )	_aspeechr  rP  r  r'  rQ  rH   rT  rU  )r[  r   r  r  r  r  r  s          r   aspeechRouter.aspeech  s     @	#7O#7O#7O*.--F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s:   B?A A
A 	B
A 
B5BBBc           	        #    Un [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nU R                  XtS9  US	   R                  5       nU R                  UUS9n	U R                  U==   S
-  ss'   [        R                  " S0 0 UEUUU	S.EUED6n
U R                  UUSS9nUb]  [        U[        R                  5      (       a>  U IS h  vN    U R                  XvS9I S h  vN   U
I S h  vN n
S S S 5      IS h  vN   O!U R                  XvS9I S h  vN   U
I S h  vN n
U R                   U==   S
-  ss'   [         R"                  " SU S35        U
$  GN3 N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [$         aG  n[         R"                  " SU S['        U5       S35        Ub  U R(                  U==   S
-  ss'   UeS nAff = f7f)NzInside _aspeech()- model: r  r  r  r  r  r  r  r   r~   )r  r  r  r  r  r  zlitellm.aspeech(model=r  r  r  r   )r   r  r"   r  r  r  r  r  r5  r  r  r  r   r'  r  r  r7  rQ  r  r  r6  )r[  r   r  r  r  r  rF  r  rS  r  r  r  r  s                r   r  Router._aspeechJ  sq    
B	!'',UG:fXF  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:/U./446D>>% ? L
 Z(A-( ""*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&(4KL OmF )
  . )=== *  	!&&(4HQPWX %
+q0+G	s   HAF2 F	B&F2 =F>F2 FF	FF F$F2 /F0F2 F.	F2 F06F2 H	F2 F2 FFF2 F+FF+'F2 0F2 2
H<AG>>HHc           
      ,  #     XS'   [         US'   U R                  US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  N! [         a:  n[
        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7fNr   r  r  r  rG  r   )
r  _arerankr  rP  r  r'  rQ  rH   rT  rU  r[  r   r  r  r  s        r   arerankRouter.arerank  s     	#7O#F7O*.--F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   BA A AA 
BA 
B5BBBc           	        #    S n [         R                  " SU SU 35        U R                  UUR                  SS 5      US9I S h  vN nU R	                  XBS9  US   R                  5       nUS   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEU R                  US	.EUED6I S h  vN nU R                  U==   S-  ss'   [         R                  " S
U S35        U$  N N9! [         aG  n[         R                  " S
U S[        U5       S35        Ub  U R                  U==   S-  ss'   UeS nAff = f7f)NzInside _rerank()- model: r  r  )r   r  r  r  r   r   r~   )r  r  zlitellm.arerank(model=r  r  r  r   )r   r  r  r  r  r  r  r5  r  r  r   r7  rQ  r  r  r6  )	r[  r   r  r  r  rS  r  r  r  s	            r   r  Router._arerank  s    
'	!''+E7*VHE  $BB$*JJ/Dd$K%  C   J
 //:/U./446DgJ>>% ? L Z(A-($__ #33* 	 H z*a/*!&&(4KL O9  	!&&(4HQPWX %
+q0+G	sO   EA D C=A>D C?6D <E=D ?D 
EAEEEis_retryis_fallbackis_asyncc                    SUS./n XS'   X&S'   UR                  SU R                  5      US'   UR                  S0 5      R                  SU05        U R	                  UUUR                  SS 5      S	9nUS
   R                  5       n	U R                  R                  5        H+  u  pX;  a  XU
'   M  U
S:X  d  M  Xj   R                  U5        M-     [        R                  " S0 0 U	EX R                  S.EUED6$ ! [         a  nUeS nAff = f)Nr  r  r   r  r   r   r  r  r  r   )r  r  r   )r~  r   r;  r  r  r  r  r   rq  r  text_completionr   r  )r[  r   r  r
  r  r  r  r  r  rS  r  r  r  s                r   r  Router.text_completion  s"    $78	#7O%8$*JJ}d>N>N$OF=!j"-44mU5KL 66!$*JJ/Dd$K 7 J ./446D3399;O !1I*_I$$Q' < **s-r-rSgSg-rkq-rss 	G	s   B,C7 8>C7 7
DDDc           
        #    UR                  SS 5      b5  U R                  UUR                  S5      U R                  X4US9I S h  vN $  XS'   X&S'   U R                  US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  NG N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)	Nr@  )r   r@  r  rh  r  r   r  r  r  rG  r   )r~  r  r  atext_completion_atext_completionr  rP  r  r'  rQ  rH   rT  rU  )	r[  r   r  r
  r  r  r  r  r  s	            r   r  Router.atext_completion  s      ::j$'3//J/"&"7"7_ 0   	#7O%8*.*@*@F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		sH   ACBC;B BB CB 
C5CCCc           	      *  #     [         R                  " SU SU 35        [        U5      nU R                  USUS./UR	                  SS 5      US9I S h  vN nU R                  XSS9  US   R                  5       nUS	   nU R                  UUS9nU R                  U==   S
-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XTS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XTS9I S h  vN   U	I S h  vN n	U R"                  U==   S
-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S
-  ss'   UeS nAff = f7f)N#Inside _atext_completion()- model: r  r  r  r  r  r  r   r   r~   r  r  r  r  zlitellm.atext_completion(model=r  r  r  r   )r   r  r"   r  r  r  r  r  r5  r  r  r   r  r   r'  r  r  r7  rQ  r  r  r6  )r[  r   r  r  rF  r  rS  r  r  r  r  r  s               r   r  Router._atext_completion  s~    C	!''5eWJvhO  AH#BB#)f=>$*JJ/Dd$K%	  C   J //:/U./446DgJ>>% ? L Z(A-(// $#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&1*=TU OoH )
  . )=== *  	!&&1%8LSQRVHT[\  &!+&G	   HAF? FB5F? 
FF? F!"F#	F!,F-F!1F? <F=F? F;	F? F=6F? HF? F? F!F!F? !F8'F*(F84F? =F? ?
H	AHHH
adapter_idc           
        #     X&S'   XS'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)	Nr   r  r  r   r   r  rG  r   )_aadapter_completionr~  r   r;  r  rP  r  r'  rQ  rH   rT  rU  )	r[  r  r   r
  r  r  r  r  r  s	            r   aadapter_completionRouter.aadapter_completion`  s     	#7O#-< *.*C*CF&'$*JJ}d>N>N$OF=!j"-44mU5KL!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   CA.A; 2A93A; 8C9A; ;
B?5B::B??Cc           	      *  #     [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nU R                  XSS9  US	   R                  5       nUS
   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XTS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XTS9I S h  vN   U	I S h  vN n	U R"                  U==   S-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S-  ss'   UeS nAff = f7f)Nz&Inside _aadapter_completion()- model: r  r  zdefault textr  r  r  r  r   r   r~   )r  r  r  r  r  r  z"litellm.aadapter_completion(model=r  r  r  r   )r   r  r"   r  r  r  r  r  r5  r  r  r   r  r   r'  r  r  r7  rQ  r  r  r6  )r[  r  r   r  rF  r  rS  r  r  r  r  r  s               r   r  Router._aadapter_completion}  s    C	!''8z&R  AH#BB#)nEF$*JJ/Dd$K%	  C   J //:/U./446DgJ>>% ? L Z(A-(22 ",#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&4ZL@WX OoH )
  . )=== *  	!&&4UG;OPSTUPVxW^_  &!+&G	r  c                 T   #    SSK Jn  UR                  " SU US.UD6I Sh  vN $  N7f)z
Helper function to make a search API call through the router with load balancing and fallbacks.
Reuses the router's retry/fallback infrastructure.
r   SearchAPIRouter)router_instancer  Nr   )&litellm.router_utils.search_api_routerr!  async_search_with_fallbacks)r[  r  r  r!  s       r   _asearch_with_fallbacksRouter._asearch_with_fallbacks  s=     
 	K$@@ 
 /
 
 
 	
 
s   (&(original_generic_functionc                 V   #    SSK Jn  UR                  " SU UUS.UD6I Sh  vN $  N7f)z
Helper function for search API calls - selects a search tool and calls the original function.
Called by async_function_with_fallbacks for each retry attempt.
r   r   )r"  r   r'  Nr   )r#  r!  "async_search_with_fallbacks_helper)r[  r   r'  r  r!  s        r   _asearch_with_fallbacks_helper%Router._asearch_with_fallbacks_helper  s@      	K$GG 
 &?
 	
 
 	
 
s    )')guardrail_namec                    #    XS'   X#S'   U R                   US'   U R                  UUSS9  [        R                  " SU SU 35        U R                  " S	0 UD6I Sh  vN nU$  N7f)
a7  
Execute a guardrail with load balancing and fallbacks.

Args:
    guardrail_name: Name of the guardrail to execute
    original_function: The guardrail's execution function (e.g., async_pre_call_hook)
    **kwargs: Additional arguments passed to the guardrail

Returns:
    Result from the guardrail execution
r   r'  r  r  r  z&Inside aguardrail() - guardrail_name: r  Nr   )_aguardrail_helperr  r   r  rP  )r[  r,  r  r  r  s        r   
aguardrailRouter.aguardrail  s     " )w.?*+&*&=&="#,, #5 	- 	

 	##4^4DJvhW	
 ;;EfEE Fs   AA%A#A%c                    #    UnU R                  US9n[        R                  " SUR                  S0 5      R                  S5       35        XSS'   U" S0 UD6I Sh  vN nU$  N7f)aE  
Helper for aguardrail - selects a guardrail deployment and executes it.
Called by async_function_with_fallbacks for each retry attempt.

Args:
    model: The guardrail_name (named 'model' for fallback system compatibility)
    original_generic_function: The guardrail's execution function
    **kwargs: Additional arguments
)r,  zSelected guardrail deployment: r   	guardrailselected_guardrailNr   )get_available_guardrailr   r  r~  )r[  r   r'  r  r,  r3  r  s          r   r.  Router._aguardrail_helper  s      !99) : 
 	##-.@.D.DEUWY.Z.^.^_j.k-lm	

 (:#$2<V<< =s   AA$A"A$r^   c           	          SSK Jn  U R                   Vs/ s H  o3R                  S5      U:X  d  M  UPM     nnU(       d  [	        SU 35      e[        U5      S:X  a  US   $ [        [        U" U UUS95      $ s  snf )z
Select a guardrail deployment using the router's load balancing strategy.

Args:
    guardrail_name: Name of the guardrail to select

Returns:
    Selected guardrail configuration dict
r   r0   r,  zNo guardrail found with name: r~   llm_router_instancer&  r   )&litellm.router_strategy.simple_shuffler1   r   r~  r  r  r   r^   )r[  r,  r1   gr&  s        r   r4  Router.get_available_guardrail  s     	J **
*!ee4D.E.WA* 	 
 #=n=MNOO"#q(&q)) $($7$
 	

s
   A8A8c           
      \  #     XS'   X#S'   U R                   US'   U R                  XSS9  [        R                  " SU SU 35        U R                  " S
0 UD6I Sh  vN nU$  N! [
         a:  n[        R                  " [        U U[        R                  " 5       US	95        UeSnAff = f7f)
Helper function to make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
r   r'  r  r  r  z3Inside ageneric_api_call_with_fallbacks() - model: r  NrG  r   )r  r  r   r  rP  r  r'  rQ  rH   rT  rU  )r[  r   r  r  r  r  s         r   rO  (Router._ageneric_api_call_with_fallbacks?  s     	#7O2C./*.*W*WF&'00CU 1  "''EeWJW]V^_ "??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B,AA% A#A% "B,#A% %
B)/5B$$B))B,r  c                     SU;   aU  US   (       aK  SSK Jn   U" UUR                  S5      UR                  S5      S9u  n    nUnUS   R	                  X'5      US'   U$ ! [         a    Un N'f = f)z
Add the deployment model to the endpoint for LLM passthrough route.

e.g for bedrock invoke users can pass endpoint as /model/special-bedrock-model/invoke
  it should be actually sent as /model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke
endpointr   )ry   rm  rJ  )r   rm  rJ  )r  ry   r~  r  replace)r[  r  r   r  ry   stripped_model_namer  replacement_model_names           r   ;_add_deployment_model_to_endpoint_for_llm_passthrough_routeBRouter._add_deployment_model_to_endpoint_for_llm_passthrough_route^  s     F:$6 1
4/?$(.

3H(I#ZZ
30,#Q1
 *=&
 "(
!3!;!;"F:   4)3&4s   -A A-,A-c           	        #    UR                  SS5      nSn [        U5      n U R                  UUUR                  SS5      UR                  SS5      S9I Sh  vN nU R                  XsUS	9  US
   R                  5       n	U	S   n
U R                  U
==   S-  ss'   U R                  X1U
S9   U	R                  S5      n[        U	S   US9u  p  nU=(       d    Un0 U	ESU R                  0EUESU
0EnUb  XS'   U" S0 UD6nU R                  UUSS9nUb]  [        U[        R                  5      (       a>  U ISh  vN    U R                  XvS9I Sh  vN   UI Sh  vN nSSS5      ISh  vN   O!U R                  XvS9I Sh  vN   UI Sh  vN nU R                   U
==   S-  ss'   ["        R$                  " SU
 S35        U$  GNn! [         a&  nU(       a  U" SSU0UD6I Sh  vN  s SnA$ UeSnAff = f! [         a    Sn GN1f = f N N N N! , ISh  vN  (       d  f       N= f N N! [         aG  n["        R$                  " SU S['        U5       S35        Ub  U R(                  U==   S-  ss'   UeSnAff = f7f)r=  passthrough_on_no_deploymentFrO  r  Nr  )r   r  r  r  r   rK  r   r~   )r  r   r  rm  r   rm  r  r  r  r  z'ageneric_api_call_with_fallbacks(model=r  r  r  r   )r  r"   r  r~  r  r  r  r5  rD  ry   r   r  r   r'  r  r  r7  r   rQ  r  r6  )r[  r   r'  r  rG  r3  rF  r  r  rS  r  rm  r  inferred_custom_llm_providerresponse_kwargsr  r  s                    r   r  /Router._ageneric_api_call_with_fallbacks_helper}  s      (.zz2PRW'X$;W	@H
#'#F#F#)#ZZ
D9(.

3H$(O	 $G $ 
 //%M 0  ./446DgJZ(A-(LLz M 

+&*hh/D&E#8Hw-(;95A
 (G+G $4//  	O #.9L 560C?CH ,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&9*E\] O[  /!:!Q!Q&!QQQQ6  +&*#+0 )
  . )=== *  	!&&9%@TUXYZU[T\\cd  &!+&G	s,  J
H6 5G
 GG
 AH6 .-G= A H6 ;H<H6 ?HH	HHH"H6 -H.H6 H2	H6 H46H6 J
G
 

G:G5(G+)G5-G:.H6 2J
3G55G::H6 =H	H6 HH6 HHH6 H/H!H/+H6 4H6 6
J AJJJ
)r   r   c                   #    SSK Jn  SSKJn  UR	                  5       n[        UR                  S5      [        5      (       a  U" US   5      US'   [        UR                  S5      [        5      (       a  U" US   5      US'   XS'   U R                  " SSU0UD6I Sh  vN nUR                  S	5      (       a(  [        Xc5      (       a  U R                  UUS
9I Sh  vN $ U$  ND N7f)aW  
_ageneric_api_call_with_fallbacks for the Responses API, with the
addition of mid-stream fallback handling.

When stream=True and the underlying call returns a
BaseResponsesAPIStreamingIterator, wrap it with
_aresponses_streaming_iterator so MidStreamFallbackError raised
during iteration triggers the Router's cross-provider fallback chain.
r   r   r  r  r   r'  r  Nr  )r  r  r   )
r  r   r  r  r  r   r~  rP  rO  r  )r[  r  r  r   r  fallback_kwargsr  s          r   $_aresponses_with_streaming_fallbacks+Router._aresponses_with_streaming_fallbacks  s    	
 	K" +1++-o))*<=tDD2@ 233O./ o))*5t<<*89T*UOJ'7H34?? 
/
39
 
 ::hJ%
 %
 <<!. =    
s$   BC'C#=C'C%C'%C'c           
         UR                   n[        SS9n [        R                  " SU SU SU 35        U R	                  UUUS9  U R                  UUR                  SS5      UR                  S	S5      US
9nU R                  XcSS9  US   R                  5       nUS   nU R                  U==   S-  ss'   SU;   a!  US   (       a  US   R                  X5      US'   XS'   U R                  US9   UR                  S5      n	[        US   U	S9u  p  n
U	=(       d    Un	U" S0 0 UEU	U R                  S.EUED6nU R                   U==   S-  ss'   [        R"                  " U SU S35        U$ ! [         a    Sn	 N^f = f! [         aI  n[        R"                  " U SU S[%        U5       S35        Ub  U R&                  U==   S-  ss'   UeSnAff = f)aV  
Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
Args:
    model: The model to use
    original_function: The handler function to call (e.g., litellm.completion)
    **kwargs: Additional arguments to pass to the handler function
Returns:
    The response from the handler function
generic_api_callr5  z&Inside _generic_api_call() - handler: z	, model: r  r  r  Nr  r  rK  r   r   r~   r@  r  rm  rH  )rm  r  z(model=r  r  r  r   )r   r5   r   r  r  r  r~  r  r  r  r5  rA  r  ry   r  r   r7  rQ  r  r6  )r[  r   r  r  handler_namer  r  rS  r  rm  r  rI  r  r  s                 r    _generic_api_call_with_fallbacks'Router._generic_api_call_with_fallbacks  sa    )11!C,"
A	!''8iPUwV`ag`hi 00'= 1 
 66J5$*JJ/Dd$K%	 7 J //%DV 0  ./446DgJZ(A-( V#z(:%+J%7%?%?%Rz"(7O 11Z1H
+&*hh/D&E#8Hw-(;95A
 (G+G $ ) +>#33 	H z*a/*!&&.
|3JK O!  +&*#+"  	!&&.w.B3q6('R  &!+&G	s>   CF 3-E0  AF 0E?<F >E??F 
GAGGc                      XS'   X$S'   U R                   US'   U R                  XS9  U R                  " S0 UD6nU$ ! [         a  nUeS nAff = f)Nr   r  r  r  r   )
_embeddingr  r  r  r[  r   r  r  r  r  r  s          r   	embeddingRouter.embeddingi  sb    	#7O#7O*.//F&'00u0L33=f=HO 	G	s   9< 
AAAc           	         S n [         R                  " SU SU 35        U R                  UUUR                  SS 5      S9nU R	                  XSS9  US   R                  5       nUS   nU R                  XSSS	9nUR                  S
S 5      nUb  Ub  XR                  :w  a  S n	OUn	U R                  U==   S-  ss'   U R                  US9  [        R                  " S0 0 UEUU R                  U	S.EUED6n
U R                  U==   S-  ss'   [         R                  " SU S35        U
$ ! [          aG  n[         R                  " SU S[#        U5       S35        Ub  U R$                  U==   S-  ss'   UeS nAff = f)NzInside embedding()- model: r  r  )r   r  r  r  r   r   syncr  r  r~   r  r  r  r  zlitellm.embedding(model=r  r  r  r   )r   r  r  r  r  r  r  r~  r  r5  r  r  rX  r   r7  rQ  r  r  r6  )r[  r  r   r  r  r  rS  r  r  r  r  r  s               r   rV  Router._embeddingz  s   
4	!''-eWJvhG 66$*JJ/Dd$K 7 J
 //:/U./446DgJ%)%5%5%& &6 &" %jjD9O+*6#'E'EE#5Z(A-( 11Z1H(( "#33*	
 H z*a/*!&&*:,6MN O 	!&&*:,6J3q6(RYZ %
+q0+G	s   D"D' '
E81AE33E8c           
      "  #     XS'   X$S'   U R                   US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R
                  " [        U U[        R                  " 5       US95        UeS nAff = f7fr  )	_aembeddingr  rP  r  r'  rQ  rH   rT  rU  rW  s          r   
aembeddingRouter.aembedding  s     	#7O#7O*.*:*:F&'00u0L!??I&IIHO J 		(,0#)(1(<(<(>'(	 G		r  c           	      &  #    S n [         R                  " SU SU 35        [        U5      nU R                  UUUR	                  SS 5      US9I S h  vN nU R                  XcS9  US   R                  5       nUS   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEUU R                  US	.EUED6n	U R                  UUS
S9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XeS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XeS9I S h  vN   U	I S h  vN n	U R"                  U==   S-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S-  ss'   UeS nAff = f7f)NzInside _aembedding()- model: r  r  )r   r  r  r  r  r   r   r~   r\  r  r  r  zlitellm.aembedding(model=r  r  r  r   )r   r  r"   r  r  r  r  r  r5  r  r`  r   r  r   r'  r  r  r7  rQ  r  r  r6  )r[  r  r   r  r  rF  r  rS  r  r  r  r  s               r   r_  Router._aembedding  s{    
B	!''/wjI  AH#BB$*JJ/Dd$K%	  C   J //:/U./446DgJ>>% ? L
 Z(A-()) "#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&+J<7NO OmF )
  . )=== *  	!&&+J<7KCPQF8SZ[ %
+q0+G	s   HAF= FB5F= F	F= F F!	F*F+F/F= :F;F= F9	F= F;6F= HF= F= FFF= F6%F(&F62F= ;F= =
HAH		HHc           
      X  #     XS'   U R                   US'   UR                  SU R                  5      US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  N! [
         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)Nr   r  r   r  rG  r   )_acreate_filer~  r   r  rP  r  r'  rQ  rH   rT  rU  r  s        r   acreate_fileRouter.acreate_file  s     
	#7O*.*<*<F&'$*JJ}d>N>N$OF=!00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B*AA# A!A#  B*!A# #
B'-5B""B''B*c                   ^ ^^#     SSK Jn  [        R                  " SU ST 35        [	        T5      mT R                  USSS./TR                  SS 5      TTS	9I S h  vN nS
[        S[        4UUU 4S jjn/ n[        U[        5      (       a  UR                  U" U5      5        O U H  nUR                  U" U5      5        M     [        R                  " U6 I S h  vN n[        U5      S:X  a  [        S5      eU" XHS9n	[        [        US   5      n
U	U
R                   S'   U
$  N NJ! [         aJ  n[        R"                  " SU ST S[%        U5       S35        Ub  T R&                  U==   S-  ss'   UeS nAff = f7f)Nr   )add_model_file_id_mappingsr  r  r  files-api-fake-textr  r  r   r  r  r  rF  r  r   c           	        >#    SSK Jn  U" T5      nTR                  U USS9  U S   R                  5       nUS   nTR	                  U US9nTR
                  U==   S-  ss'   UR                  S	5      n[        US   US
9u  px  n	U=(       d    Un[        [        [           TR                  S5      5      n
[        [        [           TR                  S5      5      nU(       a  U
(       d  [        S5      e[        U
S9nU(       a  [        UUS9nXS'   SU;   a  US   UR                  S0 5      S'   [         R"                  " S0 0 UEUTR$                  US.EUED6nTR'                  U USS9nUb^  [)        U[*        R,                  5      (       a?  U IS h  vN    TR/                  U TS9I S h  vN   UI S h  vN nS S S 5      IS h  vN   O"TR/                  U TS9I S h  vN   UI S h  vN nTR0                  U==   S-  ss'   [2        R4                  " SU S35        U$  N Nv Nn N`! , IS h  vN  (       d  f       NR= f Nc N[7f)Nr   r  rf  rK  r   r   r  r~   rm  rH  purposer  z2file and file_purpose are required for create_file)rm  )file_contentnew_model_namegcs_bucket_namer  rm  r  r  r  r  r  litellm.acreate_file(model=r  r   )r  r  r  r  r  r5  r~  ry   r   r   rT   rR   r  r7   r6   r;  r  rf  r   r  r   r'  r  r  r7  r   rQ  )r  r  kwargs_copyrS  r  r  rm  stripped_modelrI  r  rm  r  replace_model_in_jsonl_boolr  r  r  rF  r[  s                  r   create_file_for_deployment8Router._acreate_file.<locals>.create_file_for_deployment>  s    R,V433)&"0 4 
 ""2388:!']
#BB)&  C     ,1, '+hh/D&E#EUw-(;FBa (G+G $
 x(:;VZZ	=RSHY/F1CD7#L  /L#/+ /1%)'5D
 +/'%- ./  **+=rB) #// /B#'#7#7".	
 & !% 0 0)& 7 !1 ! !,!7#4#42 2  -} #II'1DT J    *2>  -}} EE#-@P F    &.~H"":.!3.%**1*=TU  )  -
 $2  -}}}  .s   FIH!IH),H#-	H)6H%7H);IH'II 	I)I*8I#H)%H)'I)I /H20I <IINo healthy deployments found.)r&  r  model_file_id_mappingrr  , r  r  r~   )!litellm.router_utils.common_utilsri  r   r  r"   async_get_healthy_deploymentsr  rP  rS   r   r3  r'  r*  r  r  r   ro  r#  r  r6  )r[  r   r  ri  r&  rv  tasksr  r  ry  returned_responser  rF  s   ` `         @r   re  Router._acreate_file*  s    
I	T!''5eWJvhO  AH(,(J(J#)6KLM$*JJ/Dd$K%!1 )K ) #] T ] FV ]  ] ~ E-t4478KLM"5JLL!;J!GH #6 &nne44I9~" ?@@$>$7%! !%%5y| D% ,,-DE %$w#^ 5  	!++-eWBvh>RSVWXSYRZZab  &!+&G	sP   FAD+ D'A?D+ D)AD+ &F'D+ )D+ +
E?5AE::E??Fc           	        #     Uc&  SSK Jn  U R                  USS9nU" S0 UD6I Sh  vN $ SSKJn  [	        U5      nU R                  USSS./UR                  S	S5      US
9I Sh  vN nUS   R                  5       nUS   n	U R                  UUSS9  U R                  UUS9n
U R                  U	==   S-  ss'   UR                  S5      n[        US   US9u  p  nU=(       d    UnU" S0 0 UEUU R                  U
S.EUED6nU R                  UUSS9nUb\  [        U[         R"                  5      (       a=  U ISh  vN   U R%                  XvS9I Sh  vN   UI Sh  vN nSSS5      ISh  vN   O!U R%                  XvS9I Sh  vN   UI Sh  vN nU R&                  U	==   S-  ss'   [(        R*                  " SU	 S35        U$  GN GNe N N{ Ns Ne! , ISh  vN  (       d  f       NX= f Ni Na! [,         aG  n[(        R.                  " SU S[1        U5       S35        Ub  U R2                  U==   S-  ss'   UeSnAff = f7f)z
Create a vector store for a specific model.

Args:
    model: Model name from router config
    **kwargs: Vector store creation parameters

Returns:
    VectorStoreCreateResponse
Nr   r  avector_store_creater  r  zvector-store-api-fake-textr  r  r  r   r   rK  r  r~   rm  rH  rq  r  r  r  z#litellm.avector_store_create(model=r  r  r  r   )r  r  r  litellm.vector_storesr"   r  r  r  r  r  r5  r~  ry   r   r  r   r'  r  r  r7  r   rQ  r  r#  r  r6  )r[  r   r  r  
factory_fnavector_store_create_sdkrF  r  rS  r  r  rm  r  rI  r  r  r  s                    r   r  Router.avector_store_create  s    T	}> "22'= 3 
 (1&111Q@H#BB#)6RST$*JJ/Dd$K%	  C   J ./446DgJ//%4 0   >>% ? L Z(A-( #'((+@"A4D7m$751AQ #6"U9U/ +>#33*	
 H !,,%3 - M (Zw00. . )=EE#- F    &.~H	 )== AA) B    "*>z*a/*!&&5j\AXY OI 2
^ )  .	 )=== *  	!++5eW<PQTUVQWPXX_`  &!+&G	s   I	$G5 G	G5 I	:G5 'G(CG5 >G?G5 GG	GG G$G5 /G0G5 G1	G5 G36G5 I		G5 G5 G5 GGG5 G.G G.*G5 3G5 5
I?AIII	c                     g)a  
Override factory-generated vector store methods with router-aware implementations.
This is called after _initialize_vector_store_endpoints() to ensure our custom
methods that handle deployment selection and credential injection are used instead
of the generic factory-generated ones.
Nr   r  s    r   r  0Router._override_vector_store_methods_for_router   s     	r   c           
      n  #     XS'   U R                   US'   UR                  SU R                  5      US'   [        SS9nU R	                  UUUS9  U R
                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)	Nr   r  r   _acreate_batchr5  r  rG  r   )r  r~  r   r5   r  rP  r  r'  rQ  rH   rT  rU  r[  r   r  r  r  r  s         r   acreate_batchRouter.acreate_batch+  s     
	#7O*.*=*=F&'$*JJ}d>N>N$OF=!%G.&" 00'= 1 
 "??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   B5A!A. %A,&A. +B5,A. .
B285B--B22B5c                   #     [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nUS   R                  5       nUS	   nU R                  XBS
S9  U R                  UUS9nU R                  U==   S-  ss'   UR                  S5      n[        US	   US9u  p  n	U=(       d    U
n[        R                  " S0 0 UEUU R                  US.EUED6nU R                  UUSS9nUb]  [        U[         R"                  5      (       a>  U IS h  vN    U R%                  XCS9I S h  vN   UI S h  vN nS S S 5      IS h  vN   O!U R%                  XCS9I S h  vN   UI S h  vN nU R&                  U==   S-  ss'   [         R(                  " SU S35        U$  GNp N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [*         aJ  n[         R,                  " SU SU S[/        U5       S35        Ub  U R0                  U==   S-  ss'   UeS nAff = f7f)Nz Inside _acreate_batch()- model: r  r  rj  r  r  r  r   r   r  rK  r  r~   rm  rH  rq  r  r  r  zlitellm.acreate_batch(model=r  zlitellm._acreate_batch(model=rz  r  r  r   )r   r  r"   r  r  r  r  r  r5  r~  ry   r  r  r   r  r   r'  r  r  r7  rQ  r  r#  r  r6  )r[  r   r  rF  r  rS  r  r  rm  r  rI  r  r  r  s                 r   r  Router._acreate_batchJ  s    
N	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K%	  C   J ./446DgJ//%DT 0   >>% ? L Z(A-( #'((+@"A4D7m$751AQ #6"U9U,, +>#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&.zl:QR OE\ )
  . )=== *  	!++/wb@TUXYZU[T\\cd  &!+&G	s   IAG- GC#G- 8G9G- <GG		GGGG- *G+G- G)	G- G+6G- IG- G- 	GGG- G&GG&"G- +G- -
I7AH<<IIc                   ^ ^^	#     [        T5      nUb0  T R                  USSS./TR                  SS5      TUS9I Sh  vN nOT R                  5       nUc  [	        S5      e/ m	S[
        4UU	U 4S	 jjnUbh  [        U[        5      (       aS  [        U5      S
:  aD  [        R                  " U Vs/ s H  nU" [        [
        U5      5      PM     snSS06I Sh  vN nOBUb4  [        U[        5      (       a  U" [        [
        U5      5      I Sh  vN nO[	        S5      eUbN  [        U[        5      (       a  U$ [        U[        5      (       a"  U H  n[        U[        5      (       d  M  Us  $    T	(       a  T	S
   e[	        SR                  T	5      5      e GNZs  snf  N N! [         a:  n[        R                  " [!        T T["        R$                  " 5       US95        UeSnAff = f7f)zh
Iterate through all models in a model group to check for batch

Future Improvement - cache the result.
Nr  zretrieve-api-fake-textr  r  rk  Router not yet initialized.r  c                 4  >#     SSK Jn  U S   R                  S5      nU S   R                  5       nUR                  S5      nUc  [	        SU  35      eU(       d  [        US9u  pT  nU" T	5      nTR                  [        [        U 5      USS	9  UR                  SS 5        UR                  SS 5        [        R                  " S
0 0 UESU0EUED6I S h  vN $  N! [         a0  nSS KnUR                  " 5         T
R                  U5         S nAg S nAff = f7f)Nr   r  r   r   rm  z2Model not found in litellm_params for deployment: r  aretrieve_batchrK  r   )r  r  r~  r  r  ry   r  r   rP  r  r  r  rT  	print_excr3  )r  r  r   rS  rm  r  
new_kwargsr  rT  r  receieved_exceptionsr[  s            r   try_retrieve_batch2Router.aretrieve_batch.<locals>.try_retrieve_batch  sC    $ V&'78<<WEE%&67<<>D*.((3H*I'}'PQ[P\] 
 /7G"'841 "0!7J77#'j#9)&7 8 
 NN#8$?HH2D9!(!8!8 ""13F )"    !  $'')(//2 sA   DCC CC DC 
D%&DDDDr   r%  Trx  z7Unable to find batch in any model. Received errors - {}rG  )r"   r|  r  r  r  r]   r   rA  r  r'  r*  r   rP  ro   rR  rQ  rH   rT  rU  )
r[  r   r  rF  filtered_model_listr  resultsr  r  r  s
   ` `      @r   r  Router.aretrieve_batch  s    l	@H  <<'-:RST(.

3H$(O#)%5 =   $ '+&9&9&;#"* =>>#% % 5H %  % R $/2D99+,q0 ' &9%8E +40CU+KL%8!
 '+!  %0Z#T6 6 !3,.AB!    ?@@ "g|44"N..")%fl;;#)M #*
 $*1-- IPP( sx2  		(,0#)(1(<(<(>'(	 G		s|   G,8F% F A0F% 0 F
F% F!6F% F#)F% ;G,<-F% -F% 0G,1)F% F% #F% %
G)/5G$$G))G,c           
      n  #     XS'   U R                   US'   UR                  SU R                  5      US'   [        SS9nU R	                  UUUS9  U R
                  " S	0 UD6I Sh  vN nU$  N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeSnAff = f7f)
zJ
Cancel a batch through the router with proper model-to-provider mapping.
r   r  r   _acancel_batchr5  r  NrG  r   )r  r~  r   r5   r  rP  r  r'  rQ  rH   rT  rU  r  s         r   acancel_batchRouter.acancel_batch  s     	#7O*.*=*=F&'$*JJ}d>N>N$OF=!%G.&" 00'= 1 
 "??I&IIHO J  		(,0#)(1(<(<(>'(	 G		r  c                 Z  #     [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nUR                  S5      =(       d    0 R                  S	5      nUS
   R                  5       nU R                  U=(       d    US9nUb  UR                  U5        UR	                  SS 5        US   nU R                  XBSS9  U R                  UUS9n	U R                  U==   S-  ss'   UR                  S5      n
[        US   U
S9u  p  nU
=(       d    Un
[        R                  " S0 0 UEU
U R                  U	S.EUED6nU R!                  UUSS9nUb]  [#        U[$        R&                  5      (       a>  U IS h  vN    U R)                  XCS9I S h  vN   UI S h  vN nS S S 5      IS h  vN   O!U R)                  XCS9I S h  vN   UI S h  vN nU R*                  U==   S-  ss'   [         R,                  " SU S35        U$  GN N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [.         aJ  n[         R0                  " SU SU S[3        U5       S35        Ub  U R4                  U==   S-  ss'   UeS nAff = f7f)Nz Inside _acancel_batch()- model: r  r  zbatch-api-fake-textr  r  r  r*  r  r   model_idrN  r   r  rK  r  r~   rm  rH  rq  r  r  r  zlitellm.acancel_batch(model=r  zlitellm._acancel_batch(model=rz  r  r  r   )r   r  r"   r  r  r~  r  (get_deployment_credentials_with_providerr  r  r  r5  ry   r  r  r   r  r   r'  r  r  r7  rQ  r  r#  r  r6  )r[  r   r  rF  r  selected_deployment_idrS  resolved_credentialsr  r  rm  r  rI  r  r  r  s                   r   r  Router._acancel_batch9  s/    
U	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K%	  C   J '1nn\&B&Hb%M%Md%S"./446D#'#P#P/85 $Q $  $/01HH.5gJ//%DT 0   >>% ? L Z(A-( #'((+@"A4D7m$751AQ #6"U9U,, +>#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&.zl:QR OSj )
  . )=== *  	!++/wb@TUXYZU[T\\cd  &!+&G	s   J+AI H+E
I H. I #H67H08	H6H2H6I H4I )I*	I 3I46I *J++I .I 0H62H64I 6I<H?=I	I I 
J(AJ##J((J+c                   ^#    U R                  US9nUc  [        S5      eS[        4U4S jjn[        R                  " U Vs/ s H
  o" U5      PM     sn6 I Sh  vN nS/ SSSS.nU Hr  nUc  M  US	   c   [        US	5      (       a  [        US	5      US	'   [        US
5      US
'   US   R                  UR                  5        [        USS5      SL d  Mm  SUS'   Mt     U$ s  snf  N7f)zA
Return all the batches across all deployments of a model group.
r  Nr  r   c                 |   >#     [         R                  " S0 0 U S   ETED6I S h  vN $  N! [         a     g f = f7f)Nr   r   )r  alist_batchesr  r  s    r   r  0Router.alist_batches.<locals>.try_retrieve_batch  sS     $22 ;/0;F;     s(   < , *, <, 
9<9<rA  F)objectrS  first_idlast_idhas_morer  r  rS  r  T)	r  r  r]   r'  r*  r_  r  extendrS  )r[  r   r  r  r  r  final_resultsr  s     `     r   r  Router.alist_batches  s     #11U1C&9::	,? 	  5HI5HE '5HI
 

 
 F! ,49T9T07
0KM*-+269+Ei(f%,,V[[9 6:u5=04M*-  / J
s+   A C)C"C)C'C)2A!C)C)rm  c                   #    U R                  UR                  SS5      US9  UR                  S5      (       aq  U R                  US   S9(       aZ  U R                  US   US9I S h  vN nUS   S   US'   US   R	                  5       nU R                  UUS9  UR                  U5        U" S0 UD6I S h  vN $  NS N7f)	Nr   r  r  r  )r   r  r   r  r   )r  r~  r  r  r  r  r  )r[  r  rm  r  r  rS  s         r   )_pass_through_moderation_endpoint_factory0Router._pass_through_moderation_endpoint_factory  s      	,,**Wb) 	- 	
 ::g4#6#6&/#6#R#BBWo%  C   J ))9:7CF7O./446D//% 0  MM$&0000 1s%   A$B>&B:'AB>5B<6B><B>r  )n
assistantsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rB  rA  rC  r  r  r  r  r  r  r%  r'  r)  r+  r-  r/  r  r  r  r  r  r  r&  r(  r*  r,  r.  r0  rH  rI  r  r  r  rT  r]  rV  r_  rX  ra  rP  rY  rW  r`  rQ  rZ  rU  r^  rR  r[  rS  r\  rf  rj  rh  rl  ri  rm  rg  rk  aupload_container_fileupload_container_filealist_container_fileslist_container_filesaretrieve_container_fileretrieve_container_fileadelete_container_filedelete_container_filerx  ry  rz  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  c                   ^ ^^ TS;   a)    SS[         [           S[         [           4UU 4S jjjnU$ TS;   a)    SS[         [           S[         [           4UU 4S jjjnU$ TS;   a(    SS[         [           S[         [           4U4S jjjnU$ TS	;   a(    SS[         [           S[         [           4U4S
 jjjnU$   SS[         [           S[         [           4UUU 4S jjjnU$ )z
Creates appropriate wrapper functions for different API call types.

Returns:
    - A synchronous function for synchronous call types
    - An asynchronous function for asynchronous call types
)r  rB  rC  r  r  rI  r  r]  r_  ra  rY  r`  rj  rl  rm  rk  rm  r  c                 ,   > TR                   " SST0UD6$ )Nr  r   )rS  rm  r  r  r  r[  s      r   sync_wrapper-Router.factory_function.<locals>.sync_wrapperq  s(    
 << &7;A r   )r  r  r  r  c                    > U (       a
  SU;  a  XS'   UR                  S5      (       a  TR                  " SST0UD6$ T" S0 UD6$ )Nrm  r   r  r   )r~  rS  r  s      r   vector_store_sync_wrapper:Router.factory_function.<locals>.vector_store_sync_wrapper  s[    
 '+@+N4G01::g&&@@ *;?E  )2622r   )r&  r(  r*  r,  r.  r0  c                    > T" SU US.UD6$ )Nrm  r  r   r   rm  r  r  r  s      r   vector_store_file_sync_wrapper?Router.factory_function.<locals>.vector_store_file_sync_wrapper  s'    
 ) (;!  r   )r  r  r  r  r  c                 L   > U (       a
  SU;  a  XS'   SU;  a  SUS'   T" S0 UD6$ )Nrm  geminir   r   r  s      r   managed_agents_sync_wrapper<Router.factory_function.<locals>.managed_agents_sync_wrapper  s:    
 '+@+N4G01(64<F01(2622r   c                   >#    TS:X  a  TR                   " STU US.UD6I S h  vN $ TS:X  a  TR                  " SST0UD6I S h  vN $ TS;   a  TR                  " SST0UD6I S h  vN $ TS;   a  TR                  " STU S.UD6I S h  vN $ TS:X  a  TR                  " SST0UD6I S h  vN $ TS	;   a  TR
                  " SST0UD6I S h  vN $ TS
;   a  TR                  " STU S.UD6I S h  vN $ TS:X  a  TR
                  " STSS.UD6I S h  vN $ TS;   a  TR                  " SST0UD6I S h  vN $ TS;   a  TR                  " STU S.UD6I S h  vN $ TS;   a  TR
                  " STU US.UD6I S h  vN $ TS;   a  TR                  " STU S.UD6I S h  vN $ TS;   a  TR                  " STU S.UD6I S h  vN $ g  GN GN GNh GNH GN) GN
 N N N N Ne ND N#7f)Nr  )r  rm  r  r  r  rJ  )r%  r'  r)  r+  r-  r/  )r  rm  r  )r  r  r  r  r  r  r  r  r  r  rA  rH  rI  rT  rV  rX  rP  rW  rQ  rU  rR  rS  rx  ry  rz  r{  )	rf  rh  ri  rg  r  r  r  r   aretrieve_container_file_contentr  T)r  rG  )r  r  r  r  r  )r  r  r  r  r  r  )r  r  )r  r  r  r  r  )r  r  r  r  r  r   )
)_pass_through_assistants_endpoint_factoryr  r%   _init_vector_store_api_endpointsrN  rO  _init_containers_api_endpoints_init_responses_api_endpoints _init_interactions_api_endpoints"_init_managed_agents_api_endpoints)rm  r  r  r  r  r[  s      r   async_wrapper.Router.factory_function.<locals>.async_wrapper  s:    
 L(!KK &7(;! 	   l*!KK &7;A   33!99 &7      "BB &7(;   
 l*!FF &7     8 "CC &7    
 
 "@@ &7(;   
 66!CC &715   
    "?? &7      "BB &7(;   
 ??!CC &7(;! 	      "BB &7(;   
    "DD &7(;   E@s    G)G	!G)G!G)'G("G)
G!G),G-!G)G"G)1G2"G)G!G)6G7"G)G!#G)=G#>"G) G%!"G)G'G)G)G)G)G)G)G)G)G)!G)#G)%G)'G)NN)r   r  r   )r[  r  r  r  r  r  r  r  s   ```     r   r  Router.factory_function  s9   x  
 
( 6:(,%-c]     
 
 6:(,3%-c]3 3 3 -, 
 
 6:(,	%-c]	 	 	 21 
 
 6:(,	3%-c]	3 	3 	3 /. 26$(S	!)#S	SMS	 S	j r   c                    #    U(       a
  SU;  a  X#S'   UR                  S5      (       a  U R                  " SSU0UD6I Sh  vN $ U" S0 UD6I Sh  vN $  N N7f)z
Initialize the Vector Store API endpoints on the router.

If a model is provided in kwargs, use model-based routing to get
the deployment credentials. Otherwise, call the original function directly.
rm  r   r  Nr   r~  rO  r[  r  rm  r  s       r   r  'Router._init_vector_store_api_endpointsU  sq      #8#F,?() ::g?? "3   '0000 1s#   ?AAAAAAc                 ~  #    U(       a
  SU;  a  X#S'   SSK Jn  UR                  S5      nUR                  S5      n[        U[        5      (       a  UR                  U5      nUR                  SU5      nX:w  a  XS'   UR                  S5      n	U	(       a  UR                  S5      S:X  a  XS'   UR                  S5      =(       d<    [        U[        5      (       a%  UR                  5       (       a  UR                  5       OSn
U
(       a   XS	'   U R                  " SS
U0UD6I Sh  vN $ U" S0 UD6I Sh  vN $  N N7f)a  
Initialize the Containers API endpoints on the router.

LiteLLM-managed container IDs (``cntr_...``) encode ``model_id`` and provider
metadata. When present, decode the ID, replace ``container_id`` with the
upstream value, and route through ``_ageneric_api_call_with_fallbacks`` so
deployment credentials (e.g. regional ``api_base`` for Azure) match
:meth:`_init_responses_api_endpoints`. Otherwise call the handler directly.
rm  r   ResponsesAPIRequestUtilscontainer_idr  response_idr+  Nr   r  r   )litellm.responses.utilsr  r~  r   r  _decode_container_idr  rO  )r[  r  rm  r  r  r  _forwarded_model_iddecodedoriginal_iddecoded_providerr  s              r   r  %Router._init_containers_api_endpointsn  s:     #8#F,?()Dzz.1$jj4lC((.CCLQG!++m\BK*)4~&&{{+@AFJJ/D$E$Q0@,- {{:. 1377<O<U<U<W<W $))+ 
 "*w!CC &7  
 '0000
 1s$   D"D=$D9%D=4D;5D=;D=c                    #    SSK Jn  UR                  UR                  S5      5      nUb  XBS'   U R                  " SSU0UD6I Sh  vN $  N7f)z
Initialize the Responses API endpoints on the router.

GET, DELETE, CANCEL Responses API Requests encode the model_id in the response_id, this function decodes the response_id and sets the model to the model_id.
r   r  r  Nr   r  r   )r  r  get_model_id_from_response_idr~  rO  )r[  r  r  r  r  s        r   r  $Router._init_responses_api_endpoints  sc      	E+IIJJ}%
 &7O;; 
/

 
 	
 
s   AAAAc                 L  #    U(       a
  SU;  a  X#S'   SU;  a  SUS'   UR                  S5      (       a)  UR                  S5      UR                  S5      :X  a  SUS'   UR                  S5      (       a  U R                  " SSU0UD6I Sh  vN $ U" S0 UD6I Sh  vN $  N N7f)z
Initialize the Interactions API endpoints on the router.

GET, DELETE, CANCEL Interactions API Requests don't need model-based routing,
so we call the original function directly with the custom_llm_provider.
rm  r  agentr   Nr  r   r  r  s       r   r  'Router._init_interactions_api_endpoints  s      #8#F,?() .,4F()::g6::g#6&**W:M#M"F7O ::g?? "3   '0000	 1s$   B	B$B B$B"B$"B$c                 f   #    U(       a
  SU;  a  X#S'   SU;  a  SUS'   U" S0 UD6I Sh  vN $  N7f)z
Initialize the Managed Agents API endpoints on the router (v1beta/agents).

CRUD operations for Gemini managed agents don't need model-based routing,
so we call the original function directly with the custom_llm_provider.
rm  r  Nr   r   r  s       r   r  )Router._init_managed_agents_api_endpoints  sB      #8#F,?() .,4F()&00000s   (1/1r  c                    #    UcF  U R                   b.  U R                   S   nUR                  U R                   S   5        O[        S5      eU" SX#S.UD6I Sh  vN $  N7f)z@Internal helper function to pass through the assistants endpointNrm  r   z'custom_llm_provider' must be set. Either via:
 `Router(assistants_config={'custom_llm_provider': ..})` 
or
 `router.arun_thread(custom_llm_provider=..)`r  r   )r   r  r  )r[  r  rm  r  r  s        r   r  0Router._pass_through_assistants_endpoint_factory  s      &%%1&*&<&<=R&S#d445EFG s  ' 
 3
FL
 
 	
 
s   AA AA original_model_groupall_deploymentsr  c                 ~  #    U R                  U5      u  pxUS:w  a  g[        USS5      n	U	(       d  gU R                  U5      n
UR                  U
5      nUc  0 nXU
'   [	        U[
        5      (       d  g[        UR                  S5      =(       d    / 5      nX1-  nU Vs1 s HX  nUR                  S5      =(       d    0 R                  S5      c  M/  UR                  S5      =(       d    0 R                  S5      iMZ     nn[        [        U SS9I Sh  vN 5      nUU-
  U-
  nU(       d  g[        R                  " SU< S	[        U5       S
U< 35        [        U5      US'   U[        U5      S.n0 UEU/US.En [        U0 UD6I Sh  vN $ s  snf  N N! [        R                  [        [         4 a     gf = f7f)zQSame-model-group retry after a failed deployment; returns None if not applicable.r   Nr)  _failover_excluded_idsr*  r  rH  rF  zWeighted failover: exclude=z, remaining=z for model_group=)r   _excluded_deployment_idsfallback_model_groupr  )r  r  '_get_metadata_variable_name_from_kwargsr~  r   rP  r  r?   r   r  r  rA  rE   r+  APIErrorrg   rh   )r[  r#  r  r  rh  r  r  r  r  	failed_idr  metaprev_excludedexcludeddall_idscooldown_ids	remainingentryfailover_kwargss                       r   _maybe_run_weighted_failover#Router._maybe_run_weighted_failover  s     //0DE''#*96Ld#S	!%!M!Mf!Uzz01<D-1)*$%%DHH%=>D"E ;. %
$l#)r..t4 2QUU< &B++D1$ 	 
 1(,t 

 |+x7	##)(\#i.AQ R368	

 *.h%& *(,X


%*G$8


	+TE_EEEQ
: F!57PQ 	 	s[   BF=.F-F>F=FA(F=<F 
FF F=F F:7F=9F::F=r  r  r  c	                 P  #    [         R                  " S[        R                  " 5        35        Un	Sn
UR	                  S5      nSnUSL d  Uc  UeU U	S.UEnSU;  a  U R
                  US'   SU;  a  S	US'   [        U[        R                  [        R                  45      nUR	                  S
0 5      =(       d    0 R	                  S5      nU R                  XS9=(       d    / nU Vs1 s HD  n[        R                  R                  U5      c  M%  [        R                  R                  U5      iMF     nn[        U5      n[        U5      S:  a  U(       d  UR	                  S5      nUb  UOUS	   nU Vs/ s H  nUU:  d  M  UUS.PM     nnSnUb=  Ub:  [        US9(       a  UnO)[!        U[#        [$        U5      S9u  nnUc  Ub  UU   S   nUU=(       d    / -   nU(       a)  UR'                  UUS.5        [)        U0 UD6I Sh  vN nU$ U R*                  (       a+  U(       d$  Ub!  U R-                  UUUUUUS9I Sh  vN nUb  U$  [         R.                  " S5        [        US9nU(       a)  UR'                  UUS.5        [)        U0 UD6I Sh  vN nU$ [        U[        R                  5      (       a  Ub>  U R1                  UUS9nUc  U	eUR'                  UUS.5        [)        U0 UD6I Sh  vN nU$ SR3                  XdU5      n[         R.                  " SR3                  U5      S9  [        R4                  (       a$  U=R6                  SR3                  U5      -  sl        O[        U[        R                  5      (       a  Ub>  U R1                  UUS9nUc  U	eUR'                  UUS.5        [)        U0 UD6I Sh  vN nU$ SR3                  XeU5      n[         R.                  " SR3                  U5      S9  [        R4                  (       a$  U=R6                  SR3                  U5      -  sl        Ub  Ub  [         R                  " SU 35        [!        U[#        [$        U5      S9u  n
n U
c  U b  UU    S   n
U
c_  [         R.                  " SU SU 35        [9        U	S5      (       a0  [        R4                  (       a  U	=R6                  SU SU 3-  sl        U	eUR'                  U
US.5        [)        U0 UD6I Sh  vN nU$ [9        U	S5      (       am  [        R4                  (       aX  U	=R6                  S"R3                  UU
5      -  sl        [        U5      S	:  a$  U	=R6                  S#R3                  U5      -  sl        U	es  snf s  snf  GN GNW GN GN GN N! [:         au  n![=        U5      n"[         R>                  " S R3                  [%        U!5      [        R                  " 5       [A        U U"S!9I Sh  vN  5      5        [%        U!5      n Sn!A!GNSn!A!ff = f7f)$z4
Common utilities for async_function_with_fallbacks
	TracebackNr   r  T)litellm_routerrJ  r   fallback_depthr   r   user_api_key_team_idr  team_idr~   _target_order)r   r  )r   )r   r  r   r  )r#  r  r  rh  r  r  zTrying to fallback b/w modelszmodel={}. context_window_fallbacks={}. fallbacks={}.

Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContextWindowExceededError'. No context_window_fallback set. Defaulting                         to fallbacks, if available.{})msgz
{}zmodel={}. content_policy_fallback={}. fallbacks={}.

Set 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting                         to fallbacks, if available.{}zinside model fallbacks: z7No fallback model group found for original model_group=z. Fallbacks=r  zlitellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}
{}

Debug Information:
Cooldown Deployments={}r  z<. Received Model Group={}
Available Model Group Fallbacks={}z
Error doing the fallback: {})!r   r  rT  rU  r~  r   r   r  ContextWindowExceededErrorr  r  utils_get_deployment_ordersortedr  rC   rD   r   r  r  rE   r   r	  rQ  (_get_fallback_model_group_from_fallbacksrR  r  r  r_  r  r"   r5  r@   )#r[  r  r  r   r   r   r  rh  r  rJ  r  r  fallback_failure_exception_strr  _skip_order_fallback_request_team_idr  r  
_order_setorder_valuescurrent_target
skip_up_tooorder_fallback_entriesexternal_fallback_groupgeneric_idxcombined_fallbacksr  is_non_standard_fallback_format#context_window_fallback_model_grouperror_message#content_policy_fallback_model_groupgeneric_fallback_idxnew_exceptionrF  s#                                      r   r  1Router.async_function_with_fallbacks_common_utilsG  s4     	##i	0D0D0F/G$HI#.4jj.A)+&$(<(DG #"4
 
 ,.,0,>,>L)</-.L)*  *//1T1TU 
 ,2::j"+E+K*P*P"+
 /     	 	 %
$}}2215 3GMM//2$ 	 

 $J/|q )=#ZZ8N"0"<,q/  &,%Az> D.C% # , 7;#$)@6K /8+;S"+$(k$:<8+[ /6;;R2;K2H2M/ "8'-2" "##0B0D "4"""    ))($0!>>%9 /) ?  H #Q	@!&&'FG /R#/+ /##090D "4""" 
  !W??@@+7EE&>(3 F  8 ;B00 ''4W4H &8&&&  H $O %m  %t  %t#y%M *..77=v)8 <<		V]]=%AA	AwBBCC+7EE&>(3 F  8 ;B00 ''4W4H &8&&&  H $O %l  %s  %s#y%M *..77=v)8 <<		V]]=%AA	$)@%++.Fyk,RS -' $S+ 6(( (/4H4T+45I+J3+O('/)..QR]Q^^jktjuv   2I>>#AA*228op{o|  }I  JS  IT  7U  U2,,##0D0D "4""" 
    &	2255 &&*i*p*p$+ & 12Q6"**4;;6* ! O
,@62 J b  	@@H!'' k  r  r&((*I04)9  	 .1-?*	@s   C%X&'$V	#V	2AX&4
V	VA;X&V8X&?V 	X&
AV$ VV$ X&AV$ 2V3V$ 8X&9CV$ VV$ X&D:V$ V"V$ BX&X&V$ V$ V$ "V$ $
X#.AX?X XX&X##X&c                   #    UR                  S5      nUR                  SS5      nUR                  SU R                  5      nUR                  SU R                  5      nUR                  SU R                  5      nUR                  SS5      n U R                  UUUUUS	9  Ub  U R                  " U0 UDSU0D6I Sh  vN n	OU R                  " U0 UD6I Sh  vN n	[        R                  " [        R                  5      (       a  [        R                  " S
U	 35        [        U	SS9n	U	$  Nm NT! [         a+  n
U R                  U
UUUUUUU5      I Sh  vN  s Sn
A
$ Sn
A
ff = f7f)zg
Try calling the function_with_retries
If it fails after num_retries, fall back to another model group
r   r  Fr   r   r   mock_timeoutN)r  r  r   r   r   zAsync Response: r   )r  attempted_fallbacks)r~  r  r   r   r   _handle_mock_testing_fallbacksasync_function_with_retriesr   isEnabledForr  r   r  r3   r  r  )r[  rh  r  r  r  r   r   r   r-  r  r  s              r   rP  $Router.async_function_with_fallbacksc  s     &,ZZ%8,2JJ7JE,R$*JJ{DNN$K	39::&(E(E4
  4:::&(E(E4
  zz.$7 	//'#)A)A 0  '!%!A!A"#"2>"  "&!A!A4!R6!RR$11'--@@%++.>xj,IJ7!$%H O S  
	HH!((	 	 	 	
	sg   B
E&/D. <D*=D. D,AD. )E&*D. ,D. .
E#8EEEE#E&E##E&c                    [         R                  " U5      nUR                  b+  UR                  SL a  [        R                  " USSU SU 3S9eUR
                  b+  UR
                  SL a  [        R                  " USSU SU 3S9eUR                  b,  UR                  SL a  [        R                  " USSU SU 3S9egg)	a  
Helper function to raise a litellm Error for mock testing purposes.

Raises:
    litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params
    litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params
    litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params
NTr  #This is a mock exception for model=z#, to trigger a fallback. Fallbacks=)r   r  r  zF, to trigger a fallback.                     Context_Window_Fallbacks=zF, to trigger a fallback.                     Context_Policy_Fallbacks=)	r`   from_kwargsmock_testing_fallbacksr  InternalServerErrormock_testing_context_fallbacksr  %mock_testing_content_policy_fallbacksr  )r[  r  r  r   r   r   mock_testing_paramss          r   r/  %Router._handle_mock_testing_fallbacks  s     6AA&I66B#::dB--!=k]Jmnwmxy   >>J#BBdJ44!=k] K..F-GI   EEQ#IITQ55!=k] K..F-GI  R Rr   c                   #    [         R                  " S5        UR                  S5      nUR                  SU R                  5      n[	        U5      nUR                  SU R
                  5      nUR                  SU R                  5      nUR                  SU R                  5      nUR                  S5      n	UR                  S5      n
UR                  S	UR                  S
5      5      =(       d    0 nSU;   aI  [        US   [        5      (       a1  U R                  US   S9nUb  UR                  S[        U5      05        [         R                  " SU SU
 35        SUS'   U
US'    U R                  XS9  U R                  " U/UQ70 UD6I S h  vN n[!        USS S9nU$  N! ["         Ga  nS nUn[%        USS 5      nUb  [        U[&        5      (       a  Un
 U R)                  UR                  S5      =(       d    SUS9I S h  vN  u  nnSnU R*                  c  UbO  U	=(       d)    UR                  S5      =(       d    UR                  S5      n[-        UUUU R*                  S9nUb  Un
SnU(       d  U R/                  UUUUUUS9  XS'   U
S:  a  U R1                  UUS9nOe [         R                  " SU
 35        U R3                  UU
U
UUS9n[4        R6                  " U5      I S h  vN    [9        U
5       GH-  n US-   US'   XS'   U R                  " U/UQ70 UD6I S h  vN  n[:        R<                  " U5      (       a  UI S h  vN  n[!        UUS-   U
S9nUs  s S nA$ ! ["         a  nUnU R1                  X.S9nX-
  S-
  nUR                  S5      nUb  U R)                  UUS9I S h  vN  u  nnO/ nU(       d&   U R/                  UUUUUUS9  O! ["         a    Uef = fU R3                  UUU
UUS9n[4        R6                  " U5      I S h  vN     S nAGM(  S nAff = f   [?        U5      [@        RB                  ;   a$  [E        USU
5        Ub  US-   OU
n[E        USU5        UeS nAff = f7f)Nz#Inside async function with retries.r  r   r   r   r   r   r   r  r   r  r  model_group_sizez/async function w/ retries: original_function - z, num_retries - r   attempted_retriesr   )r  r  )r  r>  r   r  r  Fr#  r  r   r   T)r5  r&  r  r   regular_fallbacksr   )r  r  z#Retrying request with num_retries: )r  remaining_retriesr   r&  r  r~   )#r   r  r  r   r"   r   r   r   r~  r   r  r  r  r  %_handle_mock_testing_rate_limit_error	make_callr4   r  r  r4  r  r   "_get_num_retries_from_retry_policyshould_retry_this_error	log_retry_time_to_sleep_before_retryr'  r  r}  r$   is_async_callabler   r  LITELLM_EXCEPTION_TYPESr  )r[  rh  r  r  r   rF  r   r   r   r  r   	_metadatar   r  r  current_attemptrJ  deployment_num_retriesr  _all_deployments_retry_policy_applies_model_group_for_retry_policy_retry_policy_retriesr   rA  _modelr  _timeoutactual_retries_attempteds                                r   r0  "Router.async_function_with_retries  sE    ##$IJ"JJ':;JJ{DNN;	<VD#)::&(E(E$
  $*::&(E(E$
  $*::&(E(E$
  &,ZZ%8jj/ !**%7J9OPVTV	I%*Y}5Ms*S*S,,	-8P,QJ%  "4c*o!FG##=>O=PP`al`mn	
 *+	%& 	- b	%66' 7  "^^,=OOOOH4!QDH O	 P
  X	%"O!"%,Qt%D"%1j&7 7 5 99jj)/R!1 :   $  %*!  ,0H0T
  V9==#?V6::gCV . )K0 =-E!%!2!2	)% )4"7K,0) ),,(<$4-E&/-E -  (3m$ Qv9KL!''5k]C ::$"-'$8 0 ; K --,,,#(#5>25Dq5HI12/:m,%)^^4E%W%WPV%WWWH(::   *2>><!)*9A*=$/ H
 $O  ,2 *+& "^^6^?F(3(E(I%,2JJw,?F) #'"E"E"(-= #F #  0 02, 1
$ 88&'4H0@9Q2;9Q 9   ) $"#G$  $??*;$/,@(8  @  H "--1111Y,2' $6B &'7+J+JJ*M;G ,;+FOa'K ) *M;ST$$qX	%s   E$Q9'(F# F!F#  Q9!F# #Q6.AQ1HC%Q1+K.,Q1%M)&L)'%M)MM)!Q1#Q6$Q9)
P)3?P$2N5
3P$OP$O(	(/P$P
P$Q1$P))AQ11Q66Q9c                    #    UR                  S5      nU" U0 UD6n[        R                  " U5      (       d  [        R                  " U5      (       a
  UI Sh  vN nU R                  XTUS9I Sh  vN nU$  N N7f)zN
Handler for making a call to the .completion()/.embeddings()/etc. functions.
r   N)r  r  r  )r~  r$   rH  rf  isawaitableset_response_headers)r[  r  rh  r  r  r  s         r   rC  Router.make_call  s      jj)$d5f5..x88G<O<O=
 =
 &~H22v 3 
 
  &
s$   AA:A6A:/A80A:8A:c                 @   UR                  SS5      nU R                  US9nSnUb;  [        U5      S:X  a,  [        [        [
           US   S   R                  S5      5      nUb:  USL a4  [        R                  " S	U 35        [        R                  " US
SU S3US9egg)z
Helper function to raise a mock litellm.RateLimitError error for testing purposes.

Raises:
    litellm.RateLimitError error when `mock_testing_rate_limit_error=True` passed in request params
mock_testing_rate_limit_errorNr  r~   r   r   r   TzTlitellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=r  r4  z , to trigger a rate limit error.)r   r  r  r   )r  r  r  r   r   r4  r~  r   rQ  r  RateLimitError)r[  r  r  rZ  available_modelsr   s         r   rB  ,Router._handle_mock_testing_rate_limit_error  s     9?

+T9
%  ..+.F%)'C0@,AQ,F/23CDHHWK
 *5-5!&&fgrfst ((!=k]Jjk'	  6 6r   r5  r@  c                    SnUb   [        U[        5      (       a  [        U5      nSnUb   [        U[        5      (       a  [        U5      n[        U[        R                  5      (       a  Ub  Ue[        U[        R
                  5      (       a  Ub  Ue[        USS5      n	U	b#  [        R                  " U	5      (       d  U	S;  a  Ue[        U[        R                  5      (       a  Ue[        U[        R                  5      (       a  US::  a  Ub  [        U5      S:  a  Ue[        U[        R                  5      (       a	   US::  a  UeUS::  a  Ueg)aE  
1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None

2. raise an exception for RateLimitError if
    - there are no fallbacks
    - there are no healthy deployments in the same model group
r   Nstatus_code)i  i  r~   T)r   rA  r  r  r  r  r  _should_retryNotFoundErrorr+  r[  AuthenticationError)
r[  r5  r&  r  r   r   r@  _num_healthy_deployments_num_all_deploymentsr_  s
             r   rE  Router.should_retry_this_error  sB   " $% *z:Mt/T/T'*+>'?$ &:ot+L+L#&#7  ug@@AA(4K ugAABB(4Ke]D9"7+@+@+M+M*,eW2233KeV2233(A-%1)*Q.eV7788
 %) $q(Kr   c                 4    [        U R                  /UQ70 UD6$ )zf
Sync wrapper for async_function_with_fallbacks

Wrapped to reduce code duplication and prevent bugs.
)r!   rP  )r[  rh  r  s      r   r  Router.function_with_fallbacks  s     "$"D"DVtVvVVr   c                 r    Uc  gSnU H*  n[        UR                  5       5      S   U:X  d  M%  XB   n  U$    U$ )a  
Returns the list of fallback models to use for a given model group

If no fallback model group is found, returns None

Example:
    fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}]
    model_group = "gpt-3.5-turbo"
    returns: ["gpt-4"]
Nr   )rA  keys)r[  r   r  r  r  s        r   r  /Router._get_fallback_model_group_from_fallbacks  sO     48DDIIK #{2'+'8$##	  $#r   c                     U R                   c  gU R                    HT  n[        U[        5      (       d  M  SU;   d  M"  US   n[        U[        5      (       d  M>  [	        U5      S:  d  MO  US   s  $    g)zH
Returns the first model from the default_fallbacks list, if it exists.
Nr   r   )r   r   rP  rA  r  )r[  fallbackdefault_lists      r   _get_first_default_fallback"Router._get_first_default_fallback)  sc     >>!H(D))cXo'}lD11c,6G!6K'?*	 '
 r   rA  c                    Ub  [        U5      S:X  a  O(Ub%  [        U[        5      (       a  [        U5      S:  a  gSn[        US5      (       a1  [        UR                  S5      (       a  UR                  R
                  n[        US5      (       a  UR                  nUb#  [        R                  " UUUU R                  S9nU$ [        R                  " UUU R                  S9nU$ )	z
Calculate back-off, then retry

It should instantly retry only when:
    1. there are healthy deployments in the same model group
    2. there are fallbacks for the completion call
Nr~   r   r  headerslitellm_response_headers)rA  r   response_headersmin_timeout)rA  r   rt  )
r  r   rA  r_  r  rq  rr  r  _calculate_retry_afterr   )r[  r  rA  r   r&  r  rs  r   s           r   rG  "Router._time_to_sleep_before_retry6  s    " &3+?1+D+.55'(1,481j!!gajj)&D&D zz111011 99'44"3'!1 ,,	G  44"3' ,,G r   c                   #    SSK Jn   UR                  S5      S;   a  gUR                  SS5      nUc  [        S5      eUS   R                  S	5      c  gUS   S	   R                  S
S5      nUS   S	   R                  SS5      nUS   R                  S0 5      =(       d    0 n	U	R                  SS5      n
Ub  U
c  g[	        U
[
        5      (       a  [        U
5      n
U R                  U
S9nUc  gU R                  UUS9nUR                  SS5      nUR                  SS5      nUR                  R                  nUR                  R                  nUR                  SS5      nUR                  SS5      n[        U U
S9  Uc  Uc  Uc
  Uc  Uc  Uc  g[        U5      nUR                  SS5      n[        5       nUR                  S5      n[         R"                  R$                  R'                  U
UUS9n/ nUR)                  U" UU[*        R,                  R$                  S95        [         R.                  R$                  R'                  U
UUS9nUR)                  U" US[*        R,                  R$                  S95        U R0                  R3                  UUS9I Sh  vN   U$  N! [4         a8  n[6        R8                  " SR'                  [        U5      5      5         SnAgSnAff = f7f)z7
Track remaining tpm/rpm quota for model in model_list
r   )RedisPipelineIncrementOperationr  )r  r  Nr  zstandard_logging_object is Noner   r   r  r  r*  r  r  r  received_model_nametpmrpmrH  r+  r  %H-%Mr  current_minuter   )r0  increment_valuer   r~   )increment_listrF  zOlitellm.router.Router::deployment_callback_on_success(): Exception occured - {})litellm.types.cachingrx  r~  r  r   r4  r  get_deploymentget_router_model_infor   r{  r|  rN   r"   r|   strftimerd   TPMr  rR  r3  r   r   RPMr   async_increment_cache_pipeliner  r   r  )r[  r  completion_responserD  rE  rx  r  deployment_namer  r*  r  deployment_infodeployment_model_infor{  r|  tpm_litellm_paramsrpm_litellm_paramstpm_model_inforpm_model_inforF  r  dtr  tpm_keypipeline_operationsrpm_keyr  s                              r   rD  %Router.deployment_callback_on_successi  s>     	Jw	zz+&*QQHN

)4I# '. !BCC&'++J7?"()9"::"F"J"J $# %%56zBFF!4 $$4599,KQr
^^D$/&"*C((RB #'"5"5r"5"B"*,0,F,F#2,7 -G -)
 *--eT:C)--eT:C *9)G)G)K)K&)8)G)G)K)K& &;%>%>ud%KN%:%>%>ud%KN B,0"$ K*2*2&.&.#DV#L &=&A&A.RS&T
 &'!#" *--33::. ;  NP# $**3#(4'OO11 *--33::. ;  $**3#()'OO11 jj??#6%5 @   
   	!''ahhF
 	s}   K&J! K&5J! K&A&J! <K&=2J! /K&0B J! K&DJ! JJ! K&J! !
K#+.KK&K##K&c                 0   SnUS   R                  S5      c  OoUS   S   R                  SS5      nUS   R                  S0 5      =(       d    0 nUR                  SS5      nUb  Uc  g[        U[        5      (       a  [        U5      nUb  [	        U US9nU$ g)z
Tracks the number of successes for a deployment in the current minute (using in-memory cache)

Returns:
- key: str - The key used to increment the cache
- None: if no key is found
Nr   r   r  r*  r  r}  )r~  r   r4  r  rN   )	r[  r  r  rD  rE  r  r  r*  r0  s	            r   rG  *Router.sync_deployment_callback_on_success  s     "#''
3; !12:>BB=RVWK 0155lBGM2Jd+B"bjB$$W>C(, C Jr   c                    [         R                  " S5         UR                  SS5      n[        USS5      nUR                  S0 5      nUR                  S0 5      n[        R
                  R                  R                  US9n	UR                  S	S5      n
SnU	b  [        R                  R                  U	S
9nU
b	  U
S:  a  U
nOUb	  US:  a  UnOU R                  n[        U[        5      (       a.  UR                  S5      nUc  g[        U US9  [        U UUUUS9nU$ [         R                  " S5        g! [         a  nUeSnAff = f)a(  
2 jobs:
- Tracks the number of failures for a deployment in the current minute (using in-memory cache)
- Puts the deployment in cooldown if it exceeds the allowed fails / minute

Returns:
- True if the deployment should be put in cooldown
- False if the deployment should not be put in cooldown
z1Router: Entering 'deployment_callback_on_failure'r#  Nr_  r  r   r*  )rJ  r   )rs  r   r  Fr}  rH  exception_statusrJ  r  time_to_cooldownzWRouter: Exiting 'deployment_callback_on_failure' without cooldown. No model_info found.)r   r  r~  r  r  litellm_core_utilsexception_mapping_utils_get_response_headersr  &_get_retry_after_from_exception_headerr   r   rP  rM   rB   r  )r[  r  r  rD  rE  r#  r  r   _model_infoexception_headersdeployment_cooldownheader_cooldown_time_to_cooldownr+  r  r  s                   r   rM  %Router.deployment_callback_on_failure  ss     	##$WX9	

;5I&y-D $ZZ(8"=N(,,\2>K ' : : R R h h#, !i !
 #1"4"4_d"K"O ,")--"V"V%6 #W # #.3F!3K$7! ,A1E$3!$($6$6!+t,,/:t/D ( @,0"/ 3,0%5'0,%6 %++m  	G	s$   C+D4 D4 D4 4
E>E  Er  c                 2  #    US   S   R                  SS5      nUS   S   R                  SS5      nUS   R                  S0 5      =(       d    0 nUR                  SS5      nUb  Uc  g[        U[        5      (       a  [        U5      n[	        U5      n	[        5       n
U
R                  S5      n[        R                  R                  R                  XUS	9nU R                  R                  US
U	[        R                  R                  S9I Sh  vN   g N7f)z#
Update RPM usage for a deployment
r   r   r  Nr  r*  r  r~  r  r~   )r0  r  rF  r   )r~  r   r4  r  r"   r|   r  rd   r  r  rR  r   async_increment_cacher   r   )r[  r  r  rD  rE  r  r  r*  r  rF  r  r  r  s                r   rJ  +Router.async_deployment_callback_on_failure]  s     !!12:>BB$
 -.z:>>}dS,-11,CIr
^^D$'"*C  RB<VD

 "%%++22 3 
 jj..-%%	 / 
 	
 	
s   DDDD)r   r  c                     [        U5      $ )ak  
Helper to return what the "metadata" field should be called in the request data

- New endpoints return `litellm_metadata`
- Old endpoints return `metadata`

Context:
- LiteLLM used `metadata` as an internal field for storing metadata
- OpenAI then started using this field for their metadata
- LiteLLM is now moving to using `litellm_metadata` for our metadata
)r#   )r[  r  s     r   r  .Router._get_metadata_variable_name_from_kwargs  s     6f==r   c                 .    SU;   a  SOSn[        U5      R                  [        U5      S.nUR                  5        H`  u  nnXSSS4;  a  XdU'   M  XS:X  d  M  [	        U[
        5      (       d  M2  0 XC'   X   R                  5        H  u  pxUS:w  d  M  XU   U'   M     Mb     [        U R                  5      S:  a  U R                  R                  S5        U R                  R                  U5        U R                  X   S'   U$ ! [         a  nUeS	nAff = f)
zs
When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
r  r   )exception_typeexception_stringr  r  r8     r   N)r   r   r  rq  r   rP  r  r8  r  r3  r  )	r[  r  r  _metadata_varprevious_modelr  r  
metadata_k
metadata_vs	            r   rF  Router.log_retry  s   	&8F&B"
 
 #'q'"2"2$'FN 	 J8KLL()1%'Jq$,?,?46N1282G2M2M2O.
%)::<F1-j9 3P  4''(1,$$((+  ''77;7K7KF!"34M 	G	s+   AD D /D A0D 
DDDr+  rF  c                     UnU R                   R                  X2SS9nUc  SnU R                   R                  X4SSS9  U$ US-  nU R                   R                  X4SS9  U$ )zF
Update deployment rpm for that minute

Returns:
- int: request count
T)r0  rF  
local_onlyr~   r   )r0  r  r  r   )r0  r  r  )r   	get_cache	set_cache)r[  r+  rF  r  request_counts        r   _update_usageRouter._update_usage  s      

,,t - 
  MJJ  Tr !   QMJJ  T !  r   c                     U R                   c  gU R                    H"  n[        U[        5      (       d  M  SU;   d  M"    g   g)NFr   T)r   r   rP  )r[  rl  s     r   _has_default_fallbacksRouter._has_default_fallbacks  s9    >>!H(D))(? ' r   c                    UR                   (       a7  [        UR                   5      S:  a  UR                   S   R                  S:w  a  gUR                  SU R                  5      nUb6  SnU H)  n[        UR                  5       5      S   U:X  d  M%  Xa   n  O   Ub  gOU R                  5       (       a  g[        R                  " SR                  X5      5        g)z
Determines if a content policy error should be raised.

Only raised if a fallback is available.

Else, original response is returned.
r   content_filterFr   NTzyContent Policy Error occurred. No available fallbacks. Returning original response. model={}, content_policy_fallbacks={})choicesr  finish_reasonr~  r   rA  ri  r  r   r  rR  )r[  r   r  r  r   r  r  s          r   r  )Router._should_raise_content_policy_error  s     H$4$4 5 9"004DD#)::&(E(E$
 
 $/#' 0		$Q'50+/;( 1
 $/ 0((**## H  O  O	

 r   c                    / n U R                  US9u  pC[        U[        5      (       a  / $  [	        XS9n[        U5      nU Vs/ s H  owS   S   U;  d  M  UPM     nnU R                  U5      nX4$ ! [         a     NSf = fs  snf )Nr  r  r*  r  )#_common_checks_available_deploymentr   rP  r  rA   r  _filter_blocked_deployments)	r[  r   rF  rM  r  unhealthy_deploymentsunhealthy_setr  r&  s	            r   _get_healthy_deploymentsRouter._get_healthy_deployments  s    !#	"&"J"J #K #A *D11	 2
 !:$(!
 12'%
'!\?4+@+UA' 	 %
 #>>?RS"44  		%
s   'A4 BB4
B Bc                 4  #    / n U R                  US9u  pC[        U[        5      (       a  / U4$  [	        XS9I Sh  vN n[        U5      nU Vs/ s H  nUS   S   U;  d  M  UPM     nnU R                  U5      nX4$ ! [         a     N\f = f NSs  snf 7f)z
Returns Tuple of:
- Tuple[List[Dict], List[Dict]]:
    1. healthy_deployments: list of healthy deployments
    2. all_deployments: list of all deployments
r  r  Nr*  r  )r  r   rP  r  r?   r  r  )	r[  r   rF  rM  r  r  unhealthy_deployments_setr  r&  s	            r   r  %Router._async_get_healthy_deployments  s      "$	"&"J"J #K #A *D11+++ 2
 'F$('
 !
 %((=$>! &%
%t$,EE % 	 %

 #>>?RS"44  		!

%
sI   B)B BBBB%B+B
BBBBBc                     [         R                   H+  n[        U[        5      (       d  M  UR	                  U5        M-     g)z
Mimics 'async_routing_strategy_pre_call_checks'

Ensures consistent update rpm implementation for 'usage-based-routing-v2'

Returns:
- None

Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
N)r  rw  r   r    r  )r[  r  r  s      r   r  'Router.routing_strategy_pre_call_checks4  s/     !**I)\22((4 +r   rn  c           
        #    [         R                   H4  n[        U[        5      (       d  M   UR	                  X5      I Sh  vN   M6     g N	! [         R
                   a  nUb  [        R                  " UR                  U[        R                  " 5       [        R                  " 5       S95        [        R                  " UR                  U[        R                  " 5       4S9R                  5         [!        U UR"                  UUS   S   U R$                  S9  UeSnAf[&         a  nUb  [        R                  " UR                  U[        R                  " 5       [        R                  " 5       S95        [        R                  " UR                  U[        R                  " 5       4S9R                  5         UeSnAff = f7f)  
For usage-based-routing-v2, enables running rpm checks before the call is made, inside the semaphore.

-> makes the calls concurrency-safe, when rpm limits are set for a deployment

Returns:
- None

Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
Nr#  traceback_exceptionrE  r  rh  r*  r  r  )r  rw  r   r    async_pre_call_checkr[  r'  rQ  async_failure_handlerrT  rU  rL  r  r  failure_handlerr  rB   r_  r   r  )r[  r  rF  rn  r  r  s         r   r  -Router.async_routing_strategy_pre_call_checksD  s|    " !**I)\22(#88VVV + W-- ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'-04)*+,#-l#;D#A)-);); G  ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'GsH   'GAAAGAF?"B9DF?(BF::F??Gc           
        #    Un[         R                   H6  n[        U[        5      (       d  M   UR	                  UUUUUS9I Sh  vN nM8     U$  N
! [
         a  n	Ub  [        R                  " UR                  U	[        R                  " 5       [        R                  " 5       S95        [        R                  " UR                  U	[        R                  " 5       4S9R                  5         U	eSn	A	ff = f7f)r  r   r&  r  r  rF  Nr  r  )r  rw  r   r    async_filter_deploymentsr  r'  rQ  r  rT  rU  rL  r  r  r  r  )
r[  r   r&  r  rF  r  rn  returned_healthy_deploymentsr  r  s
             r   !async_callback_filter_deployments(Router.async_callback_filter_deployments  s     ( (;$ **I)\22'@@"'0L%-+9-= A   1 +8 ,+1 ! ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'Gs;   )C7AAAC7A
C4BC//C44C7r   c                    U/nUR                  5        GH   u  pE[        U[        5      (       a  UR                  U5        OU[        U[        5      (       a&  UR                  [
        R                  " U5      5        OUR                  [        U5      5        [        U[        5      (       a  UR                  U5        M  [        U[        5      (       a'  UR                  [
        R                  " U5      5        M  UR                  [        U5      5        GM     SR                  U5      n[        R                  " UR                  5       5      nUR                  5       $ )z
Helper function to consistently generate the same id for a deployment

- create a string from all the litellm params
- hash
- use hash as id
r  )rq  r   r  r3  rP  r  dumpsjoinhashlibsha256encode	hexdigest)r[  r  r   partsr  r  
concat_strhash_objects           r   r8  Router._generate_model_id  s     "((*DA!S!!QAt$$TZZ]+SV$!S!!QAt$$TZZ]+SV$ + WWU^
nnZ%6%6%89$$&&r   r*  backend_modelc                    ^  Sn[        U 4S jU 5       5      (       a  g [        R                  " XS9nU H2  nT R	                  U5      b  M  UR	                  U5      nUc  M-  UT U'   M4     g! [         a     gf = f)a  Fill missing cache pricing on a custom-priced deployment entry from
the backend model's built-in cost map entry, so a deployment that
only spells out ``input_cost_per_token``/``output_cost_per_token``
does not silently bill cache_read/cache_creation at 0.

User-specified cache fields always win; only ``None``/missing entries
are inherited. No-op when the backend model has no canonical entry.
)cache_creation_input_token_cost)cache_creation_input_token_cost_above_1hr1cache_creation_input_token_cost_above_200k_tokenscache_read_input_token_cost-cache_read_input_token_cost_above_200k_tokensc              3   J   >#    U  H  nTR                  U5      S Lv   M     g 7fr   r  )r   fr*  s     r   r  8Router._inherit_builtin_cache_pricing.<locals>.<genexpr>  s     Clz~~a ,ls    #NrH  )r|  r  get_model_infor  r~  )r*  r  rm  cache_fieldsbackend_infofieldbackend_values   `      r   _inherit_builtin_cache_pricing%Router._inherit_builtin_cache_pricing  s    
 ClCCC	"11#L
 "E~~e$, , 0 0 7 ,(5Ju%	 "  		s   A- -
A:9A:r  _model_name_litellm_paramsr  c           
      j    [        S0 UD6n[        S0 UDUUUS.D6n[        R                  R	                  5        H2  nUR
                  R                  U5      c  M!  UR
                  U   XG'   M4     UR                  S5      b=  [        R                  UUR
                  R                  UR
                  R                  S9  UR                  R                  nUb  [        R                  " X0S9  UR
                  R                  nUR
                  R                  b  UR
                  R                  S-   U-   n[        R                  R	                  5       n	UR                  5        V
Vs0 s H  u  pX;  d  M  X_M     nn
n[!        ["        [$           [        R&                  R                  U0 5      5      =(       d    0 R                  S5      nUR                  S5      nUS:H  =(       a    US	:H  nUSL=(       a    USL nU(       d  U(       a   Ub  [(        R*                  " S
UUU5        XS'   X,0nSU;   a  UR-                  SS5      nUUU'   [        R                  " US9  U R/                  US9SLa4  [(        R*                  " SUR0                   SUR                  S    35        gUR
                  R                  S5      =(       d    / nU H  n [2        R4                  " U5        M     U R;                  US9nUR=                  SS9nU R?                  UUR                  R                  S9  U$ s  snn
f ! [2        R6                   a%  n[9        SUR0                   SU< SU 35      UeSnAff = f! [@         a7  nU RB                  (       a  [(        RD                  " SU S35         SnAgUeSnAff = f)a.  
Create a deployment object and add it to the model list

If the deployment is not active for the current environment, it is ignored

Returns:
- Deployment: The deployment object
- None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
r7  Ninput_cost_per_tokenr*  r  rm  
model_costr  moder  r:  zRouter: preserving existing mode=%s for shared backend key %s instead of the deployment-specified mode=%s (prevents alias registration from downgrading the shared backend mode).
responses/r  r  TzIgnoring deployment z% as it is not active for environment supported_environments	tag_regexz&Invalid regex in tag_regex for model 'z': u    — r   r   r  zError creating deployment: 1, ignoring and continuing with other deployments.r   )#r_   r\   rm   model_fieldsri  r   r~  r   r  r   rm  r*  r  r  register_modelrq  r   r   rP  r  r   r  rA  $deployment_is_active_for_environmentr  recompiler5  r  _add_deploymentto_json _add_model_to_list_and_index_mapr  r   r#  )r[  r  r  r  r  r   r  r  r  _custom_pricing_fieldsr  r  _shared_model_info_existing_shared_mode_deployment_mode_is_responses_to_chat_downgrade_would_clear_existing_mode_backend_alias_cost_stripped_model_name
_tag_regexpatternr  r   r  s                           r   _create_deploymentRouter._create_deployment  s    v	-;-No-NN# !&-&	J 4@@EEG,,007C)3)B)B5)IK& H 56B55*",";";"A"A(2(A(A(U(U 6  ",,//H#&&   %3399K((<<H--AACG+U  &@%L%L%Q%Q%S"!,!2!2!4"!48W!4  " Xd^W%7%7%;%;K%LMSQSc&k "  255f= &4S9IV9S , &T1N6F$6N ' /2L#/)110 .#( .C6* $/"C{*'2':':<'L$<N#$89"".AB 99Z9P &--*:+@+@*AAfgqg|g|  ~V  hW  gX  Y  $2266{CIrJ%JJw' & ---DJ&&D&9E11j&;&;&>&> 2  K"r xx $@AVAV@WWZ[bZeejknjop  	..%//1!4ef 	si   AM1 DM1 )L/8L/>D(M1 ')M1 L5'AM1 /M1 5M.	 M))M..M1 1
N2;+N-+N--N2c                    UR                   R                  S5      (       a  gUR                   R                  S5      (       a  gUR                   R                  S5      (       a  gUR                   R                  S5      (       a  gg)a  
Check if the deployment is an auto-router deployment (semantic router).

Returns True if the litellm_params model starts with "auto_router/"
but NOT "auto_router/complexity_router" or "auto_router/adaptive_router"
(which use the complexity-router and adaptive-router strategies).
auto_router/complexity_routerFauto_router/adaptive_routerauto_router/quality_routerzauto_router/Tr   r  r[  r   s     r   _is_auto_router_deployment!Router._is_auto_router_deployment|  sr     **+JKK**+HII**+GHH**>::r   c           	         SSK Jn  UR                  R                  nUR                  R                  nUc  Uc  [        S5      eUR                  R                  nUc  [        S5      eUR                  R                  nUc  [        S5      eU" UR                  UUUUU S9nUR                  U R                  ;   a  [        SUR                   S	35      eXpR                  UR                  '   g)
zy
Initialize the auto-router deployment.

This will initialize the auto-router and add it to the auto-routers dictionary.
r   )r   Nzzauto_router_config_path or auto_router_config is required for auto-router deployments. Please set it in the litellm_paramszfauto_router_default_model is required for auto-router deployments. Please set it in the litellm_paramszhauto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params)r  auto_router_config_pathauto_router_configdefault_modelembedding_modelrH  zAuto-router deployment 3 already exists. Please use a different model name.)
/litellm.router_strategy.auto_router.auto_routerr   r   r%  r&  r  auto_router_default_modelauto_router_embedding_modelr  r  )r[  r  r   r%  r&  r'  r(  autor_routers           r   init_auto_router_deployment"Router.init_auto_router_deployment  s    	O %%== 	  -7,E,E,X,X"*/A/I M 
 %%?? 	  x 
 %%AA 	 "z  $.!,,$;1'+$($
   D$5$55)**?*?)@@st  4@*//0r   c                 F    UR                   R                  S5      (       a  gg)z
Check if the deployment is a complexity-router deployment.

Returns True if the litellm_params model starts with "auto_router/complexity_router"
r  TFr   r!  s     r    _is_complexity_router_deployment'Router._is_complexity_router_deployment  s"     **+JKKr   c                    SSK Jn  UR                  R                  nUR                  R                  nUcB  U(       a;  UR                  S0 5      nUR                  S5      =(       d    UR                  S5      nUc  [        S5      eU" UR                  UU US9nUR                  U R                  ;   a  [        S	UR                   S
35      eX`R                  UR                  '   g)z
Initialize the complexity-router deployment.

This will initialize the complexity-router and add it to the complexity-routers dictionary.
r   r   NtiersMEDIUMSIMPLEzcomplexity_router_default_model is required for complexity-router deployments, or configure tiers in complexity_router_config. Please set it in the litellm_params)r  r'  rH  complexity_router_configzComplexity-router deployment r)  )	;litellm.router_strategy.complexity_router.complexity_routerr   r   r7  complexity_router_default_modelr~  r  r  r  )r[  r  r   r7  r'  r4  complexity_routers          r   !init_complexity_router_deployment(Router.init_complexity_router_deployment  s    	

 %%>> 	!
 %%EE 	
  %=,00"=E!IIh/F599X3FM f 
 /?!,,'$(%=	/
   D$;$;;/
0E0E/FFyz  :K
 5 56r   c                 8    UR                   R                  S5      $ )zUTrue when this deployment opts in via the `auto_router/adaptive_router` model prefix.r  r   r!  s     r   _is_adaptive_router_deployment%Router._is_adaptive_router_deployment  s    ##../LMMr   c           	         SSK Jn  [        R                  [        R                  [        R
                  [        R                  [        R                  4 H"  n[        R                  R                  X!5        M$     U R                  =(       d    /  GHM  n[        U[        5      (       a  UR                  S5      OUR                  nU(       a2  [        U[        5      (       a  UR                  S5      OUR                  OSnU(       a  UR!                  S5      (       d  M  [        U[        5      (       a  UR                  S5      OUR"                  nU(       a  U(       d  M  X`R$                  ;   a  M  ['        U[        U[        5      (       d  UO
[)        S0 UD6[        U[        5      (       a  UR                  S5      OUR*                  S	9nU R-                  US
9  GMP     g)zLocate every adaptive-router deployment in the finalized model_list and
build an AdaptiveRouter for each. Safe no-op when none are configured.
Idempotent: skips any deployment whose model_name is already initialized.r   AdaptiveRouterPostCallHookr   r   Nr  r  r*  r7  r  r   )-litellm.router_strategy.adaptive_router.hooksrB  r  rw  rE  rK  r@  rH  rB  remove_callbacks_by_typer   r   rP  r~  r   r   r  r  r  r\   r_   r*  init_adaptive_router_deployment)r[  rB  _cb_listr  lplp_modelr  r  s           r   '_finalize_adaptive_router_if_configured.Router._finalize_adaptive_router_if_configured  s   	

 $$$$++++
H ,,EE
 __**E eT** 		*+))  LNJr4$8$8bhhSW  !4!45R!S!S+5eT+B+B		,'HXHX  R222#%(T22B8L8L "%.. IIl+))
J 00J0G; +r   c                    SSK Jn  SSKJn  SSKJnJn  UR                  R                  nUc  [        S5      eU" S0 UD6n0 n0 n	UR                   GHJ  n
U R                  R                  U
/ 5      nU(       d  M)  U R                  =(       d    / US      n[        U[        5      (       a  UR                  S5      OUR                   n[        U[        5      (       a  UOU(       a  UR#                  5       O0 nUR                  S5      nUb
  U" S0 UD6X'   [        U[        5      (       a  UR                  S	5      OUR                  n[        U[        5      (       a  UOU(       a  UR#                  5       O0 nUR                  S
5      nUc  GM=  [%        U5      X'   GMM     UR&                  U R(                  ;   a  [        SUR&                   S35      eU" UR&                  UUU	S9nUU R(                  UR&                  '   [*        R,                  R/                  U" US95        [0        R2                  " SUR&                  [5        UR                  5      5        g)a  
Build an AdaptiveRouter instance for this deployment and register its
post-call hook. Multiple adaptive routers can coexist on a single Router,
keyed by `deployment.model_name`.

`model_to_prefs` and `model_to_cost` are derived from the OTHER models
already registered in `self.model_list` whose `model_name` appears in
`available_models`. Models not yet registered fall back to defaults.
r   r   rA  )AdaptiveRouterConfigAdaptiveRouterPreferencesNzCadaptive_router_config is required for adaptive-router deployments.r*  adaptive_router_preferencesr   r  zAdaptive-router deployment r)  )router_nameconfigmodel_to_prefsmodel_to_cost)adaptive_routerz-AdaptiveRouter[%s] initialized with %d modelsr   )7litellm.router_strategy.adaptive_router.adaptive_routerr   rC  rB  litellm.types.routerrL  rM  r   adaptive_router_configr  r\  r!  r~  r   r   rP  r*  rS  r(  r  r  r  rB  rV  r   rQ  r  )r[  r  r   rB  rL  rM  
raw_configrP  rQ  rR  rs  indicesr  mimi_dict	prefs_rawrG  lp_dictcostrS  s                       r   rE  &Router.init_adaptive_router_deployment/   s   	
	
	

  ..EE
U  &3
3?A*, ++D;;??bIG&B
3A(21d(;(;|$B T**BB   $ABI$'@'M9'M$ -7q$,?,?'(QEUEUB T**BB  ;;56D&+Dk#) ,,   D$9$99-j.C.C-DDwx  )"--)'	
 8Gj334((==&G	
 	"";!!''(	
r   c                 F    UR                   R                  S5      (       a  gg)z
Check if the deployment is a quality-router deployment.

Returns True if the litellm_params model starts with "auto_router/quality_router".
r  TFr   r!  s     r   _is_quality_router_deployment$Router._is_quality_router_deployment}   s"     **+GHHr   c                 v   SSK Jn  UR                  R                  nUR                  R                  nUc  U(       a  UR                  S5      nUc  [        S5      eU" UR                  UU US9nUR                  U R                  ;   a  [        SUR                   S35      eXPR                  UR                  '   g)	z
Initialize the quality-router deployment.

Resolves the default model from either `quality_router_default_model` or
`quality_router_config["default_model"]`, then instantiates the
QualityRouter and stores it in `self.quality_routers`.
r   r   Nr'  zquality_router_default_model is required for quality-router deployments, or set default_model in quality_router_config. Please configure it in the litellm_params)r  r'  rH  quality_router_configzQuality-router deployment r)  )	5litellm.router_strategy.quality_router.quality_routerr   r   rc  quality_router_default_modelr~  r  r  r  )r[  r  r   rc  r'  quality_routers         r   init_quality_router_deployment%Router.init_quality_router_deployment   s    	

 %%;; 	
 %%BB 	  %:155oFM k 
 )6!,,'$("7	)
   D$8$88,Z-B-B,CCvw  7EZ223r   c           	      b   UR                   b   SUR                   ;  d  UR                   S   c  g[        SS9nUc  [        S5      eU[        ;  a  [        S[         SU 35      eUR                   S    H%  nU[        ;  d  M  [        S[         SU S	U 35      e   X!R                   S   ;   a  gg
)a  
Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments

Requires `LITELLM_ENVIRONMENT` to be set in .env. Valid values for environment:
    - development
    - staging
    - production

Raises:
- ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values
- ValueError: If supported_environments is not set in model_info or not one of the valid values
r  TLITELLM_ENVIRONMENT)secret_namezPSet 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .envz#LITELLM_ENVIRONMENT must be one of z. but set as: z&supported_environments must be one of z for deployment: F)r*  r   r  rV   )r[  r  litellm_environment_envs       r   r
  +Router.deployment_is_active_for_environment   s     !!)'z/D/DD$$%=>F,9NO&b  &@@56P5QQ_`s_tu  ))*BCD55 <=W<XXfgkfll}  I  ~J  K  D "7"78P"QQr   c           	         [         R                  " U5      n/ U l        0 U l        0 U l        0 U l        0 U l        0 U l        0 U l        U R                  5         U R                  5         U GH  nUR                  S5      nUR                  S5      n[        U[        5      (       aU  UR                  5        HA  u  pg[        U[        5      (       d  M  UR!                  S5      (       d  M4  [#        U5      XV'   MC     UR                  S0 5      nSU;  a  U R%                  XE5      n	XS'   UR'                  SS 5      b=  [        US   [(        5      (       a%  US    H  n
XS'   U R+                  UUUUS9  M     GM  U R+                  UUUUS9  GM"     [,        R.                  " SU R1                  5        35        U Vs1 s H  oS   iM	     snU l        U R5                  5         g s  snf )	Nr  r   os.environ/r*  r  organization)r  r  r  r  z
Initialized Model List )r  r  r   r   r!  r"  r  r  r  "_invalidate_model_group_info_cache_invalidate_access_groups_cacher  r   rP  rq  r  r  r{   r8  r~  rA  r  r   r  get_model_namesr   rI  )r[  r   original_model_listr   r  r  r  r  r  _idorgr^  s               r   r%  Router.set_model_list   s   "mmJ702-02-02-  ""$//1,,. )E))L1K#ii(89O/400+113DA!!S))all=.I.I-7]* 4 !&		, ;K ;&--kK$'D!"">48D/J J +>:C69N3++(-$/(7$/	 ,  ; ''$) +$3 +	 ( 9 )F 	##'(<(<(>'?@	
 6@@ZlOZ@ 	446 As   9G c                    SS K nUR                  R                  nSnSU;   a*  UR                  S5      S   nU[        R
                  ;   a  SnU(       a	  UnS nS nS n	O[        R                  " UR                  R                  UR                  R                  SS 5      S9u  nnnn	U[        R                  ;  a)  [        R                  " U5      (       d  [        SU 35      eU R                  R                  UR                  R                  5        UR                  R                  c)  [        USS 5      b  [        US5      UR                  l        UR                  R                   c)  [        US	S 5      b  [        US	5      UR                  l        S
UR"                  ;   a}  U R$                  R'                  UR"                  UR)                  SS95        UR*                  R,                  (       a/  U R.                  R                  UR*                  R,                  5        UR*                  R                  S5      n
UR*                  R                  S5      nU
bZ  UbW  S
U;   aQ  XR0                  ;  a  [3        5       U R0                  U
'   U R0                  U
   R'                  XR)                  SS95        UR                  R                  S/ 5      =(       d    / nU Hq  nUR                  S0 5      nS HV  nX;   d  M
  X   R5                  S5      (       d  M$  X   R7                  SS5      nUR8                  R                  US5      X'   MX     Ms     Ub%  U R;                  UUUR                  R                  S9  U R=                  UR                  S9(       a  U R?                  US9  U RA                  UR                  S9(       a  U RC                  US9  U RE                  UR                  S9(       a  U RG                  US9  U$ )Nr   Fr  Trm  rH  zUnsupported provider - r|  r{  r   r   r  team_public_model_namedataSources
parameters)r@  r0  rp  r  )r  rm  r   )r   r  )$osr   r   r  r  r  ry   r~  provider_listr)   existsr  r  r3  r|  r  r{  r  r  add_patternr  r*  r  r  r  r   r  rA  environ'_initialize_deployment_for_pass_throughr"  r.  r1  r;  r`  rg  )r[  r  r}  r  is_prompt_management_modelr  rQ  rm  r  rJ  _team_id_team_public_model_namedata_sourcesdata_sourcer   	param_keyenv_names                    r   r  Router._add_deployment!  s    #1177%*"-"/"5"5c":1"="g&W&WW-1*% #F"&"OH (( //55$.$=$=$A$A)4%# $7+@+@@,334GHH"9:M9N OPP 	$$Z%>%>%D%DE
 %%))1
E40<,3J,FJ%%) %%))1
E40<,3J,FJ%%) *''' ++%%z'9'9t'9'L $$''44;;J<Q<Q<T<TU((,,Y7","7"7";";<T"U '3..8886H6J))(3%%h/;;');););)N
 "0044]BGM2'K __\26F0	&6+<+G+G+V+V%088KH(*

x(DF%	 1 ( *88%$7 //55 9  ***:S:S*T,,
,C
 00%44 1 
 22j2I --Z=V=V-W//:/Fr   c                 F   UR                   R                  SL Ga  SSKJn  UR                   R                  b+  [
        R                  " UR                   R                  5      nO0 nUS:X  a  UR                  S5      =(       d    UR                   R                  nUR                  S5      =(       d    UR                   R                  nUR                  S5      =(       d    UR                   R                  nUb  Uc  [        S	5      eUR                  UUUS
9  OUR                  S5      =(       d    UR                   R                  n	UR                  S5      =(       d    UR                   R                  n
U
c  [        R                   " SUU5        gUR#                  UU	U
S9   g)a  
Optional: Initialize deployment for pass-through endpoints if `deployment.litellm_params.use_in_pass_through` is True

Each provider uses diff .env vars for pass-through endpoints, this helper uses the deployment credentials to set the .env vars for pass-through endpoints
Tr   )passthrough_endpoint_routerN	vertex_aivertex_projectvertex_locationvertex_credentialsz]vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints.)
project_idlocationr  rJ  r  zSkipping pass-through credential setup for deployment model=%s, custom_llm_provider=%s; no api_key set. Providers like bedrock resolve credentials at request time.)rm  rJ  r  )r   use_in_pass_through>litellm.proxy.pass_through_endpoints.llm_passthrough_endpointsr  rN  r%   get_credential_valuesr~  r  r  r  r  add_vertex_credentialsrJ  r  r   r  set_pass_through_credentials)r[  r  rm  r   r  credential_valuesr  r  r  rJ  r  s              r   r  .Router._initialize_deployment_for_pass_through!  s    $$88D@ ((@@L$6$L$L--EE%! %'!"k1%))*:; @!00?? 
 &))*;< A!00@@  
 &))*>? D!00CC #
 ")_-D$w  ,BB-,'9 C  &))*5 :!0099 
 &)))4 9!0088  ?)// ~+
 +HH(;%# I 
 r   c                    UR                   R                  nU(       a  U R                  U5      (       a  gUR                  SS9nU R	                  US9  UR                   R                  SS9n[        R                  R                  5        H'  nUR                  R                  U5      nUc  M#  XdU'   M)     UR                  S5      b=  [        R                  UUR                  R                  UR                  R                  S9  UR                   R                  nUb  [        R                   " Xt0S9  UR                  R                  nUR                  R                  b  UR                  R                  S-   U-   n[        R                  R                  5       n	UR#                  5        V
Vs0 s H  u  pX;  d  M  X_M     nn
nX0nS	U;   a  UR%                  S	S
5      nXU'   [        R                   " US9  U R'                  X1R                   R                  S9  U R(                  R+                  UR,                  5        U R/                  US9  U$ s  snn
f )z
Parameters:
- deployment: Deployment - the deployment to be added to the Router

Returns:
- The added deployment
- OR None (if deployment already exists)
NTr   r  r  r  r   r  r  r  r  )r*  r  r  r  r  rS  rm   r  ri  r   r~  r   r  r   rm  r  r	  rq  rA  r  r   r  r  _sync_deployment_budget_config)r[  r  _deployment_model_id_deployment_model_info_dictr  field_valuer;  r  r  r  r  r  r  r  s                  r   add_deploymentRouter.add_deployment!  sB     *4477D$5$56J$K$K !((d(;
3!+!6!6!A!At!A!T/<<AACE$3377>K&*5' D
  67C11+(77==$.$=$=$Q$Q 2  )),,	 ""y.KL !//55$$88D))==CkQ  "<!H!H!M!M!O-335
5TQ9XDAD5 	 
  +?;&#.#6#6|R#H 8J 45*=> 	--(=(=(@(@ 	. 	
 	Z223++z+B
s   ,I;Ir  removal_idxc                    U R                   R                  5        H  u  p4XB:  d  M  US-
  U R                   U'   M      XR                   ;   a  U R                   U	 [        U R                  R                  5       5       Ho  u  pV/ nU H6  nXB:X  a  M
  XB:  a  UR	                  US-
  5        M%  UR	                  U5        M8     [        U5      S:  a  XpR                  U'   Mb  U R                  U	 Mq     [        U R                  R                  5       5       Ho  u  p/ nU H6  nXB:X  a  M
  XB:  a  UR	                  US-
  5        M%  UR	                  U5        M8     [        U5      S:  a  XpR                  U'   Mb  U R                  U	 Mq     g)z
Helper method to update deployment indices after a deployment has been removed from model_list.

Parameters:
- model_id: str - the id of the deployment that was removed
- removal_idx: int - the index where the deployment was removed from model_list
r~   r   N)r   rq  rA  r!  r3  r  r"  )	r[  r  r  r+  r  r  rX  updated_indicesr0  s	            r   (_update_deployment_indices_after_removal/Router._update_deployment_indices_after_removal."  st    #'"G"G"M"M"OM GJQw55mD #P <<<55h? $((M(M(S(S(U#VJ O%&#**373 $**3/  ?#a'DS55jA99*E% $W* !!F!F!L!L!NOLC O%&#**373 $**3/  ?#a'=L55c:99#>% Pr   r  c                 l   UR                  S5      =(       d    0 R                  S5      nUR                  S5      =(       d    0 R                  S5      nU(       a[  U(       aS  X44nXPR                  ;  a  / U R                  U'   X R                  U   ;  a  U R                  U   R                  U5        gggg)z
Helper to update team_model_to_deployment_indices for a single deployment.

Parameters:
- model: dict - the deployment to index
- idx: int - the index in model_list
r*  r  rz  N)r~  r"  r3  )r[  r   r  r  rz  r0  s         r   _update_team_model_indexRouter._update_team_model_indexj"  s     99\*0b55i@"'))L"9"?R!D!D$"
 -3C???=?55c:??DD55c:AA#F E	 .7r   c                    [        U R                  5      nU R                  R                  U5        U R                  5         U R	                  5         Ub  X0R
                  U'   O6UR                  S0 5      R                  S5      b  X0R
                  US   S   '   UR                  S5      nU(       a<  X@R                  ;  a  / U R                  U'   U R                  U   R                  U5        U R                  X5        g)z
Helper method to add a model to the model_list and update both indices.

Parameters:
- model: dict - the model to add to the list
- model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"]
Nr*  r  r  )	r  r   r3  rr  rs  r   r~  r!  r  )r[  r   r  r  r  s        r   r  'Router._add_model_to_list_and_index_map}"  s     $//"u%//1,,. >A11(;YY|R(,,T2>OR11%2Ed2KL YY|,
!F!FFDF55jA11*=DDSI 	%%e1r   c                 n    UR                   R                  =(       d    SnU R                  US9nUb  UR                  UR                  :X  a  UR                   UR                   :X  a  gSnUR                   R                  nU R                  nXV;   aQ  Xe   nUbJ  U R
                  R                  U5        U R                  5         U R                  5         U R                  XTS9  U R                  US9  U$ ! [         a7  nU R                  (       a  [        R                  " SU S35         SnAgUeSnAff = f)z
Add or update deployment
Parameters:
- deployment: Deployment - the deployment to be added to the Router

Returns:
- The added/updated deployment
r  r  Nr  r  r  zError upserting deployment: r  )r*  r  r  r   r   r   r  rr  rs  r  r  r  r   r   r  )r[  r  r  _deployment_on_routerr  r+  deployment_fast_mappingr  s           r   r9  Router.upsert_deployment"  sA   +	#-#8#8#;#;#Ar :>:M:M- ;N ;! %0 --1F1U1UU"--1F1Q1QQ   .2 * 5 5 8 8*.*O*O' ;"9"HK".++K8??A<<>EE%2 F 
 :6 	..%++21#5fg 	s%   A%C3 (B
C3 3
D4=+D/-D//D4r  c                 H   SnXR                   ;   a  U R                   U   n Ubn  U R                  R                  U5      nU R                  5         U R	                  5         U R                  XS9  U R                  5       nUb  UR                  US9  U$ g! [         a     gf = f)z
Parameters:
- id: str - the id of the deployment to be deleted

Returns:
- The deleted deployment
- OR None (if deleted deployment not found)
Nr  r  )	r   r   r  rr  rs  r  r  unregister_deployment_budgetr  )r[  r  deployment_idxr  _budget_limiters        r   delete_deploymentRouter.delete_deployment"  s     666!BB2FN	)**>:779446== >  #'"L"L"N".#@@"@M 		s   A0B 
B! B!c                     U R                   b  U R                   $ U R                  (       a/  U R                   H  n[        U5      [        L d  M  Xl         Us  $    g)z
Return the router's deployment-budget callback.

Uses exact-type matching so proxy subclasses (e.g. virtual-key model budgets)
registered on litellm.callbacks are not mistaken for router deployment budgets.
N)rN  r   r   r*   )r[  _cbs     r   r  ,Router._get_router_deployment_budget_limiter"  sR     $$0,,,""..9 4403-J / r   c                     UR                   R                  S5      S L=(       a<    UR                   R                  S5      S L=(       a    UR                  R                  S L$ )N
max_budgetbudget_duration)r   r~  r*  r  )r[  r  s     r   _deployment_has_budget_limits$Router._deployment_has_budget_limits#  sX    %%)),7tC 5))--.?@L5%%((4	
r   c                 ,   UR                   R                  nUc  g U R                  5       nU R                  US9(       d  Ub  UR	                  US9  g Uc   U R                  S/S9  U R                  5       nUb  UR                  UR                  SS9S9  g g )Nr  r  r   )r   Tr   )r*  r  r  r  r  rT  register_deployment_budgetr  )r[  r  r  r  s       r   r  %Router._sync_deployment_budget_config#  s    ((++DDF11Z1H*<<h<O"--*B)C .  #HHJO&66%--4-@ 7  'r   c                    XR                   ;   ax  U R                   U   nU R                  U   n[        U[        5      (       a  [	        S0 UD6$ [        U[        5      (       a  U$ [        SR                  [        U5      5      5      eg)zT
Returns -> Deployment or None

Raise Exception -> if model found in invalid format
zModel invalid format - {}Nr   )r   r   r   rP  r\   r  rR  r   )r[  r  r  r   s       r   r  Router.get_deployment"#  sy     <<<77ACOOC(E%&&!*E**E:.. ; B B4; OPPr   c                     U R                  US9nUb  U R                  U5      (       a  g[        S0 UR                  R	                  SS9D6R	                  SS9$ )z
Returns -> dict of credentials for a given model id.

Returns None if the deployment is paused via `LiteLLM_ProxyModelTable.blocked`,
so file/batch/passthrough callers that resolve credentials directly cannot keep
using a paused deployment.
r  NTr   r   )r  _is_deployment_blockedrZ   r   rS  )r[  r  r  s      r   get_deployment_credentials!Router.get_deployment_credentials5#  se     ((((;
!<!<Z!H!H& 
''222E

*$*
'	(r   model_group_namec                 &   XR                   ;   a  U R                   U   nU(       al  U R                  US      n[        U[        5      (       a  [	        S0 UD6$ [        U[        5      (       a  U$ [        SR                  [        U5      5      5      eg)z
Returns -> Deployment or None

Raise Exception -> if model found in invalid format

Optimized with O(1) index lookup instead of O(n) linear scan.
r   zModel Name invalid - {}Nr   )r!  r   r   rP  r\   r  rR  r   )r[  r  rX  r   s       r   "get_deployment_by_model_group_name)Router.get_deployment_by_model_group_nameD#  s     DDD;;<LMG
3eT**%...z22 L#$=$D$DT%[$QRRr   c                    U R                  US9nUc  U R                  US9nUch  U R                  R                  U5      =(       d    / nU(       a=  US   n[	        U[
        5      (       a  [        S0 UD6nO[	        U[        5      (       a  UnUb  U R                  U5      (       a  g[        S0 UR                  R                  SS9D6R                  SS9nUR                  R                  b  [        R                  " UR                  R                  5      nU(       d.  [        R                  " SUR                  R                   S35        UR!                  U5        UR#                  S	S5        UR                  R$                  (       a  UR                  R$                  US
'   U$ SUR                  R&                  ;   a-  UR                  R&                  R)                  S5      S   US
'   U$ SUS
'   U$ )a:  
Get API credentials and provider info from a model name in model_list.
Useful for passthrough endpoints (files, batches, etc.) that need credentials.

This method tries to find a deployment by model_id first, and if not found,
it tries to find by model_group_name (model_name).

Args:
    model_id: Model ID or model name from model_list (e.g., "gpt-4o-litellm")

Returns:
    Dictionary containing api_key, api_base, custom_llm_provider, etc.
    Returns None if model not found, or if the resolved deployment is
    paused via `LiteLLM_ProxyModelTable.blocked` (so passthrough callers
    cannot bypass an admin pause by resolving credentials directly).

Example:
    credentials = router.get_deployment_credentials_with_provider("gpt-4o-litellm")
    # Returns: {"api_key": "sk-...", "custom_llm_provider": "openai", ...}
r  N)r  r   Tr   zCredential 'z' not found in credential_listrN  rm  r  r+  r   )r  r  r  router   rP  r\   r  rZ   r   rS  rN  r%   r  r   r  r  r  rm  r   r  )r[  r  r  potential_wildcard_modelsdeployment_dictcredentialsr  s          r   r  /Router.get_deployment_credentials_with_provider\#  s   0 ((((;
 @@!) A J
 (,(;(;(A(A((K(Qr%(";A">ot44!+!>o!>J<<!0J!<!<Z!H!H . 
''222E

*$*
' 	
 $$<<H 2 H H))AA! %%--":#<#<#T#T"UUst 01OO5t< $$88))== -.  J--3331;1J1J1P1P1V1V22K-.  2:K-.r   r\   rz  c                     g r   r   r[  r  rz  r  s       r   r  Router.get_router_model_info#  s     	r   c                     g r   r   r  s       r   r  r  #  s     	r   c           	      r   Ub  U R                  US9nUb  UnUc  [        S5      eUR                  S5      =(       d    0 R                  SS5      nUc*  UR                  S5      =(       d    0 R                  SS5      nUnUR                  S5      n[        U[        5      (       a  UnOs[        U[
        5      (       a  SU;   a  [	        S0 UD6nOL[        S[        U5      R                   S	UR                  S5      =(       d    0 R                  S
S5       35      e[        R                  " UR                  US9u  p  nU
S:X  a  Uc  [        R                  " SU	 S35        OU
S:w  a  U	nSU;   a  U R                  R                  U5      nUb  U H  n UR                  S5      =(       d    0 R                  S
5      UR                  S5      =(       d    0 R                  S
5      :X  a+  UR                  S5      =(       d    0 R                  S5      n  OM     Uc  U	nUR!                  SR#                  U
5      5      (       d  SR#                  X5      nOUn[        R$                  " US9nUR                  S5      =(       d    0 nUb  UR'                  [)        [*        U5      5        U$ ! [         a     GM%  f = f)a  
For a given model id, return the model info (max tokens, input cost, output cost, etc.).

Augment litellm info with additional params set in `model_info`.

For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.

Returns
- ModelInfo - If found -> typed dict with max tokens, input cost, etc.

Raises:
- ValueError -> If model is not mapped yet
Nr  zDeployment not foundr*  
base_modelr   r   z.Deployment missing valid litellm_params. Got: z, deployment_id: r  unknownr   r   azurez Could not identify azure model 'z'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-modelsr   z{}/z{}/{}r  r   )r  r  r~  r   r_   rP  r   r   r  ry   r   r   r5  r  r  r  r  rR  r  r  r   rp   )r[  r  rz  r  r  r  r   litellm_params_datar   rQ  rm  r  potential_modelspotential_modelmodel_info_namer*  user_model_infos                    r   r  r  #  s   & >--r-:K&(
344 !nn\28b==lDQ
$..)9:@bEEdJ  )nn-=>)>::0N+T22wBU7U+B.ABN01::; <"#->>,#?#E2"J"J4QZ"[!\^ 
 -4,D,D &&)-
)Q ')j.@!''26(  ;Z  [ !G+Ee|#'#6#6#<#<=P#Q #/+;	! / 3 3L A GRLLTR *| < B!c$i ( %4$7$78H$I$OR"%#g, !& !& ( ,< = E-@ ABB%nn-@HO#O++/B
 %..6<"!d9o>?-  ) ! !s   A=J''
J65J6c                 ^    XR                   ;   a  U R                   U   nU R                  U   $ g)z
For a given model id, return the model info

Returns
- dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
- None: could not find deployment in list

Optimized with O(1) index lookup instead of O(n) linear scan.
N)r   r   )r[  r  r  s      r   r  Router.get_model_info$  s2     66677;C??3''r   c                 P    U R                  US9nUc  gUS   nU R                  US9$ )zD
Return list of all models in the same model group as that model id
)r  Nr  r  )r  r  )r[  r  r*  r  s       r   get_model_groupRouter.get_model_group,$  s>    
 ((B(/
-
""j"99r   c           	      6   SSK Jn  SnSnSn [        R                  R	                  U5      n [        R                  " US9n UbN  UR	                  SS5      nUb9  [        R                  " US9nUb"  U=(       d    0 nU" [        [        U5      U5      nUb8  Ub5  [        [        U" [        [        U5      R                  5       U5      5      nU$ Ub  UnU$ Ub  [        [        U5      nU$ ! [
         a     Nf = f! [
         a     Nf = f! [
         a     Nf = f)z
For a given model id, return the model info

1. Check if model_id is in model info
2. If not, check if litellm model name is in model info
3. If not, return None
r   )_update_dictionaryNr  r  )litellm.utilsr  r  r  r~  r  r  r   rP  rp   r  )	r[  r  r  r  r*  custom_model_infolitellm_model_name_model_infor  base_model_infos	            r   get_deployment_model_info Router.get_deployment_model_info8$  sW    	5*.
,0=A%	 ' 2 2 6 6x @	,3,B,B,T)
	 ,.22<F
)&-&<&<:&NO&2,=,C),> 7--) (-J-V "<=BBD%J  +66J  * i):;JY  		
  		"  		s5   C+ C; AD +
C87C8;
DD
DDuser_facing_model_group_namec                 
   SnSnSnSnU R                  US9nUc  gU GH  nSn	SU;   a  US   U:X  a  Sn	O$SU;   a  U R                  R                  U5      b  Sn	U	(       d  ME  [        S 0 US   D6n
U
R                  nUR                  S0 5      nUR                  S0 5      nSnUc  UR                  SS5      nUc  UR                  SS5      nUc  UR                  SS5      nSnUc  UR                  S	S5      nUc  UR                  S	S5      nUc  UR                  S	S5      n UR                  S
S5      nUb  U R                  XR                  S9nOSn Su  nn [        R                  " U
R                  U
R                  S9u  nn  nUcr  [        R$                  " UUS9nUc  / nUR                  S0 5      nUR                  SS5      nUR                  S5      nUR                  S5      n['        USSSUUUUUSS9
nUc  [)        S 0 UU/S.UED6nGOUUR*                  ;  a  UR*                  R-                  U5        UR                  SS5      b0  US   b*  UR.                  b  US   UR.                  :  a
  US   Ul        UR                  SS5      b0  US   b*  UR0                  b  US   UR0                  :  a
  US   Ul        UR                  SS5      b<  UR2                  b%  US   =(       d    SUR2                  =(       d    S:  a
  US   Ul        UR                  SS5      b<  UR4                  b%  US   =(       d    SUR4                  =(       d    S:  a
  US   Ul        UR                  SS5       b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl         UR                  SS5      b  US   b
  US   Ul!        UR                  SS5      b  Uc  UR                  S5      nUR                  S	S5      b  Uc  UR                  S	5      nUb	  Uc  SnXM-  nUc  GM  Uc  SnX^-  nGM     Ub  Ub  XCl"        Ub  XSl#        Ub  UUl        U$ ! [         a    Sn GNf = f! [        R                  R                   a9  n[        R                  " SR!                  [#        U5      5      5         SnAGNSnAff = f)!z
For a given model group name, return the combined model info

Returns:
- ModelGroupInfo if able to construct a model group
- None if error constructing model group info
Nr  Fr  Tr   r*  r{  r|  r  )r  r  )r  r  rH  z.litellm.router.py::get_model_group_info() - {}r  r:  r  output_cost_per_token)
r0  
max_tokensmax_input_tokensmax_output_tokensr  r  litellm_providerr  supported_openai_paramssupports_system_messages)r  	providersr  r  g        "supports_parallel_function_callingsupports_visionsupports_function_callingsupports_web_searchsupports_url_contextsupports_reasoningr  r   r   )$r  r  r  r_   #configurable_clientside_auth_paramsr~  r  r   r  r  ry   rm  
exceptionsBadRequestErrorr   r5  rR  r  get_supported_openai_paramsModelMapInfora   r  r3  r  r  r  r  r  r  r  r  r  r  r  r{  r|  )r[  r  r  model_group_info	total_tpm	total_rpmr  r   r   is_matchr   model_litellm_paramsmodel_info_dict_deployment_tpm_deployment_rpmr  r*  r  r  r  r  r  db_model_infor  r  r  s                             r   _set_model_group_infoRouter._set_model_group_infox$  sj    6:#'	#'	SW+((K(@
EH%%*=*L%''--k:F+Fe4D.EFN BB 0
 $)99-=r#B #iib9O .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B	"*..tT:'!%!?!?!)6J6J "@ "J "&J
 +1'M<4;4L4L(..(6(J(J51|Q !*1*M*M'\+' +2.0+ !&		, ;$((8'4'8'89O'P$(5(9(9:Q(R%)##%)&*)=*?%1,C-1
  '#1 $'C&2^ %$   '7'A'AA$..55lCNN#5t<H"#56B(99A%&89*;;< 9CCU8V$5NN#6=I"#67C(::B%&9:*<<= :DDW9X$6>>"8$?K$99A"#9:Ac'<<CE =G.=$9 >>"94@L$::B"#:;Bs'==DF >H/>$: NN#GN "#GHDPJN$GNN#4d;GJWhLimqLq7;$4NN#>EQ"#>?4GAE$>NN#8$?K"#89TA;?$8NN#94@L"#9:dB<@$9 NN#7>JzZnOoswOw:>$7NN#<dCO"#<=I?I1@$< >>%.:?V&0nnU&;O>>%.:?V&0nnU&;O*$ !I,	*$ !I,	K  L '$'0$$'0$ 3>7 !D  A  "!
" %%55 %++DKKCPQFS s0   $.S#S#.S6#S32S36U.UUc                     XR                   ;   a\  U R                   U   n[        U[        5      (       a  UnO%[        U[        5      (       a  US   SL a  gUS   nOgU R	                  UUS9$ U R	                  XS9$ )z
For a given model group name, return the combined model info

Returns:
- ModelGroupInfo if able to construct a model group
- None if error constructing model group info or hidden model group
hiddenTNr   )r  r  )r   r   r  rP  r  )r[  r  r  _router_model_groups       r   get_model_group_infoRouter.get_model_group_info`%  s     000))+6D$$$&*#D$''>T)*.w-'--/-8 .   ))# * 
 	
r   c           	      *  #    [        5       nUR                  S5      n/ n/ nU R                  US9nUc  gU H  nUR                  S0 5      R                  S5      nUS   R                  S5      n	Ub  U	c  M@  UR	                  [
        R                  R                  R                  UU	US	95        UR	                  [
        R                  R                  R                  UU	US	95        M     XE-   n
U R                  R                  U
S
9I Sh  vN nUc  gUS[        U5       nU[        U5      S nSnUb)  U H#  n[        U[        5      (       d  M  Uc  SnX-  nM%     SnUb*  U H$  n[        U[        5      (       d  M  Uc  SnUU-  nM&     UU4$  N7f)z
Returns current tpm/rpm usage for model group

Parameters:
- model_group: str - the received model name from the user (can be a wildcard route).

Returns:
- usage: Tuple[tpm, rpm]
r~  r  Nr  r*  r  r   r   )r  r   r  )ri  r   )r|   r  r  r~  r3  rd   r  r  rR  r  r   async_batch_get_cacher  r   r4  )r[  r  r  r  tpm_keysrpm_keysr   r   r  r  combined_tpm_rpm_keyscombined_tpm_rpm_valuestpm_usage_listrpm_usage_list	tpm_usager  	rpm_usages                    r   get_model_group_usageRouter.get_model_group_usage%  s     
 ! ((K(@
E %		, ; ? ? EB+01A+B+F+F,M z]2OO##))00'#1 1  OO##))00'#1 1   * !) 3(,

(H(H& )I )
 #
 #*)@3x=)Q)@X)Q $(	%#a%% ($%	NI	 $ $(	%#a%% ($%	NI	 $
 )##3#
s   D	FFAF*F?F)maxsizec                 $    U R                  U5      $ )z
Cached version of get_model_group_info, uses @lru_cache wrapper

This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request
)r  )r[  r  s     r   _cached_get_model_group_info#Router._cached_get_model_group_info%  s     ((55r   c                 Z  #    U R                  U5      nUb  UR                  b  UR                  nOS nUb  UR                  b  UR                  nOS nUc  Uc  0 $ U R                  U5      I S h  vN u  pV0 nUb  UU=(       d    S-
  US'   X7S'   Ub  UU=(       d    S-
  US'   XGS'   U$  N:7f)Nr   x-ratelimit-remaining-tokenszx-ratelimit-limit-tokensx-ratelimit-remaining-requestszx-ratelimit-limit-requests)r   r{  r|  r  )r[  r  r  	tpm_limit	rpm_limitcurrent_tpmcurrent_rpmreturned_dicts           r   get_remaining_model_group_usage&Router.get_remaining_model_group_usage%  s     <<[I',<,@,@,L(,,II',<,@,@,L(,,II!2I)-)C)CK)P#P  <E q=M89 9B45 >G q?M:; ;D67 $Qs   A,B+.B)/;B+c                 h  #    [        U[        5      (       Ga  [        US5      (       Ga  [        UR                  [        5      (       Ga  UR                  R                  S0 5        UUR                  S   S'   UR                  S   n[        U[        5      (       a  UR                  S5      =(       d    0 O0 n[        U[        5      (       a  UR                  S5      OSn[        U[        5      (       a  UR                  S5      b  [        US   5      US'   UR                  S	5      b  [        US	   5      US
'   UR                  S5      b  [        US   5      US'   UR                  S5      b  [        US   5      US'   UR                  S5      b  [        US   5      US'   SU;  a  SU;  a}  Ubz  U R                  U5      I Sh  vN nSn[        USS5      n	U	b  [        U	SS5      =(       d    SnUSS.n
UR                  5        H   u  pUc  M
  XR                  US5      -
  XK'   M"     U$  Ng7f)z
Add the most accurate rate limit headers for a given model response.

## TODO: add model group rate limit headers
# - if healthy_deployments > 1, return model group rate limit headers
# - else return the model's rate limit headers
ro  r  zx-litellm-model-groupr   quality_router_decisionNrouted_modelzx-litellm-quality-router-modelquality_tierzx-litellm-quality-router-tier
routed_viazx-litellm-quality-router-viamatched_keywordz x-litellm-quality-router-keywordcomplexity_tierz#x-litellm-quality-router-complexityr#  r$  r   r\  r  r~   )r#  r$  )r   r   r_  ro  rP  r;  r~  r  r*  r  rq  )r[  r  r  r  r  r   decisionremaining_usagein_flight_tokensr\  in_flight_deltaheaderr  s                r   rW  Router.set_response_headers%  sq     x++"233822D99##../CRH  ##$89' "*!8!89M!N nd33  ##J/52  h-- 67 
 (D))
 <</;KN 0L&'GH <</;JM 0K&'FG <<-9IL .J&'EF << 12>MP !23N&'IJ << 12>PS !23Q&'LM
 /6HH4<NN+(,(L(L) # $% '48$'.una'H'MA$4D67#
 &5%:%:%<MF(5:=P=P"A> 6*2 &=
 7#s   GH2H0	AH2H2c                 ^   U R                   R                  5         U R                  R                  5         [        U5       Hj  u  p#UR	                  S5      nU(       a<  X@R                   ;  a  / U R                   U'   U R                   U   R                  U5        U R                  X25        Ml     g)z
Build model_name -> deployment indices mapping for O(1) lookups.

This index allows us to find all deployments for a given model_name in O(1) time
instead of O(n) linear scan through the entire model_list.
r  N)r!  clearr"  r  r~  r3  r  )r[  r   r  r   r  s        r   _build_model_name_indexRouter._build_model_name_indexU&  s     	--335--335#J/JC<0J%J%JJHJD99*E55jAHHM))%5 0r   c                 r   / U l         U R                  5         U R                  5         [        U5       H  u  p#UR	                  S0 5      nUR	                  S5      nUcG  UR	                  SS5      nUR	                  S0 5      nU R                  Xg5      nSU;  a  0 US'   XSS   S'   U R                  X5S9  M     g)z
Build model index from model list to enable O(1) lookups immediately.
This is called during initialization to avoid the race condition where
requests arrive before model_id_to_deployment_index_map is populated.
r*  r  Nr  r  r   r  )r   rr  rs  r  r~  r8  r  )r[  r   r  r   r*  r  r  r   s           r   '_build_model_id_to_deployment_index_map.Router._build_model_id_to_deployment_index_maph&  s     //1,,.!*-HA<4J!~~d+H "YY|R8
!&+;R!@22:Nu,*,E,',4l#D)111Q .r   exclude_team_modelsc                 "   / nUb  XR                   ;   as  U R                   U   nU H^  nU R                  U   nSU;   d  M  SUS   ;   d  M%  U(       a  US   R                  S5      (       a  MG  UR                  US   S   5        M`     U$ U R                  R                  5        Hg  nU R                  U   nU R                  U   nSU;   d  M)  SUS   ;   d  M4  U(       a  US   R                  S5      (       a  MV  UR                  U5        Mi     U$ )z
if 'model_name' is none, returns all.

Returns list of model id's.

Optimized with O(1) or O(k) index lookup when model_name provided,
instead of O(n) linear scan.
r*  r  r  )r!  r   r~  r3  r   ri  )r[  r  r@  idsrX  r  r   r  s           r   get_model_idsRouter.get_model_ids&  s    !BBB??
K"C OOC0E#u,|9L1L.53F3J3J93U3U$

5#6t#<= #" 
 !AAFFH;;HE,5(TU<5H-H*u\/B/F/Fy/Q/Q JJx( I 
r   candidate_idc                     XR                   ;   $ )ar  
O(1) membership check for a deployment ID without allocating large lists.

Note: Call sites may pass a variable named `model` when it actually
contains a deployment ID. This helper expects the deployment ID string.

Uses the existing `model_id_to_deployment_index_map` which is kept
in sync by `_build_model_id_to_deployment_index_map` and model-list
mutation helpers.
)r   )r[  rE  s     r   r  Router.has_model_id&  s     DDDDr   c                    U(       d  gXR                   ;   d  U R                  U5      (       a  U$ U R                  SS9nU(       d  gU H  nUR                  S0 5      nUR                  S5      nXQ:H  =(       dA    U=(       a    UR	                  SU 35      =(       d    U=(       a    UR	                  SU 35      nU(       d  M{  UR                  S5      nU(       d  M  Us  $    g)a  
Resolve model_name from model_id.

This method attempts to find the correct model_name to use with the router
so that litellm_params can be automatically injected from the model config.

Strategy:
1. First, check if model_id directly matches a model_name or deployment ID
2. If not, search through router's model_list to find a match by litellm_params.model
3. Return the model_name if found, None otherwise

Args:
    model_id: The model_id extracted from decoded video_id
             (could be model_name or litellm_params.model value)

Returns:
    model_name if found, None otherwise. If None, the request will fall through
    to normal flow using environment variables.
Nr  r   r   r  :r  )r   r  r  r~  endswith)r[  r  
all_modelsr  r   actual_modelmatchesr  s           r    resolve_model_name_from_model_id'Router.resolve_model_name_from_model_id&  s    ,  '''4+<+<X+F+FO ((D(9
$J'^^,<bAN)--g6L
 ( L J\%:%:Qxj>%JL J\%:%:Qxj>%J  w'^^L9
:%% %$ r   team_model_namer  c                    U R                  XS9nU(       d  gU Hj  nUR                  S0 5      R                  S5      U:X  d  M*  Uc;  UR                  S0 5      R                  S5      =(       d    UR                  S5      s  $ Us  $    g)a`  
Check if team_model_name resolves to team-specific deployments.

Returns the public model name (unchanged) so the router can find all
sibling deployments via team_id filtering, instead of collapsing to a
single internal model_name.

When team_model_name is None (e.g. vector store / file endpoints that
don't include a model in their request), returns the first matching
team deployment's team_public_model_name so the router can inject BYOK
credentials from the team-scoped deployment.

Returns:
- str: the team_model_name if team deployments exist for this team
- None: if no team-specific model is found
r  Nr*  r  rz  r  )r  r~  )r[  rP  r  r  r   s        r   map_team_modelRouter.map_team_model&  s    & $$$QEyyr*..y9WD"* !99\26::0 1<01 '&  r   c                 B   Ub[  UR                  S5      =(       d    0 R                  S5      U:X  a.  UUR                  S5      =(       d    0 R                  S5      :X  a  gUb>  US   U:X  a5  UR                  S5      =(       d    0 R                  S5      nUb  Ub  XC:X  a  gg)zE
Get the team-specific model name if team_id matches the deployment.
r*  r  rz  Tr  Fr  )r[  r  r   r  model_team_ids        r   should_include_deployment Router.should_include_deployment'  s     <(.B33I>'I		,'-2223KLM #l(;z(I #YY|4:??	JM ( +r   model_aliasc                 2   / nUb  X14nXPR                   ;   a  U R                   U   nU Hd  nU R                  U   nU R                  XUS9(       d  M)  Ub'  UR                  5       n	X)S'   UR	                  U	5        MS  UR	                  U5        Mf     U(       a  U$ XR
                  ;   a{  U R
                  U   nU Hd  nU R                  U   nU R                  XUS9(       d  M)  Ub'  UR                  5       n	X)S'   UR	                  U	5        MS  UR	                  U5        Mf     U$ Ubp  [        U R                  5       HW  u  pxU R                  XUS9(       d  M  Ub'  UR                  5       n	X)S'   UR	                  U	5        MF  UR	                  U5        MY     U$ )a  
Return all deployments of a model name

Used for accurate 'get_model_list'.

if team_id specified, only return team-specific models

Optimized with O(1) index lookup instead of O(n) linear scan.

Note: when team_id is provided, O(1) lookup in
`team_model_to_deployment_indices` only applies when `model_name` is the
team public model name. If a caller passes an internal deployment model
name (for example, `model_name_<team_id>_<uuid>`), this method falls back
to the standard model-name index / scan path.
)r  r   r  r  )r"  r   rV  r  r3  r!  r  )
r[  r  rX  r  returned_modelsr0  rX  r  r   alias_models
             r   _get_all_deploymentsRouter._get_all_deployments)'  s   * 68 'C;;;??D"C OOC0E99#-G :  !".&+jjl4?L1'..{;'..u5 # #** >>>;;JGG ,11) 2   #. ',jjl4?L1'..{;'..u5 : !   (8
11) 2   #.&+jjl4?L1'..{;'..u5 9 r   c                 4   U R                  5       =(       d    / n/ nU Hv  nUR                  S5      nU R                  U5      (       a+  U R                  XAS9nU(       a  UR	                  U5        MS  MU  UR	                  UR                  SS5      5        Mx     U$ )u   
Returns all possible model names for the router, including models defined via model_group_alias.

If a team_id is provided, only deployments configured with that team_id (i.e. team‐specific models)
will yield their team public name.
r*  )r  r  r  r  )r  r~  _is_team_specific_model_get_team_specific_modelr3  )r[  r  deploymentsr   r  r*  rP  s          r   rt  Router.get_model_namesy'  s     ))+1r%J#5J++J77"&"?"?) #@ # #&&7 # "":>>,#CD & r   c                    U R                  5       =(       d    / n0 nU Hk  nUR                  S5      =(       d    SnU(       d  M&  UR                  S5      =(       d    0 R                  S5      SL nXB;   a  X$   =(       a    UX$'   Mg  XRU'   Mm     UR                  5        VVs1 s H  u  pFU(       d  M  UiM     snn$ s  snnf )a3  
Returns the set of model_names where every backing deployment has `blocked=True`.

Used by `/v1/models` to hide paused models from client listings while still
surfacing them on admin endpoints (e.g. `/model/info`). A model with at least
one non-blocked deployment is still serviceable and remains visible.
r  r  r*  blockedT)r  r~  rq  )r[  ra  blocked_by_namer  rs  
is_blockedfully_blockeds          r   get_fully_blocked_model_names$Router.get_fully_blocked_model_names'  s     ))+1r+-%J>>,/52D$..6<"AA)LPTTJ&(7(=(L*%(2% & -<,A,A,C
,C(T}D,C
 	
 
s    B;1B;ra  c                 P    [        U 5      S:  =(       a    [        S U  5       5      $ )Nr   c              3   x   #    U  H0  nUR                  S 5      =(       d    0 R                  S5      SL v   M2     g7f)r*  rd  TNr  )r   r  s     r   r  6Router._are_all_deployments_blocked.<locals>.<genexpr>'  s5      ,
)
 ^^L)/R44Y?4G)s   8:)r  r|  ra  s    r   _are_all_deployments_blocked#Router._are_all_deployments_blocked'  s/     ;!# 
 ,
),
 )
 	
r   c                 P    U R                  US9=(       d    / nU R                  US9$ )Nr  rm  )r  rn  )r[  r   ra  s      r   _is_model_fully_blockedRouter._is_model_fully_blocked'  s.    ))U);Ar00[0IIr   c                   #    U R                   b
  [        5       $ U R                  R                  5       I Sh  vN nU(       d
  [        5       $ U R	                  5       =(       d    / n0 nU H  nUR                  S5      =(       d    0 nUR                  S5      =(       d    S/nUR                  S5      nU(       a  UR                  U5        UR                  S5      U;   nU H&  n	U	(       d  M  X;   a  X9   =(       a    UX9'   M"  XU	'   M(     M     UR                  5        V	V
s1 s H  u  pU
(       d  M  U	iM     sn
n	$  GNs  sn
n	f 7f)u  
Returns the set of model names where every backing deployment is currently
marked unhealthy by background health checks (and the health state is not stale).

Used by `/v1/models?healthy_only=true` to hide models that cannot serve any
request. A model with at least one healthy (or unknown-health) deployment
remains visible. Returns an empty set when no health state is available, so
callers fail open to the unfiltered listing.

Notes:
    - Mirrors `_async_filter_health_check_unhealthy_deployments`: when
      `allowed_fails_policy` is set, cooldown is the sole routing exclusion
      mechanism, so nothing is hidden here either.
    - Team-specific public model names (`team_public_model_name`) are
      aggregated alongside `model_name`, so team aliases of fully-unhealthy
      deployments are hidden too (unlike `get_fully_blocked_model_names`,
      which matches `model_name` only).
    - Wildcard routes (e.g. `openai/*`) are matched by their literal
      deployment name only; models expanded from a wildcard route are not
      hidden (fail open).
    - Intentionally diverges from the routing-time safety net (which
      bypasses the health filter when every candidate is unhealthy and
      still attempts the request): hiding here is presentation-only —
      it answers "should this model be advertised?", not "should a
      request for it still be attempted?". A hidden model can still be
      called directly.
Nr*  r  r  rz  r  )r   r  r)  "async_get_unhealthy_deployment_idsr  r~  r3  rq  )r[  unhealthy_idsra  unhealthy_by_namer  r*  namesrz  is_unhealthyrs  fully_unhealthys              r   %async_get_fully_unhealthy_model_names,Router.async_get_fully_unhealthy_model_names'  s*    8 $$05L))LLNN 	 5L))+1r-/%J#5;J^^L17R8E%/^^4L%M"%34%>>$/=@L,.?.E.V,%+.:d+  &  *;)@)@)B
)B% )B
 	
) O(
s(   5ED=C*E"E 3E 9E Ec                     UR                  S5      =(       d    0 nUc  gX#R                  S5      :X  a  UR                  S5      $ g)az  
Get the team-specific model name if team_id matches the deployment.

Args:
    deployment: DeploymentTypedDict - The model deployment
    team_id: Optional[str] - If passed, will return router models set with a `team_id` matching the passed `team_id`.

Returns:
    str: The `team_public_model_name` if team_id matches
    None: If team_id doesn't match or no team info exists
r*  Nr  rz  r  )r[  r  r  r*  s       r   r`  Router._get_team_specific_model'  sB     &0^^L%A%GR
nnY//>>":;;r   c                 H    [        U=(       a    UR                  S5      5      $ )z
Check if model info contains team-specific configuration.

Args:
    model_info: Model information dictionary

Returns:
    bool: True if model has team-specific configuration
r  )r  r~  )r[  r*  s     r   r_  Router._is_team_specific_model(  s     J<:>>)#<==r   c                    / nUb#  XR                   ;  a  U$ XR                   U   4/nO#[        U R                   R                  5       5      nU Hm  u  pE[        U[        5      (       a  UnO2[        U[
        5      (       a  [        S0 UD6nUS   SL a  MG  US   nOMO  UR                  U R                  XdS95        Mo     U$ )zt
Helper function to get model list from model alias.

Used by `.get_model_list` to get model list from model alias.
r  Tr   )r  rX  r   )	r   rA  rq  r   r  rP  rf   r  r\  )r[  r  rZ  alias_itemsrX  model_value_router_model_name_model_values           r   get_model_list_from_model_alias&Router.get_model_list_from_model_alias(  s     68!!7!77&&&(>(>z(JKLKt55;;=>K(3$K+s++*5"K..8G;G)T1)5g)>&""))1 *  )4$ r   c                    / nUb  UR                  U R                  XS95        UR                  U R                  US95        [        U5      S:X  a  U R                  R                  U5      =(       d    / nUbG  X R                  ;   a8  U R                  U   R                  U5      =(       d    / nUR                  U5        Ub,  Ub)  U H#  n[        S0 UD6nXS'   UR                  U5        M%     Uc  X0R                  -  nU$ )zk
Includes router model_group_alias'es as well

if team_id specified, returns matching team-specific models
r  r  r   r  r   )
r  r\  r  r  r  r  r  r]   r3  r   )r[  r  r  rZ  r  #potential_team_only_wildcard_modelsr^  deployment_typed_dicts           r   r  Router.get_model_list4(  s    68!""))Z)Q 	00J0G	
 1$(,(;(;(A(A*(M(SQS% "w2K2K'K--g6<<ZHNB 4 *001TU%*C*O2A,?,D!,D):D,7#**+@A 3
 .Or   c                 l    U R                   R                  5         U R                  R                  5         g)ur  Invalidate the cached model group info.

Call this whenever self.model_list is modified to ensure the cache is rebuilt.
Also clears the auth-layer zero-cost cache, which depends on the same
``ModelGroupInfo`` data — without this, an in-place pricing update on
an existing deployment (same model count) would keep a stale ``True``
result and bypass budget enforcement.
N)r   cache_clearr$  r:  r  s    r   rr  )Router._invalidate_model_group_info_cache^(  s(     	))557##%r   c                     SU l         g)zuInvalidate the cached access groups.

Call this whenever self.model_list is modified to ensure the cache is rebuilt.
N)r#  r  s    r   rs  &Router._invalidate_access_groups_cachej(  s    
 %)!r   model_access_groupc                    USL =(       a    USL =(       a    USL nU(       a  U R                   b  U R                   $ SSKJn  U" [        5      nU R	                  XS9nU(       a  U H{  nUR                  S5      n	U	(       d  M  U	R                  S/ 5      =(       d    /  H?  n
Ub!  X:X  a  US   nXj   R                  U5        M%  M'  US   nXj   R                  U5        MA     M}     U(       a  [        U5      U l         U R                   $ U$ )a  
If model_name is provided, only return access groups for that model.

Parameters:
- model_name: Optional[str] - the received model name from the user (can be a wildcard route). If set, will only return access groups for that model.
- model_access_group: Optional[str] - the received model access group from the user. If set, will only return models for that access group.
- team_id: Optional[str] - the team id, to resolve team-specific models
Nr   r   r  r*  r?  r  )r#  collectionsr   rA  r  r~  r3  rP  )r[  r  r  r  
_use_cacher   r?  r   r^  r  r  s              r   get_model_access_groupsRouter.get_model_access_groupsq(  s
     $Q#5#=Q'T/ 	
 $33?,,,+#D)((J(P
eeL1;!,"!E!K!K-9$:-.|_
 - 4 ; ;J G  ; *+<J)077
C "L   (,](;D%,,,r   c                     U R                  US9n[        U5      S:X  a  gUR                  U/ 5      nU H  nU R                  R	                  US9c  M    g   g)z7
Return True if model access group is a wildcard route
)r  r   Fr  T)r  r  r~  r  r  )r[  r  r?  r  r   s        r   )_is_model_access_group_for_wildcard_route0Router._is_model_access_group_for_wildcard_route(  sm     441 5 
 }"""#5r:E""(((7C 
 r   c                 n   [        U 5      n0 n/ SQnU H]  nXA;   a  X   X$'   US:X  d  M  U R                  S:X  d  M(  U R                  c  M7  U R                  R                  R	                  5       X$'   M_     U R
                  R                  5        Vs/ s H  oUR                  5       PM     snUS'   U$ s  snf )z
Get router settings method, returns a dictionary of the settings and their values.
For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after
)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )varsr   r  r  r  r  r  rS  )r[  	_all_vars_settings_to_returnvars_to_includevarr  s         r   get_settingsRouter.get_settings(  s    
 J	 
" #C+4>#(..))-DD--9+/+D+D+Q+Q+V+V+X#( # -1,@,@,G,G,I1
,I5,I1
,- #"1
s   B2c                 >   / SQn/ SQnU R                  5       nSnU H  nXb;   a  Xc;   a  [        X   5      n[        XU5        M(  US:X  a  X   U l        SnM;  X   nUS:X  a<  U R	                  U5      nUS   U:w  a"  U R                  UUR                  S0 5      S9  Sn[        XU5        M  [        R                  " S	R                  U5      5        M     U(       a  U R                  U R                  5        [        R                  " S
U R                  5        35        g)z
Update the router settings.
)r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   Fr   Tr   r   r   zSetting {} is not allowedzUpdated Router settings: N)r  r4  r  r1  r0  r=  r~  r   r  rR  r>  )	r[  r  _allowed_settings_int_settings_existing_router_settingsrebuild_routing_groupsr  _casted_valuer  s	            r   update_settingsRouter.update_settings(  s'   

"
 %)$5$5$7!!&C''$'$4MD}5,,17D.-1*"KE00 $ 8 8 ?45GHEQ 66166<jj$;R7" 7  6:2Du-%++,G,N,Ns,ST/ 2 "%%d&@&@A##&?@Q@Q@S?T$UVr   c                 X   US   S   n[        U5      nUS:X  a`  SR                  U5      nU R                  R                  USUS9nUc/  [        R
                  " XS9  U R                  R                  USUS9nU$ US:X  aX  UR                  S	5      SL a"  U S
3nU R                  R                  USUS9nU$ U S3nU R                  R                  USUS9nU$ UR                  S	5      SL a   U S3nU R                  R                  XeS9nU$ U S3nU R                  R                  XeS9nU$ )at  
Returns the appropriate client based on the given deployment, kwargs, and client_type.

Parameters:
    deployment (dict): The deployment dictionary containing the clients.
    kwargs (dict): The keyword arguments passed to the function.
    client_type (str): The type of client to return.

Returns:
    The appropriate client based on the given client_type and kwargs.
r*  r  r  z{}_max_parallel_requests_clientTr0  r  rF  )rH  r   ra  r  _stream_async_client_async_client_stream_client)r0  rF  _client)r"   rR  r   r  r8    set_max_parallel_requests_clientr~  )r[  r  r  r  r  rF  	cache_keyr  s           r   r  Router._get_client)  s    l+D1+LV+T119@@JIZZ))$AQ * F ~%FF,0 --!dEU .  MG#zz(#t+'j(<=	--!dEU .  'j6	--!dEU .  zz(#t+'j7	--! .  'j0	--! .  r   c           	      	   [         R                  " SU 35        [        U5      n[        5       n [        R
                  " US9nSn	Sn
Sn[        U5      n[        5       nUR                  S5      nU SU 3nU R                  R                  US	US
9=(       d    0 n[        U5       GH  u  nnUR!                  S0 5      nUR!                  S0 5      nSn UR!                  SS5      nUc  UR!                  SS5      nU R#                  UUS9nU=(       d    UR!                  SS5      n[%        U[&        5      (       ab  UR!                  SS5      bO  [%        US   [(        5      (       a7  UUS   :  a.  UR+                  U5        S	n	U
SR                  UUS   U5      -  n
M  UR!                  SS5      nU R                  R                  US	US
9=(       d    Sn[%        U[&        5      (       a  U R.                  S:w  a  UR!                  US5      UU'   [1        UUU   5      n[%        U[&        5      (       aJ  UR!                  SS5      b7  [%        US   [(        5      (       a  US   U::  a  UR+                  U5        S	nGM  UbR  UR!                  S5      b@  UR!                  S5      nUb,  [3        [5        S!0 UD6US9(       d  UR+                  U5        GM*  Uc  GM0  [        R6                  SL d  GMF  U=(       d    Un[        R8                  " U[5        S!0 UD6S9u  nnnn[        R:                  " UUS9nUc  GM  [        R<                  R?                  US9n S/n!U RA                  5        H@  u  n"n#U"U;  d  M  U"U!;   d  M  [         R                  " SU" 35        UR+                  U5        MB     GM     [C        U5      [C        U5      :X  a9   US	L a	  [E        US9eU	S	L a%  [        RF                  " SR                  U
5      USS 9e[C        U5      S:  a(  [        U5       V$V%s/ s H  u  n$n%U$U;  d  M  U%PM     nn$n%U$ ! [         a:  n[         R                  " SR                  [        U5      5      5        Us SnA$ SnAff = f! [         a9  n[         R,                  " SR                  [        U5      5      5         SnAGN SnAff = fs  sn%n$f )"a  
Filter out model in model group, if:

- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
- filter models above rpm limits
- if region given, filter out models not in that region / unknown region
- [TODO] function call and model doesn't support function calling
z2Starting Pre-call checks for deployments in model=)r  zllitellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {}NFr  r~  z:rpm:Tr  r   r*  r  ry  r   r  z%Model={}, Max Input Tokens={}, Got={}zAn error occurs - {}r  r   r   r|  allowed_model_region)r   r  r  rH  )passed_paramsresponse_formatz1INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=r  z~litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.
{}r  r   )$r   r  rA  r  r  token_counterr  r5  rR  r  r"   r|   r  r   r  r  r~  r  r   rP  r4  r  r#  r   maxr}   r_   drop_paramsry   r   r  get_non_default_paramsrq  r  rh   r  )&r[  r   r&  r  r  _returned_deploymentsinvalid_model_indicesr  r  _context_window_error_potential_error_str_rate_limit_errorrF  r  r  r  model_group_cacher  r  r  r  _deployment_modelr  r*  r  current_request_cache_localcurrent_requestr  _dep_model_for_paramsrm  r  r  r  special_paramsr  r  ir  s&                                         r   _pre_call_checksRouter._pre_call_checksR)  sW     	##@H	
 !%%8 9 #	)"00(CL !&!!<^L W-G5 01JJ  ?O !   	 	  ))>?OC(nn-=rBO$..r:K !%W(__\4@
%!0!4!4\4!HJ!77)u 8 
 %/$T/2E2Egt2T! z400"'94@L #:.@#A3GG(:6H+II-11#604-,CJJ 1 *+= > ,, ! #tR0H 

$$ TDT %   	 ( ,d33))-EE.?.C.CHa.P!(+"%/1B81L#
 55'++E48D #?5#93??+E2oE-11#6,0)  *"&&'=>J'5'9'9:P'Q$'3,'5'H'H-A .11#6  )g.A.AU.J(9(BU% ,,/#1#DO#D)' +2*M*M/(;+'
 +2 *1)M)M&4 *N *& '8%8N 2 8 8 :1$;;^@S177"STUSV W 255c: !;a  @p $%-B)CC !D(/  '$.88 ^  e  e,  !#  $%) &&;<%<DAq11 < " % %$W  	)!''~  F  FF
 )(	)r  W%//0F0M0McRSf0UVVWX%sB   P8 CQ?S-S8
Q</Q71Q<7Q<?
S	.R==Sc                     XR                   ;  a  gU R                   U   n[        U[        5      (       a  UnU$ US   nU$ )zt
Get the model from the alias.

Returns:
- str, the litellm model name
- None, if model is not in model group alias
Nr   )r   r   r  )r[  r   _items      r   r!  Router._get_model_from_alias*  sL     ...&&u-eS!!E  'NEr   c                 `    U R                    Vs/ s H  o"S   S   U:X  d  M  UPM     sn$ s  snf )z&
Get the deployment by litellm model.
r   r   )r   )r[  r   r^  s      r    _get_deployment_by_litellm_model'Router._get_deployment_by_litellm_model.*  s0      ??T?a0@.A'.Je.S?TTTs   ++request_team_idc                    XR                   ;   a  gUb  U R                  XS9nU(       a  X4$ U R                  R                  US9nU(       a  X4$ Ub5  X R                  ;   a&  U R                  U   R                  US9nU(       a  X4$ U R
                  bD  U R
                  R                  5       nU R
                  S   R                  5       US'   XS   S'   X4$ g)z
When ``model`` is not in ``self.model_names``, try team routes, pattern routes,
team pattern routers, then default deployment. Returns None if none apply.
Nr  r  r   r   )r   r\  r  get_deployments_by_patternr  r  r  )r[  r   r  team_deploymentspattern_deploymentsupdated_deployments         r   5_try_early_resolve_deployments_for_model_not_in_names<Router._try_early_resolve_deployments_for_model_not_in_names4*  s    $$$ &#88   9    .."11LL M 
 --&?>W>W+W"&";";#(( )   
 #11"".!%!8!8!=!=!?373J3J 4df /0 =B/09,,r   r  c                    SnUb]  UR                  S5      =(       d    0 nUR                  S5      =(       d    0 nUR                  S5      =(       d    UR                  S5      nUSL a  XR                  US94$ U R                  U5      (       aG  U R                  US9n	U	b&  U	R                  R
                  n
XR                  SS94$ [        S	U S
35      eU R                  US9nUb  UnU R                  XS9nUb  U$ U R                  XS9n[        U5      nU R                  UUUUS9nUS:  =(       a    [        U5      S:H  n[        U5      S:X  aG  US:X  aA  U R                  US9nU R                  UUUUS9n[        U5      S:  a  [        U5      S:X  a  Sn[        R                  " [        R                   5      (       a  [        R"                  " SU 35        [        U5      S:X  a  U R%                  5       (       a^  U(       dW  U R'                  5       nU(       a@  [        R(                  " SU SU S35        UnU R                  XS9nU R                  UUUUS9n[        U5      S:X  aQ  U R+                  US9c  SU S3R-                  U5      nOSU S3R-                  U5      n[.        R0                  " UUSS9e[.        R2                  (       a'  U[.        R2                  ;   a  [.        R2                  U   nX4$ )a  
Common checks for 'get_available_deployment' across sync + async call.

If 'healthy_deployments' returned is None, this means the user chose a specific deployment

Returns
- str, the litellm model name
- List, if multiple models chosen
- Dict, if specific model chosen
Nr   r  r  Tr  r  r   zBLiteLLM Router: Trying to call specific deployment, but Model ID :z does not exist in Model ID map)r   r  r  )r   r&  r  r  r   zinitial list of deployments: Model 'z7' not found. Attempting to use default fallback model 'r  r  zYou passed in model=z+. There is no 'model_name' with this stringz1. There are no healthy deployments for this modelr  r  )r~  r  r  r  r   r   rS  r  r!  r  r\  r  *_filter_deployments_by_model_access_groupsr   r1  r  r   r  r  rn  rQ  r  rR  r  r  model_alias_map)r[  r   r  r  r  r  r  r   r  r  deployment_model_model_from_aliasearlyr&  "_pre_model_access_group_filter_len'_access_group_filter_emptied_candidates_litellm_model_deploymentsfallback_modelr  s                      r   r  *Router._common_checks_available_deploymentb*  s   & *.%%))*5;H-112DEK&ll& >!%%&<=  $&??e?LLLu%%,,e,<J%#-#<#<#B#B ')>)>D)>)QQQTUZT[[z{  !66U6C(%EJJ K 
 L #77 8 
 .11D-E*"MM 3)+	 N 
 /2Ts;N7OST7T 	0 "#q( 2Q6-1-R-R .S .* '+&U&U(B#1$3	 'V '# 23a7/0A5>B; --gmm<<!''/0C/DE "#q( ++--?!%!A!A!C!)..!%(_`n_ooqr +E*.*C*C#( +D +' GG"'0C+9,;	 H  ( &'1,&&%&8@ 4UG;fgnnG !5UG;lmttG --#!#  ""u0G0G'G++E ))r   c                    U(       a  Uc  U$ UR                  S5      =(       d    0 nUR                  S5      =(       d    0 nUR                  S5      =(       d    UR                  S5      nUc  U$ [        [        US/ 5      =(       d    / 5      n[        [        US/ 5      =(       d    / 5      n	X-  n
U
(       d  U$ X;   d  SU
;   d  SU
;   a  U$ U R                  XS9n[	        U5      S	:X  a  U$ [        UR                  5       5      U
-  nU(       d  U$ / nU H^  nUR                  S
5      =(       d    0 n[        UR                  S/ 5      =(       d    / 5      nUU-  (       d  MM  UR                  U5        M`     U$ )a  
Restrict candidate deployments to caller-authorized model access groups.

This is only applied when:
- request metadata includes `user_api_key_auth`, and
- caller permissions for this model are access-group-only
  (no explicit model, wildcard, or all-proxy grants).
r   r  user_api_key_authr  team_modelsr   zall-proxy-modelsr  r   r*  r?  )r~  r  r  r  r  ri  r3  )r[  r   r&  r  r  r   r  r  object_modelsobject_team_modelsallowed_modelsaccess_groups_for_modelallowed_access_groupsfiltered_deploymentsr  r  deployment_access_groupss                    r   r  1Router._filter_deployments_by_model_access_groups*  s    #n&<&&!%%j17R)--.@AGR$LL)<= 
AQAUAUB
 $&&G$5xDJK ):M2!N!TRTU&;&&
 #n$!^3&&"&">"> #? #
 &'1,&& #$;$@$@$B Cn T$ '&!-J$.NN<$@$FB!'*%))/2>D"($ (*???$++J7 . $#r   c           	      z  #    U R                  UUUUUS9u  p[        UUS9n[        R                  " [        R
                  5      (       a  [        R                  " SU 35        [        UUS9n[        R                  " [        R
                  5      (       a  [        R                  " SU 35        [        U[        5      (       aH  UR                  S5      =(       d    0 R                  S5      SL a  [        R                  " SU S	3US
S9eU$ U R                  UUS9I Sh  vN n[        XS9I Sh  vN n[        R                  " [        R
                  5      (       a  [        R                  " SU 35        Un	U R                  UUS9nU(       d6  U R                   (       a%  U R"                  b  [        R$                  " S5        U	nU R'                  U5      nU R)                  UUUb  [+        [,        [.           U5      OSUUS9I Sh  vN nU R0                  (       a*  Ub'  U R3                  U[+        [,        [4           U5      UUS9n[7        U UUUU R9                  U5      S9I Sh  vN nU=(       d    0 R;                  SS5      n
[        R<                  R?                  [+        [,        [4           U5      U
S9nU=(       d    0 R;                  SS5      n[        R<                  RA                  [+        [,        [4           U5      US9n[C        U5      S:X  a  [E        U UUS9I Sh  vN nUeU$  GN GN  GN, N N7f)z
Get the healthy deployments for a model.

Returns:
- List[Dict], if multiple models chosen
*OR*
- Dict, if specific model chosen
r   r  r  r  r  )r&  r  z'healthy_deployments after team filter: z-healthy_deployments after web search filter: r*  rd  Tr  1' is currently paused and cannot accept requests.r  r  r&  rF  Nr  cooldown deployments: r&  cooldown_deploymentsOAll deployments in cooldown via health-check routing, bypassing cooldown filterr  r   r&  r  r  )r8  r   r  r&  r  r  target_orderr  excluded_deployment_idsr   rH  r   rF  )#r  r;   r   r1  r  r   r  r<   r   rP  r~  r  ServiceUnavailableError0_async_filter_health_check_unhealthy_deploymentsr?   _filter_cooldown_deploymentsr   r   r  r  r  r   r   rQ   r   r  r
   r2   r  r  r  _get_order_filtered_deployments"_get_excluded_filtered_deploymentsr  rG   )r[  r   r  r  r  r  rF  r&  r  _pre_cooldown_deploymentsr  r  r#  s                r   r|  $Router.async_get_healthy_deployments4+  sx    $ &*%M%M 3) &N &
" 7 3)

 !--gmm<<!''9:M9NO < 3)

 !--gmm<<!''?@S?TU )400#''5;@@KtS55%eW,]^!# 
 '& GG$7!1 H   	 &E$(&
  
 !--gmm<<!''*@AU@V(WX$7!"?? 3!5 @ 
 $00))5!))a #<">>?RS$($J$J 3:B:NT*+X6TX)- %K %
 
 &&8+?"&"7"7$(d5H$I!-	 #8 # %< $) 3#'#O#O$%
 
 (-222?DI%mmKKd01 L 
 %3$8b#=#=&$
  &mmNNd01$< O 

 "#q(A(,!1 I
 O""i 
2
$
8s]   DL; L.!L;1L12CL;L4	AL;'L7(B=L;%L9&	L;1L;4L;7L;9L;c                 <  #    U R                   S:w  aS  U R                   S:w  aC  U R                   S:w  a3  U R                   S:w  a#  U R                   S:w  a  U R                  UUUUUS9$  [        U5      nU R                  UUUUUS9I Sh  vN nUb  UR                  nUR
                  nU R                  U5      u  pU R                  UUUUUUS	9I Sh  vN n
[        U
[        5      (       a  U
$ UR                  S
5      (       a  [        U
5      S:X  a  U
S   $ [        R                  " 5       nUS:X  a  [        U U
US9$ U R                  UU	UU
UUUS9I Sh  vN nUc  [        U UUS9I Sh  vN nUe[         R"                  " SU SU R%                  U5       SU 35        [        R                  " 5       nX-
  n[&        R(                  " U R*                  R-                  [.        R0                  USUUUS95        U$  GNt GN- N N! [2         a  n[4        R6                  " 5       nUbj  UR                  SS5      nUbU  [8        R:                  " UR<                  UU4S9R?                  5         [&        R(                  " URA                  UU5      5        UeSnAff = f7f)z
Async implementation of 'get_available_deployments'.

Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps).
r   r   r   r   r   r  r   r  r  r  r  Nr   r  r  r  r  rF  "_encrypted_content_affinity_pinnedr~   r   r7  r  r  r   r&  r  r  r  r  $get_available_deployment for model: , Selected deployment:  for model: 2<routing_strategy>.async_get_available_deploymentsrB  rC  r  rF  rD  rE  r  r  )!r   r  r"   async_pre_routing_hookr   r  r  r|  r   rP  r~  r  rL  r1   r  rG   r   rQ  r  r'  rQ  r  rR  rl   rS  r  rT  rU  r  r  r  r  r  )r[  r   r  r  r  r  rF  pre_routing_hook_responser  strategy_selectorr&  rD  r  r#  rE  rW  r  r  rn  s                      r   r  %Router.async_get_available_deployment+  s!     !!%==%%)99%%)==%%)@@%%500!$7- 1  c	@P /3.I.I-!$7 /J / )% )41774== +/*C*CE*J'H(,(J(J-!$7!1 )K ) # -t44** ""#GHH+,1*1--J++%(,(; 
  $<<!*$7!-  =   J !"E,0%5# 	
  !&&6ug=TUYUjUjkuUvTw  xD  EJ  DK  L yy{H -I''BB(//&R%5)% C 	 W)"#20  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G!	s   A$J'!H G<	A	H G?H -J.)H J%H =J>H HH ,H-BH ;J<H ?H H H 
JBJJJc                   #     [        U5      nU R                  UUUUUS9I Sh  vN nUb  UR                  nUR                  nU R	                  UUUUUUS9I Sh  vN n[        U[        5      (       a  UR                  S5      =(       d    0 R                  S5      SL a  [        R                  " SU S3US	S
9eUR                  S0 5      n	U	R                  S5      (       a  U$ [        R                  " SUR                  S0 5      R                  S5       S3US	S
9eU R                  US9n
[        U
5      S:X  a  [        R                  " SU S3US	S
9e[        R                  " 5       nU R                  U5      u  pUS:X  a  [!        U U
US9$ U R#                  UUUU
UUUS9I Sh  vN nUc  [%        U UUS9I Sh  vN nUe[&        R(                  " SU SU R+                  U5       35        [        R                  " 5       nUU-
  n[,        R.                  " U R0                  R3                  [4        R6                  USUUUS95        U$  GN' GN N N! [8         a  n[:        R<                  " 5       nUbj  UR                  SS5      nUbU  [>        R@                  " URB                  UU4S9RE                  5         [,        R.                  " URG                  UU5      5        UeSnAff = f7f)z
Async version of get_available_deployment_for_pass_through

Only returns deployments configured with use_in_pass_through=True
r	  Nr	  r*  rd  Tr  r  r  r  r   r  Deployment r  C does not support pass-through endpoint (use_in_pass_through=False)r&  r   Model z has no deployments configured with use_in_pass_through=True. Please add use_in_pass_through: true to the deployment configurationr   r7  r	  r  z7async_get_available_deployment_for_pass_through model: , selected deployment: r
	  r	  r  r  )$r"   r	  r   r  r|  r   rP  r~  r  r  r   _filter_pass_through_deploymentsr  rL  perf_counterr  r1   r  rG   r   rQ  r  r'  rQ  r  rR  rl   rS  r  rT  rU  r  r  r  r  r  )r[  r   r  r  r  r  rF  r	  r&  r   pass_through_deploymentsrD  r  r	  r  r#  rE  rW  r  r  rn  s                        r   /async_get_available_deployment_for_pass_through6Router.async_get_available_deployment_for_pass_throughE,  s    p	@P /3.I.I-!$7 /J / )% )41774== )-(J(J-!$7!1 )K ) # -t44'++L9?RDDYOSWW!99")%0a b#%' 
 "5!8!89I2!N!%%&;<<..!11"-.A.E.ElTV.W.[.[\`.a-b  cf  !g#%'  (,'L'L$7 (M ($ +,1--$UG  ,n  o!#  **,J*.*C*CE*J'H++%(,(@ 
  $<<!*$<!-  =   J !"E,0%5# 	
  !&&I%Pghlh}h}  I  iJ  hK  L ((*H :-I''BB(//&R%5)% C 	 )#`2  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G	s   K,!I I6I IBI %K,&B)I K,I (I)I >I?BI K,I I I I 
K) BK$$K))K,c                   #    XR                   ;   a(  U R                   U   R                  UUUUUS9I Sh  vN $ XR                  ;   a(  U R                  U   R                  UUUUUS9I Sh  vN $ U R                  R	                  U5      nUb  UR                  UUUUUS9I Sh  vN $ XR
                  ;   a(  U R
                  U   R                  UUUUUS9I Sh  vN $ g N Nw N@ N7f)z
This hook is called before the routing decision is made.

Used for the litellm auto-router to modify the request before the routing decision is made.
r	  N)r  r	  r  r  r~  r  )r[  r   r  r  r  r  rS  s          r   r	  Router.async_pre_routing_hook,  s<      %%%**51HH-!$7 I    +++007NN-!$7 O    //33E:&(??-!$7 @    (((--e4KK-!$7 L    [sE   3C*C"6C*,C$-8C*%C&&6C*C(C*$C*&C*(C*c           
      d   U R                  UUUUUS9u  p[        U[        5      (       aH  UR                  S5      =(       d    0 R                  S5      SL a  [        R
                  " SU S3USS9eU$ [        U5      nU R                  UUS	9n[        XS
9nUn	U R                  UUS9nU(       d6  U R                  (       a%  U R                  b  [        R                  " S5        U	nU R                  U5      nU R                  (       a  Ub  U R!                  UUUUS9nU=(       d    0 R#                  SS5      n
[        R$                  R'                  XjS9nU=(       d    0 R#                  SS5      n[        R$                  R)                  UUS9n[+        U5      S:X  aG  U R-                  US9nU R.                  R1                  XS9n[        XS
9n[3        UUU R                  US9eU R5                  U5      u  nnUS:X  a  [7        U UUS9$ U R9                  UUUUUUUS9nUca  [        R:                  " SU S35        U R-                  US9nU R.                  R1                  XS9n[        XS
9n[3        UUU R                  US9e[        R:                  " SU SU R=                  U5       SU 35        U$ )z2
Returns the deployment based on routing strategy
r  r*  rd  Tr  r  r  r  r  r  r  Nr  r  r  r  r  r  r   r  	model_idsrF  r   r   r   cooldown_listr   r7  r	  r	  z, No deployment availabler	  r		  )r  r   rP  r~  r  r  r"   *_filter_health_check_unhealthy_deploymentsrA   r  r   r   r   r  r  r   r  r  r  r  r  r  rC  r'  get_min_cooldownrg   r  r1   r  rQ  r  )r[  r   r  r  r  r  r&  rF  r  r 	  r  r  r	  _cooldown_time_cooldown_listr  r	  r  s                     r   r  Router.get_available_deployment-  s    &*%M%M 3) &N &
" )400#''5;@@KtS55%eW,]^!# 
 '&+L,

 #MM 3- N 

  9$( 
 %8!"?? 3!5 @ 

 $00))5!))a #<">>?RS &&8+?"&"7"7$7!-	 #8 # (-222?DI%mmKK L 
 %3$8b#=#=&$
  &mmNN$< O 

 "#q(**e*<I!00AA# B N 7(,N ','+'B'B,	  '+&?&?&F##'' "$($7 
 11& 3) 2 

 !&&6ug=VW **e*<I!00AA# B N 7(,N ','+'B'B,	  	""25'9PQUQfQfgqQrPss  AF  @G  H	
 r   c           
      ,   U R                  UUUUS9u  p[        U[        5      (       a  UR                  S5      =(       d    0 R                  S5      SL a  [        R
                  " SU S3USS9eUR                  S	0 5      nUR                  S
5      (       a  U$ [        R                  " SUR                  S0 5      R                  S5       S3USS9eU R                  US9n[        U5      S:X  a  [        R                  " SU S3USS9e[        U5      n	U R                  UU	S9n[        X	S9n
U R                  UU
S9nU R                  U5      nU R                  (       a  Ub  U R                  UUUUS9n[        U5      S:X  aG  U R!                  US9nU R"                  R%                  XS9n[        X	S9n['        UUU R                  US9eU R)                  U5      u  pUS:X  a  [+        U UUS9$ U R-                  UUUUUUUS9nUca  [.        R0                  " SU S35        U R!                  US9nU R"                  R%                  XS9n[        X	S9n['        UUU R                  US9e[.        R0                  " SU SU R3                  U5       35        U$ )ai  
Returns deployments available for pass-through endpoints (based on load balancing strategy)

Similar to get_available_deployment, but only returns deployments with use_in_pass_through=True

Args:
    model: Model name
    messages: Optional list of messages
    input: Optional input data
    specific_deployment: Whether to find a specific deployment
    request_kwargs: Optional request parameters

Returns:
    Dict: Selected deployment configuration

Raises:
    BadRequestError: If no deployment is configured with use_in_pass_through=True
    RouterRateLimitError: If no pass-through deployments are available
)r   r  r  r  r*  rd  Tr  r  r  r  r   r  r	  r  r	  r	  r   r	  z has no deployment configured with use_in_pass_through=True. Please add use_in_pass_through: true in the deployment configurationr  r  r  r  r  r	  r 	  r   r7  r	  z1get_available_deployment_for_pass_through model: z, no available deploymentsr	  )r  r   rP  r~  r  r  r  r	  r  r"   r"	  rA   r  r  r   r  rC  r'  r#	  rg   r  r1   r  r   rQ  r  )r[  r   r  r  r  r  r&  r   r	  rF  r  r	  r$	  r%	  r  r	  r  s                    r   )get_available_deployment_for_pass_through0Router.get_available_deployment_for_pass_through-  s   8 &*%M%M 3	 &N &
" )400#''5;@@KtS55%eW,]^!# 
 1445ErJN!!"788** --)*=*A*A,PR*S*W*WX\*])^  _b  c!#  $(#H#H 3 $I $
  '(A-))   (i  j  ,M,
 $(#R#R 8- $S $
   9$( 
 $(#D#D 8!5 $E $
  $(#C#C$$
 
 &&8+?'+'<'<$<!-	 (= ($ '(A-**e*<I!00AA# B N 7(,N ','+'B'B,	  '+&?&?&F#''!$($< 
 11& 8) 2 

 !&&CE7Jde **e*<I!00AA# B N 7(,N ','+'B'B,	  	""?wF]^b^s^st~^  ^A  B	
 r   r  c                     [         R                  " [        R                  5      (       a  [         R                  " SU 35        [        U5      nU Vs/ s H  nUS   S   U;  d  M  UPM     sn$ s  snf )a  
Filters out the deployments currently cooling down from the list of healthy deployments

Args:
    healthy_deployments: List of healthy deployments
    cooldown_deployments: List of model_ids cooling down. cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]

Returns:
    List of healthy deployments
r  r*  r  )r   r1  r  r   r  r  )r[  r&  r  cooldown_setr  s        r   r  #Router._filter_cooldown_deployments*.  su     !--gmm<<!''*@AU@V(WX/0 2
1
,'-\A 1
 	
 
s   A/&A/c                     U Vs/ s H3  nUR                  S5      =(       d    0 R                  S5      SLd  M1  UPM5     sn$ s  snf )u  
Filters out deployments that an admin has paused via `LiteLLM_ProxyModelTable.blocked`.

Applied alongside the cooldown filter on every routing entry point that calls
`_common_checks_available_deployment` directly — the primary sync/async path,
the sync pass-through path, and the retry / health-check helpers — so paused
deployments never serve a request. The async pass-through path inherits this
filter through its delegation to `async_get_healthy_deployments`.
r*  rd  Tr  )r[  r&  r  s      r   r  "Router._filter_blocked_deploymentsA.  sJ     2
1
|,277	B$N 1
 	
 
s
   0AAc                 B    [        U SS5      nUc  g[        USS5      SL $ )z
Returns True when a `Deployment` Pydantic instance carries the admin-paused
flag. Used by credential-lookup helpers so passthrough file / batch endpoints
cannot bypass the pause by resolving credentials directly.
r*  NFrd  T)r  )r  r*  s     r   r  Router._is_deployment_blockedS.  s/     Zt<
z9d3t;;r   c                 0  #    U R                   (       d  U$ U R                  b  U$ U R                  R                  US9I Sh  vN nU(       d  U$ U Vs/ s H  oDS   S   U;  d  M  UPM     nnU(       d  [        R
                  " S5        U$ U$  NMs  snf 7f)z
Filter out deployments marked unhealthy by background health checks.
No-op when enable_health_check_routing is False.
Returns all deployments if health state is unavailable, stale, or would
exclude every candidate (safety net).
NrF  r*  r  JAll deployments marked unhealthy by health checks, bypassing health filter)r   r   r)  rt  r   r  r[  r&  rF  ru  r  filtereds         r   r  7Router._async_filter_health_check_unhealthy_deployments_.  s      //&&
 $$0&& ))LL!1 M   	
 && +
*!od.C=.XA* 	 
 !))\ '&#
s)   ?BBBB&B,$BBc                    U R                   (       d  U$ U R                  b  U$ U R                  R                  US9nU(       d  U$ U Vs/ s H  oDS   S   U;  d  M  UPM     nnU(       d  [        R
                  " S5        U$ U$ s  snf )zASync version of _async_filter_health_check_unhealthy_deployments.r2	  r*  r  r3	  )r   r   r)  get_unhealthy_deployment_idsr   r  r4	  s         r   r"	  1Router._filter_health_check_unhealthy_deployments.  s     //&&$$0&&//LL- M 
 && +
*!od.C=.XA* 	 
 !))\ '&
s   	BBc                    [         R                  " S[        U5       S35        U Vs/ s H.  nUR                  S0 5      R                  SS5      (       d  M,  UPM0     nn[         R                  " S[        U5       S35        U$ s  snf )z
Filter out deployments configured with use_in_pass_through=True

Args:
    healthy_deployments: List of healthy deployments

Returns:
    List[Dict]: Only includes a list of deployments that support pass-through
z%Filter pass-through deployments from z healthy deploymentsr   r  FzFound z& deployments with pass-through enabled)r   r  r  r~  )r[  r&  r  r	  s       r   r	  'Router._filter_pass_through_deployments.  s     	##3C8K4L3MMab	
 2$
1
~~.3778MuU 1 	! $
 	##S1233YZ	
 ('$
s   +BBc                      UR                  S0 5      R                  SS5      nUc  Ub  U R                  XB5        ggg! [         a,  n[        R                  " S[        U5       35         SnAgSnAff = f)z'
Tracks successful requests rpm usage.
r*  r  Nz$Error in _track_deployment_metrics: )r~  r  r  r   r5  r  )r[  r  rF  r  r  r  s         r   r   Router._track_deployment_metrics.  s~    		Y!~~lB7;;D$GH'&&  (    	Y!''*NsSTvh(WXX	Ys   9> 
A4"A//A4c                 B    [        UUU R                  U R                  S9$ )Nr?  )rD  r   r   )r[  r#  r  s      r   rF   (Router.get_num_retries_from_retry_policy.  s)     2#%)%B%B**	
 	
r   c                 V   U R                   nUc  g[        U[        R                  5      (       a  UR                  b  UR                  $ [        U[        R
                  5      (       a  UR                  b  UR                  $ [        U[        R                  5      (       a  UR                  b  UR                  $ [        U[        R                  5      (       a  UR                  b  UR                  $ [        U[        R                  5      (       a  UR                  b  UR                  $ gg)z
BadRequestErrorRetries: Optional[int] = None
AuthenticationErrorRetries: Optional[int] = None
TimeoutErrorRetries: Optional[int] = None
RateLimitErrorRetries: Optional[int] = None
ContentPolicyViolationErrorRetries: Optional[int] = None
N)r   r   r  rb  AuthenticationErrorAllowedFailsr  TimeoutErrorAllowedFailsr[  RateLimitErrorAllowedFailsr  'ContentPolicyViolationErrorAllowedFailsr  BadRequestErrorAllowedFails)r[  r#  r   s      r   get_allowed_fails_from_policy$Router.get_allowed_fails_from_policy.  s     >B=V=V' y'"="=>>$DDP'GGGy'//22$==I'@@@y'"8"899$??K'BBBy'"E"EFF$LLX'OOOy'"9"9::$@@L'CCC M ;r   c                 F   SSK Jn  U R                  c  g U R                  nU" UR                  S/UR                  S9nX0l        [        R                  R                  U5        [        R                  R                  UR                  5        [        R                  " S5        g )Nr   )SlackAlertingslack)alerting_thresholdalertingdefault_webhook_urlz2[94m
Initialized Alerting for litellm.Router[0m
)1litellm.integrations.SlackAlerting.slack_alertingrI	  r   rK	  webhook_urlslack_alerting_loggerr  rB  rV  rF  !response_taking_too_long_callbackr   rQ  )r[  rI	  router_alerting_config_slack_alerting_loggers       r   rW  Router._initialize_alerting/  s    S'151E1E!.5HHY 6 B B"
 &<"((==>TU((EE"DD	
 	""H	
r   CustomRoutingStrategyc                 `    [        U SUR                  5        [        U SUR                  5        g)z
Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router

Use this to set your custom routing strategy

Args:
    CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase
r  r  N)r  r  r  )r[  rU	  s     r   set_custom_routing_strategy"Router.set_custom_routing_strategy/  s4     	&!::	

 	,!@@	
r   c                 N    S [         l        U R                  R                  5         g r   )r  r   flush_cacher  s    r   rZ	  Router.flush_cache3/  s    

 r   c                     / [         l        / [         l        / [         l        / [         l        S U l        U R                  5         g r   )r  rE  r@  rK  rH  r   rZ	  r  s    r   resetRouter.reset7/  s6    #% *,'#% *,' r   )r#  r  r  r.  r  r  r  r1  r$  r  r  r  r  r  r  r?  r  r  r  rf  r  r  rx  r  r  r  r  rg  r  r  r{  r  r  r   r  rA  r  r  r  r  r  rz  r  r  r   r  r  rh  r  r  ry  r  r   r   r  r  rH  r  ri  r  r  r  r   r  r  r+  r%  r/  r'  r)  r-  r  r  r  r  rP  rQ  rR  rS  rT  rU  rV  rW  rX  r   r   r  r:  r   r  r   r   r'  r   r  rj  r  r   r  r   r   r   r  rk  r  r   r  r  r<  r   r   r   r   r   r6  r*  r   rB  rC  r  r  r   r   r)  r&  r   r   r  r  rl  r  r  r   r  r   r   r   r   r   r   r!  r   r   rI  r   r  r8  r   r  r  r  rm  r   r   rN  r   r   r   r  r  r   r  r   rP	  r   r7  r   r"  r  r   r5  r  r  r,  r&  r0  r(  r*  r.  r  r  r  r  rY  rZ  r[  r\  r]  r^  r_  r`  ra  )T)F)rX  r   r  r   r   N)r   r   )FFF)r  r  )NNNN)NNNNN)r   N)NF)NNN)NNFN)NNF(-  r   r   r   r   r  r   __annotations__r   r   r  r   r4  tenacityr   r+   r   r.   r   r   r   r    r	   r  re   r]   r
   r   rY   rk   r^   rP  r~  r(  r   rf   rc   rX   rb   ri   rn   rW   rc  staticmethodrm  rZ  ry  r   r   r  r  r  rj   r0  r  r  r  r=  r>  r   r  rA  r  r  rX  r  r  r>  rE  rM  rc  ru  r|  r  r  r  rY  r2  r  rT  r  rv   rt   r  r  r  r   r   rQ   r&  rq   rs   rb  r  r  r  r  r  r  r	  rK  r  r  r	  r
  r1  r\   r=  rG  r  r  rd  rh  rU  r  r  r  rO  r  rM  rN  r  r  r  r  rR   r  r  r  r  r  r  r  r  r  r  r  r%  r*  r/  r.  r4  rO  rD  r  rN  rS  ru   rX  rV  r`  r_  rS   rf  re  r  r  ro   r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r	  r  r&   wraprP  r/  r0  rC  rB  rE  r  r  rn  rG  rD  rG  rM  rJ  r  rF  r   r  r  r  r  r  r  r  r  r  r8  r  r  r_   r"  r.  r1  r;  r>  rI  rE  r`  rg  r
  r%  r  r  r  r  r  r  r9  r  r*   r  r  r  r  r  r  r  r  r  r  r  rp   r  ra   r  r  r  r   r   r   r*  rW  r;  r>  rC  r  rN  rR  rV  r\  rt  r   rh  rn  rq  rz  r`  r_  r  r  rr  rs  r  r  r  r  r  r  r!  r  r  r  r  r|  r  r	  r   r	  r  r(	  r  r  r  r  r"	  r	  r  rF   rF	  rW  r[   rW	  rZ	  r]	  r   r   r   r   r   r      sB'   uK&+OXd^+&11H:>h67>:>h67>MQeL(C,G&H!IJQ ;?<@=A#'$($((,"&*/ ,0*.%) #'*. 7;!06 )+)+ ',%*(,     ,0 DH&(DHDH48 "#/3FJ+0,1:>5:).u{&$*+T$sCx.-AAB
{& $$78{& t$789{& !&8!9:{& C={& SM{& SM{& !{&  3-!{&" "$#{&$ %{&& !K
'{&, -{&0 #5/1{&2 #3-3{&6 c]7{&8  
9{&> %?{&@ !A{&B !)!
C{&H (0}I{&J K{&L _-M{&N $I
O{&T U{&V #'W{&X #'Y{&Z $eC!::;;<
[{&` !%a{&b #c{&d "&e{&f g{&h +t#$
i{&n #'#
o{&t  
u{&z '
{{&@  
A{&F $D>G{&H "&
I{&X #++@"AY{&Z  $[{&\ !eL$,>&?!@A]{&^ !))@ A_{&` ".1a{&b "*!"
c{&h *-i{&j &.d3S	>.B%Ck{&l %)m{&n &*o{&p +33-q{&r /3s{&t #'u{&v 
w{&z DI   D .38n.	z,,	-. ."+ +$ )1"7!71:&S#X  d23	#  %osD&@ A	0 $(	0,-0  $0 !	0
 
#0dd3i D &* %os&: ;*TX*BLtE,*<$=>?L 
L\"# "%xPS}8T2U ">3 3-3 3-	3
 3 "3 4S#X/03 c4i()3 !3 
#3j+ 3-+ 3-	+
 + "+ 4S#X/0+ c4i()+ !+ 
#+Z	FD
P0
dF
P
,
HB
H+MZ
"
H$
L41$ ;M&*;M	;MzeM(01F(GeMN4 2$(c3h$8	}11	2&dd$(c3h$8d	}11	2dL. .`11+/91j $()9$:DKDM	 
 UZ$()9$:DKEN	 
 kp$()9$:DI'RV-Y`afYgJgDh	"M1	2  	33 '(3 	3j 8*8(08 
8 8(N>+N> tCH~&N> 	N>
 
N>` J<J	$	%J JX '
8'
)'
 
'
 '
R :?E#';";<=:?:? 
:? :?xkG5kG S#XkG 
-	kGZB+B tCH~&B 	B
 
BB+/9@oo$(c3h$8o		
oj 1;	

 
 !)	

 

*"04	*"04	, EOPP4<SMP	P, NR''(,'=Ec]'	'@ > >t > > >. (,	T
T
 T
  }	T

 
T
l t 4

"&
	%s
#	$
"&	%s
#	$$	4 	t 	uczAR8S 	]#S	]# T#s(^,d4S#X3G.HHI]#~++$(.>)?$@+^ $(c3h$8BI$-	  SX$(c3h$8BI%.	  	M,M, tCH~&M, 	M,f di$()9$:FISZ[`Sa	 
 $()9$:FISZ[_S`	  @@ '(@ 	@D?? ? $	?
 CHo? S#X?B
X 
X 
XaBaB '(aB S#X	aBFs 3 , ,C ,\c # *Fs F3 FP( (3 (TD) DC DL23 2s 23 2hDC D DC DL3 *)C )^ $)&+#(## # 4.	#
 d^# 4.#R $)&+#(## # 4.	#
 d^# 4.#JDS D# DT $)&+#(  4.	
 d^ 4.:DS D DL
x 


5=
  $> $,:!
!
 
!
F-5>38n-0>A	c3h>``5=`D5!)5585	J	K5nQQ-5Qn $)	 S$Y 4.	 
"6c4i 0 6 6x $(	 S$Y 4.	 
2DuS$Y'7 D DN 
	0NN 
	NbcS$YcJ	 
	>SS 
	Sn  $v}v 
	vp   
	 DZZ 
	Zx//l .21#1 &c]1X eq#q m#o
ql .21#1 &c]18 .2-1#-1 &c]-1^
#
2 .21#1 &c]1@ .21#1 &c]1* .2(,	
#
 &c]
 %	
,MM "M 12	M
 M M M 
#M^Z!Z! $D>Z! D>	Z!
 #+4.Z! #+4.Z! c]Z! Z! Z!x [[]1 1l &*$(3737-- c]- D>	-
 #+4.- #+4.-^ [[]C% C%J $ :>!!)1#!L /3*.3737,0FF &d^F "$	F
 #+4.F #+4.F $D>FPW &*$S$s)^,-$ c]$ 
$s)		$2Xc] $ /3*.// / 	/
 &d^/ "$/ 
sEz	/fCJ! 
#!FJ 
JX 
+3C= 
D>>	/	0> # # #t #J 4<TN	6 $$$1$;?$	$L5c 5Xd^ 5,55,4TN5	tDz4:%	&5@54 5( 15	;; #4.; n-	;F *.041,1, "$Z1, 4 012	1,
 #4.1, !1, n-1,f'c '4 '> 66),6CKC=6	6 6@FF F 	F
 F 
*	FP D $-@j -@^~ RV +KJ +KZN^ NPT N5HnL
* L
 L
\N t (E (ET&z &d &P<7 <7|I* I IVA$A;>AGJAFG G8L GR:?:?*-:?	:?xGd G G G( 6:22%-c]2	2>4J 48J;O 4lC HZ,@ >	&	'&

 
t 
  .s x
/C &(3 (8D> ( #	*	0MM	$sCx.	!M^ 
 	$,- ! 	
 
  58>A	  !	`U4#567` !` SM	`
 
`D $  
:# 
:(4. 
:>>),>	)	>@f f >Af 	.	!f P
 
8P 
>G$G$	x}hsm+	,G$R 1266	.	!6 36 cSVh F &*)-	`` c]` !	`
 
`D6$ 64 6&R$ R8 MR""3-"EI"	c"HE E E4 4	#4l#'}#7:#	##L FJ&*5=c]	: &*!%	NN c]N #	N
 
!	"N`x} S	 .
s3x 
. 
-.
	
 
JS JT J7
SX 7
t IM-8@	#*
>(4. 
>T 
> +/$"3-$	!	"$N JN("3-(9A#(	$*+	,(T
&) %),0!%	.SM. %SM. #	.
 
c49n	.`"%	.%#N;Wz4v *.G%G% "G% tCH~&	G%
 !G%R3 8C= &Uc Ud U,,+3C=,	%U4:../	0,b 48,0.3)-P*P* 4S#X/0P* c4i()	P*
 &d^P* !P* 
sE$*%%	&P*d>$>$ ">$ !	>$
 "#>$ 
>$H 48,0.3+/O#O# O# 4S#X/0	O#
 c4i()O# &d^O# #4.O# 
tDz4	 O#j 48,0.3~~ ~ 4S#X/0	~
 c4i()~ &d^~H 48,0.3}} } 4S#X/0	}
 c4i()} &d^}F 48,0.3>> > 4S#X/0	>
 c4i()> &d^> 
(	)>F 48,0.3)-MM 4S#X/0M c4i()	M
 &d^M !Md 48,0.3)-UU 4S#X/0U c4i()	U
 &d^U !Un
#':
EI#Y
	d
.
#':
	d
$ 	<< 	<D 	< 	< ,0&!$Z& #4.& 
d	&V ,0!$Z #4. 
d	<(#':(	d(: FJY,4TNY$ BF
"
19#
&Dy &DP
0
%>
.! r   r   )r'  r  enumr  rf  r  r  r  r  rL  rT  r  r   	functoolsr   typingr   r   r   r	   r
   r   r   r   r   r   r   r   r   r  httpxr+  r   pydanticr   typing_extensionsr   r  litellm.litellm_core_utils2litellm.litellm_core_utils.exception_mapping_utilsr   litellm._loggingr   litellm._uuidr   litellm.caching.cachingr   r   r   r   litellm.constantsr   r   r   "litellm.integrations.custom_loggerr    #litellm.litellm_core_utils.asyncifyr!   r  r"   r#   ,litellm.litellm_core_utils.coroutine_checkerr$   .litellm.litellm_core_utils.credential_accessorr%   %litellm.litellm_core_utils.dd_tracingr&   *litellm.litellm_core_utils.litellm_loggingr'   r  0litellm.litellm_core_utils.sensitive_data_maskerr(   $litellm.llms.openai_like.json_loaderr)   &litellm.router_strategy.budget_limiterr*   "litellm.router_strategy.least_busyr+   #litellm.router_strategy.lowest_costr,   &litellm.router_strategy.lowest_latencyr-   &litellm.router_strategy.lowest_tpm_rpmr.   )litellm.router_strategy.lowest_tpm_rpm_v2r/   r9  r1   )litellm.router_strategy.tag_based_routingr2   /litellm.router_utils.add_retry_fallback_headersr3   r4    litellm.router_utils.batch_utilsr5   r6   r7   /litellm.router_utils.client_initalization_utilsr8   2litellm.router_utils.clientside_credential_handlerr9   r:   r{  r;   r<   #litellm.router_utils.cooldown_cacher=   &litellm.router_utils.cooldown_handlersr>   r?   r@   rA   rB   ,litellm.router_utils.fallback_event_handlersrC   rD   rE   *litellm.router_utils.get_retry_from_policyrF   rD  !litellm.router_utils.handle_errorrG   rH   'litellm.router_utils.health_state_cacherI   >litellm.router_utils.pre_call_checks.deployment_affinity_checkrJ   ;litellm.router_utils.pre_call_checks.model_rate_limit_checkrK   Dlitellm.router_utils.pre_call_checks.prompt_caching_deployment_checkrL   >litellm.router_utils.router_callbacks.track_deployment_metricsrM   rN   litellm.schedulerrO   rP   r  rQ   rR   rS   rT   rU  rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   litellm.types.servicesrl   litellm.types.utilsrm   rn   ro   rp   r  rq   rr   rs   r  rt   ru   rv   rw   rx   ry   rz   r{   r|   r}   &router_utils.pattern_match_deploymentsr   opentelemetry.tracer   _Spanr*  r   r   r8  r   rT  r   rd  r   r  r   litellm.types.llms.baser   r   r   r   Enumr   r   r   r   r   <module>r	     s          	    #           &  ! 9 " 2   
 < B K M 8 P P E G F H N J P A M 
 R >  
 J 2       2 0 
 * 9 
   G1 H  DDJNM $)) Sy Syr   