
    j                    @   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlm Z  d dl!Z!d dl"Z!d dl#Z!d dl!m$Z$ d d	l%m&Z& d d
l'm(Z( d dl)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8 d dl9m:Z: d dl;m<Z< d dl=m>Z? d dl@mAZA d dlBmCZC d dlDmEZE d dlFmGZG d dlHmIZI d dlJmKZK d dlLmMZM d dlNmOZO d dlPmQZQ d dlRmSZSmTZT d dlUmVZVmWZWmXZX d dlYmZZZ d d l[m\Z\m]Z] d d!l^m_Z_ d d"l`maZambZbmcZcmdZdmeZe d d#lfmgZgmhZhmiZi d d$ljmkZl d d%lmmnZnmoZo d d&lpmqZq d d'lrmsZs d d(ltmuZumvZv d d)lwmxZxmyZy d d*lzm{Z{m|Z|m}Z}m~Z~ d d+lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d d,lmZ d d-lmZmZmZ d d.lmZ d d.lmZ d d/lmZmZmZ d d0lmZmZmZmZmZmZmZmZmZmZ d1d2lmZ erd d3lmZ d d4lmZmZ eeef   ZneZeZeZ G d5 d6ejf                        Z G d7 d8      Zy)9    Ndefaultdict)	lru_cache)TYPE_CHECKINGAnyAsyncGeneratorCallableDictListLiteralOptionalTupleUnioncast)AsyncOpenAI)	BaseModel)overload)get_secret_str)verbose_router_logger)uuid)	DualCacheInMemoryCache
RedisCacheRedisClusterCache)DEFAULT_MAX_LRU_CACHE_SIZE)CustomLogger)run_async_function)!_get_parent_otel_span_from_kwargs&get_metadata_variable_name_from_kwargs)coroutine_checker)CredentialAccessor)tracer)Logging)SensitiveDataMasker)RouterBudgetLimiting)LeastBusyLoggingHandler)LowestCostLoggingHandler)LowestLatencyLoggingHandler)LowestTPMLoggingHandler)LowestTPMLoggingHandler_v2)simple_shuffle)get_deployments_for_tag) add_fallback_headers_to_responseadd_retry_headers_to_response)"_get_router_metadata_variable_namereplace_model_in_jsonlshould_replace_model_in_jsonl)InitalizeCachedClient)get_dynamic_litellm_paramsis_clientside_credential)CooldownCache)DEFAULT_COOLDOWN_TIME_SECONDS_async_get_cooldown_deployments/_async_get_cooldown_deployments_with_debug_info_get_cooldown_deployments_set_cooldown_deployments)#_check_non_standard_fallback_formatget_fallback_model_grouprun_async_fallback)!get_num_retries_from_retry_policy)#async_raise_no_deployment_exceptionsend_llm_exception_alert)PromptCachingDeploymentCheck)ResponsesApiDeploymentCheck)0increment_deployment_failures_for_current_minute1increment_deployment_successes_for_current_minute)FlowItem	Scheduler)AllMessageValues	FileTypesOpenAIFileObjectOpenAIFilesPurpose)#CONFIGURABLE_CLIENTSIDE_AUTH_PARAMSVALID_LITELLM_ENVIRONMENTSAlertingConfigAllowedFailsPolicyAssistantsTypedDictCredentialLiteLLMParamsCustomRoutingStrategyBase
DeploymentDeploymentTypedDictLiteLLM_ParamsMockRouterTestingParamsModelGroupInfoOptionalPreCallChecksRetryPolicyRouterCacheEnumRouterGeneralSettingsRouterModelGroupAliasItemRouterRateLimitErrorRouterRateLimitErrorBasicRoutingStrategySearchToolTypedDict)ServiceTypes)CustomPricingLiteLLMParamsGenericBudgetConfigTypeLiteLLMBatch)	ModelInfo)ModelResponseStreamStandardLoggingPayloadUsage)
CustomStreamWrapperEmbeddingResponseModelResponseRulesfunction_setupget_llm_provider!get_non_default_completion_params
get_secretget_utc_datetimeis_region_allowed   )PatternMatchRouter)Span)
AutoRouterPreRoutingHookResponsec                       e Zd ZdZy)RoutingArgs<   N)__name__
__module____qualname__ttl     g/Users/manta/Documents/Projects/TheRoad-I1/backend/.venv/lib/python3.12/site-packages/litellm/router.pyrx   rx      s    
Cr   rx   c            W       ,   e Zd ZU  e       Zeed<   dZee   ed<   dZ	e
ed<   dZdZee   ed<   dZee   ed<   dZeeeeeef         ed	<   ddddddddi dddddddddddd
dg g g i ddddi ddddddi dd e       df*deeee   eeeef      f      dee   deee      dee   dee   dee
   dee   dee   dedeee      de
dee   dee
   dee
   dee
   dee   dee   dee   dee
   ded ed!   d"eee      d#ed$ed%ed&eeeeee f   f      d'ed(ed)e
d*eee!ef      d+eee!f   d,ee
   d-ee"   d.ee   d/ee   d0ed1   d2ee#   d3ed4ee$   d5ee%   d6ee   d7ed8dfVd9Z&d: Z'd; Z(e)d<eeef   d8ee*e+f   fd=       Z,d>e*fd?Z-d0ee.ef   d3efd@Z/dA Z0dB Z1dC Z2dD Z3dEee   fdFZ4d2ee#   fdGZ5dHefdIZ6dJedKeeeef      d8ee7e8f   fdLZ9dJedKeeeef      d8ee7e8f   fdMZ:e;dJedKee<   dNedO   d8e8fdP       Z=e;	 d!dJedKee<   dNed   d8e7fdQ       Z=e;	 d!dJedKee<   dNeedO   ed   f   d8ee8e7f   fdR       Z=	 d!dJedKee<   dNefdSZ=dTe8dKeeeef      dUed8e8fdVZ>dJedKeeeef      d8ee7e8f   fdWZ?	 d"dJedXedYee   d8dfdZZ@	 d"dXedYee   d8dfd[ZA	 d#dHedXed\ee   d8eBfd]ZC	 d#dHedXed\ee   d8dfd^ZDdHedXefd_ZEdXed`ed8eeee
f      fdaZFdXed`ed8eeee
f      fdbZGdXed`ed8eeee
f      fdcZHddee   dKeeeeef      eeeeef         f   fdeZIdJedKeee<      fdfZJe;dJedKeeeef      dNedO   d8e8fdg       ZKe;	 d!dJedKeeeef      dNed   d8e7fdh       ZK	 d!dJedKeeeef      dNefdiZKe;	 d!dJedKee<   dje
dNed   d8e7f
dk       ZLe;dJedKee<   dje
dNedO   d8e8f
dl       ZL	 d!dJedKee<   dje
fdmZLdJedje
dnedoeMedpf   dXeeef   f
dqZNdJed8efdrZOdJedKee<   dXeeef   fdsZPdtedJefduZQdtedJefdvZRdtedJefdwZSdtedJefdxZTdyeUdJefdzZVdyeUdJefd{ZWdJed|ed}efd~ZXdJefdZYdJefdZZ	 	 	 d$dJedtedee   dee   dee   f
dZ[	 	 	 d$dJedtedee   dee   dee   f
dZ\dJedtefdZ]	 	 	 d$dedJedee   dee   dee   f
dZ^dedJefdZ_dnefdZ`dJedefdZadJednefdZbdXeeef   dJeded8eeef   fdZcdJedefdZddJednefdZe	 d!dJed|eeef   dee   d8effdZgd|eeef   dJefdZh	 d%dJed|eeef   dee   d8effdZid|eeef   dJefdZjdJed8ekfdZldJed8ekfdZmdJed8enfdZodJed8enfdZp	 d#dJee   d8enfdZqdJefdZr	 d#dnedee   fdZs	 d&dneded   fdZt	 d#dnedee   fdZudnefdZv	 	 d'dnedee   deew   fdZxdeydee   d#ee   d$ee   d%ee   dee   doedXefdZz e{j                         d        Z}	 	 	 	 d(dXedee   d#ee   d$ee   d%ee   f
dZ~ e{j                         d        ZdnefdZ	 d#dXedee   fdZ	 	 	 	 	 d)deydee   dee   d$ee   d%ee   dee   fdZd Z	 d#d#eeeee   f      dee   d8eee      fdZd8ee   fdZ	 	 d'deyde
de
dee   dee   d8ee
ef   fdZd Zd8ee   fdZd8efdZdee   fdZdXed8ed   fdZdXedeyd8efdZdedee   d8e
fdZd8efdÄZdJede7dXed8efdńZdJedee   fdƄZdJedee   d8eMee   ee   f   fdǄZdHefdȄZ	 d#dHedee   dee   fdʄZ	 	 d'dJedee   dKeee<      dee   dee   dee   fd̄Zdedefd΄Zdedededed8eeB   f
dӄZded8efdԄZdHeBfdՄZdHeBd8efdքZdefdׄZdHeBd8eBfd؄ZdHeBdedJefdلZdHeBd8eeB   fdڄZdede
d8dfd݄Z	 d#dJedee   d8dfdބZdHeBd8eeB   fd߄Zded8eeB   fdZded8eeB   fdZded8ee   fdZded8eeB   fdZe;	 d#dHededdd8efd       Ze;dHddeded8efd       Z	 d#dHee   dedee   d8efdZded8ee   fdZded8ee   fdZdeded8ee   fdZdeded8ee   fdZded8ee   fdZded8eMee
   ee
   f   fdZ ee      ded8ee   fd       Zded8eee
f   fdZ	 d#dedee   d8efdZded8dfdZdefdZ	 d*dee   ded8ee   fdZded8efdZdeded8ee   fdZ	 d#dedJedee   d8efdZ	 	 d'dedee   dee   d8ee   fd Zd#dee   d8ee   fdZ	 d#dHedee   d8ee   fdZdee   d8efdZ	 d#dee   d8ee   fdZ	 d'dee   dee   d8eee      fdZ	 	 	 d+dee   dee   dee   d8eeee   f   fdZĐded8efd	ZŐd
 ZƐd Zǐd#dZ	 d#dJededKeeeef      dee   fdZdJed8ee   fdZdJed8efdZ	 	 	 	 d,dJedKeeeeef         d|eeeef      dee   dee   d8eMeeeef   f   fdZ	 	 	 	 d,dJededKeeeeef         d|eeeef      dee   dee   d8eee   ef   fdZ	 	 	 d-dJededKeeeeef         d|eeeef      dee   f
dZ	 	 	 d-dJededKeeeeef         d|eeeef      dee   d8ee   fdZ	 	 	 	 d,dJedKeeeeef         d|eeeef      dee   dee   f
dZdee   dee   d8ee   fdZ	 d#dee   fdZ	 d#deydee   fdZԐdeyfdZՐd Z֐defdZؐd Zِd  Zy(.  Routermodel_namesFcache_responsesi  default_cache_time_secondsNleastbusy_loggerlowesttpm_loggeroptional_callbacksINFOr   simple-shuffle
model_listassistants_configsearch_tools	redis_url
redis_host
redis_portredis_passwordcache_kwargscaching_groups
client_ttlpolling_intervaldefault_prioritynum_retriesmax_fallbackstimeoutstream_timeoutdefault_litellm_paramsdefault_max_parallel_requestsset_verbosedebug_level)DEBUGr   default_fallbacks	fallbackscontext_window_fallbackscontent_policy_fallbacksmodel_group_aliasenable_pre_call_checksenable_tag_filteringretry_afterretry_policymodel_group_retry_policyallowed_failsallowed_fails_policycooldown_timedisable_cooldownsrouting_strategy)r   
least-busyusage-based-routinglatency-based-routingcost-based-routingusage-based-routing-v2optional_pre_call_checksrouting_strategy_argsprovider_budget_configalerting_configrouter_general_settingsignore_invalid_deploymentsreturnc+                    || _         |*| _        || _        || _        || _        ddlm}+  |+       | _        dt        _	        | j                   du rQ|dk(  r$t        j                  t        j                         n(|dk(  r#t        j                  t        j                         |)xs
 t               | _        || _        |xs g | _        g | _        i | _        d},d}-i }.|| _        ||K|Id},|||.d	<   |||.d
<   |t-        |      |.d<   |||.d<   |.j/                  |	       | j1                  |.      }-|r7t        j2                   t        j4                  d d|,i|.t        _        || _        t9        |-t;                     | _        t=        ||-      | _        || _         d| _!        || _"        g | _#        tI               | _%        i | _&        i | _'        |xs i | _(        i | _)        i | _*        |G| jW                  |       | jX                  | _-        |D ]  }/d|/d   v sd| j(                  |/d   d   <   ! ng | _,        | | | _.        nt        j\                  | _.        |"xs t^        | _0        tc        | j2                  | j`                        | _2        |#| _3        t;               | _4        ||| _5        n;t        jj                  t        jj                  | _5        ntl        jn                  | _5        ||| _8        n;t        jp                  t        jp                  | _8        nt        jr                  | _8        |xs t        jt                  | _;        || _<        || _=        |$| _>        |xs t        j~                  }0| j                  |0       |0| _?        |t        j                  H|xs t        j                  }0| j~                  | j~                  j                  d|0i       n
d|0ig| _?        |xs t        j                  | _C        |xs t        j                  }1| j                  |1       |1| _D        t        t              | _G        t        t              | _H        t        t              | _I        g | _J        |xs i }t        j                  ||       | _L        || _M        | j                  j                  d|       | j                  j                  dd       | j                  j                  di       j/                  d|
i       i | _O        	 | j                  |$|&       d| _Q        t        t        j                  t              r*t        j                  j                  | j                         n)t        j                  j                  | j                         t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _X        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _[        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _^        |&| _a        |'| _b        d| _c        t        j                  || j                        r|%|%j                  d       ndg}%d| _f        |vt        |t              rt        d i || _f        nt        |t              r|| _f        t        j                  dj                  | j                  j                  d                   || _l        d| _m        |!vt        |!t              rt        d i |!| _m        nt        |!t              r|!| _m        t        j                  dj                  | j                  j                  d                   |(| _o        |%| j                  |%       | j                  | j                          | j                          | j                          | j                          y)!ah  
        Initialize the Router class with the given parameters for caching, reliability, and routing strategy.

        Args:
            model_list (Optional[list]): List of models to be used. Defaults to None.
            redis_url (Optional[str]): URL of the Redis server. Defaults to None.
            redis_host (Optional[str]): Hostname of the Redis server. Defaults to None.
            redis_port (Optional[int]): Port of the Redis server. Defaults to None.
            redis_password (Optional[str]): Password of the Redis server. Defaults to None.
            cache_responses (Optional[bool]): Flag to enable caching of responses. Defaults to False.
            cache_kwargs (dict): Additional kwargs to pass to RedisCache. Defaults to {}.
            caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None.
            client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600.
            polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms.
            default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None.
            num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2.
            timeout (Optional[float]): Timeout for requests. Defaults to None.
            default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}.
            set_verbose (bool): Flag to set verbose mode. Defaults to False.
            debug_level (Literal["DEBUG", "INFO"]): Debug level for logging. Defaults to "INFO".
            fallbacks (List): List of fallback options. Defaults to [].
            context_window_fallbacks (List): List of context window fallback options. Defaults to [].
            enable_pre_call_checks (boolean): Filter out deployments which are outside context window limits for a given prompt
            model_group_alias (Optional[dict]): Alias for model groups. Defaults to {}.
            retry_after (int): Minimum time to wait before retrying a failed request. Defaults to 0.
            allowed_fails (Optional[int]): Number of allowed fails before adding to cooldown. Defaults to None.
            cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1.
            routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle".
            routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}.
            alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None.
            provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None.
            ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
        Returns:
            Router: An instance of the litellm.Router class.

        Example Usage:
        ```python
        from litellm import Router
        model_list = [
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-1>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-2>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "openai-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "gpt-3.5-turbo",
                "api_key": <your-api-key>,
            },
        ]

        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
        ```
        r   )ServiceLoggingTr   r   localNredisurlhostportpasswordtype)redis_cachein_memory_cache)r   r   modellitellm_params)cachedefault_cooldown_time)fallback_param*)params
router_objr   max_retriesmetadatar   r   r   )r   r   router_budget_limitingz+[32mRouter Custom Retry Policy Set:
{}[0mexclude_nonez3[32mRouter Custom Allowed Fails Policy Set:
{}[0mr~   )ur   r   r   r   r   litellm._service_loggerr   service_logger_objlitellmsuppress_debug_infor   setLevelloggingr   r   rZ   r   r   r   deployment_namesdeployment_latency_mapr   strupdate_create_redis_cacher   Cacher   r   r   rF   	schedulerr   default_deploymentr   provider_default_deployment_idsrs   pattern_routerteam_pattern_routersauto_routersr    model_id_to_deployment_index_map model_name_to_deployment_indicesset_model_listr   healthy_deploymentsr   r6   r   r5   cooldown_cacher   failed_callsr   openaiDEFAULT_MAX_RETRIESr   ROUTER_MAX_FALLBACKSrequest_timeoutr   r   r   r   r   validate_fallbacksr   appendr   r   r   inttotal_calls
fail_callssuccess_callsprevious_modelsChatchatr   
setdefaultdeployment_statsrouting_strategy_initaccess_groups
isinstance_async_success_callbacklistlogging_callback_manager"add_litellm_async_success_callbackdeployment_callback_on_successsuccess_callbackadd_litellm_success_callback#sync_deployment_callback_on_success_async_failure_callback"add_litellm_async_failure_callback$async_deployment_callback_on_failurefailure_callbackadd_litellm_failure_callbackdeployment_callback_on_failurer   r   router_budget_loggerr%   !should_init_router_budget_limiterr   dictrX   infoformat
model_dumpr   r   rN   r   add_optional_pre_call_checks_initialize_alertinginitialize_assistants_endpointinitialize_router_endpointsapply_default_settings)2selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
cache_typer   cache_configm
_fallbacks_content_policy_fallbackss2                                                     r   __init__zRouter.__init__   s   l '*D'&&<#$8!:2@2B&*#t#f$%..w||<'%..w}}=#>'<'> 	$ "3(.B 	 ')#  	 ')$ Z%;
@V J$&/U#%'1V$%'*:V$)+9Z( -22<@K}}$ ' N: N N#2D #]_


 #-;
 !1"&-J*:<,02  	! 68 # 	
 AC- GI-!
+-1__D$a 011PQD//2B0CG0LM  
  O $!.D!(!6!6D*K.K+**D4F4F
 "3O 	 "*D  ,&22D%99D$!.D"".!(!6!6D!(!=!=D9'"9"9,& 0 3'"3"3
z:#(G,E,E,Q*Gg.G.GJ~~)%%sJ&78#&
"3!4 %H(H(H 	%
 %H(H(H 	" 	/HI(A%(3)
 (3(
 +6+
  	
 "8!=2LL(>4P	 '=###..y'B##..}a@##..z2>EE~.	
 ')	 	""-"7 	# 	
 "g55t<,,OO33 ,,OO33 g..5,,II88 )-(P(P'QG$g55t<,,OO99
 99/G+ g..5,,II33 )-(K(K'LG$%:"&<#DH!AA!$:U:U
 (3(//0HI,D+E(37#,-$/$?,$?!L+6$0!!&&DKK%%00d0C % 	% CG!+.5,>,VAU,V)02DE,@)!&&LSS--88d8K :I#/--.FG+%%'++-((*##%r   c                 *    g }| j                  |       y)z;
        Apply the default settings to the router.
        N)r  )r  default_pre_call_checkss     r   r  zRouter.apply_default_settings{  s    
 :<))*ABr   c                 $   t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j
                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        | j                  B| j                  D ]2  }t         j                  j                  t         j                  |d       4 yy)z
        Pseudo-destructor to be invoked to clean up global data structures when router is no longer used.
        For now, unhook router's callbacks from all lists
        NF)require_self)r   r   #remove_callback_from_list_by_objectr   r  r  r	  input_callbackservice_callback	callbacksr   )r  callbacks     r   discardzRouter.discard  s3   
 	((LL++T	
 	((LL$$d	
 	((LL++T	
 	((LL$$d	
 	((LL""D	
 	((LL$$d	
 	((LLt	

 "". 3300TT%%xe U  4 /r   r  c                 P    | j                  d      rt        di | S t        di | S )za
        Initializes either a RedisCache or RedisClusterCache based on the cache_config.
        startup_nodesr~   )getr   r   )r  s    r   r   zRouter._create_redis_cache  s.     O,$4|44---r   r   c                 T    | j                   j                  || j                   _        yy)z
        Update the redis cache for the router, if none set.

        Allows proxy user to just do
        ```yaml
        litellm_settings:
            cache: true
        ```
        and caching to just work.
        N)r   r   )r  r   s     r   _update_redis_cachezRouter._update_redis_cache  s%     ::!!)%*DJJ" *r   c                    t        j                  d|        |t        j                  j                  k(  s|t        j                  k(  rt        | j                        | _        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _
        t        t        j                  t              r*t        j                  j                  | j                         y y |t        j                   j                  k(  s|t        j                   k(  ret#        | j                  |      | _        t        t        j                  t              r*t        j                  j                  | j$                         y y |t        j&                  j                  k(  s|t        j&                  k(  ret)        | j                  |      | _        t        t        j                  t              r*t        j                  j                  | j*                         y y |t        j,                  j                  k(  s|t        j,                  k(  ret/        | j                  |      | _        t        t        j                  t              r*t        j                  j                  | j0                         y y |t        j2                  j                  k(  s|t        j2                  k(  ret5        | j                  i       | _        t        t        j                  t              r*t        j                  j                  | j6                         y y y )NzRouting strategy: )router_cache)r.  routing_args)r   r  r^   
LEAST_BUSYvaluer&   r   r   r   r   r#  r   r   r%  r   add_litellm_callbackUSAGE_BASED_ROUTINGr)   r   USAGE_BASED_ROUTING_V2r*   lowesttpm_logger_v2LATENCY_BASEDr(   lowestlatency_logger
COST_BASEDr'   lowestcost_logger)r  r   r   s      r   r   zRouter.routing_strategy_init  sn    	""%78H7I#JK : : @ @@?#=#==$;$TD!'00$7&&--d.C.CD*.*?*?)@&'++T200EEdF[F[\ 3  C C I II?#F#FF$;!ZZ2%D! '++T200EEdF[F[\ 3  F F L LL?#I#II'A!ZZ2(D$ '++T200EEdF^F^_ 3  = = C CC?#@#@@(C!ZZ2)D% '++T200EEdF_F_` 3  : : @ @@?#=#==%=!ZZ&D" '++T200EEdF\F\] 3 r   c                 D   | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j
                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _	        y N)
factory_functionr   acreate_assistantsadelete_assistantaget_assistantsacreate_threadaget_threada_add_messageaget_messagesarun_threadr  s    r   r  z%Router.initialize_assistants_endpoint  s    "&"7"78R8R"S!%!6!6w7P7P!Q#44W5L5LM"33G4J4JK001D1DE!2273H3HI!2273H3HI001D1DEr   c                    | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j
                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _	        | j                  t        j                  d	      | _
        | j                  t        j                  d
      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                   d      | _        | j                  t        j"                  d      | _        | j                  t        j$                  d      | _        | j                  t        j&                  d      | _        | j                  t        j(                  d      | _        | j                  t        j*                  d      | _        | j                  t        j,                  d      | _        y)z+Helper to initialize core router endpoints.
moderation	call_typeanthropic_messagesagenerate_contentaadapter_generate_content
aresponsesafile_deleteafile_content	responsesaget_responsesacancel_responsesadelete_responsesalist_input_items
_arealtimeacreate_fine_tuning_jobacancel_fine_tuning_jobalist_fine_tuning_jobsaretrieve_fine_tuning_jobalist_filesaimage_editallm_passthrough_routeN)r<  r   amoderationrJ  aanthropic_messagesrK  rL  rM  rN  rO  rP  rQ  rR  rS  rT  rU  rV  rW  rX  rY  
afile_listr[  r\  rE  s    r   _initialize_core_endpointsz!Router._initialize_core_endpoints  s   00< 1 
 $(#8#8&&2F $9 $
  "&!6!6%%1D "7 "
 *.)>)>--9T *? *
& //, 0 
 !11  N 2 
 "22!!_ 3 
 ..w/@/@K.X"33"".> 4 
 "&!6!6%%1D "7 "
 "&!6!6%%1D "7 "
 "&!6!6%%1D "7 "
 //, 0 
 (,'<'<++7P (= (
$ (,'<'<++7P (= (
$ '+&;&;**6N '< '
# *.)>)>--9T *? *
& //- 0 
  00= 1 
 '+&;&;**6N '< '
#r   c                    ddl m}m}m}m} | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _	        ddl
m}m}m}m}m}	m}
m}m}m}m}m}m} | j                  |d	      | _        | j                  |d
      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |	d      | _        | j                  |d      | _        | j                  |
d      | _        | j                  |d      | _         ddl!m"}m#}m$}m%} | j                  |d      | _"        | j                  |d      | _$        | j                  |d      | _#        | j                  |d      | _%        ddl&m'}m(} | j                  |d      | _'        | j                  |d      | _(        ddl)m}m} | j                  |d      | _        | j                  |d      | _        dd l*m+}m,}m-}m.}m/}m0}m1}m2}m3}m4}  | j                  |d!      | _,        | j                  |d"      | _1        | j                  |d#      | _-        | j                  |d$      | _2        | j                  |d%      | _/        | j                  | d&      | _4        | j                  |d'      | _+        | j                  |d(      | _0        | j                  |d)      | _.        | j                  |d*      | _3        dd+l5m6}!m7}"m8}#m9}$m:}%m;}&m<}'m=}( | j                  |!d,      | _6        | j                  |%d-      | _:        | j                  |#d.      | _8        | j                  |'d/      | _<        | j                  |$d0      | _9        | j                  |(d1      | _=        | j                  |"d2      | _7        | j                  |&d3      | _;        y4)5z`Helper to initialize specialized router endpoints (vector store, OCR, search, video, container).r   )acreateasearchcreatesearchavector_store_searchrH  avector_store_createvector_store_searchvector_store_create)rb  alist	aretrievearetrieve_contentaupdateadeleterd  r   retrieveretrieve_contentr   deleteavector_store_file_createvector_store_file_createavector_store_file_listvector_store_file_listavector_store_file_retrievevector_store_file_retrieveavector_store_file_contentvector_store_file_contentavector_store_file_updatevector_store_file_updateavector_store_file_deletevector_store_file_delete)rK  agenerate_content_streamgenerate_contentgenerate_content_streamrK  r  r~  r  )aocrocrr  r  rc  re  rc  re  )
avideo_contentavideo_generationavideo_listavideo_remixavideo_statusvideo_contentvideo_generation
video_listvideo_remixvideo_statusr  r  r  r  r  r  r  r  r  r  )acreate_containeradelete_containeralist_containersaretrieve_containercreate_containerdelete_containerlist_containersretrieve_containerr  r  r  r  r  r  r  r  N)>litellm.vector_stores.mainrb  rc  rd  re  r<  rf  rg  rh  ri  litellm.vector_store_files.mainrj  rk  rl  rm  rn  r   ro  rp  r   rq  rr  rs  rt  ru  rv  rw  rx  ry  rz  r{  r|  r}  litellm.google_genairK  r~  r  r  litellm.ocrr  r  litellm.searchlitellm.videosr  r  r  r  r  r  r  r  r  r  litellm.containersr  r  r  r  r  r  r  r  ))r  rb  rc  rd  re  avector_store_file_create_fnavector_store_file_list_fnavector_store_file_retrieve_fnavector_store_file_content_fnavector_store_file_update_fnavector_store_file_delete_fnvector_store_file_create_fnvector_store_file_list_fnvector_store_file_retrieve_fnvector_store_file_content_fnvector_store_file_update_fnvector_store_file_delete_fnrK  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s)                                            r   !_initialize_specialized_endpointsz(Router._initialize_specialized_endpointsC  sN   OO$($9$95 %: %
! %)$9$95 %: %
! $(#8#83 $9 $
  $(#8#83 $9 $
 	
 	
 	
 	
 *.)>)>(4O *? *
& )-(=(='3M )> )
% (,'<'<&2K (= (
$ '+&;&;%1I '< '
# ,0+@+@*6S ,A ,
( +/*?*?)5Q +@ +
' +/*?*?)5Q +@ +
' *.)>)>(4O *? *
& *.)>)>(4O *? *
& )-(=(='3M )> )
% *.)>)>(4O *? *
& )-(=(='3M )> )
%	
 	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 )-(=(=$0J )> )
% (,'<'<#/H (= (
$ 	*))$&)A	(((>2,,W	,J++Fh+G	
 	
 	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
  000V//
l/S!22_ 3 
 !11N 2 
 #33&6 4 
 "22_ 3 
 !11N 2 
  000V		
 		
 		
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 !% 5 5(: !6 !
  $44'8  5  
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
 "&!6!6)< "7 "
 !% 5 5(: !6 !
r   c                 D    | j                          | j                          y r;  )r`  r  rE  s    r   r  z"Router.initialize_router_endpoints  s    '')..0r   r   c           	          |y|D ]J  }t        |t              st        d| d      t        |      dk7  s1t        d| dt        |       d       y)z3
        Validate the fallbacks parameter.
        NzItem 'z' is not a dictionary.rr   zDictionary 'z%' must have exactly one key, but has z keys.)r   r  
ValueErrorlen)r  r   fallback_dicts      r   r   zRouter.validate_fallbacks  sl     !+MmT2 6-8N!OPP=!Q& "=/1VWZ[hWiVjjpq 	 ,r   c                 v   ||D ]  }d }|dk(  rt        | j                        }nA|dk(  r-t        | j                  | j                  | j                        }n|dk(  r
t               }|e| j                  g | _        | j                  j                  |       t        j                  j                  |        y y )Nprompt_caching)r   r   )
dual_cacher   r   responses_api_deployment_check)rA   r   r%   r   r   rB   r   r   r   r   r2  )r  r   pre_call_check	_callbacks       r   r  z#Router.add_optional_pre_call_checks  s     $/":48	!%55 <4:: NI#'?? 4#'::/3/J/J#'??!I
 $'GG ; =I(..624/++229=44II)T! #; 0r   
deploymentc                 .   	 t        j                  |      }|d   }t        j                  r#t	        dd      }|j                  |      |d<   |S d|v r|d   dd dz   |d<   |S # t        $ r(}t        j                  dt        |              |d}~ww xY w)	z
        returns a copy of the deployment with the api key masked

        Only returns 2 characters of the api key and masks the rest with * (10 *).
        r      r   )visible_prefixvisible_suffixapi_keyNz
**********z+Error occurred while printing deployment - )
copydeepcopyr   redact_user_api_key_infor$   	mask_dict	Exceptionr   debugr   )r  r  _deployment_copyr   maskeres         r   print_deploymentzRouter.print_deployment  s    	#}}Z8#34D#EN//,AaP5;5E5En5U !12 $# n,,:9,Ebq,IH,Ty)## 	!''=c!fXF G		s   AA# A# #	B,#BBr   messagesc                     	 t        j                  d| d       ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di |}|S # t
        $ r}|d}~ww xY w)	z
        Example usage:
        response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}]
        zrouter.completion(model=z,..)r   r  original_functionr   kwargsNr~   )r   r  _completion_update_kwargs_before_fallbacksfunction_with_fallbacksr  )r  r   r  r  responser  s         r   
completionzRouter.completion0  s    
	!''*B5'(NO#F7O!)F:*.*:*:F&'00uV0L3t33=f=HO 	G	s   AA 	A+$A&&A+c           	         d }	 | j                  |||j                  dd       |      }| j                  ||       |d   }|d   }| j                  ||      }|j	                  dd       }||||j
                  k7  rd }	n|}	| j                  |      s| j                  |       t        j                  di i ||| j                  |	d|}
t        j                  d	| d
       t        |
t              r.| j                  ||
|      }|rt        j                   d|d      |
S # t"        $ r,}t        j                  d	| dt%        |       d       |d }~ww xY w)Nspecific_deploymentr   r  r  request_kwargsr  r  r   r   r  r  r  cachingclientzlitellm.completion(model=)[32m 200 OK[0mr   r  r  Response output was blocked. messager   llm_provider)[31m Exception [0mr~   )get_available_deploymentpop_update_kwargs_with_deployment_get_clientr*  r  has_model_id routing_strategy_pre_call_checksr   r  r   r   r  r   rj   "_should_raise_content_policy_errorContentPolicyViolationErrorr  r   )r  r   r  r  
model_namer  datapotential_model_clientdynamic_api_keymodel_clientr  _should_raiser  s                r   r  zRouter._completionC  s    
>	66!$*JJ/Dd$K%	 7 J //:f/U ./DgJ%)%5%5%f &6 &" %jjD9O+*6#'='E'EE#5 $$U+555L))  (#33*	
 H "&&+J<7NO
 (M2 $ G G(6 !H ! !!== >#%'  O 	!&&+J<7KCPQF8SZ[ G		s   D#D( (	E1'EEstreamTc                    K   y wr;  r~   r  r   r  r  r  s        r   acompletionzRouter.acompletion        	   c                    K   y wr;  r~   r  s        r   r  zRouter.acompletion  r  r  c                    K   y wr;  r~   r  s        r   r  zRouter.acompletion  r  r  c                 &  K   	 ||d<   ||d<   ||d<   | j                   |d<   | j                  ||       |j                  d      xs | j                  }t	        j
                         }| j                  |      }|r| j                  |||       d {   S |+t        |t              r | j                  di | d {   }n | j                  di | d {   }t	        j
                         }	|	|z
  }
t        j                  | j                  j                  t         j"                  |
d||	t%        |      	             |S 7 7 7 n# t&        $ r;}t        j                  t)        | |t+        j,                         |
             |d }~ww xY ww)Nr   r  r  r  r  priority)r   r  r  r  servicedurationrI  
start_timeend_timeparent_otel_spanlitellm_router_instancer  error_traceback_stroriginal_exceptionr~   )_acompletionr  r*  r   timeperf_counter_is_prompt_management_model_prompt_management_factoryr   r   schedule_acompletionasync_function_with_fallbacksasynciocreate_taskr   async_service_success_hookr`   ROUTERr   r  r@   	traceback
format_exc)r  r   r  r  r  request_priorityr  r  r  r  	_durationr  s               r   r  zRouter.acompletion  s    ,	#F7O!)F:%F8*.*;*;F&'00uV0L%zz*5N9N9N**,J*.*J*J5*Q'*!<<%! =   
  +
;KS0Q!:!:!:!DV!DD!C!C!C!Mf!MM((*H :-I''BB(//&+)%%Fv%N C 	 O- EM  		(,0#)(1(<(<(>'(	 G		sl   FBE
 EE
 F'E
 >E?E
 EA)E
 FE
 E
 E
 
	F6F		FFmodel_responseinitial_kwargsc                 n    K   ddl m  G fddt              } fd} | |             S w)z
        Helper to iterate over a streaming response.

        Catches errors for fallbacks using the router's fallback system
        r   )MidStreamFallbackErrorc                   4     e Zd Zdef fdZd Zd Z xZS )ERouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapperasync_generatorc                 v    t         |   |j                  j                  j                         || _        y )N)completion_streamr   custom_llm_providerlogging_obj)superr  r   r  r  _async_generator)r  r  	__class__r  s     r   r  zNRouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__init__  s=     &5(..(6(J(J . : :	 !  )8%r   c                     | S r;  r~   rE  s    r   	__aiter__zORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__aiter__  s    r   c                 R   K   | j                   j                          d {   S 7 wr;  )r  	__anext__rE  s    r   r#  zORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__anext__  s!     !22<<>>>>s   '%')rz   r{   r|   r   r  r!  r#  __classcell__)r  r  s   @r   FallbackStreamWrapperr    s    8 8?r   r%  c                 L  K   	 2 3 d {   } |  7 
6 y # $ r}ddl m}  |j                        }t        t        t
           t        |dd             }	 t        t        j                  d            }j                  dj                        }j                  dj                        }j                  dj                        }j                  d	<   d
ddd|j                  ddgz   d<   j                  |       j                  |d||||d       d {  7  }	t!        |	d      r|	2 3 d {  7  }
|
rt#        |
t$              rt!        |
d      rzddlm} t        t        t
           t        |
dd             }||g}ng }|)t!        |d      r|j*                  |j-                  |       |j/                  |      }t1        |
d|       |
 6 nd  n+# t2        $ r}t5        j6                  d|        |d }~ww xY wY d }~y d }~ww xY ww)Nr   )stream_chunk_builder)chunksusager   r   r   r   r  systema  You are a helpful assistant. You are given a message and you need to respond to it. You are also given a generated content. You need to respond to the message in continuation of the generated content. Do not repeat the same content. Your response should be in continuation of this text: rolecontent	assistantT)r,  r-  prefixr  r  Fr~   )r  disable_fallbacksr   r   r   model_groupargsr  r!  )BaseTokenUsageProcessor)usage_objectszFallback also failed: )litellm.mainr'  r(  r   r   rg   getattrr   r*  r   r   r   r  generated_contentr  *async_function_with_fallbacks_common_utilshasattrr   re   litellm.cost_calculatorr3  r)  r   combine_usage_objectssetattrr  r   error)itemr  r'  complete_response_objectcomplete_response_object_usager1  r   r   r   fallback_responsefallback_itemr3  r)  r4  combined_usagefallback_errorr  r  r  r  r  s                   r   stream_with_fallbackszERouter._acompletion_streaming_iterator.<locals>.stream_with_fallbacks  s    a)"0 $J.) ^)=+?)00,( 26UO4gtD2.T)"&sN,>,>w,G"HK0>0B0B#T^^1I @N?Q?Q2D4Q4Q@, @N?Q?Q2D4Q4Q@, ;?:K:KN#6719$, (I
 %0'(':':&*
= 
2N:. 88). 9  #MM.3&/5M5M(3!##1 N 	 	 	 & 0+>3D  0  0- -$.}>Q$R$+M7$C!" )-$,UO$+M7D$I)" $)#45:GM46M %C$N(/0NPW(X(F(L(L(X$1$8$89W$X %<$Q$Q6C %R %& !/
 !(w O"//A 4EF #
  ))//00@A )()s^)s   H$   H$H!;HCG,(D+)G,<G$ EG$B&G,+H,	H5HHHH$H!!H$)litellm.exceptionsr  rh   )r  r  r  r  r%  rE  r  s   ````  @r   _acompletion_streaming_iteratorz&Router._acompletion_streaming_iterator  s7      	>	?$7 	?"b	) b	)H %%:%<==s   .5c                 @  K   d}i }	 |j                         }||d<   t        |      }t        j                         }| j	                  |||j                  dd      |       d{   }	|	}t        j                         }
|
|z
  }t        j                  | j                  j                  t        j                  |d||
t        |                   | j                  |	|       | j                  |	|       |	d	   }|d   }| j                  |	|      }| j                  |xx   d
z  cc<   i ||| j                   |d|}t#        j$                  di |}|j'                  dd      }| j)                  |	|d      }|]t+        |t        j,                        rC|4 d{    	 | j/                  |	||       d{    | d{   }ddd      d{    n&| j/                  |	||       d{    | d{   }t+        t0              r.| j3                  |||      }|rt#        j4                  d|d      | j6                  |xx   d
z  cc<   t9        j:                  d| d       | j                  |	||       t+        |t<              r| j?                  |||       d{   S |S 7 K7 7 7 7 # 1 d{  7  sw Y   xY w7 7 7 +# t"        j@                  $ rf}|j'                  d	i       j'                  dd      }|j'                  d	i       j'                  dd      }|xjB                  d| d| z  c_!        |d}~wtD        $ rE}t9        j:                  d| dtG        |       d       || jH                  |xx   d
z  cc<   |d}~ww xY ww)z
        - Get an available deployment
        - call it with a semaphore over the call
        - semaphore specific to it's rpm
        - in the semaphore,  make a check against it's local rpm before running
        Nr   r  r  async_get_available_deploymentr  r  r  r  r   rr   r  litellm_logging_objmax_parallel_requestsr  r  client_type)r  r  r  r  r  r  r  zlitellm.acompletion(model=r  )r  r  r  )r  r  r  r   r   z$

Deployment Info: request_timeout: z

timeout: r  r  r~   )%r  r   r  r  rI  r  r
  r  r   r  r`   r  _track_deployment_metricsr  _get_async_openai_model_clientr   r   r   r  r*  r  r   	Semaphore&async_routing_strategy_pre_call_checksrj   r  r  r   r   r  rh   rG  Timeoutr  r  r   r   )r  r   r  r  r  _timeout_debug_deployment_dict#input_kwargs_for_streaming_fallbackr  r  r  r  r  r  r  input_kwargs	_responser  rpm_semaphorer  r  r   deployment_request_timeout_paramdeployment_timeout_params                          r   r  zRouter._acompletionW  sX     
 	'D	28++-/;@/8@H**,J#BB!$*JJ/Dd$K%	  C   J .8*((*H :-I''BB(//&>)%%Fv%N C 	 **%8H +  //:f/U./DgJ>>% ? L Z(A-($//&	
 L  ++;l;I4:JJ%t5K !,,%3 - M
 (Zw00. )= EE#-$/)9 F   
 &/H )== AA) +%5 B    "+? (M2 $ G G(6 !H ! !!== >#%'  z*a/*!&&,ZL8OP **%!!1 +  ($78!AA#+%#F B    O]x )

  / )=== +4  	/M/Q/Q "0c#T* - (F'I'I "(c)T" % IIABbAccn  pH  oI  J  JIG 	!&&,ZL8LSQRVHT[\ %
+q0+G	s   NAK $J/%D7K J2K  J;9J5:	J;J7J;K J9K 0K1	K :K;B-K (K)K ,N-K .N/K 2K 5J;7J;9K ;KKK	K K K N)A!M

NA NNNr  metadata_variable_namec                    |j                  d| j                        |d<   |j                  dt        t	        j
                                      d}| j                  |      r|}|j                  |i       j                  ||d       y)zm
        Adds/updates to kwargs:
        - num_retries
        - litellm_trace_id
        - metadata
        r   litellm_trace_idNr   )r1  r   )r*  r   r   r   r   uuid4_get_model_from_aliasr   )r  r   r  r[  r   s        r   r  z&Router._update_kwargs_before_fallbacks  s}     !'

=$:J:J K},c$**,.?@+/%%E%2 %0"5<<!8IJ	
r   c                     | j                   j                         }|j                  di       xs i }|j                         D ]  \  }}|	|j	                  ||        |j	                  |i       j                  |       y)z
        Adds default litellm params to kwargs, if set.

        Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name
        r   N)r   r  r  itemsr   r   )r  r  r[  defaultsmetadata_defaultskeyr1  s          r   *_update_kwargs_with_default_litellm_paramsz1Router._update_kwargs_with_default_litellm_params  s|     ..335$LLR8>B #..*JC}c5) + 	0"5<<=NOr   function_namec                    |j                  di       j                         }|d   j                         }t        ||      }t        |      }|j                  |i       j                  d      }| j	                  ||      }	|j                  d      }
|	|d<   |
|d<   t        |t        di ||	      }| j                  |
       |S )z.
        Handle clientside credential
        
model_infor   )r   r  rg  r1  )r1  r   idoriginal_model_idr  r   ri  r  r~   )r*  r  r3   r/   _generate_model_idrR   rT   upsert_deployment)r  r  r  rg  ri  r   dynamic_litellm_paramsr[  r1  	_model_idrl  deployment_pydantic_objs               r   _handle_clientside_credentialz$Router._handle_clientside_credential  s      ^^L"5::<
#$45::<!;)&"
 "D'"
 jj!7<@@O++#4J , 
	 'NN40$
4*;
&'",")C,BC!#

 	. 	 	
 '&r   c                    |j                  di       j                         }|d   d   }|d   j                  d      }|d   }t        |      rZ| j                  |||      }|j                  j                         }|j                  j                  }|j                  j                  }t        |      }	|j                  |	i       j                  ||||d	       ||d<   | j                  ||d   
      |d<   | j                  ||	       y)z
        2 jobs:
        - Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
        - Adds default litellm params to kwargs, if set.
        ri  r   r   api_baser  )r  r  r  rg  rj  )r  ri  ru  deployment_model_namer  r  r   )r  r[  N)r*  r  r4   rs  ri  r  r   r   ru  r/   r   r   _get_timeoutrf  )
r  r  r  rg  ri  deployment_litellm_model_namedeployment_api_baserw  rr  r[  s
             r   r  z%Router._update_kwargs_with_deployment4  s9     ^^L"5::<
(23C(DW(M%()9:>>zJ *< 8#6:&*&H&H%fM 'I '# 1;;FFHJ,C,R,R,X,X)"9"H"H"Q"Q!C'"
 	0"5<<;(/)>		
  *| --
+; < . 
y 	772H 	8 	
r   c                     | j                  ||d      }|j                  dd      }||||j                  k7  rd}|S |}|S )a  
        Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment

        The same OpenAI client is re-used to optimize latency / performance in production

        If dynamic api key is provided:
            Do not re-use the client. Pass model_client=None. The OpenAI/ AzureOpenAI client will be recreated in the handler for the llm provider
        asyncrM  r  N)r  r*  r  )r  r  r  r  r  r  s         r   rP  z%Router._get_async_openai_model_clienta  sh     "&!1!1!&g "2 "

 !**Y5'&2#9#A#AAL  2Lr   r  c                     |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      S )z=Helper to get stream timeout from kwargs or deployment paramsr   N)r*  r   r   )r  r  r  s      r   _get_stream_timeoutzRouter._get_stream_timeout{  s_    
 JJ'. Gxx $G ""	G
 **../?F	
r   c                     |j                  dd      xsf |j                  dd      xsR |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      }|S )zAHelper to get non-stream timeout from kwargs or deployment paramsr   Nr   )r*  r   r   r  r  r  r   s       r   _get_non_stream_timeoutzRouter._get_non_stream_timeout  s    
 JJy$' 	@zz+T2	@xx4	@
 xx!4	@ ||	@ **..y$? 	 r   c                 ~    d}|j                  dd      r| j                  ||      }|| j                  ||      }|S )z6Helper to get timeout from kwargs or deployment paramsNr  Frx  )r*  r  r  r  s       r   ry  zRouter._get_timeout  sP    /3::h&..f4.HG?22D 3 G r   modelsc                    K   dt         dt        t           f fd}dt         dt        t           dt        f fd}t	        |t
              rQt        d |D              r?g }|D ]  }|j                   |d||d|        t        j                  |  d{   }|S t	        |t
              rt        d	 |D              rg }t        |      D ]'  \  }	}
|D ]  }|j                   |d||	|
d
|        ) t        j                  |  d{   }t        t        |            D cg c]  }g  }}|D ]A  }t	        |t              r||d      j                  |d          .|d   j                  |       C |S yy7 7 qc c}w w)a6  
        Async Batch Completion. Used for 2 scenarios:
        1. Batch Process 1 request to N models on litellm.Router. Pass messages as List[Dict[str, str]] to use this
        2. Batch Process N requests to M models on litellm.Router. Pass messages as List[List[Dict[str, str]]] to use this

        Example Request for 1 request to N models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    {"role": "user", "content": "is litellm becoming a better product ?"}
                ],
                max_tokens=15,
            )
        ```


        Example Request for N requests to M models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    [{"role": "user", "content": "is litellm becoming a better product ?"}],
                    [{"role": "user", "content": "who is this"}],
                ],
            )
        ```
        r   r  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwzs
            Wrapper around self.async_completion that catches exceptions and returns them as a result
            r   r  Nr~   r  r  r   r  r  r  r  s       r   _async_completion_no_exceptionszARouter.abatch_completion.<locals>._async_completion_no_exceptions  D     -T--WEHWPVWWWW 0   <$ "$ <$ 	949<9<idxc                    K   	  j                   d| |d| d{   |fS 7 # t        $ r}||fcY d}~S d}~ww xY wwr  r  )r   r  r  r  r  r  s        r   *_async_completion_no_exceptions_return_idxzLRouter.abatch_completion.<locals>._async_completion_no_exceptions_return_idx  sS     *$**TTVTT T  #vs4   A & $& A & 	=8=A =A c              3   <   K   | ]  }t        |t                y wr;  )r   r  .0r  s     r   	<genexpr>z+Router.abatch_completion.<locals>.<genexpr>  s     -T8ajD.A8   r  Nc              3   <   K   | ]  }t        |t                y wr;  )r   r   r  s     r   r  z+Router.abatch_completion.<locals>.<genexpr>  s     /VX
1d0CXr  )r   r  r  rr   r   r~   )r   r   rG   r   r   r   allr   r
  gather	enumerateranger  tuple)r  r  r  r  r  r  _tasksr   r  r  r  rP  _final_responsess   `             r   abatch_completionzRouter.abatch_completion  s    H				"&'7"8				+,	 	& h%#-T8-T*TF=gET\g`fgh   %^^V44HO$'C/VX/V,VF )( 3W#EMMB "'S7FL $ !4 &nnf55I<A#h-<P/Q<Pq<PO/Q%h.#HQK077D#A&--h7	 &
 #"# -W' 5 6/Qs8   BFE;A4FE=F&	E?/AF=F?Fc           	          K   dt         dt        t           f fd}g }|D ]  }|j                   |d||d|        t	        j
                  |  d{   }|S 7 w)a  
        Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router

        Use this for sending multiple requests to 1 model

        Args:
            model (List[str]): model group
            messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
            **kwargs: additional kwargs
        Usage:
            response = await self.abatch_completion_one_model_multiple_requests(
                model="gpt-3.5-turbo",
                messages=[
                    [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
                    [{"role": "user", "content": "hello good mornign"}],
                ]
            )
        r   r  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwr  r  r  s       r   r  z]Router.abatch_completion_one_model_multiple_requests.<locals>._async_completion_no_exceptions  r  r  r  Nr~   )r   r   rG   r   r
  r  )r  r   r  r  r  r  message_requestr  s   `       r   -abatch_completion_one_model_multiple_requestsz4Router.abatch_completion_one_model_multiple_requests  sv     ,				"&'7"8		 'OMM/ /=C  ( !00 1s   AAAAc                    K   y wr;  r~   r  s        r   "abatch_completion_fastest_responsez)Router.abatch_completion_fastest_response0  r  r  c                    K   y wr;  r~   r  s        r   r  z)Router.abatch_completion_fastest_response8  r  r  c                    K   |j                  d      D cg c]  }|j                          }}dt        dt        t        t        t        f      dt
        dt        dt        t        t        t        f   f
 fd}g dt        j                  ffd	}|D ]2  }t        j                   |d|||d
|      }	j                  |	       4 r_t        j                  t        j                          d{   \  }
|
D ]'  } ||       d{   }|d|j"                  d<   |c S  r_t        d      c c}w 7 F7 0w)z
        model - List of comma-separated model names. E.g. model="gpt-4, gpt-3.5-turbo"

        Returns fastest response from list of model names. OpenAI-compatible endpoint.
        ,r   r  r  r  r   c                    K   	  j                   d| ||d| d{   }|S 7 # t        j                  $ r& t        j                  dj                  |               t        $ r}|cY d}~S d}~ww xY ww)zn
            Wrapper around self.acompletion that catches exceptions and returns them as a result
            r   r  r  Nz4Received 'task.cancel'. Cancelling call w/ model={}.r~   )r  r
  CancelledErrorr   r  r  r  )r   r  r  r  resultr  r  s         r   r  zRRouter.abatch_completion_fastest_response.<locals>._async_completion_no_exceptionsN  s~     	/t//hehW]haghh i)) %++JQQRWX  s>   A6' %' A6' A A3'A.(A3)A6.A33A6taskc                   K   	 |  d {   }t        |t        t        f      r@t        j                  d       D ]  }|j                           |	 j                  |        S 	 	 j                  |        y 7 n# t        $ r Y S w xY w# t        $ r Y /w xY w# t        $ r Y y w xY w# 	 j                  |        w # t        $ r Y w w xY wxY ww)Nz=Received successful response. Cancelling other LLM API calls.)	r   rj   rh   r   r  cancelremoveKeyErrorr  )r  r  tpending_taskss      r   check_responsezARouter.abatch_completion_fastest_response.<locals>.check_responsea  s     #f}6I&JK)//W +
 +!!((. L!((. $         !((. s   CB
 A9AB
 A;$C%B( 'B 8C9B
 ;	BCBC
	BB( BB( 	B%"C$B%%C(C*B<;C<	CCCCCr  )return_whenNT!fastest_response_batch_completionzAll tasks failedr~   )splitstripr   r   r
   boolr   r   rj   rh   r  r
  Taskr  r   waitFIRST_COMPLETED_hidden_params)r  r   r  r  r  r  r  r  r  r  donecompleted_taskr  r  s   `            @r   r  z)Router.abatch_completion_fastest_response@  sV     &+[[%56%5!'')%56		"&tCH~"6	@D	PS	="5y@A	" 	w|| 	, E&&/ (6EKD
   &  (/7+B+B) #D- #'-n==%QUF))*MN!M #'	  *++C 7j# >s:   D?D6CD?4D;5D?D=D?D?+D?=D?r  c                    K   y wr;  r~   r  r   r  r  r  r  s         r   r  zRouter.schedule_acompletion  r  r  c                    K   y wr;  r~   r  s         r   r  zRouter.schedule_acompletion  r  r  c                   K   t        |      }t        t        j                               }t	        ||d      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                   |       d {    t        j                         }
|
|	k  r|r]	  | j"                  d|||d| d {   }|j$                  j'                  di        |j$                  d   j)                  d	d
i       |S t/        j0                  d|d      7 `7 7 7 7 c# t*        $ r}t-        |d|       |d }~ww xY ww)Nzgpt-3.5-turbor  
request_idr  requestFr   r  rk  r  health_deploymentsr  additional_headers%x-litellm-request-prioritization-usedTr  %Request timed out while polling queuer   r  r~   )r   r   r   r_  rE   r   add_requestr  	monotonicr   r   _async_get_healthy_deploymentspollr  r  r
  sleepr  r  r   r   r  r<  r   rS  )r  r   r  r  r  r  r  _request_idr>  r  	curr_timepoll_intervalmake_request_healthy_deploymentsr  rW  r  s                    r   r  zRouter.schedule_acompletion  s     =VD$**,'"&
 nn(((666 >>#dll2NN$	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 NN,	 (" "2$"2"2 #(6#EK# 	 ((334H"M(()=>EE<dC ! 
 //?% K 	7' 3
  :x0s   AG#F8A,G#F;<G#F=G#"F?#G#G#G GAG G#;G#=G#?G#G 	G GG  G#r  r2  .c                   K   t        |      }t        t        j                               }t	        |||      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                   |       d {    t        j                         }
|
|	k  r|ri	  ||i | d {   }t#        |j$                  t&              r<|j$                  j)                  di        |j$                  d   j+                  ddi       |S t1        j2                  d
|d      7 l7 7 7 7 ~# t,        $ r}t/        |d	|       |d }~ww xY ww)Nr  r  Fr  r  r  r  Tr  r  r   r  )r   r   r   r_  rE   r   r  r  r  r   r   r  r  r  r  r
  r  r   r  r  r   r   r  r<  r   rS  )r  r   r  r  r2  r  r  r  r>  r  r  r  r  r  r  rW  r  s                    r   _schedule_factoryzRouter._schedule_factory  s     =VD$**,'"
 nn(((666 >>#dll2NN$	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 NN,	 (" 
"3T"DV"DD	i66=,,778LbQ,,-ABII@$G ! 
 //?% I 	7' 3
 E  :x0s   AG0GA,G0G<G0G
G0"G#G0G0G GAG +G0G0
G0G0G 	G-G((G--G0c                     | j                  |      }|t        |      dk7  ry|d   d   j                  dd       }|d|vry|j                  d      d   }|t        j
                  v S )Nr  rr   Fr   r   r   /)get_model_listr  r*  r  r   )_known_custom_logger_compatible_callbacks)r  r   r   litellm_modelsplit_litellm_models        r   r  z"Router._is_prompt_management_model  s|    ((E(:
ZA!5"1&67;;GTJ C}$<+11#6q9"g&W&WWWr   c                   K   |j                  dd       }|$t        di dt               t               d|\  }}t	        t
        |      }| j                  |dddg|j                  dd             }| j                  ||	       |d
   j                         }|j                  dd       }|j                  d      xs |d
   j                  dd       }|j                  d      xs |d
   j                  dd       }	|j                  dd       xs |d
   j                  dd       }
|t        |t              st        d| dt        |             |	*t        |	t              st        d|	 dt        |	             |j                  ||t!        |      ||	|
      \  }}}h d}|j#                         D ci c]  \  }}||vs|| }}}i |||}||d<   ||d<   ||d<   ||d<   |	|d<   |
|d<   | j%                  |      }|t'        |      dk(  r.|j                  d       t)        j*                  di | d {   S  | j,                  di | d {   S c c}}w 7 $7 w)NrK  r  )r  	rules_objr  userpromptr+  r  r   r  r  r  r   r   	prompt_idprompt_variablesprompt_labelz*Prompt ID is not set or not a string. Got=z, type=z2Prompt variables is set but not a dictionary. Got=)r  )r   r  non_default_paramsr  r  r  >   r  r  prompt_versionbitbucket_configdotprompt_configr  r  r  r   r  r~   )r*  rl   rk   rp   r   LiteLLMLoggingr  r  r  r  r   r   r  r   r  get_chat_completion_promptrn   rb  r  r  r   r  r	  )r  r   r  r  litellm_logging_objectprompt_management_deploymentr  r  r  r  r  optional_paramsprompt_management_paramskvfiltered_data_model_lists                    r   r  z!Router._prompt_management_factory(  s     "(,A4!H!)-; .)6!&"2"4 	.*"F "&n6L!M'+'D'D%(;< &

+@$ G (E (
$ 	++3F 	, 	
 ,,<=BBD$/JJ{+ !/K0

#k4
  	 "::
 
)*:;??
 	
 zz.$7 $;W<

#nd
# 	 Jy#$><YKwtT]N_`  '
;KT0RDEUDVV]^bcs^t]uv  #==@O-% > 
		
$
  "ZZ\
)TQQ6N-NAqD\ 	 
 @M?V??w%z(>$%'{%5!"!-~))U);#k"2a"7JJ*+ ,,6v6667T77A&AAA#
 7As7   F6I'8II
A4I'>I#?I'I%I'%I'r  c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   r  r  r   r   r1  r~   )_image_generationr*  r   r   r   r  r  r  r  r   r  r  r  s         r   image_generationzRouter.image_generation  s    
	#F7O%F8*.*@*@F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	   A.A1 1	B:A<<Bc           	         d}	 t        j                  d| d|        | j                  |dddg|j                  dd             }| j	                  ||	       |d
   j                         }| j                  ||	      }| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |d|}| j                  |xx   dz  cc<   t        j                  d| d       |S # t        $ rE}	t        j                  d| dt        |	       d       || j                   |xx   dz  cc<   |	d }	~	ww xY w)Nr  #Inside _image_generation()- model: 
; kwargs: r  r  r+  r  r  r  r   rr   r  r  r  r  zlitellm.image_generation(model=r  r  r  r~   )r   r  r  r  r  r  rP  r   r  r   r  r   r   r  r  r   r   )
r  r  r   r  r  r  r  r  r  r  s
             r   r  zRouter._image_generation  s   
*	!''5eWJvhO 66#)h?@$*JJ/Dd$K 7 J
 //:f/U./446D>>% ? L
 Z(A-( 11Z1H// $#33*	
 H z*a/*!&&1*=TU O 	!&&1*=QRUVWRXQYY`a %
+q0+G	s   C?D 	EA EEc           	      l  K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   r  r  r   r  r  r~   )_aimage_generationr*  r   r  r	  r  r
  r  r@   r  r  r  s         r   aimage_generationzRouter.aimage_generation  s     	#F7O%F8*.*A*AF&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B4A A- $A+%A- *B4+A- -	B166B,,B11B4c           	      6  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nr  r  r  r  r+  r  r  r  r   r   rr   r  rL  rM  rJ  z litellm.aimage_generation(model=r  r  r  r~   )r   r  r   rI  r  r  r  rP  r   r   r  r   r  r   r
  rQ  rR  r   r  r  r   r   )r  r  r   r  r  r  r  r  r  r  rX  r  s               r   r  zRouter._aimage_generation  s    
D	!''5eWJvhO  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L
 Z(A-(00 $#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&2:,>UV OqJ )
  . )=== *  	!&&2:,>RSVWXSYRZZab %
+q0+G	s   HAG F$B8G F'G F/-F).	F/7F+8F/<G F-G #G$	G -G.5G #H$G 'G )F/+F/-G /G5F86G=G G 	HA HHHfilec           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d{   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "whisper",
                "litellm_params": {
                    "model": "whisper-1",
                },
            },
        ])

        audio_file = open("speech.mp3", "rb")
        transcript = await client.atranscription(
        model="whisper",
        file=audio_file
        )

        ```
        r   r  r  r  Nr  r~   )	_atranscriptionr  r	  r  r
  r  r@   r  r  )r  r  r   r  r  r  s         r   atranscriptionzRouter.atranscription	  s     .	#F7O!F6N*.*>*>F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   BAA AA BA 	B6BBBc           	      ,  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 G7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)Nz!Inside _atranscription()- model: r  r  r  r+  r  r  r  r   rr   )r  r  r  rL  rM  rJ  zlitellm.atranscription(model=r  r  r  r~   )r   r  r   rI  r  r  r  rP  r   r   r  r   r  r   r
  rQ  rR  r   r  r  r   r   )r  r  r   r  r  r  r  r  r  r  rX  r  s               r   r  zRouter._atranscriptionF	  s    
B	!''3E7*VHM  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:f/U./446D>>% ? L
 Z(A-(--  #33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&/
|;RS OmF )
  . )=== *  	!&&/
|;OPSTUPVxW^_ %
+q0+G	s   HAG FB3G F"G F*(F$)	F*2F&3F*7G F(G F?	G (G)5G HG "G $F*&F*(G *F<0F31F<8G G 	HA HHHinputvoicec           	        K   	 ||d<   ||d<   | j                  |dddg|j                  dd      |       d{   }| j                  ||	       |d
   j                         }|d    | j                  j                         D ])  \  }}||vr|||<   |dk(  s||   j                  |       + | j                  ||d      }	|j                  dd      }
|
|	|
|	j                  k7  rd}n|	}t        j                  di i |d|i| d{   }|S 7 7 # t        $ r;}t        j                  t        | |t!        j"                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "tts",
                "litellm_params": {
                    "model": "tts-1",
                },
            },
        ])

        async with client.aspeech(
            model="tts",
            voice="alloy",
            input="the quick brown fox jumped over the lazy dogs",
            api_base=None,
            api_key=None,
            organization=None,
            project=None,
            max_retries=1,
            timeout=600,
            client=None,
            optional_params={},
        ) as response:
            response.stream_to_file(speech_file_path)

        ```
        r
  r  r  r  r+  r  Nr  r  r   r   r   r}  rM  r  r  r  r~   )rI  r  r  r  r   rb  r   r  r*  r  r   aspeechr  r
  r  r@   r  r  )r  r   r
  r  r  r  r  r  r  r  r  r  r  r  s                 r   r  zRouter.aspeech	  s    @4	#F7O#F7O#BB#)h?@$*JJ/Dd$K%	  C   J 00uV0L./446DM3399;1VO !F1I*_1I$$Q' < &*%5%5%f' &6 &" %jjD9O+*6#'='E'EE#5$__ l  H OM>  		(,0#)(1(<(<(>'(	 G		sR   E 6D DAD A4D DD E D D 	E"6EEE c           	      6  K   	 ||d<   t         |d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwNr   r
  r  r  r  r~   )
r
  _arerankr  r	  r  r
  r  r@   r  r  r  r   r  r  r  s        r   arerankzRouter.arerank	  s     	#F7O#F7O*.--F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   BAA 	A
A BA 	B6BBBc           	        K   d }	 t        j                  d| d|        | j                  ||j                  dd       |       d {   }| j	                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i || j                  |d	| d {   }| j                  |xx   dz  cc<   t        j                  d
| d       |S 7 7 8# t        $ rE}t        j                  d
| dt        |       d       || j                  |xx   dz  cc<   |d }~ww xY ww)NzInside _rerank()- model: r  r  )r   r  r  r  r   r   rr   )r  r  zlitellm.arerank(model=r  r  r  r~   )r   r  rI  r  r  r  rP  r   r   r  r   r   r  r  r   r   )	r  r   r  r  r  r  r  r  r  s	            r   r  zRouter._arerank	  s    
'	!''+E7*VHE  $BB$*JJ/Dd$K%  C   J
 //:f/U./446DgJ>>% ? L Z(A-($__ #33* 	 H z*a/*!&&(4KL O9  	!&&(4HQPWX %
+q0+G	sO   EAD D	BD D5D ED D 	EA EEEis_retryis_fallbackis_asyncc                    d|dg}	 ||d<   ||d<   |j                  d| j                        |d<   |j                  di       j                  d|i       | j	                  |||j                  dd       	      }|d
   j                         }	| j                  j                         D ])  \  }
}|
|vr|||
<   |
dk(  s||
   j                  |       + t        j                  di i |	|| j                  d|S # t        $ r}|d }~ww xY w)Nr  r+  r   r  r   r   r1  r  r  r   )r  r  r~   )r*  r   r   r   r  r  r  r   rb  r   text_completionr   r  )r  r   r  r  r  r  r  r  r  r  r  r  r  s                r   r  zRouter.text_completion"
  s1    $78	#F7O%F8$*JJ}d>N>N$OF=!j"-44mU5KL 66!$*JJ/Dd$K 7 J ./446D3399;1VO !F1I*_1I$$Q' < **s-r-rSWSgSg-rkq-rss 	G	s   B2C9 ;=C9 9	D	DD	c           	        K   |j                  dd       9| j                  ||j                  d      | j                  ||f|       d {   S 	 ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 M7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr  )r   r  r  r2  r  r   r  r  r  r  r~   )r*  r  r  atext_completion_atext_completionr  r	  r  r
  r  r@   r  r  )	r  r   r  r  r  r  r  r  r  s	            r   r  zRouter.atext_completionG
  s      ::j$'3//J/"&"7"7V_ 0   	#F7O%F8*.*@*@F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		sI   AC"	B
C"AB BB C"B 	C$6CCC"c           	      2  K   	 t        j                  d| d|        t        |      }| j                  |d|dg|j	                  dd       |       d {   }| j                  ||       |d   j                         }|d	   }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)N#Inside _atext_completion()- model: r  r  r+  r  r  r  r   r   rr   r  rL  rM  rJ  zlitellm.atext_completion(model=r  r  r  r~   )r   r  r   rI  r  r  r  rP  r   r   r  r   r  r   r
  rQ  rR  r   r  r  r   r   )r  r   r  r  r  r  r  r  r  r  rX  r  s               r   r  zRouter._atext_completionl
  s    C	!''5eWJvhO  AH#BB#)f=>$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L Z(A-(// $#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&1*=TU OoH )
  . )=== *  	!&&1%8LSQRVHT[\  &!+&G	   HAG F"B8G F%G F-+F',	F-5F)6F-:G F+G !G"	G +G,5G !H"G %G 'F-)F-+G -F?3F64F?;G G 	HA HHH
adapter_idc           	        K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   r  r  r   r   r1  r  r~   )_aadapter_completionr*  r   r   r   r	  r  r
  r  r@   r  r  )	r  r  r   r  r  r  r  r  r  s	            r   aadapter_completionzRouter.aadapter_completion
  s     	#F7O#-F< *.*C*CF&'$*JJ}d>N>N$OF=!j"-44mU5KL?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   CA0A= 4A;5A= :C;A= =	C6B<<CCc           	      2  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nz&Inside _aadapter_completion()- model: r  r  zdefault textr+  r  r  r  r   r   rr   )r  r  r  rL  rM  rJ  z"litellm.aadapter_completion(model=r  r  r  r~   )r   r  r   rI  r  r  r  rP  r   r   r"  r   r  r   r
  rQ  rR  r   r  r  r   r   )r  r  r   r  r  r  r  r  r  r  rX  r  s               r   r!  zRouter._aadapter_completion
  s    C	!''8z&R  AH#BB#)nEF$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L Z(A-(22 ",#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&4ZL@WX OoH )
  . )=== *  	!&&4UG;OPSTUPVxW^_  &!+&G	r  c                 T   K   ddl m}  |j                  d| |d| d{   S 7 w)z
        Helper function to make a search API call through the router with load balancing and fallbacks.
        Reuses the router's retry/fallback infrastructure.
        r   SearchAPIRouter)router_instancer  Nr~   )&litellm.router_utils.search_api_routerr&  async_search_with_fallbacks)r  r  r  r&  s       r   _asearch_with_fallbackszRouter._asearch_with_fallbacks  s@     
 	K@_@@ 
 /
 
 
 	
 
s   (&(original_generic_functionc                 V   K   ddl m}  |j                  d| ||d| d{   S 7 w)z
        Helper function for search API calls - selects a search tool and calls the original function.
        Called by async_function_with_fallbacks for each retry attempt.
        r   r%  )r'  r   r+  Nr~   )r(  r&  "async_search_with_fallbacks_helper)r  r   r+  r  r&  s        r   _asearch_with_fallbacks_helperz%Router._asearch_with_fallbacks_helper"  sC      	KG_GG 
 &?
 	
 
 	
 
s    )')c           	      f  K   	 ||d<   ||d<   | j                   |d<   | j                  ||d       t        j                  d| d|         | j                  d
i | d{   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |	             |d}~ww xY ww)
        Helper function to make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
        r   r+  r  litellm_metadatar   r  r[  z3Inside ageneric_api_call_with_fallbacks() - model: r  Nr  r~   )(_ageneric_api_call_with_fallbacks_helperr  r   r  r	  r  r
  r  r@   r  r  )r  r   r  r  r  r  s         r   !_ageneric_api_call_with_fallbacksz(Router._ageneric_api_call_with_fallbacks2  s     	#F7O2CF./*.*W*WF&'00FCU 1  "''EeWJW]V^_ @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B1AA* !A("A* 'B1(A* *	B.36B))B..B1r  c                     d|v rT|d   rOddl m} 	  |||j                  d      |j                  d            \  }}}}|}|d   j	                  ||      |d<   |S # t        $ r |}Y 'w xY w)a  
        Add the deployment model to the endpoint for LLM passthrough route.

        e.g for bedrock invoke users can pass endpoint as /model/special-bedrock-model/invoke
          it should be actually sent as /model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke
        endpointr   )rm   r  ru  )r   r  ru  )r   rm   r*  r  replace)r  r  r   r  rm   stripped_model_namer  replacement_model_names           r   ;_add_deployment_model_to_endpoint_for_llm_passthrough_routezBRouter._add_deployment_model_to_endpoint_for_llm_passthrough_routeQ  s     F:$6 1
4/?$(.

3H(I#ZZ
30,#Q1
 *=&
 "(
!3!;!;-"F:   4)3&4s   0A A)(A)c           	      ~  K   |j                  dd      }d}	 t        |      }	 | j                  |||j                  dd      |j                  dd             d{   }| j                  |||	       |d
   j                         }	|	d   }
| j                  |
xx   dz  cc<   | j                  |||
        |di i |	d| j                  i|}| j                  ||d      }|\t        |t        j                        rB|4 d{    	 | j                  ||       d{    | d{   }ddd      d{    n%| j                  ||       d{    | d{   }| j                  |
xx   dz  cc<   t!        j"                  d|
 d       |S 7 ?# t        $ r!}|r |dd|i| d{  7  cY d}~S |d}~ww xY w7 7 7 7 # 1 d{  7  sw Y   |xY w7 7 # t        $ rE}t!        j"                  d| dt%        |       d       || j&                  |xx   dz  cc<   |d}~ww xY ww)r0  passthrough_on_no_deploymentFr4  r  Nr  )r   r  r  r  r   rv  r   rr   )r  r   r  r  rL  rM  rJ  z'ageneric_api_call_with_fallbacks(model=r  r  r  r~   )r  r   rI  r*  r  r  r  r   r:  r   r  r   r
  rQ  rR  r   r   r  r   r   )r  r   r+  r  r<  rg  r  r  r  r  r  r  rX  s                r   r3  z/Router._ageneric_api_call_with_fallbacks_helperp  s     (.zz2PRW'X$;F	@H
#'#F#F#)#ZZ
D9(.

3H$(O	 $G $ 
 //%fM 0  ./446DgJZ(A-(LLUz M  1 t33 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&9*E\] Oy  /!:!Q!Q&!QQQQD )
  . )=== *  	!&&9%@TUXYZU[T\\cd  &!+&G	s  H=G, 8F FF !B'G, G	G, G$G%	G.G/G3G, >G?G, G(	G, $G*%5G, H=F 	G'G6F97G;G<G,  H=GGG, GGG, G%GG%!G, *G, ,	H:5A H55H::H=c           
      F   |j                   }	 t        j                  d| d| d|        | j                  ||j	                  dd      |j                  dd            }| j                  ||d	       |d
   j                         }|d   }| j                  |xx   dz  cc<   d|v r|d   r|d   j                  ||      |d<   ||d<   | j                  |       	 t        |d         \  }}	}} |di i ||	| j                  d|}
| j                  |xx   dz  cc<   t        j                  | d| d       |
S # t        $ r d}	Y \w xY w# t        $ rG}t        j                  | d| dt!        |       d       || j"                  |xx   dz  cc<   |d}~ww xY w)a  
        Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
        Args:
            model: The model to use
            original_function: The handler function to call (e.g., litellm.completion)
            **kwargs: Additional arguments to pass to the handler function
        Returns:
            The response from the handler function
        z&Inside _generic_api_call() - handler: z	, model: r  r  Nr  r  generic_api_callrv  r   r   rr   r6  r  r^  )r  r  z(model=r  r  r  r~   )rz   r   r  r  r*  r  r  r  r   r7  r  rm   r  r   r   r  r   r   )r  r   r  r  handler_namer  r  r  r  r  r  r  s               r    _generic_api_call_with_fallbacksz'Router._generic_api_call_with_fallbacks  s    )114	!''8iPUwV`ag`hi 66J5$*JJ/Dd$K 7 J
 //%fDV 0  ./446DgJZ(A-( V#z(:%+J%7%?%?z%Rz"(F7O 11Z1H+/?d7m/T,&1 ) +>#33 	H z*a/*!&&.
|3JK O!  +&*#+"  	!&&.w.B3q6('R  &!+&G	s>   CE D? 0AE ?E
E EE 	F AFF c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   r
  r  r   r   r1  r~   )
_embeddingr*  r   r   r   r  r  r  r   r
  r  r  r  r  s          r   	embeddingzRouter.embedding  s    		#F7O#F7O*.//F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	r  c           	         d }	 t        j                  d| d|        | j                  |||j                  dd             }| j	                  ||       |d   j                         }|d   }| j                  ||d	      }|j                  d
d       }||||j                  k7  rd }	n|}	| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |	d|}
| j                  |xx   dz  cc<   t        j                  d| d       |
S # t         $ rE}t        j                  d| dt#        |       d       || j$                  |xx   dz  cc<   |d }~ww xY w)NzInside embedding()- model: r  r  )r   r
  r  r  r   r   syncrM  r  rr   r  r
  r  r  zlitellm.embedding(model=r  r  r  r~   )r   r  r  r  r  r  r  r*  r  r   r  r   rD  r   r   r  r  r   r   )r  r
  r   r  r  r  r  r  r  r  r  r  s               r   rB  zRouter._embedding  s   
4	!''-eWJvhG 66$*JJ/Dd$K 7 J
 //:f/U./446DgJ%)%5%5%f& &6 &" %jjD9O+*6#'='E'EE#5Z(A-( 11Z1H(( "#33*	
 H z*a/*!&&*:,6MN O 	!&&*:,6J3q6(RYZ %
+q0+G	s   D+D0 0	E>9A E99E>c           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d }~ww xY wwr  )	_aembeddingr  r	  r  r
  r  r@   r  r  rC  s          r   
aembeddingzRouter.aembeddingN  s     	#F7O#F7O*.*:*:F&'00uV0L?T??I&IIHO J 		(,0#)(1(<(<(>'(	 G		r  c           	      .  K   d }	 t        j                  d| d|        t        |      }| j                  |||j	                  dd       |       d {   }| j                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d	|}	| j                  ||d
      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)NzInside _aembedding()- model: r  r  )r   r
  r  r  r  r   r   rr   rG  rL  rM  rJ  zlitellm.aembedding(model=r  r  r  r~   )r   r  r   rI  r  r  r  rP  r   r   rJ  r   r  r   r
  rQ  rR  r   r  r  r   r   )r  r
  r   r  r  r  r  r  r  r  rX  r  s               r   rI  zRouter._aembeddingg  s    
B	!''/wjI  AH#BB$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L
 Z(A-()) "#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&+J<7NO OmF )
  . )=== *  	!&&+J<7KCPQF8SZ[ %
+q0+G	s   HAG F B8G F#G F+)F%*	F+3F'4F+8G F)G G  	G )G*5G H G #G %F+'F+)G +F=1F42F=9G G 	HA HHHc           	      b  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   r  r   r  r  r~   )_acreate_filer*  r   r  r	  r  r
  r  r@   r  r  r  s        r   acreate_filezRouter.acreate_file  s     
	#F7O*.*<*<F&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B/AA( A& A( %B/&A( (	B,16B''B,,B/c                    K   	 ddl m} t        j                  d| d        t	               j                  |dddgj                  dd       	       d {   }d
t        dt        f fd}g }t        |t              r|j                   ||             n|D ]  }|j                   ||              t        j                  |  d {   }t        |      dk(  rt        d       |||      }	t        t        |d         }
|	|
j                   d<   |
S 7 7 M# t        $ rH}t        j"                  d| d dt%        |       d       | j&                  |xx   dz  cc<   |d }~ww xY ww)Nr   )add_model_file_id_mappingsr  r  r  files-api-fake-textr+  r  r   r  r  r  r  r  r   c           	        K   ddl m}  |      }j                  | |d       | d   j                         }|d   }j	                  | |      }j
                  |xx   dz  cc<   t        |d   	      \  }}}}t        t        t           j                  d
            }	t        t        t           j                  d            }
|
r|	st        d      t        |	      }|rt        |
|      }
|
|d<   t        j                   di i ||j"                  |d|}j%                  | |d      }|\t'        |t(        j*                        rB|4 d {    	 j-                  |        d {    | d {   }d d d       d {    n%j-                  |        d {    | d {   }j.                  |xx   dz  cc<   t1        j2                  d| d       |S 7 7 x7 p7 b# 1 d {  7  sw Y   LxY w7 ]7 Uw)Nr   safe_deep_copyrN  rv  r   r   r  rr   r^  purposer  z2file and file_purpose are required for create_file)rV  )file_contentnew_model_namer  r  r  rL  rM  rJ  litellm.acreate_file(model=r  r~   )'litellm.litellm_core_utils.core_helpersrU  r  r  rP  r   rm   r   r   rJ   r*  rH   r  r1   r0   r   rN  r   r  r   r
  rQ  rR  r   r   r  )r  rU  kwargs_copyr  r  r  stripped_modelr  r  rV  r  replace_model_in_jsonl_boolr  rX  r  r  r  s                 r   create_file_for_deploymentz8Router._acreate_file.<locals>.create_file_for_deployment  sn    R,V433)&"0 4 
 ""2388:!']
#BB)&  C     ,1, =Mw-=9 3Q
 x(:;VZZ	=RSHY/F1CD7#L  /L#/+ /1%)'5D
 +/K'"// /B#'#7#7".	
 & !% 0 0)& 7 !1 ! !,!7#4#42  -} #II'1DT J    *2>  -}} EE#-@P F    &.~H"":.!3.%**1*=TU  )  -
 $2  -}}}  .s   EH
G)H
G12G+3	G1<G-=G1H
G/H
(H)	H
2H37H
+G1-G1/H
1H7G:8H?H
H
No healthy deployments found.)r   rP  model_file_id_mappingrZ  , r  r  rr   )!litellm.router_utils.common_utilsrP  r   r  r   async_get_healthy_deploymentsr  r  rI   r   r   r
  r  r  r  r   r  	exceptionr   r   )r  r   r  rP  r   r_  tasksr  rP  ra  returned_responser  r  s   ` `         @r   rM  zRouter._acreate_file  s    
}	T!''5eWJvhO  AH(,(J(J#)6KLM$*JJ/Dd$K%!1 )K ) #Q T Q FV Q f E-t478KLM"5JLL!;J!GH #6 &nne44I9~" ?@@$>$79%! !%%5y| D & ,,' %$_#F 5  	!++-eWBvh>RSVWXSYRZZab  &!+&G	sP   E:AD&  D"!A5D& D$A
D& !E:"D& $D& &	E7/AE22E77E:c           	      |  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   t        d      }| j	                  |||        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   r  r   _acreate_batchrj  r2  r  r~   )ri  r*  r   r/   r  r	  r  r
  r  r@   r  r  )r  r   r  r[  r  r  s         r   acreate_batchzRouter.acreate_batchJ  s     
	#F7O*.*=*=F&'$*JJ}d>N>N$OF=!%G.&" 00'= 1 
 @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B<A(A5 ,A3-A5 2B<3A5 5	B9>6B44B99B<c                 b  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }|d   j                         }|d	   }| j                  ||d
       | j                  ||      }| j                  |xx   dz  cc<   t        |d	         \  }}	}}t        j                  di i ||	| j                  |d|}
| j                  ||d      }|\t        |t        j                         rB|4 d {    	 | j#                  ||       d {    |
 d {   }
d d d       d {    n%| j#                  ||       d {    |
 d {   }
| j$                  |xx   dz  cc<   t        j&                  d| d       |
S 7 a7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t(        $ rH}t        j*                  d| d| dt-        |       d       || j.                  |xx   dz  cc<   |d }~ww xY ww)Nz Inside _acreate_batch()- model: r  r  rQ  r+  r  r  r   r   ri  rv  r  rr   r^  rY  rL  rM  rJ  zlitellm.acreate_batch(model=r  zlitellm._acreate_batch(model=rb  r  r  r~   )r   r  r   rI  r  r  r  rP  r   rm   r   rj  r   r  r   r
  rQ  rR  r   r  r  re  r   r   )r  r   r  r  r  r  r  r  r  r  r  rX  r  s                r   ri  zRouter._acreate_batchi  s    
I	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K%	  C   J ./446DgJ//%fDT 0   >>% ? L Z(A-( ,<$w-+P(A"Aq,, +>#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&.zl:QR O{R )
  . )=== *  	!++/wb@TUXYZU[T\\cd  &!+&G	s   H/AG F7CG $F:%G (G F<	G
F>GG G G 6G7	G  G5G 6H/7G :G <G>G G GG	GG G 	H,$AH''H,,H/c                    	K   	 t              }|3 j                  |dddgj                  dd      |       d{   }n j                         }|t	        d      g 	dt
        f	 fd	}|^t        |t              rNt        |      d
kD  r@t        j                  |D cg c]  } |t        t
        |             c}ddi d{   }n<|/t        |t              r |t        t
        |             d{   }nt	        d      |=t        |t              r|S t        |t              r|D ]  }t        |t              s|c S  	r	d
   t	        dj                  	            7 /c c}w 7 7 z# t        $ r;}t        j                  t!         t#        j$                         |             |d}~ww xY ww)z
        Iterate through all models in a model group to check for batch

        Future Improvement - cache the result.
        Nr  zretrieve-api-fake-textr+  r  rR  Router not yet initialized.r  c                 2  K   	 ddl m} | d   j                  d      }| d   j                         }|j                  d      }|t	        d|        |st        |      \  }}}} |	      }j                  t        t        |       |d	       |j                  dd        |j                  dd        t        j                  d
i i |d|i| d {   S 7 # t        $ r0}dd l} |j                          
j                  |       Y d }~y d }~ww xY ww)Nr   rT  r   r   r  z2Model not found in litellm_params for deployment: r^  aretrieve_batchrv  r~   )r[  rU  r*  r  r  rm   r  r   r  r  r   ro  r  	print_excr   )r  rU  r   r  r  r  
new_kwargsr  r  r  receieved_exceptionsr  s            r   try_retrieve_batchz2Router.aretrieve_batch.<locals>.try_retrieve_batch  sI    $ V&'78<<WEE%&67<<>D*.((3H*I'}'PQ[P\] 
 /7G"'84.1 "0!7J77#'j#9)&7 8 
 NN#8$?HH2D9!(!8!8 ""13F )"    !  $'I'')(//2 sA   DCC CC DC 	D$&D
DDDr   return_exceptionsTr`  z7Unable to find batch in any model. Received errors - {}r  )r   rd  r  r  r  rS   r   r   r  r
  r  r   r  rc   r  r  r@   r  r  )
r  r   r  r  filtered_model_listrs  resultsr  r  rr  s
   ` `      @r   ro  zRouter.aretrieve_batch  s    l	@H  <<'-:RST(.

3H$(O#)%5 =   $ '+&9&9&;#"* =>>#% % 5H % R $/2D9+,q0 ' &9%8E +40CU+KL%8!
 '+!  %0Z#T6 !3,.AB!    ?@@ "g|4"N.")%fl;#)M #*
 $*1-- IPP( sx2  		(,0#)(1(<(<(>'(	 G		s}   G:E< E0A%E< 'E3
E< E80E< ?E: #E< #G$%E< 
E< G#E< 3E< :E< <	G 6F;;G  Gc                   K   | j                  |      }|t        d      dt        ffd}t        j                  |D cg c]
  } ||       c}  d{   }dg dddd}|D ]g  }||d	   t        |d	      rt        |d	      |d	<   t        |d
      |d
<   |d   j                  |j                         t        |dd      du scd|d<   i |S c c}w 7 w)zQ
        Return all the batches across all deployments of a model group.
        r  Nrm  r   c                 x   K   	 t        j                  di i | d    d {   S 7 # t        $ r Y y w xY ww)Nr   r~   )r   alist_batchesr  r  s    r   rs  z0Router.alist_batches.<locals>.try_retrieve_batch>  sT     $22 ;/0;F;     s(   :+ )+ :+ 	7:7:r   F)objectr  first_idlast_idhas_morer{  r|  r  r}  T)	r  r  rS   r
  r  r9  r6  extendr  )r  r   r  ru  rs  rv  final_resultsr  s     `     r   ry  zRouter.alist_batches1  s     #11U1C&9::	,? 	  5HI5HE '5HI
 

 
 F! ,49T07
0KM*-+269+Ei(f%,,V[[9 6:u5=04M*-  / J
s*   ?CCCCC-AC
Cr  c                   K   | j                  |j                  dd      |       |j                  d      ru| j                  |d         r`| j                  |d   |       d {   }|d   d   |d<   |d   j	                         }| j                  ||       |j                  |        |di | d {   S 7 V7 w)	Nr   r  r  r  )r   r  r   r  r~   )r  r*  r  rI  r  r  r   )r  r  r  r  r  r  s         r   )_pass_through_moderation_endpoint_factoryz0Router._pass_through_moderation_endpoint_factoryd  s      	,,**Wb) 	- 	
 ::g4#6#6&/#6#R#BBWo%  C   J ))9:7CF7O./446D//% 0  MM$&0000 1s%   A#C %B<&AC 7B>8C >C rI  )>
assistantsrG  rJ  rM  rR  rP  rQ  rS  rN  rO  rU  rV  rW  rX  rY  rZ  r[  r\  rT  rK  r  r~  r  rf  rg  rr  rt  rv  rx  rz  r|  rh  ri  rs  ru  rw  ry  r{  r}  r  r  rc  re  rL  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  c                      dv r&	 	 ddt         t           dt         t           f fd}|S dv r%	 	 ddt         t           dt         t           ffd}|S 	 	 ddt         t           dt         t           f fd}|S )	z
        Creates appropriate wrapper functions for different API call types.

        Returns:
            - A synchronous function for synchronous call types
            - An asynchronous function for asynchronous call types
        )rP  r  r  rh  ri  r  re  r  r  r  r  r  r  r  r  r  r  r  c                 ,     j                   ddi|S )Nr  r~   )r@  )r  r  r  r  r  s      r   sync_wrapperz-Router.factory_function.<locals>.sync_wrapper  s*    
 =t<< &7;A r   )rs  ru  rw  ry  r{  r}  c                      d| |d|S )Nr  r  r~   r~   )r  r  r  r  s      r   vector_store_file_sync_wrapperz?Router.factory_function.<locals>.vector_store_file_sync_wrapper  s'    
 ) (;!  r   c                   K   dk(  r j                   d| |d| d {   S dk(  r j                  ddi| d {   S dv r j                  ddi| d {   S dv r j                  d| d| d {   S dv r j                  ddi| d {   S d	k(  r j                  dd
d| d {   S dv r j
                  ddi| d {   S dv r j                  d| d| d {   S dv r j                  d| |d| d {   S y 7 7 7 7 7 7 s7 U7 67 w)Nr  )r  r  r  rG  r  r  )rr  rt  rv  rx  rz  r|  )r  r  )rJ  rM  rU  rV  rW  rX  rY  rZ  r[  rK  r~  r  r  r  r  r  r  r  r  r  r  r  r\  T)r  r<  )rQ  rR  rS  rT  )rf  rg  )rN  rO  r~   ))_pass_through_assistants_endpoint_factoryr  r*   _init_vector_store_api_endpointsr4  _init_responses_api_endpoints)r  r  r  rI  r  r  s      r   async_wrapperz.Router.factory_function.<locals>.async_wrapper  sY    
 L(KTKK &7(;! 	   l*KTKK &7;A   339T99 &7      CTBB &7(;   
   0 DTCC &7   66CTCC &715   
    @T?? &7      CTBB &7(;   
 ??CTCC &7(;! 	   @c:
s   ED/ ED2E#D4$ ED6E$D8%!ED:E&D<' ED>!E)E *E2E4E6E8E:E<E>E ENN)r   r   r   )r  r  rI  r  r  r  s   ```   r   r<  zRouter.factory_function~  s    X  
 
& 6:(,%-c]    
 
 6:(,	%-c]	 	 21 26$(]	!)#]	SM]	~ r   c                 D   K   |r	d|vr||d<    |di | d{   S 7 w)zJ
        Initialize the Vector Store API endpoints on the router.
        r  Nr~   r~   )r  r  r  r  s       r   r  z'Router._init_vector_store_api_endpoints_  s2      #8#F,?F()&00000s     c                    K   ddl m} |j                  |j                  d            }|||d<    | j                  dd|i| d{   S 7 w)z
        Initialize the Responses API endpoints on the router.

        GET, DELETE, CANCEL Responses API Requests encode the model_id in the response_id, this function decodes the response_id and sets the model to the model_id.
        r   )ResponsesAPIRequestUtilsresponse_idNr   r  r~   )litellm.responses.utilsr  get_model_id_from_response_idr*  r4  )r  r  r  r  model_ids        r   r  z$Router._init_responses_api_endpointsl  sh      	E+IIJJ}%
 &F7O;T;; 
/

 
 	
 
s   AAAAr  c                    K   |E| j                   .| j                   d   }|j                  | j                   d          nt        d       |d||d| d{   S 7 w)z@Internal helper function to pass through the assistants endpointNr  r   z'custom_llm_provider' must be set. Either via:
 `Router(assistants_config={'custom_llm_provider': ..})` 
or
 `router.arun_thread(custom_llm_provider=..)`r  r~   )r   r   r  )r  r  r  r  r  s        r   r  z0Router._pass_through_assistants_endpoint_factory  s      &%%1&*&<&<=R&S#d445EFG s  ' 
 3F
FL
 
 	
 
s   AAAAr  r0  r1  c	                   K   t        j                  dt        j                                 |}	d}
|j	                  d      }d}|du s||| |	d|}d|vr| j
                  |d<   d|vrd	|d<   	 t        j                  d
       t        |      }|r)|j                  ||d       t        |i | d{   }|S t        |t        j                        r|@| j                  ||      }||	|j                  ||d       t        |i | d{   }|S dj                  |||      }t        j                  dj                  |             |xj                  dj                  |      z  c_        nt        |t        j                         r|@| j                  ||      }||	|j                  ||d       t        |i | d{   }|S dj                  |||      }t        j                  dj                  |             |xj                  dj                  |      z  c_        ||t        j                  d|        t#        |t%        t&        |            \  }
}|

|||   d   }
|
Dt        j                  d| d|        t)        |	d      r|	xj                  d| d| z  c_        |	|j                  |
|d       t        |i | d{   }|S t)        |	d      rW|	xj                  dj                  ||
      z  c_        t3        |      d	kD  r$|	xj                  dj                  |      z  c_        |	7 7 97 7 t# t*        $ ru}t-        |      }t        j.                  dj                  t'        |      t        j                         t1        | |       d{  7               t'        |      }Y d}~d}~ww xY ww)zD
        Common utilities for async_function_with_fallbacks
        	TracebackNr   r  T)litellm_routerr  r   fallback_depthr   zTrying to fallback b/w models)r   )fallback_model_grouporiginal_model_group)r   r1  zmodel={}. context_window_fallbacks={}. fallbacks={}.

Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContextWindowExceededError'. No context_window_fallback set. Defaulting                         to fallbacks, if available.{})msgz
{}zmodel={}. content_policy_fallback={}. fallbacks={}.

Set 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting                         to fallbacks, if available.{}zinside model fallbacks: r   z7No fallback model group found for original model_group=z. Fallbacks=r  zlitellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}
{}

Debug Information:
Cooldown Deployments={}r   r  z<. Received Model Group={}
Available Model Group Fallbacks={}z
Error doing the fallback: {})r   r  r  r  r*  r   r  r;   r   r=   r   r   ContextWindowExceededError(_get_fallback_model_group_from_fallbacksr  r  r  r<   r   r   r9  r  r   r=  r8   r  )r  r  r0  r   r   r   r1  r2  r  r  r  r  fallback_failure_exception_strrV  is_non_standard_fallback_formatr  #context_window_fallback_model_grouperror_message#content_policy_fallback_model_groupgeneric_fallback_idxnew_exceptionr  s                         r   r8  z1Router.async_function_with_fallbacks_common_utils  s     	##i	0D0D0F/G$HI#.4jj.A)+&$(<(DG #"4
 
 ,.,0,>,>L)</-.L)*L	@!&&'FG /R#/+ /##090D "4""" 
  !W??@+7 EE":$/ F  8 ;B00 ''4W4H &8&&&  H $O %m  %t  %t#%=y%M *..77=v)8 II}!==IAwBBC+7 EE":$/ F  8 ;B00 ''4W4H &8&&&  H $O %l  %s  %s#%=y%M *..77=v)8 II}!==I$)@%++.Fyk,RS -' $S+ 6(( (/4H4T+45I+J3+O('/)..QR]Q^^jktjuv 19=*228op{o|  }I  JS  IT  7U  U2,,##0D0D "4""" 
   %y1&&*i*p*p$+ & 12Q6"**4;;6* ! W2 H Z  	@@H!'' k  r  r&((*I04)9  	 .1-?*	@s   A-O0AM 5M6M ;O<AM M
M OB2M 
MM OD
M MM !A&OM 
M M M 	OAO-N0.OOOOc                   K   |j                  d      }|j                  dd      }|j                  d| j                        }|j                  d| j                        }|j                  d| j                        }|j                  dd      }	 | j                  |||||	       | | j                  |i |d|i d{   }	n | j                  |i | d{   }	t        j                  d
|	        t        |	d      }	|	S 7 F7 -# t        $ r+}
| j                  |
|||||||       d{  7  cY d}
~
S d}
~
ww xY ww)z
        Try calling the function_with_retries
        If it fails after num_retries, fall back to another model group
        r   r0  Fr   r   r   mock_timeoutN)r  r1  r   r   r   zAsync Response: r   )r  attempted_fallbacks)r*  r  r   r   r   _handle_mock_testing_fallbacksasync_function_with_retriesr   r  r-   r  r8  )r  r2  r  r1  r0  r   r   r   r  r  r  s              r   r	  z$Router.async_function_with_fallbacksW  sy     &,ZZ%8,2JJ7JE,R$*JJ{DNN$K	39::&(E(E4
  4:::&(E(E4
  zz.$7	//'#)A)A 0  '!A!A!A"#"2>"  "B!A!A4!R6!RR!''*:8*(EF7!$%H O S  
	HH!((	 	 	 	
	sf   B
E 1D	 >D?D	 D*D	 E D	 D	 		D=D8-D0.D82D=3E 8D==E c                    t        j                  |      }|j                  ,|j                  du rt        j                  |dd| d|       |j
                  ,|j
                  du rt        j                  |dd| d|       |j                  -|j                  du rt        j                  |dd| d|       yy)	a  
        Helper function to raise a litellm Error for mock testing purposes.

        Raises:
            litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params
            litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params
            litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params
        NTr  #This is a mock exception for model=z#, to trigger a fallback. Fallbacks=)r   r  r  zF, to trigger a fallback.                     Context_Window_Fallbacks=zF, to trigger a fallback.                     Context_Policy_Fallbacks=)	rU   from_kwargsmock_testing_fallbacksr   InternalServerErrormock_testing_context_fallbacksr  %mock_testing_content_policy_fallbacksr  )r  r  r1  r   r   r   mock_testing_paramss          r   r  z%Router._handle_mock_testing_fallbacks  s     6AA&I66B#::dB--!=k]Jmnwmxy   >>J#BBdJ44!=k] K..F-GI   EEQ#IITQ55!=k] K..F-GI  R Rr   c           
        K   t        j                  d       |j                  d      }|j                  d| j                        }t	        |      }|j                  d| j
                        }|j                  d| j                        }|j                  d      }|j                  d      }	|j                  d|j                  d	            xs i }
d
|
v rFt        |
d
   t              r3| j                  |
d
         }||
j                  dt        |      i       t        j                  d| d|	        	 | j                  ||        | j                  |g|i | d {   }t        |dd       }|S 7 # t         $ rO}d }|}t#        |dd       }|t        |t$              r|}		 | j'                  |j                  d      xs d|       d {  7  \  }}| j)                  ||||||       | j*                  | j,                  &| j/                  ||j                  d            }||}	|	dkD  r| j1                  ||      }n t        j                  d|	        | j3                  ||	|	||      }t5        j6                  |       d {  7   t9        |	      D ]  }	  | j                  |g|i | d {  7  }t;        j<                  |      r| d {  7  }t        ||dz   |	      }|c cY d }~S # t         $ r}| j1                  ||      }|	|z
  }|j                  d      }| | j'                  ||       d {  7  \  }}ng }| j3                  |||	||      }t5        j6                  |       d {  7   Y d }~d }~ww xY w t?        |      t@        jB                  v rtE        |d|	       tE        |d|       |d }~ww xY ww)Nz#Inside async function with retries.r  r   r   r   r   r   r1  r   r1  r  model_group_sizez/async function w/ retries: original_function - z, num_retries - )r1  r  r   )r  attempted_retriesr   r  r  )r=  r   all_deploymentsr   regular_fallbacksr   )re  r1  )r  r  z#Retrying request with num_retries: )r  remaining_retriesr   r   r  rr   r   )#r   r  r  r   r   r   r   r*  r   r   r  r   r  %_handle_mock_testing_rate_limit_error	make_callr.   r  r6  r   r  should_retry_this_errorr   r   r>   	log_retry_time_to_sleep_before_retryr
  r  r  r    is_async_callabler   r   LITELLM_EXCEPTION_TYPESr<  )r  r2  r  r  r   r  r   r   r1  r   	_metadatar   r  r  current_attemptr  deployment_num_retriesr  _all_deployments_retry_policy_retriesr   r  _modelr  _timeouts                            r   r  z"Router.async_function_with_retries  sN    ##$IJ"JJ':;JJ{DNN;	<VD#)::&(E(E$
  $*::&(E(E$
  &,ZZ%8jj/ !**%7J9OPVTV	I%*Y}5Ms*S,,	-8P,QJ%  "4c*o!FG##=>O=PP`al`mn	
r	%66' 7  ,T^^,=OOOOH4!QDH O	 P
  h	%"O!"%,Qt%D"%1j&7 5 99jj)/R!1 :   $  (($8 0)A"+)A )  !!-00< )-(N(N0fjj>Q )O )% )4"7KQv9KL!''5k]C ::$"-'$8 0 ; K --,,,#(#5%2%3T^^4E%W%WPV%WWWH(::  *2>><!)*9A*=$/ H
 $O  2!^^6Q^?F(3o(E%,2JJw,?F) #'"E"E"(-= #F #  0 02,#??,*;$/,@(8  @  H "--111-2! $6P &'7+J+JJ*M;G*M?K$$Qh	%s   D2O5,E8 !E6"E8 5O6E8 8
OAOGB=OJO$K==K >K=K K=5O7O8O=	NANM

9NN
NON;OOOc                    K   |j                  d      } ||i |}t        j                  |      st        j                  |      r
| d{   }| j                  ||       d{   }|S 7 !7 w)z^
        Handler for making a call to the .completion()/.embeddings()/etc. functions.
        r   N)r  r1  )r*  r    r  inspectisawaitableset_response_headers)r  r  r2  r  r1  r  s         r   r  zRouter.make_callF  s}      jj)$d5f5..x8G<O<O=
 &~H22; 3 
 
  &
s$   A	A1A-A1&A/'A1/A1c                 @   |j                  dd      }| j                  |      }d}|:t        |      dk(  r,t        t        t
           |d   d   j                  d            }|:|du r5t        j                  d	|        t        j                  |d
d| d|      yy)z
        Helper function to raise a mock litellm.RateLimitError error for testing purposes.

        Raises:
            litellm.RateLimitError error when `mock_testing_rate_limit_error=True` passed in request params
        mock_testing_rate_limit_errorNr  rr   r   r   r   TzTlitellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=r  r  z , to trigger a rate limit error.)r   r  r  r   )r  r  r  r   r   r   r*  r   r  r   RateLimitError)r  r  r1  r  available_modelsr   s         r   r  z,Router._handle_mock_testing_rate_limit_errorW  s     9?

+T9
%  ..+.F%)'C0@,AQ,F/23CDHHWK
 *5-5!&&fgrfst ((!=k]Jjk'	  6 6r   r=  r   r  r  c                    d}|t        |t              rt        |      }d}|t        |t              rt        |      }t        |t        j                        r||t        |t        j
                        r||t        |t        j                        r|t        |t        j                        r|dk  r|t        |      dkD  r|t        |t        j                        r	 |dk  r||dk  r|y)au  
        1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
        2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None

        2. raise an exception for RateLimitError if
            - there are no fallbacks
            - there are no healthy deployments in the same model group
        r   rr   T)
r   r   r  r   r  r  NotFoundErrorr   r  AuthenticationError)	r  r=  r   r  r   r   r  _num_healthy_deployments_num_all_deploymentss	            r   r  zRouter.should_retry_this_errorz  s    " $% *z:Mt/T'*+>'?$ &:ot+L#&#7  ug@@A(4K ugAAB(4KeW223KeV223(A-%1)*Q.eV778
 %) $q(Kr   c                 4    t        | j                  g|i |S )z~
        Sync wrapper for async_function_with_fallbacks

        Wrapped to reduce code duplication and prevent bugs.
        )r   r	  )r  r2  r  s      r   r  zRouter.function_with_fallbacks  s     "$"D"DVtVvVVr   c                 l    |yd}|D ])  }t        |j                               d   |k(  s#||   } |S  |S )a@  
        Returns the list of fallback models to use for a given model group

        If no fallback model group is found, returns None

        Example:
            fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}]
            model_group = "gpt-3.5-turbo"
            returns: ["gpt-4"]
        Nr   )r   keys)r  r   r1  r  r>  s        r   r  z/Router._get_fallback_model_group_from_fallbacks  sQ     48DDIIK #{2'+K'8$##	  $#r   c                     | j                   y| j                   D ]C  }t        |t              sd|v s|d   }t        |t              s/t	        |      dkD  s>|d   c S  y)zX
        Returns the first model from the default_fallbacks list, if it exists.
        Nr   r   )r   r   r  r   r  )r  fallbackdefault_lists      r   _get_first_default_fallbackz"Router._get_first_default_fallback  s]     >>!H(D)cXo'}lD1c,6G!6K'?*	 '
 r   r  c                    |t        |      dk(  rn!|t        |t              rt        |      dkD  ryd}t        |d      r,t        |j                  d      r|j                  j
                  }t        |d      r|j                  }|%t        j                  |||| j                        }|S t        j                  ||| j                        }|S )	z
        Calculate back-off, then retry

        It should instantly retry only when:
            1. there are healthy deployments in the same model group
            2. there are fallbacks for the completion call
        Nrr   r   r  headerslitellm_response_headers)r  r   response_headersmin_timeout)r  r   r  )
r  r   r   r9  r  r  r  r   _calculate_retry_afterr   )r  r  r  r   r   r  r  r   s           r   r  z"Router._time_to_sleep_before_retry  s    " &3+?1+D+.5'(1,481j!gajj)&D zz11101 99'44"3'!1 ,,	G  44"3' ,,G r   c                   K   ddl m} 	 |j                  dd      }|t        d      |d   j                  d      y|d   d   j                  dd      }|d   d   j                  d	d      }|d   j                  d
i       xs i }	|	j                  dd      }
||
yt	        |
t
              rt        |
      }
| j                  |
      }|y| j                  |j                         |      }|j                  dd      }|j                  dd      }|j                  j                  }|j                  j                  }|j                  dd      }|j                  dd      }t        | |
       ||	||||yt        |      }|j                  dd      }t               }|j!                  d      }t"        j$                  j&                  j)                  |
||      }g }|j+                   |||t,        j.                  j&                               t"        j0                  j&                  j)                  |
||      }|j+                   ||dt,        j.                  j&                               | j2                  j5                  ||       d{    |S 7 # t6        $ r7}t9        j:                  dj)                  t        |                   Y d}~yd}~ww xY ww)zG
        Track remaining tpm/rpm quota for model in model_list
        r   )RedisPipelineIncrementOperationstandard_logging_objectNzstandard_logging_object is Noner   r   r  r1  ri  rk  r  r  received_model_nametpmrpmr   deployment_idtotal_tokens%H-%Mrk  current_minuter   )re  increment_valuer}   rr   )increment_listr  zOlitellm.router.Router::deployment_callback_on_success(): Exception occured - {})litellm.types.cachingr  r*  r  r   r   r   get_deploymentget_router_model_infor  r   r  r  rD   r   rp   strftimerY   TPMr1  r  r   rx   r}   RPMr   async_increment_cache_pipeliner  r   r  )r  r  completion_responser  r  r  r  deployment_namer1  ri  rk  deployment_infodeployment_model_infor  r  tpm_litellm_paramsrpm_litellm_paramstpm_model_inforpm_model_infor  r  dtr  tpm_keypipeline_operationsrpm_keyr  s                              r   r  z%Router.deployment_callback_on_success  s0     	Jt	HN

)4I# '. !BCC&'++J7?"()9"::"F"J"J $# %%56zBFF!4 $$4599,KQr
^^D$/&"*C(RB #'"5"5r"5"B"*,0,F,F#2#=#=#?,7 -G -)
 *--eT:C)--eT:C *9)G)G)K)K&)8)G)G)K)K& &;%>%>ud%KN%:%>%>ud%KN B,0"$ K*2*2&.&.#DV#L &=&A&A.RS&T
 &'!#" *--33::. ;  NP# $**3#(4'OO11 *--33::. ;  $**3#()'OO11 jj??#6%5 @   
   	!''ahhF
 	sq   K3J KAJ K/J KB.J <K=DJ JJ KJ 	K%-KKKKc                    d}|d   j                  d      nc|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }|t	        | |      }|S y)z
        Tracks the number of successes for a deployment in the current minute (using in-memory cache)

        Returns:
        - key: str - The key used to increment the cache
        - None: if no key is found
        Nr   r   r1  ri  rk  r  )r*  r   r   r   rD   )	r  r  r  r  r  rk  r1  ri  re  s	            r   r  z*Router.sync_deployment_callback_on_success  s     "#''
3; !12:>BB=RVWK 0155lBGM2Jd+B"bjB$W>C(, C Jr   c                    t        j                  d       	 |j                  dd      }t        |dd      }|j                  di       }|j                  di       }t        j
                  j                  j                  |      }	|j                  d	d      }
d}|	 t        j                  j                  |	
      }|
|
dk\  r|
}n||dk\  r|}n| j                  }t        |t              r3|j                  d      }|yt        | |       t        | ||||      }|S t        j                  d       y# t        $ r}|d}~ww xY w)a`  
        2 jobs:
        - Tracks the number of failures for a deployment in the current minute (using in-memory cache)
        - Puts the deployment in cooldown if it exceeds the allowed fails / minute

        Returns:
        - True if the deployment should be put in cooldown
        - False if the deployment should not be put in cooldown
        z1Router: Entering 'deployment_callback_on_failure're  Nstatus_coder  r   ri  )r  r   )r  r   rk  Fr  r   exception_statusr  r  time_to_cooldownzWRouter: Exiting 'deployment_callback_on_failure' without cooldown. No model_info found.)r   r  r*  r6  r   litellm_core_utilsexception_mapping_utils_get_response_headersutils&_get_retry_after_from_exception_headerr   r   r  rC   r:   r  )r  r  r  r  r  re  r
  r   _model_infoexception_headersdeployment_cooldownheader_cooldown_time_to_cooldownr  r  r  s                   r   r  z%Router.deployment_callback_on_failure  sp     	##$WX9	

;5I&y-D $ZZ(8"=N(,,\2>K ' : : R R h h#, !i !
 #1"4"4_d"K"O ,")--"V"V%6 #W # #.3F!3K$7! ,A1E$3!$($6$6!+t,/:t/D ( @,0"/ 3,0%5'0,%6 %++m  	G	s$   C&D3 >D3 D3 3	E<D>>Er  c                 (  K   |d   d   j                  dd      }|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }t	        |      }	t               }
|
j                  d      }t        j                  j                  j                  |||	      }| j                  j                  |d
|	t        j                  j                         d{    y7 w)z3
        Update RPM usage for a deployment
        r   r   r  Nr1  ri  rk  r  r  rr   )re  r1  r  r}   )r*  r   r   r   r   rp   r  rY   r  r1  r  r   async_increment_cacherx   r}   )r  r  r  r  r  r  r1  ri  rk  r  r  r  r  s                r   r  z+Router.async_deployment_callback_on_failure  s     !!12:>BB$
 -.z:>>}dS,-11,CIr
^^D$'"*C RB<VD

 "%%++22. 3 
 jj..-%%	 / 
 	
 	
s   DD
DD)r   r1  c                     t        |      S )a  
        Helper to return what the "metadata" field should be called in the request data

        - New endpoints return `litellm_metadata`
        - Old endpoints return `metadata`

        Context:
        - LiteLLM used `metadata` as an internal field for storing metadata
        - OpenAI then started using this field for their metadata
        - LiteLLM is now moving to using `litellm_metadata` for our metadata
        )r   )r  r  s     r   '_get_metadata_variable_name_from_kwargsz.Router._get_metadata_variable_name_from_kwargs0  s     6f==r   c                    	 d|v rdnd}t        |      j                  t        |      d}|j                         D ]W  \  }}||ddfvr|||<   ||k(  st	        |t
              s*i ||<   ||   j                         D ]  \  }}|dk7  s|||   |<    Y t        | j                        dkD  r| j                  j                  d       | j                  j                  |       | j                  ||   d<   |S # t        $ r}|d	}~ww xY w)
z
        When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
        r1  r   )exception_typeexception_stringr  r  r      r   N)r   rz   r   rb  r   r  r  r   r  r   r  )	r  r  r  _metadata_varprevious_modelr  r  
metadata_k
metadata_vs	            r   r  zRouter.log_retry@  s+   	&8F&B"
 
 #'q'"2"2$'FN 	 ]J8KLL()N1%-'Jq$,?46N=1282G2M2M2O.
J%)::<FN1-j9 3P  4''(1,$$((+  ''77;7K7KF=!"34M 	G	s+   AC7 C7 %#C7 	A-C7 7	D DDr  r  c                     |}| j                   j                  ||d      }|#d}| j                   j                  ||dd       |S |dz  }| j                   j                  ||d       |S )zf
        Update deployment rpm for that minute

        Returns:
        - int: request count
        T)re  r  
local_onlyrr   ry   )re  r1  r#  r}   )re  r1  r#  )r   	get_cache	set_cache)r  r  r  r  request_counts        r   _update_usagezRouter._update_usagee  s      

,,*:t - 
  MJJ  =Tr !   QMJJ  =T !  r   c                 n    | j                   y| j                   D ]  }t        |t              sd|v s y y)NFr   T)r   r   r  )r  r  s     r   _has_default_fallbackszRouter._has_default_fallbacks  s6    >>!H(D)(? ' r   r  c                    |j                   r5t        |j                         dkD  r|j                   d   j                  dk7  ry|j                  d| j                        }|2d}|D ](  }t        |j                               d   |k(  s#||   } n |y| j                         ryt        j                  dj                  ||             y)z
        Determines if a content policy error should be raised.

        Only raised if a fallback is available.

        Else, original response is returned.
        r   content_filterFr   NTzyContent Policy Error occurred. No available fallbacks. Returning original response. model={}, content_policy_fallbacks={})choicesr  finish_reasonr*  r   r   r  r)  r   r  r  )r  r   r  r  r   r  r>  s          r   r  z)Router._should_raise_content_policy_error  s     H$4$4 5 9"004DD#)::&(E(E$
 
 $/#' 0		$Q'50+/;( 1
 $/((*## H  O  O/	

 r   c                     g }	 | j                  |      \  }}t        |t              rg S 	 t	        | |      }g }|D ]  }|d   d   |v r|j                  |         ||fS # t        $ r Y Aw xY w)Nr^  r  ri  rk  )#_common_checks_available_deploymentr   r  r  r9   r   )r  r   r  r  r  unhealthy_deploymentsr   r  s           r   _get_healthy_deploymentszRouter._get_healthy_deployments  s    !#	"&"J"J #K #A *D1	 2
 !:$(;K!
 %'*J,'-1FF#**:6	 + #$444  		s   &A" "	A.-A.c                   K   g }	 | j                  |      \  }}t        |t              rg |fS 	 t	        | |       d{   }t        |      }g }|D ]  }|d   d   |vs|j                  |         ||fS # t        $ r Y Tw xY w7 Gw)z
        Returns Tuple of:
        - Tuple[List[Dict], List[Dict]]:
            1. healthy_deployments: list of healthy deployments
            2. all_deployments: list of all deployments
        r^  r  Nri  rk  )r/  r   r  r  r7   setr   )	r  r   r  r  r  r0  unhealthy_deployments_setr   r  s	            r   r  z%Router._async_get_healthy_deployments  s      "$	"&"J"J #K #A *D1+++ 2
 'F$(;K'
 !
 %((=$>!$&*J,'-5NN#**:6 + #$444  		!
s9   B
(A9 B
 B B
"B
9	BB
BB
c                 r    t         j                  D ]$  }t        |t              s|j	                  |       & y)a  
        Mimics 'async_routing_strategy_pre_call_checks'

        Ensures consistent update rpm implementation for 'usage-based-routing-v2'

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        N)r   r%  r   r   r  )r  r  r  s      r   r  z'Router.routing_strategy_pre_call_checks  s,     !**I)\2((4 +r   r  c           
      |  K   t         j                  D ].  }t        |t              s	 |j	                  ||       d{    0 y7 # t         j
                  $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          t!        | |j"                  ||d   d   | j$                         |d}~wt&        $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          |d}~ww xY ww)O  
        For usage-based-routing-v2, enables running rpm checks before the call is made, inside the semaphore.

        -> makes the calls concurrency-safe, when rpm limits are set for a deployment

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        Nre  traceback_exceptionr  targetr2  ri  rk  r	  )r   r%  r   r   async_pre_call_checkr  r
  r  async_failure_handlerr  r  r  	threadingThreadfailure_handlerstartr:   r  r   r  )r  r  r  r  r  r  s         r   rR  z-Router.async_routing_strategy_pre_call_checks  s}    " !**I)\2(#88EUVVV + W-- ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'-04)*+,#-l#;D#A)-);); G  ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'GsF   $F<AAAF<AF9B<DF9"BF44F99F<r  c           
        K   |}t         j                  D ]2  }t        |t              s	 |j	                  |||||       d{   }4 |S 7 # t
        $ r}	|t        j                  |j                  |	t        j                         t        j                                      t        j                  |j                  |	t        j                         f      j                          |	d}	~	ww xY ww)r7  r   r   r  r  r  Nr8  r:  )r   r%  r   r   async_filter_deploymentsr  r
  r  r=  r  r  r  r>  r?  r@  rA  )
r  r   r   r  r  r  r  returned_healthy_deploymentsr  r  s
             r   !async_callback_filter_deploymentsz(Router.async_callback_filter_deployments1  s     ( (;$ **I)\2'@@"'0L%-+9-= A   1 +8 ,+1 ! ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'Gs;   &C1AAAC1A	C.BC))C..C1r   c                    |g}|j                         D ]  \  }}t        |t              r|j                  |       nOt        |t              r%|j                  t        j                  |             n|j                  t        |             t        |t              r|j                  |       t        |t              r%|j                  t        j                  |             |j                  t        |              dj                  |      }t        j                  |j                               }|j                         S )z
        Helper function to consistently generate the same id for a deployment

        - create a string from all the litellm params
        - hash
        - use hash as id
        r  )rb  r   r   r   r  jsondumpsjoinhashlibsha256encode	hexdigest)r  r1  r   partsr  r  
concat_strhash_objects           r   rn  zRouter._generate_model_idd  s     "((*DAq!S!QAt$TZZ]+SV$!S!QAt$TZZ]+SV$ + WWU^
nnZ%6%6%89$$&&r   r  _model_name_litellm_paramsr  c                    	 t        di |}t        di ||||d}t        j                  j	                         D ]0  }|j
                  j                  |      |j
                  |   ||<   2 |j                  j                  }|t        j                  ||i       |j
                  j                  }|j
                  j                  |j
                  j                  dz   |z   }t        j                  ||i       | j                  |      dur3t        j                  d|j                    d|j                  d	           y| j#                  |      }|j%                  d
      }	| j'                  |	|j                  j                         |S # t(        $ r1}
| j*                  rt        j,                  d|
 d       Y d}
~
y|
d}
~
ww xY w)a^  
        Create a deployment object and add it to the model list

        If the deployment is not active for the current environment, it is ignored

        Returns:
        - Deployment: The deployment object
        - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
        rm  N)
model_costr  r  TzIgnoring deployment z% as it is not active for environment supported_environmentsr   r   r  zError creating deployment: 1, ignoring and continuing with other deployments.r~   )rT   rR   ra   model_fieldsr  r   r*  ri  rk  r   register_modelr   r  $deployment_is_active_for_environmentr   warningr  _add_deploymentto_json _add_model_to_list_and_index_mapr  r   re  )r  r  rR  rS  r  r   r  fieldr  r   r  s              r   _create_deploymentzRouter._create_deployment  s    ;	-;-No-NN# !&-&	J 4@@EEG,,007C)3)B)B5)IK& H
 ",,//H#&& +  %3399K((<<H--AACG+U  "" 99Z9P &--*:+@+@*AAfgqg|g|  ~V  hW  gX  Y ---DJ&&D&9E11j&;&;&>&> 2   	..%//1!4ef 	s,   AF C*F AF 	G%GGGc                 <    |j                   j                  d      ryy)z
        Check if the deployment is an auto-router deployment.

        Returns True if the litellm_params model starts with "auto_router/"
        zauto_router/TF)r   
startswith)r  r   s     r   _is_auto_router_deploymentz!Router._is_auto_router_deployment  s     **>:r   c                    ddl m} |j                  j                  }|j                  j                  }||t        d      |j                  j                  }|t        d      |j                  j                  }|t        d       ||j                  |||||       }|j                  | j                  v rt        d|j                   d	      || j                  |j                  <   y)
z
        Initialize the auto-router deployment.

        This will initialize the auto-router and add it to the auto-routers dictionary.
        r   )ru   Nzzauto_router_config_path or auto_router_config is required for auto-router deployments. Please set it in the litellm_paramszfauto_router_default_model is required for auto-router deployments. Please set it in the litellm_paramszhauto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params)r  auto_router_config_pathauto_router_configdefault_modelembedding_modelr   zAuto-router deployment z3 already exists. Please use a different model name.)
/litellm.router_strategy.auto_router.auto_routerru   r   rf  rg  r  auto_router_default_modelauto_router_embedding_modelr  r   )r  r  ru   rf  rg  rh  ri  autor_routers           r   init_auto_router_deploymentz"Router.init_auto_router_deployment  s!    	O %%== 	  -7,E,E,X,X"*/A/I M  %%?? 	  x  %%AA 	 "z  $.!,,$;1'+$($
   D$5$55)**?*?)@@st  4@*//0r   c           	      P   |j                   d|j                   vs|j                   d   yt        d      }|t        d      |t        vrt        dt         d|       |j                   d   D ]"  }|t        vst        dt         d| d	|        ||j                   d   v ryy
)a  
        Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments

        Requires `LITELLM_ENVIRONMENT` to be set in .env. Valid values for environment:
            - development
            - staging
            - production

        Raises:
        - ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values
        - ValueError: If supported_environments is not set in model_info or not one of the valid values
        rV  TLITELLM_ENVIRONMENT)secret_namezPSet 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .envz#LITELLM_ENVIRONMENT must be one of z. but set as: z&supported_environments must be one of z for deployment: F)ri  r   r  rL   )r  r  litellm_environment_envs       r   r[  z+Router.deployment_is_active_for_environment	  s     !!)'z/D/DD$$%=>F,9NO&b  &@@56P5QQ_`s_tu  ))*BCD55 <=W<XXfgkfll}  I  ~J  K  D *"7"78P"QQr   c                    t        j                  |      }g | _        i | _        i | _        |D ]
  }|j                  d      }|j                  d      }t        |t              rI|j                         D ]6  \  }}t        |t              s|j                  d      s)t        |      ||<   8 |j                  di       }d|vr| j                  ||      }	|	|d<   |j                  dd       8t        |d   t              r%|d   D ]  }
|
|d<   | j                  ||||        | j                  ||||        t!        j"                  d| j%                                 |D ch c]  }|d   	 c}| _        y c c}w )	Nr  r   os.environ/ri  rk  organization)r  rR  rS  r  z
Initialized Model List )r  r  r   r   r   r  r   r  rb  r   rc  ro   rn  r*  r   ra  r   r  get_model_namesr   )r  r   original_model_listr   rR  rS  r  r  r  _idorgr  s               r   r   zRouter.set_model_list1  s   "mmJ702-02- )E))L1K#ii(89O/40+113DAq!!S)all=.I-7]* 4 !&		, ;K ;&--k?K$'D!"">48D/J +>:C69ON3++(-$/(7$/	 ,  ; ''$) +$3 +	 ( 9 )F 	##'(<(<(>'?@	
 6@@ZAlOZ@@s   %E:c                    dd l }|j                  j                  }d}d|v r(|j                  d      d   }|t        j
                  v rd}|r	|}d }d }d }	njt	        j                  |j                  j                  |j                  j                  dd             \  }}}}	|t        j                  vrt        d|       | j                  j                  |j                  j                         |j                  j                  (t        |dd       t        |d      |j                  _        |j                  j                  (t        |d	d       t        |d	      |j                  _        d
|j                  v r{| j                   j#                  |j                  |j%                  d             |j&                  j(                  r/| j*                  j                  |j&                  j(                         |j&                  j                  d      }
|j&                  j                  d      }|
Z|Xd
|v rT|
| j,                  vrt/               | j,                  |
<   | j,                  |
   j#                  ||j%                  d             |j                  j                  dg       xs g }|D ]i  }|j                  di       }dD ]P  }||v s||   j1                  d      s||   j3                  dd      }|j4                  j                  |d      ||<   R k |(| j7                  |||j                  j                         | j9                  |j                        r| j;                  |       |S )Nr   Fr  Tr  r   r  zUnsupported provider - r  r  r   r   team_idteam_public_model_namedataSources
parameters)r6  re  ru  r  )r  r  r   )r   r  )osr   r   r  r   r  rm   r*  provider_listr  r   r   r  r6  r  r  r   add_patternr^  ri  rk  r   r   rs   rc  r7  environ'_initialize_deployment_for_pass_throughrd  rn  )r  r  r  r  is_prompt_management_modelr  r  r  r  ru  _team_id_team_public_model_namedata_sourcesdata_sourcer   	param_keyenv_names                    r   r]  zRouter._add_deploymentc  sW    #1177%*"-"/"5"5c":1"="g&W&WW-1*% #F"&"OH (( //55$.$=$=$A$A)4%# #'*?*??"9:M9N OPP 	$$Z%>%>%D%DE
 %%))1
E40<,3J,FJ%%) %%))1
E40<,3J,FJ%%) *''' ++%%z'9'9t'9'L $$''44;;J<Q<Q<T<TU((,,Y7","7"7";";<T"U '3..t8886H6J))(3%%h/;;');););)N
 "0044]BGM2'K __\26F0	&6)+<+G+G+V%i088KH(*

x(DF9%	 1 ( *88%$7 //55 9  ***:S:S*T,,
,Cr   c                    |j                   j                  du rSddlm} |j                   j                  *t        j                  |j                   j                        }ni }|dk(  r|j                  d      xs |j                   j                  }|j                  d      xs |j                   j                  }|j                  d      xs |j                   j                  }||t        d	      |j                  |||
       nf|j                  d      xs |j                   j                  }	|j                  d      xs |j                   j                  }
|j                  ||	|
       	 y)a*  
        Optional: Initialize deployment for pass-through endpoints if `deployment.litellm_params.use_in_pass_through` is True

        Each provider uses diff .env vars for pass-through endpoints, this helper uses the deployment credentials to set the .env vars for pass-through endpoints
        Tr   )passthrough_endpoint_routerN	vertex_aivertex_projectvertex_locationvertex_credentialsz]vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints.)
project_idlocationr  ru  r  )r  ru  r  )r   use_in_pass_through>litellm.proxy.pass_through_endpoints.llm_passthrough_endpointsr  litellm_credential_namer!   get_credential_valuesr*  r  r  r  r  add_vertex_credentialsru  r  set_pass_through_credentials)r  r  r  r   r  credential_valuesr  r  r  ru  r  s              r   r  z.Router._initialize_deployment_for_pass_through  s    $$88D@ ((@@L$6$L$L--EE%! %'!"k1%))*:; @!00?? 
 &))*;< A!00@@  
 &))*>? D!00CC #
 ")_-D$w  ,BB-,'9 C  &))*5 :!0099 
 &)))4 9!0088  ,HH(;%# I 
 r   c                 :   |j                   j                  }|r| j                  |      ry|j                  d      }| j	                  |       | j                  ||j                   j                         | j                  j                  |j                         |S )z
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added deployment
        - OR None (if deployment already exists)
        NTr   r  rW  )	ri  rk  r  r^  r]  r_  r   addr  )r  r  _deployment_model_id_deployments       r   add_deploymentzRouter.add_deployment  s      *4477D$5$56J$K !((d(;
3 	--
(=(=(@(@ 	. 	
 	Z223r   r  removal_idxc                    | j                   j                         D ]  \  }}||kD  s|dz
  | j                   |<    || j                   v r| j                   |= t        | j                  j                               D ]y  \  }}||v r|j	                  |       g }|D ]-  }||kD  r|j                  |dz
         |j                  |       / t        |      dkD  r|| j                  |<   m| j                  |= { y)a"  
        Helper method to update deployment indices after a deployment has been removed from model_list.

        Parameters:
        - model_id: str - the id of the deployment that was removed
        - removal_idx: int - the index where the deployment was removed from model_list
        rr   r   N)r   rb  r   r   r  r   r  )r  r  r  r  r  r  indicesupdated_indicess           r   (_update_deployment_indices_after_removalz/Router._update_deployment_indices_after_removal/  s     #'"G"G"M"M"OM3[ GJQw55mD #P t<<<55h? $((M(M(S(S(U#VJg%{+ !O$#**373#**3/	  ?#a'DS55jA99*E# $Wr   c                    t        | j                        }| j                  j                  |       ||| j                  |<   n6|j	                  di       j	                  d      || j                  |d   d   <   |j	                  d      }|r<|| j
                  vrg | j
                  |<   | j
                  |   j                  |       yy)a!  
        Helper method to add a model to the model_list and update both indices.

        Parameters:
        - model: dict - the model to add to the list
        - model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"]
        Nri  rk  r  )r  r   r   r   r*  r   )r  r   r  r  r  s        r   r_  z'Router._add_model_to_list_and_index_mapU  s     $//"u% >AD11(;YY|R(,,T2>ORD11%2Ed2KL YY|,
!F!FFDF55jA11*=DDSI r   c                    	 |j                   j                  xs d}| j                  |      }|w|j                  |j                  k(  ryd}|j                   j                  }| j                  }||v r5||   }|.| j
                  j                  |       | j                  ||       | j                  |       |S # t        $ r1}| j                  rt        j                  d| d       Y d}~y|d}~ww xY w)z
        Add or update deployment
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added/updated deployment
        r  r  Nr  r  r  zError upserting deployment: rX  )ri  rk  r  r   r   r   r  r  r  r  r   r   r  )r  r  r  _deployment_on_routerr  r  deployment_fast_mappingr  s           r   ro  zRouter.upsert_deploymento  s   &	#-#8#8#;#;#Ar :>:M:M- ;N ;! %0,,0E0T0TT .2 * 5 5 8 8*.*O*O' $;;"9-"HK".++K8EE%2 F 
 :6 	..%++21#5fg 	s%   AB; 
A0B; ;	C5%C0.C00C5rk  c                     d}|| j                   v r| j                   |   }	 |0| j                  j                  |      }| j                  ||       |S y# t        $ r Y yw xY w)z
        Parameters:
        - id: str - the id of the deployment to be deleted

        Returns:
        - The deleted deployment
        - OR None (if deleted deployment not found)
        Nr  )r   r   r  r  r  )r  rk  deployment_idxr>  s       r   delete_deploymentzRouter.delete_deployment  sz     666!BB2FN	)**>:==^ >   		s   1A 	A A c                     || j                   v rn| j                   |   }| j                  |   }t        |t              rt	        di |S t        |t              r|S t        dj                  t        |                  y)zl
        Returns -> Deployment or None

        Raise Exception -> if model found in invalid format
        zModel invalid format - {}Nr~   )r   r   r   r  rR   r  r  r   )r  r  r  r   s       r   r  zRouter.get_deployment  su     t<<<77ACOOC(E%&!*E**E:. ; B B4; OPPr   c                     | j                  |      }|yt        di |j                  j                  d      j                  d      S )zE
        Returns -> dict of credentials for a given model id
        r  NTr   r~   )r  rP   r   r  )r  r  r  s      r   get_deployment_credentialsz!Router.get_deployment_credentials  sU     ((((;
& 
''222E

*$*
'	(r   model_group_namec                    || j                   v rs| j                   |   }|rb| j                  |d      }t        |t              rt	        di |S t        |t              r|S t        dj                  t        |                  y)z
        Returns -> Deployment or None

        Raise Exception -> if model found in invalid format

        Optimized with O(1) index lookup instead of O(n) linear scan.
        r   zModel Name invalid - {}Nr~   )r   r   r   r  rR   r  r  r   )r  r  r  r   s       r   "get_deployment_by_model_group_namez)Router.get_deployment_by_model_group_name  s}     tDDD;;<LMG
3eT*%...z2 L#$=$D$DT%[$QRRr   r  c                      y r;  r~   r  r  r  rk  s       r   r  zRouter.get_router_model_info       	r   c                      y r;  r~   r  s       r   r  zRouter.get_router_model_info  r  r   c                    |&| j                  |      }||j                  d      }|t        d      |j                  di       j                  dd      }|"|j                  di       j                  dd      }|}t	        j
                  |j                  di       j                  d	d
      t        di |j                  di             \  }}}	}	|dk(  r|t        j                  d| d       n|dk7  r|}| j                  j                  |      }
d|v rp|
n|
D ]i  }	 |j                  di       j                  d      |j                  di       j                  d      k(  r#|j                  di       j                  d	      } nk ||}|j                  dj                  |            sdj                  ||      }n|}t	        j                  |      }|j                  di       }|j                  |       |S # t        $ r Y w xY w)a  
        For a given model id, return the model info (max tokens, input cost, output cost, etc.).

        Augment litellm info with additional params set in `model_info`.

        For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.

        Returns
        - ModelInfo - If found -> typed dict with max tokens, input cost, etc.

        Raises:
        - ValueError -> If model is not mapped yet
        Nr  Tr   zDeployment not foundri  
base_modelr   r   r  r   r   azurez Could not identify azure model 'z'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-modelsr   rk  z{}/z{}/{}r^  r~   )r  r  r  r*  r   rm   rT   r   r=  r   router  rc  r  get_model_infor   )r  r  r  rk  r  r  r   r  r  r  potential_modelspotential_modelmodel_info_nameri  user_model_infos                  r   r  zRouter.get_router_model_info  sC   & >--r-:K&(333F
344  ^^L"599,M
#(8"=AA,PTUJ -4,D,D..!126::7BG)QJNN;KR,PQ-
)#Q ')j.@!''26(  ;Z  [ !G+E#22889LMe| 0 <'7O	*..|R@DD '^^L"=AA$GH %4$7$78H"$M$Q$Q '%E "H (8 = E-@ AB%nn-@%HO#O++/B
 %..r:/*+ % s   "A$G>>	H
	H
c                 \    || j                   v r| j                   |   }| j                  |   S y)a%  
        For a given model id, return the model info

        Returns
        - dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
        - None: could not find deployment in list

        Optimized with O(1) index lookup instead of O(n) linear scan.
        N)r   r   )r  rk  r  s      r   r  zRouter.get_model_infoN  s4     66677;C??3''r   c                 Z    | j                  |      }|y|d   }| j                  |      S )zT
        Return list of all models in the same model group as that model id
        )rk  Nr  r  )r  r  )r  rk  ri  r  s       r   get_model_groupzRouter.get_model_group^  s>    
 ((B(/
-
""j"99r   c           	         ddl m} d}d}d}	 t        j                  j	                  |      }	 t        j                  |      }	 |I|j	                  dd      }|5t        j                  |      }||xs i } |t        t        |      |      }|7|5t        t         |t        t        |      j                         |            }|S ||}|S # t
        $ r Y w xY w# t
        $ r Y w xY w# t
        $ r Y hw xY w)z
        For a given model id, return the model info

        1. Check if model_id is in model info
        2. If not, check if litellm model name is in model info
        3. If not, return None
        r   )_update_dictionaryNr^  r  )litellm.utilsr  r   rU  r*  r  r  r   r  rd   r  )	r  r  r  r  ri  custom_model_infolitellm_model_name_model_infor  base_model_infos	            r   get_deployment_model_infoz Router.get_deployment_model_infoj  s6    	5*.
,0=A%	 ' 2 2 6 6x @	,3,B,B,T)
	 ,.22<F
)&-&<&<:&NO&2,=,C),> 7--) (-J-V"<=BBD%J  +66JI  		
  		"  		s5   C C AC- 	CC	C*)C*-	C98C9user_facing_model_group_namec                 	   d}d}d}d}| j                  |      }|y|D ]'  }d}	d|v r|d   |k(  rd}	n!d|v r| j                  j                  |      d}	|	s9t        di |d   }
|
j                  }|j                  di       }|j                  di       }d}||j                  dd      }||j                  dd      }||j                  dd      }d}||j                  d	d      }||j                  d	d      }||j                  d	d      }	 |j                  d
d      }|| j                  ||
j                        }nd}d\  }}	 t        j                  |
j                  |
j                        \  }}}}|Tt        j$                  ||      }|g }|j                  di       }|j                  dd      }t'        |ddddd|||d
      }|t)        di ||gd|}nG||j*                  vr|j*                  j-                  |       |j                  dd      -|d   (|j.                  |d   |j.                  kD  r
|d   |_        |j                  dd      -|d   (|j0                  |d   |j0                  kD  r
|d   |_        |j                  dd      (|j2                  |d   |j2                  kD  r
|d   |_        |j                  dd      (|j4                  |d   |j4                  kD  r
|d   |_        |j                  dd      	 |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_         |j                  dd      |d   
|d   |_!        |j                  dd      ||j                  d      }|j                  d	d      ||j                  d	      }|	|d}||z  }||d}||z  }* ||||_"        |||_#        |||_        |S # t        $ r d}Y 0w xY w# t        j                  j                  $ r8}t        j                  dj!                  t#        |                   Y d}~Nd}~ww xY w) z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info
        Nr  Fr  Tr   ri  r  r  rk  )r  r  )r  r  r|  z.litellm.router.py::get_model_group_info() - {}moder   r   )
re  
max_tokensmax_input_tokensmax_output_tokensinput_cost_per_tokenoutput_cost_per_tokenlitellm_providerr  supported_openai_paramssupports_system_messages)r1  	providersr  r  r  r  "supports_parallel_function_callingsupports_visionsupports_function_callingsupports_web_searchsupports_url_contextsupports_reasoningr  r~   )$r  r   r  rT   #configurable_clientside_auth_paramsr*  r  r   r  r   rm   r  
exceptionsBadRequestErrorr   r=  r  r   get_supported_openai_paramsModelMapInforV   r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  )r  r1  r  model_group_info	total_tpm	total_rpmr  r   r   is_matchr   model_litellm_paramsmodel_info_dict_deployment_tpm_deployment_rpmr  ri  r  r  r  r  r  db_model_infor  s                           r   _set_model_group_infozRouter._set_model_group_info  s<    6:#'	#'	SW+((K(@
EH%%*=*L%''--k:F+Fe4D.EFN BB 0
 $)99-=r#B #iib9O .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B	"*..tT:'!%!?!?!)n6J6J "@ "J "&J
 +1'M<4;4L4L(..(6(J(J51|Q !*1*M*M'\+' +2.0+ !&		, ;$((8)##%)&*)**+%1,C-1
  '#1 $'C&2^ %$   '7'A'AA$..55lCNN#5t<H"#56B(99A%&89*;;< 9CCU8V$5NN#6=I"#67C(::B%&9:*<<= :DDW9X$6>>"8$?K$99A!"89&;;< =G.=$9 >>"94@L$::B!"9:&<<= >H/>$: NN#GN "#GHDPJN$GNN#4d;G"#45=7;$4NN#>EQ"#>?4GAE$>NN#8$?K"#89TA;?$8NN#94@L"#9:dB<@$9 NN#7>J"#78D@:>$7NN#<dCO"#<=I?I1@$< >>%.:?V&0nnU&;O>>%.:?V&0nnU&;O*$ !I_,	*$ !I_,	K  L '$'0 $$'0 $ 3>7 !D  A  "!
" %%55 %++DKKCPQFS s*   4Q*0Q<*Q98Q9<S-SSc                     || j                   v rT| j                   |   }t        |t              r|}nt        |t              r|d   du ry|d   }ny| j	                  ||      S | j	                  ||      S )z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info or hidden model group
        hiddenTNr   )r1  r  )r   r   r   r  r  )r  r1  r>  _router_model_groups       r   get_model_group_infozRouter.get_model_group_info  s     $000))+6D$$&*#D$'>T)*.w-'--/-8 .   ))#+ * 
 	
r   c                   K   t               }|j                  d      }g }g }| j                  |      }|y|D ]  }|j                  di       j                  d      }|d   j                  d      }	||	=|j	                  t
        j                  j                  j                  ||	|	             |j	                  t
        j                  j                  j                  ||	|	              ||z   }
| j                  j                  |

       d{   }|y|dt        |       }|t        |      d }d}|!|D ]  }t        |t              s|d}||z  } d}|!|D ]  }t        |t              s|d}||z  } ||fS 7 qw)z
        Returns current tpm/rpm usage for model group

        Parameters:
        - model_group: str - the received model name from the user (can be a wildcard route).

        Returns:
        - usage: Tuple[tpm, rpm]
        r  r  Nr  ri  rk  r   r   )rk  r   r  )r  r   )rp   r  r  r*  r   rY   r  r1  r  r  r   async_batch_get_cacher  r   r   )r  r1  r  r  tpm_keysrpm_keysr   r   rk  r  combined_tpm_rpm_keyscombined_tpm_rpm_valuestpm_usage_listrpm_usage_list	tpm_usager  	rpm_usages                    r   get_model_group_usagezRouter.get_model_group_usage  s     
 ! ((K(@
E %		, ; ? ? EB+01A+B+F+F,M z]2OO##))00'#1 1  OO##))00'#1 1   * !)8 3(,

(H(H& )I )
 #
 #*)@3x=)Q)@X)Q $(	%#a% ($%	NI	 $ $(	%#a% ($%	NI	 $
 )##3#
s   DFF<F$F5F)maxsizec                 $    | j                  |      S )z
        Cached version of get_model_group_info, uses @lru_cache wrapper

        This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request
        )r  )r  r1  s     r   _cached_get_model_group_infoz#Router._cached_get_model_group_info  s     ((55r   c                 <  K   | j                  |      }||j                  |j                  }nd }||j                  |j                  }nd }||i S | j                  |       d {   \  }}i }|||xs dz
  |d<   ||d<   |||xs dz
  |d<   ||d<   |S 7 1w)Nr   x-ratelimit-remaining-tokenszx-ratelimit-limit-tokensx-ratelimit-remaining-requestszx-ratelimit-limit-requests)r  r  r  r  )r  r1  r  	tpm_limit	rpm_limitcurrent_tpmcurrent_rpmreturned_dicts           r   get_remaining_model_group_usagez&Router.get_remaining_model_group_usage  s     <<[I',<,@,@,L(,,II',<,@,@,L(,,II!2I)-)C)CK)P#P [ <E q=M89 9BM45 >G q?M:; ;DM67 $Qs   A&B(B)2Bc                 ~  K   t        |t              rt        |d      rt        |j                  t              r|j                  j                  di        ||j                  d   d<   |j                  d   }d|vr?d|vr;|9| j                  |       d{   }|j                         D ]  \  }}|	|||<    |S 7 &w)a  
        Add the most accurate rate limit headers for a given model response.

        ## TODO: add model group rate limit headers
        # - if healthy_deployments > 1, return model group rate limit headers
        # - else return the model's rate limit headers
        r  r  zx-litellm-model-groupr   r  N)r   r   r9  r  r  r   r  rb  )r  r  r1  r  remaining_usageheaderr1  s          r   r  zRouter.set_response_headers  s      x+"23822D9##../CRH  ##$89' "*!8!89M!N /6HH4<NN+(,(L(L) # &5%:%:%<MFE(5:*62 &= #s   BB=B;B=2
B=c                     | j                   j                          t        |      D ]T  \  }}|j                  d      }|s|| j                   vrg | j                   |<   | j                   |   j	                  |       V y)z
        Build model_name -> deployment indices mapping for O(1) lookups.

        This index allows us to find all deployments for a given model_name in O(1) time
        instead of O(n) linear scan through the entire model_list.
        r  N)r   clearr  r*  r   )r  r   r  r   r  s        r   _build_model_name_indexzRouter._build_model_name_indexA  sq     	--335#J/JC<0JT%J%JJHJD99*E55jAHHM 0r   c                 6   g | _         t        |      D ]  \  }}|j                  di       }|j                  d      }|G|j                  dd      }|j                  di       }| j                  ||      }d|vri |d<   ||d   d<   | j	                  ||        y)z
        Build model index from model list to enable O(1) lookups immediately.
        This is called during initialization to avoid the race condition where
        requests arrive before model_id_to_deployment_index_map is populated.
        ri  rk  Nr  r  r   rW  )r   r  r*  rn  r_  )r  r   r  r   ri  r  r  r   s           r   '_build_model_id_to_deployment_index_mapz.Router._build_model_id_to_deployment_index_mapQ  s     !*-HAu<4J!~~d+H "YY|R8
!&+;R!@22:~Nu,*,E,',4l#D)111Q .r   exclude_team_modelsc                    g }|p|| j                   v r`| j                   |   }|D ]L  }| j                  |   }d|v sd|d   v s|r|d   j                  d      r6|j                  |d   d          N |S | j                  j                         D ]U  }| j                  |   }| j                  |   }d|v s&d|d   v s.|r|d   j                  d      rE|j                  |       W |S )z
        if 'model_name' is none, returns all.

        Returns list of model id's.

        Optimized with O(1) or O(k) index lookup when model_name provided,
        instead of O(n) linear scan.
        ri  rk  r}  )r   r   r*  r   r   r  )r  r  r  idsr  r  r   r  s           r   get_model_idszRouter.get_model_idsj  s    !TBBB??
K"C OOC0E#u,|9L1L.53F3J3J93U$

5#6t#<= #" 
 !AAFFH;;HE,5(TU<5H-H*u\/B/F/Fy/Q JJx( I 
r   candidate_idc                     || j                   v S )a  
        O(1) membership check for a deployment ID without allocating large lists.

        Note: Call sites may pass a variable named `model` when it actually
        contains a deployment ID. This helper expects the deployment ID string.

        Uses the existing `model_id_to_deployment_index_map` which is kept
        in sync by `_build_model_id_to_deployment_index_map` and model-list
        mutation helpers.
        )r   )r  r  s     r   r  zRouter.has_model_id  s     tDDDDr   team_model_namer}  c                     | j                  ||      }|sy|D ]9  }|j                  di       j                  d      |k(  s(|j                  d      c S  y)z
        Map a team model name to a team-specific model name.

        Returns:
        - deployment id: str - the deployment id of the team-specific model
        - None: if no team-specific model name is found
        r  r}  Nri  r}  r  )r  r*  )r  r  r}  r  r   s        r   map_team_modelzRouter.map_team_model  s[     $$$QEyyr*..y9WDyy.. 
 r   c                 |    |/|d   j                  d      |k(  r||d   j                  d      k(  ry|	|d   |k(  ryy)zU
        Get the team-specific model name if team_id matches the deployment.
        ri  r}  r~  Tr  Fr*  )r  r  r   r}  s       r   should_include_deploymentz Router.should_include_deployment  sV     l#''	2g=eL1556NOO#l(;z(Ir   model_aliasc                    g }|| j                   v rv| j                   |   }|D ]`  }| j                  |   }| j                  |||      s'|'|j                         }||d<   |j	                  |       P|j	                  |       b |S |lt        | j                        D ]T  \  }}| j                  |||      s|'|j                         }||d<   |j	                  |       D|j	                  |       V |S )z
        Return all deployments of a model name

        Used for accurate 'get_model_list'.

        if team_id specified, only return team-specific models

        Optimized with O(1) index lookup instead of O(n) linear scan.
        )r  r   r}  r  )r   r   r  r  r   r  )	r  r  r  r}  returned_modelsr  r  r   alias_models	            r   _get_all_deploymentszRouter._get_all_deployments  s    68 >>>;;JGG ,11) 2  #. ',jjl4?L1'..{;'..u5 : !   (8
U11) 2  #.&+jjl4?L1'..{;'..u5 9 r   c                    | j                         xs g }g }|D ]m  }|j                  d      }| j                  |      r(| j                  ||      }|s;|j	                  |       M|j	                  |j                  dd             o |S )u  
        Returns all possible model names for the router, including models defined via model_group_alias.

        If a team_id is provided, only deployments configured with that team_id (i.e. team‐specific models)
        will yield their team public name.
        ri  )r  r}  r  r  )r  r*  _is_team_specific_model_get_team_specific_modelr   )r  r}  deploymentsr   r  ri  r  s          r   rw  zRouter.get_model_names  s     ))+1r%J#5J++J7"&"?"?)7 #@ # #&&7"":>>,#CD & r   c                 ~    |j                  d      xs i }|y||j                  d      k(  r|j                  d      S y)a  
        Get the team-specific model name if team_id matches the deployment.

        Args:
            deployment: DeploymentTypedDict - The model deployment
            team_id: Optional[str] - If passed, will return router models set with a `team_id` matching the passed `team_id`.

        Returns:
            str: The `team_public_model_name` if team_id matches
            None: If team_id doesn't match or no team info exists
        ri  Nr}  r~  r  )r  r  r}  ri  s       r   r$  zRouter._get_team_specific_model	  sD     &0^^L%A%GR
jnnY//>>":;;r   ri  c                 >    t        |xr |j                  d            S )z
        Check if model info contains team-specific configuration.

        Args:
            model_info: Model information dictionary

        Returns:
            bool: True if model has team-specific configuration
        r}  )r  r*  )r  ri  s     r   r#  zRouter._is_team_specific_model  s     J<:>>)#<==r   c                    g }| j                   j                         D ]l  \  }}|||k7  rt        |t              r|}n*t        |t              rt        di |}|d   du rD|d   }nK|j                  | j                  ||             n |S )z
        Helper function to get model list from model alias.

        Used by `.get_model_list` to get model list from model alias.
        r  Tr   )r  r  r~   )r   rb  r   r   r  r[   r~  r!  )r  r  r  r  model_value_router_model_name_model_values          r   get_model_list_from_model_aliasz&Router.get_model_list_from_model_alias*  s     68(,(>(>(D(D(F$K%+*C+s+*5"K.8G;G)T1)5g)>&""))1{ *  )G( r   c                    g }|"|j                  | j                  ||             |j                  | j                  |             t        |      dk(  r| j                  j                  |      xs g }|A|| j                  v r3| j                  |   j                  |      xs g }|j                  |       |*|(|D ]#  }t        di |}||d<   |j                  |       % ||| j                  z  }|S )z
        Includes router model_group_alias'es as well

        if team_id specified, returns matching team-specific models
        r  r  r   r  r~   )
r~  r!  r,  r  r   r  r   rS   r   r   )r  r  r}  r  potential_wildcard_models#potential_team_only_wildcard_modelsr  deployment_typed_dicts           r   r  zRouter.get_model_listI  s     68!""))Z)Q 	00J0G	
 1$(,(;(;(A(A*(M(SQS% "w$2K2K'K--g6<<ZHNB 4 *001TU%*C*O2A,?,D!,D):D),7#**+@A 3
 t.Or   model_access_groupc                 6   ddl m}  |t              }| j                  ||      }|rq|D ]l  }|j	                  d      }|s|j	                  dg       xs g D ]=  }	| |	|k(  s|d   }||	   j                  |       %|d   }||	   j                  |       ? n |S )a  
        If model_name is provided, only return access groups for that model.

        Parameters:
        - model_name: Optional[str] - the received model name from the user (can be a wildcard route). If set, will only return access groups for that model.
        - model_access_group: Optional[str] - the received model access group from the user. If set, will only return models for that access group.
        - team_id: Optional[str] - the team id, to resolve team-specific models
        r   r   r  ri  r   r  )collectionsr   r   r  r*  r   )
r  r  r1  r}  r   r   r   r  r  groups
             r   get_model_access_groupszRouter.get_model_access_groupss  s     	,#D)((J(P
eeL1!,"!E!K!K-9$(::-.|_
 -e 4 ; ;J G)*<J)%077
C "L   r   c                     | j                  |      }t        |      dk(  ry|j                  |g       }|D ]   }| j                  j	                  |        y y)zG
        Return True if model access group is a wildcard route
        )r1  r   Fr  T)r5  r  r*  r   r  )r  r1  r   r  r   s        r   )_is_model_access_group_for_wildcard_routez0Router._is_model_access_group_for_wildcard_route  sm     441 5 
 }"""#5r:E""(((7C 
 r   c                     t        |       }i }g d}|D ]K  }||v r||   ||<   |dk(  s| j                  dk(  s%| j                  j                  j	                         ||<   M |S )a  
        Get router settings method, returns a dictionary of the settings and their values.
        For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after
        )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )varsr   r7  r/  rH  )r  	_all_vars_settings_to_returnvars_to_includevars        r   get_settingszRouter.get_settings  s}    
 J	 
  #Ci+4S>#C(..))-DD+/+D+D+Q+Q+V+V+X#C( # #"r   c                    g d}g d}| j                         }|D ]  }||v rg||v rt        ||         }t        | ||       '|dk(  r1|d   ||   k7  r&| j                  ||   |j	                  di              t        | |||          nt        j                  dj                  |              t        j                  d| j                                 y)	z-
        Update the router settings.
        )r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   zSetting {} is not allowedzUpdated Router settings: N)r>  r   r<  r   r*  r   r  r  )r  r  _allowed_settings_int_settings_existing_router_settingsr=  _casted_values          r   update_settingszRouter.update_settings  s    


 %)$5$5$7!C''-'$'s$4MD#}5 1156HIVTW[X22-3C[28** 73 3  D#vc{3%++,G,N,Ns,ST' ( 	##&?@Q@Q@S?T$UVr   c                 |   |d   d   }t        |      }|dk(  rhdj                  |      }| j                  j                  |d|      }|5t	        j
                  | |       | j                  j                  |d|      }|S |dk(  r]|j                  d	      du r%| d
}| j                  j                  |d|      }|S | d}| j                  j                  |d|      }|S |j                  d	      du r$| d}| j                  j                  ||      }|S | d}| j                  j                  ||      }|S )a  
        Returns the appropriate client based on the given deployment, kwargs, and client_type.

        Parameters:
            deployment (dict): The deployment dictionary containing the clients.
            kwargs (dict): The keyword arguments passed to the function.
            client_type (str): The type of client to return.

        Returns:
            The appropriate client based on the given client_type and kwargs.
        ri  rk  rL  z{}_max_parallel_requests_clientTre  r#  r  )r   r   r}  r  _stream_async_client_async_client_stream_client)re  r  _client)r   r  r   r$  r2    set_max_parallel_requests_clientr*  )r  r  r  rN  r  r  	cache_keyr  s           r   r  zRouter._get_client   s    l+D1+LV+T119@@JIZZ))$AQ * F ~%FF,0
 --!dEU .  MG#zz(#t+'j(<=	--!dEU .  'j6	--!dEU .  zz(#t+'j7	--!4D .  'j0	--!4D .  r   c           	      V	   t        j                  d|        t        |      }t               }	 t	        j
                  |      }d}	d}
d}t        |      }t               }|j                  d      }| d| }| j                  j                  |d	|
      xs i }t        |      D ]  \  }}|j!                  di       }|j!                  di       }	 |j!                  dd      }||j!                  dd      }| j#                  ||      }|xs |j!                  dd      }t%        |t&              rZ|j!                  dd      Ht%        |d   t(              r5||d   kD  r-|j+                  |       d	}	|
dj                  ||d   |      z  }
|j!                  dd      }| j                  j                  |d	|
      xs d}t%        |t&              r| j.                  dk7  rv|j!                  |d      ||<   t1        |||         }t%        |t&              rB|j!                  dd      0t%        |d   t(              r|d   |k  r|j+                  |       d	}|M|j!                  d      <|j!                  d      }|)t3        t5        d!i ||      s|j+                  |       |t        j6                  du st	        j8                  |t5        d!i |      \  }}}}t	        j:                  ||      }|Ut        j<                  j?                  |      }dg}|jA                         D ]8  \  } }!| |vs| |v st        j                  d|         |j+                  |       :  tC        |      tC        |      k(  r<	 |d	u rtE        |      |	d	u r't	        jF                  dj                  |
      |d       tC        |      dkD  r!t        |      D "#cg c]  \  }"}#|"|vr|# }}"}#tC        |      dkD  rt        j<                  jI                  |      }|S # t        $ r9}t        j                  dj                  t        |                   |cY d}~S d}~ww xY w# t        $ r8}t        j,                  dj                  t        |                   Y d}~d}~ww xY wc c}#}"w )"a  
        Filter out model in model group, if:

        - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
        - filter models above rpm limits
        - if region given, filter out models not in that region / unknown region
        - [TODO] function call and model doesn't support function calling
        z2Starting Pre-call checks for deployments in model=)r  zllitellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {}NFr  r  z:rpm:TrF  r   ri  r  r  r   r  z%Model={}, Max Input Tokens={}, Got={}zAn error occurs - {}rk  r   r   r  allowed_model_region)r   rN  r  r|  )passed_paramsresponse_formatz1INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=r^  z~litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.
{}r  r~   )%r   r  r   r3  r   token_counterr  r=  r  r   r   rp   r  r   r$  r  r*  r  r   r  r   r  re  r   maxrq   rT   drop_paramsrm   r  r  get_non_default_paramsrb  r  r]   r  _get_order_filtered_deployments)$r  r   r   r  r  _returned_deploymentsinvalid_model_indicesinput_tokensr  _context_window_error_potential_error_str_rate_limit_errorr  r  r  r  model_group_cacher  r  rS  r  r  ri  r  current_request_cache_localcurrent_requestrN  r  r  r  r  special_paramsr  r  ids$                                       r   _pre_call_checkszRouter._pre_call_checks6  sR     	##@H	
 !%%8 9 #	)"00(CL !&!!<^L W-G5 01JJ  ?O !   	 	  ))>?OC(nn-=rBO$..r:KW(__\4@
%!0!4!4\4!HJ!77)u 8 
 #Ho&9&9'4&H z40"'94@L #:.@#A3G(:6H+II-11#604-,CJJ %z2D'E|,
 ! #tR0H 

$$ TDT %   	 ( ,d3))-EE.?.C.CHa.P!(+"%/1B81L#
 5'++E48D #?5#93?+E2oE-11#6,0)  *"&&'=>J'5'9'9:P'Q$'3,'5'H'H-A .11#6  )g.A.AU.J3:3K3K0Q0Q40*Aq +2*M*M5H+' +2 *1)M)M&4 *N *& '8%8N 2 8 8 :1$;;^@S177"STUSV W 255c: !;K  @Z $%-B)CC !D(/  '$.88 ^  e  e,  !#  $%) &&;<%<DAq11 < " % $%)$+MM$Q$Q%%! %$M  	)!''~  F  FF
 )(	)l  W%//0F0M0McRSf0UVVWH%s<   P B8Q!R%	Q%.QQQ!	R"*-RR"c                 t    || j                   vry| j                   |   }t        |t              r|}|S |d   }|S )z
        Get the model from the alias.

        Returns:
        - str, the litellm model name
        - None, if model is not in model group alias
        Nr   )r   r   r   )r  r   _items      r   r`  zRouter._get_model_from_alias  sK     ...&&u-eS!E  'NEr   c                 X    | j                   D cg c]  }|d   d   |k(  s| c}S c c}w )z6
        Get the deployment by litellm model.
        r   r   )r   )r  r   r  s      r    _get_deployment_by_litellm_modelz'Router._get_deployment_by_litellm_model  s2      ??T?aa0@.A'.Je.S?TTTs   ''r  c                    d}|N|j                  d      xs i }|j                  d      xs i }|j                  d      xs |j                  d      }|du r|| j                  |      fS | j                  |      rM| j                  |      }	|	*|	j                  j
                  }
|
|	j                  d      fS t        d	| d
      | j                  |      }||}|| j                  vr| j                  j                  |      }|r||fS |3|| j                  v r%| j                  |   j                  |      }|r||fS | j                  F| j                  j                         }| j                  d   j                         |d<   ||d   d<   ||fS | j                  |      }t!        |      dk(  r| j                  |      }t#        j$                  d|        t!        |      dk(  r| j'                         rB| j)                         }|r0t#        j*                  d| d| d       |}| j                  |      }t!        |      dk(  rU| j-                  |      d| dj/                  |      }nd| dj/                  |      }t1        j2                  ||d      t0        j4                  r%|t0        j4                  v rt0        j4                  |   }||fS )aF  
        Common checks for 'get_available_deployment' across sync + async call.

        If 'healthy_deployments' returned is None, this means the user chose a specific deployment

        Returns
        - str, the litellm model name
        - List, if multiple models chosen
        - Dict, if specific model chosen
        Nr   r1  user_api_key_team_idTr^  r  r   zBLiteLLM Router: Trying to call specific deployment, but Model ID :z does not exist in Model ID mapr   r   r  r   zinitial list of deployments: zModel 'z7' not found. Attempting to use default fallback model 'z'.zYou passed in model=z+. There is no 'model_name' with this stringz1. There are no healthy deployments for this modelr  r  )r*  rf  r  r  r   r   r  r  r`  r   r   get_deployments_by_patternr   r   r  r!  r  r   r  r)  r  r  r  r  r   r  model_alias_map)r  r   r  r
  r  r  request_team_idr   r1  r  deployment_model_model_from_aliaspattern_deploymentsupdated_deploymentr   fallback_modelr  s                    r   r/  z*Router._common_checks_available_deployment  s   & *.%%))*5;H-112DEK&ll& >!%%&<=  $&$??e?LLLu%,,e,<J%#-#<#<#B#B ')>)>D)>)QQQTUZT[[z{  !66U6C(%E((("&"5"5"P"P #Q # #111  +#t'@'@@&*&?&?#',, -  $
 ' "555 &&2%)%<%<%A%A%C"7;7N7N$8$& ##34 AF"#34W=000 #7757I"#q("&"G"Ge"G"T##+,?+@A	
 "#q(**,!%!A!A!C!)..!%(_`n_ooqr +E*.*C*Cu*C*U' &'1,&&%&8@ 4UG;fgnnG !5UG;lmttG --#!#  ""u0G0G'G++E )))r   c           	        K   ddl m} | j                  |||||      \  }} |||      }t        |t              r|S t        | |       d{   }	t        j                  d|	        t        j                  d|	        | j                  ||		      }| j                  |||t        t        t           |      nd||
       d{   }| j                  r,|*| j                  |t        t        t           |      ||      }t!        | |||| j#                  |             d{   }t%        |      dk(  rt'        | ||       d{   }
|
|S 7 7 7 17 w)z
        Get the healthy deployments for a model.

        Returns:
        - List[Dict], if multiple models chosen
        *OR*
        - Dict, if specific model chosen
        r   )filter_team_based_modelsr   r  r
  r  r  )r   r  r  Nzasync cooldown deployments: zcooldown_deployments: r   cooldown_deploymentsrC  r   r   r  r  )llm_router_instancer   r  r   r[  r   r   r  )rc  rr  r/  r   r  r7   r   r  _filter_cooldown_deploymentsrF  r   r   rG   r   rb  r
   r,   r  r  r?   )r  r   r  r  r
  r  r  rr  r   ru  re  s              r   rd  z$Router.async_get_healthy_deployments  s    " 	O%)%M%M 3) &N &
"" 7 3)

 )40&&%D$(;K&
  
 	##*+?*@A	
 	##&<=Q<R$ST"?? 3!5 @ 

 %)$J$J 3:B:NT*+X6TX)- %K %
 
 &&8+?"&"7"7$(d5H$I!-	 #8 # %< $) 3#'#O#O$%
 
 "#q(A(,!1 I
 O""a 

$
sJ   AEEA9EE	AE'E(#EE	EEEEc           
        K   | j                   dk7  rR| j                   dk7  rC| j                   dk7  r4| j                   dk7  r%| j                   dk7  r| j                  |||||      S 	 t        |      }| j                  |||||       d{   }||j                  }|j
                  }| j                  ||||||	       d{   }t        |t              r|S t        j                         }	| j                   dk(  r4| j                  (| j                  j                  ||||
       d{   }
n| j                   dk(  r4| j                  (| j                  j                  ||||
       d{   }
n| j                   dk(  r5| j                  )| j                  j                  |||||       d{   }
n`| j                   dk(  rt        | ||      S | j                   dk(  r2| j                   &| j                   j                  ||       d{   }
nd}
|
t#        | ||       d{   }|t%        j&                  d| d| j)                  |
       d|        t        j                         }||	z
  }t+        j,                  | j.                  j1                  t2        j4                  |d||	|             |
S 7 27 7 7 W7 7 7 # t6        $ r}t9        j:                         }|j|j=                  dd      }|Vt?        j@                  |jB                  ||f      jE                          t+        j,                  |jG                  ||             |d}~ww xY ww)z
        Async implementation of 'get_available_deployments'.

        Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps).
        r   r   r   r   r   rs  r   r  r  r
  r  N)r   r  r  r
  r  r  r1  r   r  r
  )r1  r   r  r
  r  rw  r   r   r1  r   rx  $get_available_deployment for model: , Selected deployment:  for model: z2<routing_strategy>.async_get_available_deployments)r  r  rI  r  r  r  rK  r:  )$r   r  r   async_pre_routing_hookr   r  rd  r   r  r  r  r5  async_get_available_deploymentsr9  r7  r+   r   r?   r   r  r  r
  r  r   r  r`   r  r  r  r  r*  r>  r?  r@  rA  r=  )r  r   r  r  r
  r  r  pre_routing_hook_responser   r  r  re  r  r  r  r9  r  s                    r   rI  z%Router.async_get_available_deployment  s     !!%==%%)99%%)==%%)@@%%500!$7- 1  	@P /3.I.I-!$7 /J / )% )41774== )-(J(J-!$7!1 )K ) # -t4****,J%%)AA,,8 22RR$),?!)#	 S    %%)==**6 00PP$),?!)#	 Q    %%)@@--9 33SS$),?!)#'5 T    &&*::%(,(;  %%5))5 //OO$),? P    "
!"E,0%5# 	
  !&&6ug=TUYUjUjkuUvTw  xD  EJ  DK  L ((*H :-I''BB(//&R%5)% C 	 O)#"&0  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G!	s   A"M!%$K 	J;
8K J>K M!AK *K+AK -K.AK 1K2!K M!;K K
K *K+BK :M!;K >K K K K 
K K 	MBMMM!c                    K   || j                   v r+| j                   |   j                  |||||       d{   S y7 w)z
        This hook is called before the routing decision is made.

        Used for the litellm auto-router to modify the request before the routing decision is made.
        r{  N)r   r  )r  r   r  r  r
  r  s         r   r  zRouter.async_pre_routing_hook  sX       D%%%**51HH-!$7 I    s   5?=?c                    | j                  ||||      \  }}t        |t              r|S t        |      }t	        | |      }| j                  ||      }| j                  r|| j                  ||||      }t        |      dk(  rU| j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      | j                  d
k(  r*| j                  | j                  j                  ||      }n| j                  dk(  rt!        | ||      S | j                  dk(  r+| j"                  | j"                  j                  |||      }nx| j                  dk(  r,| j$                   | j$                  j                  ||||      }n=| j                  dk(  r,| j&                   | j&                  j                  ||||      }nd}|nt)        j*                  d| d       | j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      t)        j*                  d| d| j-                  |       d|        |S )zB
        Returns the deployment based on routing strategy
        )r   r  r
  r  r  rt  Nrv  r   r  )	model_idsr  )r   r   r   cooldown_listr   r~  r   r}  r   )r1  r   r  r   r|  r   r  z, No deployment availabler  r  )r/  r   r  r   r9   ry  r   rb  r  r  r   get_min_cooldownr\   r   r   get_available_deploymentsr+   r7  r   r5  r   r  r  )r  r   r  r
  r  r  r   r  ru  r  _cooldown_time_cooldown_listr  s                r   r  zRouter.get_available_deployment  s8    &*%M%M 3	 &N &
"" )40&&+L,
  9$(;K 
 #?? 3!5 @ 
 &&8+?"&"7"7$7!-	 #8 # "#q(**e*<I!00AA#6F B N 7(,?ON ','+'B'B,	    L0T5J5J5V..HH!7J I J ""&66 "$($7  !!%<<))522LL!$7- M J !!%::%%1..HH!$7!	 I J !!%==((411KK!$7!	 L J J!&&6ug=VW **e*<I!00AA#6F B N 7(,?ON ','+'B'B,	  	""25'9PQUQfQfgqQrPss  AF  @G  H	
 r   ru  c                     t        j                  d|        t        |      }|D cg c]  }|d   d   |vr| c}S c c}w )a  
        Filters out the deployments currently cooling down from the list of healthy deployments

        Args:
            healthy_deployments: List of healthy deployments
            cooldown_deployments: List of model_ids cooling down. cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]

        Returns:
            List of healthy deployments
        zcooldown deployments: ri  rk  )r   r  r3  )r  r   ru  cooldown_setr  s        r   ry  z#Router._filter_cooldown_deployments  s`     	##&<=Q<R$ST/0 2
1
,'-\A 1
 	
 
s   >c                     	 |j                  di       j                  dd      }||| j                  ||       yyy# t        $ r+}t        j                  dt        |              Y d}~yd}~ww xY w)z7
        Tracks successful requests rpm usage.
        ri  rk  Nz$Error in _track_deployment_metrics: )r*  r'  r  r   r=  r   )r  r  r  r  r  r  s         r   rO  z Router._track_deployment_metrics-  s    		Y!~~lB7;;D$GH'&& "2 (    	Y!''*NsSTvh(WXX	Ys   8= 	A1!A,,A1re  c                 H    t        ||| j                  | j                        S )N)re  r1  r   r   )"_get_num_retries_from_retry_policyr   r   )r  re  r1  s      r   r>   z(Router.get_num_retries_from_retry_policy>  s)     2#%)%B%B**	
 	
r   c                    | j                   }|yt        |t        j                        r|j                  |j                  S t        |t        j
                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S yy)a  
        BadRequestErrorRetries: Optional[int] = None
        AuthenticationErrorRetries: Optional[int] = None
        TimeoutErrorRetries: Optional[int] = None
        RateLimitErrorRetries: Optional[int] = None
        ContentPolicyViolationErrorRetries: Optional[int] = None
        N)r   r   r   r  BadRequestErrorAllowedFailsr  AuthenticationErrorAllowedFailsrS  TimeoutErrorAllowedFailsr  RateLimitErrorAllowedFailsr  'ContentPolicyViolationErrorAllowedFails)r  re  r   s      r   get_allowed_fails_from_policyz$Router.get_allowed_fails_from_policyH  s     >B=V=V' y'"9"9:$@@L'CCCy'"="=>$DDP'GGGy'//2$==I'@@@y'"8"89$??K'BBBy'"E"EF$LLX'OOO Y Gr   c                 J   ddl m} | j                  y | j                  } ||j                  dg|j                        }|| _        t        j                  j                  |       t        j                  j                  |j                         t        j                  d       y )Nr   )SlackAlertingslack)alerting_thresholdalertingdefault_webhook_urlz2[94m
Initialized Alerting for litellm.Router[0m
)1litellm.integrations.SlackAlerting.slack_alertingr  r   r  webhook_urlslack_alerting_loggerr   r   r2  r  !response_taking_too_long_callbackr   r  )r  r  router_alerting_config_slack_alerting_loggers       r   r  zRouter._initialize_alertingp  s    S'151E1E!.5HHY 6 B B"
 &<"((==>TU((EE"DD	
 	""H	
r   CustomRoutingStrategyc                 `    t        | d|j                         t        | d|j                         y)a  
        Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router

        Use this to set your custom routing strategy

        Args:
            CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase
        r  rI  N)r<  r  rI  )r  r  s     r   set_custom_routing_strategyz"Router.set_custom_routing_strategy  s4     	&!::	

 	,!@@	
r   c                 N    d t         _        | j                  j                          y r;  )r   r   flush_cacherE  s    r   r  zRouter.flush_cache  s    

 r   c                     g t         _        g t         _        g t         _        g t         _        d | _        | j                          y r;  )r   r  r   r	  r  r   r  rE  s    r   resetzRouter.reset  s6    #% *,'#% *,' r   )F)r   r;  )FFF)T)r  r  )NNNN)NNNNN)NF)NNN)NNFN)NNF)rz   r{   r|   r3  r   __annotations__r   r   r  r   r   tenacityr   r&   r   r)   r   r   r   r   r	   r   rZ   rS   r
   r   rO   r_   r  r  floatr   r[   rX   rN   rW   rb   rM   r  r  r'  staticmethodr   r   r   r,  r^   r   r  r`  r  r  r   r  r  rj   rh   r  r  r   rG   r  rG  r  r  rf  rR   rs  r  rP  r  r  ry  r  r  r  r  r   r  r  r  r  r  r  r  rH   r  r  r  r  r  r  r  r  r"  r!  r*  r.  r4  r:  r3  r@  ri   rD  rB  rJ  rI  rI   rN  rM  rc   rj  ri  ro  ry  r  r<  r  r  r   r  r  r8  r"   wrapr	  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rt   r'  r)  r  r1  r  r  r  rR  rF  rn  ra  rT   rd  rn  r[  r   r   r]  r  r  r  r_  ro  r  r  r  r  r  r  r  r  rd   r  rV   r  r  r  r   r   r  r  r  r  r  r  r  r  r  r!  rw  r$  r#  r,  r  r5  r7  r>  rD  r  rb  r`  rf  r/  rd  rI  rv   r  r  ry  rO  r>   r  r  rQ   r  r  r  r~   r   r   r   r      s'   uK&+OXd^+&11H:>h67>:>h67>MQeL(C,G&H!IJQ ;?<@#'$($((,*/ ,0*.%) #'*. 7;!06 )+)+ ',%*     ,0 DH&(DH48 "#+0_m&$*+T$sCx.-AAB
m& $$78m& t$789m& C=m& SMm& SMm& !m& "$m& m&  !K
!m&& 'm&* #5/+m&, #3--m&0 c]1m&2  
3m&8 %9m&: !;m&< !)!
=m&B (0}Cm&D Em&F _-Gm&H $I
Im&N Om&P #'Qm&R #'Sm&T $eC!::;;<
Um&Z !%[m&\ #]m&^ _m&` +t#$
am&f #'#
gm&l  
mm&r '
sm&x  
ym&~ $D>m&@ "&
Am&P #++@"AQm&R  $Sm&T !))@ AUm&V ".1Wm&X "*!"
Ym&^ %)_m&` 
am&^ D 	.38n	.	z,,	-	. 	.+ +9 %os&: ;9TX9v	F;
zj
X1$ U(01F(GU,4 2$(c3h$8	}11	2&BB$(c3h$8B	}11	2BL $()9$:DKDM	 
 UZ$()9$:DKEN	 
 kp$()9$:DI'RV-Y`afYgJgDh	"M1	2  	33 '(3 	3jB>+B> tCH~&B> 	B>
 
B>HQQ$(c3h$8Q	}12	3Qn 1;	

 
 !)	

 

, EOPP4<SMP	P, NR''(,'=Ec]'	'H (,	+
+
 +
  }	+

 
+
Z t 4

"&
	%s
#	$
"&	%s
#	$$	4 	t 	uczAR8S 	Z#S	Z# T#s(^,d4S#X3G.HHIZ#x++$(.>)?$@+^ $(c3h$8BI$-	  SX$(c3h$8BI%.	  	M,M, tCH~&M, 	M,f di$()9$:FISZ[`Sa	 
 $()9$:FISZ[_S`	  << '(< 	<|;; ; $	;
 CHo; S#X;z
X 
X 
XZBZB '(ZB S#X	ZBxs 3 , ,C ,\c # *Fs F3 FP( (3 (TD) DC DLT3 Ts T3 Tl3 *)C )^ $)&+#(## # 4.	#
 d^# 4.#R $)&+#(## # 4.	#
 d^# 4.#JDS D# DT $)&+#(  4.	
 d^ 4.:DS D DL
x 


5=
 -5>38n-0>A	c3h>OO5=ObAA-5AN $)	 S$Y 4.	 
$6c4i 0 6 6x $(	 S$Y 4.	 
2DuS$Y'7 D DN 
	0BB 
	BH 
	>NN 
	Nd  $v}v 
	vp//l .21#1 &c]1x E_#_ = ?
_H .21#1 &c]1
#
2 .2(,	
#
 &c]
 %	
,}!}! $D>}! D>	}!
 #+4.}! #+4.}! c]}! }! }!~ V[[]0 0j &*$(3737-- c]- D>	-
 #+4.- #+4.-^ V[[]J% J%X $ :>!!)1#!L /3*.3737,0@@ &d^@ "$	@
 #+4.@ #+4.@ $D>@DW &*$S$s)^,-$ c]$ 
$s)		$2Xc] $ /3*.// / 	/
 &d^/ "$/ 
sEz	/f@D! 
#!FJ 
JX 
+3C= 
D>>	/	0> # # #t #J 4<TN	6 $$$1$;?$	$L5c 5Xd^ 5.55,4TN5	tDz4:%	&5<54 5( 15	;; #4.; n-	;F *.041,1, "$Z1, 4 012	1,
 #4.1, !1, n-1,f'c '4 '>KK K 	K
 K 
*	KZ D -@j -@^&z &d &P-A -Ads* s sj:$:;>:GJ:x 8L 6$F$F*-$F	$FN 6:JJ%-c]J	J4/J /8J;O /bC HZ,@ 4s x
/C &	(3 	(8D> 	( #	*	0 EI58>B	 
 58>A	  !	PTNP !P SM	P
 
Pd $  
:# 
:(4. 
:66),6	)	6pf f >Af 	.	!f P
 
8P 
>G$G$	x}hsm+	,G$R 1266	.	!6 36 cSVh B ;?""*23-"	"HN$ N4 N R$ R4 MR""3-"EI"	c"HE E Ec C HSM & FJ&*5=c]	& &*!%	33 c]3 #	3
 
!	"3jx} S	 0 IM-8@	#*
>(4. 
>T 
> +/"3-	!	"@ JN("3-(9A#(	$*+	,(X %),0!%	 SM  %SM  #	 
 
c49n	 D"%	.#B1Wf4v *.B%B% "B% tCH~&	B%
 !B%H3 8C= &Uc Ud U 48,0.3)-w*w* 4S#X/0w* c4i()	w*
 &d^w* !w* 
sE$*%%	&w*z 48,0.3+/U#U# U# 4S#X/0	U#
 c4i()U# &d^U# #4.U# 
tDz4	 U#v 48,0.3ZZ Z 4S#X/0	Z
 c4i()Z &d^Z@ 48,0.3  4S#X/0	
 c4i() &d^ 
(	)< 48,0.3)-{{ 4S#X/0{ c4i()	{
 &d^{ !{z
#':
EI#Y
	d
. FJY,4TNY$ BF
"
19#
&Py &PP
0
%>
.!r   r   )r
  r  enumrK  r  rH  r   r>  r  r  r3  r   	functoolsr   typingr   r   r   r	   r
   r   r   r   r   r   r   httpxr   r   pydanticr   typing_extensionsr   r   litellm.litellm_core_utils2litellm.litellm_core_utils.exception_mapping_utilsr   litellm._loggingr   litellm._uuidr   litellm.caching.cachingr   r   r   r   litellm.constantsr   "litellm.integrations.custom_loggerr   #litellm.litellm_core_utils.asyncifyr   r[  r   r   ,litellm.litellm_core_utils.coroutine_checkerr    .litellm.litellm_core_utils.credential_accessorr!   %litellm.litellm_core_utils.dd_tracingr"   *litellm.litellm_core_utils.litellm_loggingr#   r  0litellm.litellm_core_utils.sensitive_data_maskerr$   &litellm.router_strategy.budget_limiterr%   "litellm.router_strategy.least_busyr&   #litellm.router_strategy.lowest_costr'   &litellm.router_strategy.lowest_latencyr(   &litellm.router_strategy.lowest_tpm_rpmr)   )litellm.router_strategy.lowest_tpm_rpm_v2r*   &litellm.router_strategy.simple_shuffler+   )litellm.router_strategy.tag_based_routingr,   /litellm.router_utils.add_retry_fallback_headersr-   r.    litellm.router_utils.batch_utilsr/   r0   r1   /litellm.router_utils.client_initalization_utilsr2   2litellm.router_utils.clientside_credential_handlerr3   r4   #litellm.router_utils.cooldown_cacher5   &litellm.router_utils.cooldown_handlersr6   r7   r8   r9   r:   ,litellm.router_utils.fallback_event_handlersr;   r<   r=   *litellm.router_utils.get_retry_from_policyr>   r  !litellm.router_utils.handle_errorr?   r@   Dlitellm.router_utils.pre_call_checks.prompt_caching_deployment_checkrA   Clitellm.router_utils.pre_call_checks.responses_api_deployment_checkrB   >litellm.router_utils.router_callbacks.track_deployment_metricsrC   rD   litellm.schedulerrE   rF   litellm.types.llms.openairG   rH   rI   rJ   litellm.types.routerrK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   litellm.types.servicesr`   litellm.types.utilsra   rb   rc   rd   r  re   rf   rg   r  rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   &router_utils.pattern_match_deploymentsrs   opentelemetry.tracert   _Spanrj  ru   rv   Enumrx   r   r~   r   r   <module>r     s             #         &  ! 9 " 2   9 ; B K M 8 P P G F H N J P A M 
 R >  
 2      . 0 
 * 9 
   G1
 DDJ $)) gw gwr   