
    i*                       d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZ d dlZd dlZd dlZd dlmZ d dl m!Z! d dl"m#Z# d dl$Z$d dl%Z$d dl&Z$d dl$m'Z' d d	l(m)Z) d d
l*m+Z+ d dl,m-Z-m.Z.m/Z/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8m9Z9 d dl:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZB d dlCmDZD d dlEmFZF d dlGmHZH d dlImJZJ d dlKmLZL d dlMmNZN d dlOmPZP d dlQmRZR d dlSmTZT d dlUmVZV d dlWmXZXmYZY d dlZm[Z[m\Z\m]Z] d d l^m_Z_ d d!l`maZambZb d d"lcmdZdmeZe d d#lfmgZg d d$lhmiZimjZjmkZkmlZlmmZm d d%lnmoZompZpmqZq d d&lrmsZt d d'lumvZvmwZw d d(lxmyZy d d)lzm{Z{ d d*l|m}Z} d d+l~mZmZ d d,lmZmZ d d-lmZmZmZmZ d d.lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d d/lmZ d d0lmZmZmZ d d1lmZ d d1lmZ d d2lmZmZmZ d d3lmZmZmZmZmZmZmZmZmZmZ d4d5lmZ erd d6lmZ d d7lmZmZ d d8lmZ eeef   ZneZeZeZeZ G d9 d:ej                        Z G d; d<      Zy)=    Ndefaultdict)	lru_cache)TYPE_CHECKINGAnyAsyncGeneratorCallableDict	GeneratorListLiteralOptionalTupleUnioncast)AsyncOpenAI)	BaseModel)overload)get_secret_str)verbose_router_logger)uuid)	DualCacheInMemoryCache
RedisCacheRedisClusterCache)DEFAULT_MAX_LRU_CACHE_SIZE)CustomLogger)run_async_function)!_get_parent_otel_span_from_kwargs&get_metadata_variable_name_from_kwargs)coroutine_checker)CredentialAccessor)tracer)Logging)SensitiveDataMasker)JSONProviderRegistry)RouterBudgetLimiting)LeastBusyLoggingHandler)LowestCostLoggingHandler)LowestLatencyLoggingHandler)LowestTPMLoggingHandler)LowestTPMLoggingHandler_v2simple_shuffle)get_deployments_for_tag) add_fallback_headers_to_responseadd_retry_headers_to_response)"_get_router_metadata_variable_namereplace_model_in_jsonlshould_replace_model_in_jsonl)InitalizeCachedClient)get_dynamic_litellm_paramsis_clientside_credential)filter_team_based_modelsfilter_web_search_deployments)CooldownCache)DEFAULT_COOLDOWN_TIME_SECONDS_async_get_cooldown_deployments/_async_get_cooldown_deployments_with_debug_info_get_cooldown_deployments_set_cooldown_deployments)#_check_non_standard_fallback_formatget_fallback_model_grouprun_async_fallback)!get_num_retries_from_retry_policy)#async_raise_no_deployment_exceptionsend_llm_exception_alert)DeploymentAffinityCheck)ModelRateLimitingCheck)PromptCachingDeploymentCheck)0increment_deployment_failures_for_current_minute1increment_deployment_successes_for_current_minute)FlowItem	Scheduler)AllMessageValues	FileTypesOpenAIFileObjectOpenAIFilesPurpose)#CONFIGURABLE_CLIENTSIDE_AUTH_PARAMSVALID_LITELLM_ENVIRONMENTSAlertingConfigAllowedFailsPolicyAssistantsTypedDictCredentialLiteLLMParamsCustomRoutingStrategyBase
DeploymentDeploymentTypedDictGuardrailTypedDictLiteLLM_ParamsMockRouterTestingParamsModelGroupInfoOptionalPreCallChecksRetryPolicyRouterCacheEnumRouterGeneralSettingsRouterModelGroupAliasItemRouterRateLimitErrorRouterRateLimitErrorBasicRoutingStrategySearchToolTypedDict)ServiceTypes)CustomPricingLiteLLMParamsGenericBudgetConfigTypeLiteLLMBatch)	ModelInfo)ModelResponseStreamStandardLoggingPayloadUsage)
CustomStreamWrapperEmbeddingResponseModelResponseRulesfunction_setupget_llm_provider!get_non_default_completion_params
get_secretget_utc_datetimeis_region_allowed   )PatternMatchRouter)Span)
AutoRouterPreRoutingHookResponseComplexityRouterc                       e Zd ZdZy)RoutingArgs<   N)__name__
__module____qualname__ttl     _/Users/manta/Documents/Projects/TheRoad-I1/.venv/lib/python3.12/site-packages/litellm/router.pyr   r      s    
Cr   r   c            a          e Zd ZU  e       Zeed<   dZee   ed<   dZ	e
ed<   dZdZee   ed<   dZee   ed<   dZeeeeeef         ed	<   ddddddddddi dddddddddddd
dg g g i dddddi ddddddi dd e       dddf/deeee   eeeef      f      dee   deee      deee      dee   dee   dee
   dee   dee
   dee   dedeee      de
dee   dee
   dee
   dee
   dee   dee   d ee   d!ee
   d"ed#e d$   d%eee      d&ed'ed(ed)eeeeee!f   f      d*ed+ed,ed-e
d.eee"ef      d/eee"f   d0ee
   d1ee#   d2ee   d3ee   d4e d5   d6ee$   d7ed8ee%   d9ee&   d:ee   d;e
d<eeeee   f      d=ed>df`d?Z'e(d>ee   fd@       Z)dA Z*dB Z+e(dCeeef   d>ee,e-f   fdD       Z.dEe,fdFZ/d4ee0ef   d7efdGZ1dH Z2dI Z3dJ Z4dK Z5dL Z6dM Z7dN Z8dO Z9dP Z:dQ Z;dR Z<dS Z=dTee   fdUZ>d6ee$   fdVZ?dWefdXZ@dYedZeeeef      d>eeAeBf   fd[ZCdYedZeeeef      d>eeAeBf   fd\ZDd>efd]ZEd^edZee   fd_ZFeGdYedZeeH   d`e d   d>eBfda       ZIeG	 dNdYedZeeH   d`e d   d>eAfdb       ZIeG	 dNdYedZeeH   d`ee d   e d   f   d>eeBeAf   fdc       ZI	 dNdYedZeeH   d`efddZIe(deeJdfeeK   d>dfdg       ZLdheBdZeeeef      died>eBfdjZMdheBdZeeeef      died>eBfdkZNd^edZee   fdlZOdYedZeeeef      d>eeAeBf   fdmZP	 dOdYednedoee   d>dfdpZQdqeRdWed>dfdrZS	 dOdnedoee   d>dfdsZT	 dPdWednedtee   d>eUfduZVe(dWedned>dfdv       ZW	 dPdWednedtee   d>dfdwZXdWednefdxZYdnedyed>eeee
f      fdzZZdnedyed>eeee
f      fd{Z[dnedyed>eeee
f      fd|Z\d}ee   dZeeeeef      eeeeef         f   fd~Z]dYedZeeeH      fdZ^eGdYedZeeeef      d`e d   d>eBfd       Z_eG	 dNdYedZeeeef      d`e d   d>eAfd       Z_	 dNdYedZeeeef      d`efdZ_eG	 dNdYedZeeH   de
d`e d   d>eAf
d       Z`eGdYedZeeH   de
d`e d   d>eBf
d       Z`	 dNdYedZeeH   de
fdZ`dYede
dedeaedf   dneeef   f
dZbdYed>efdZcdYedZeeH   dneeef   fdZddedYefdZededYefdZfdedYefdZgdedYefdZhdeidYefdZjdeidYefdZkdYededefdZldYefdZmdYefdZn	 	 	 dQdYededee   dee   dee   f
dZo	 	 	 dQdYededee   dee   dee   f
dZpdYedefdZq	 	 	 dQdedYedee   dee   dee   f
dZrdedYefdZsdefdZtdYedefdZudedefdZvdYedefdZwded>dfdZxdYedefdZydneeef   dYeded>eeef   fdZzdYedefdZ{dYedefdZ|	 dNdYedeeef   dee   d>e}fdZ~deeef   dYefdZ	 dRdYedeeef   dee   d>e}fdZdeeef   dYefdZdYed>efdZdYed>efdZdYeedf   fdZd ZdYed>efdZdYed>efdZ	 dPdYee   d>efdZdYed>efdZdYed>efdZdYefdZ	 dPdedee   fdZ	 dSdede d   fdZ	 dPdedee   fdÄZ	 dPdedee   fdĄZdefdńZ	 dPdedee   fdƄZ	 	 dTdedee   dee   fdȄZdeRdee   d&ee   d'ee   d(ee   dee   dednefd̄Z ej0                         d̈́        Z	 	 	 	 dUdnedee   d&ee   d'ee   d(ee   f
d΄Z ej0                         dτ        ZdefdЄZ	 dPdnedee   fdфZ	 	 	 	 	 dVdeRdee   dee   d'ee   d(ee   dee   fdքZdׄ Z	 dPd&eeeee   f      dee   d>eee      fd؄Zd>ee   fdلZ	 	 dTdeRde
de
dee   dee   d>ee
ef   fdۄZd܄ Zd>ee   fd݄Zd>efdބZdee   fdZdned>e d   fdZdnedeRd>efdZdedee   d>e
fdZd>efdZdYedeAdned>efdZdYedee   fdZdYedee   d>eaee   ee   f   fdZdWefdZ	 dPdWedee   dee   fdZ	 	 dTdYedee   dZeeeH      dee   dee   dee   fdZdedefdZdedededed>eeU   f
dZded>efdZdWeUfdZded>efdZdWeUfdZdWeUd>efdZdefdZdWeUd>eUfdZdWeUdedYefdZdWeUd>eeU   fd Zdede
d>dfdZ	 dPdYedee   d>dfdZdWeUd>eeU   fdZded>eeU   fdZÐded>eeU   fdZĐded>ee   fd	ZŐd
ed>eeU   fdZƐded>eeeef      fdZeG	 dPdWeedf   deddd>efd       ZeGdWddeded>efd       Z	 dPdWeeedf      dedee   d>efdZɐded>ee   fdZʐded>ee   fdZːdeded>ee   fdZdeded>ee   fdZded>ee   fdZded>eaee
   ee
   f   fdZ eeӐ      ded>ee   fd       Zded>eee
f   fdZ	 dPdedee   d>efdZded>dfdZdefdZ	 dWdee   ded>ee   fd Zِd!ed>efd"Zڐdee   d>ee   fd#Zېd$ed%ed>ee   fd&Z	 dPdedYed%ee   d>efd'Z	 	 dTded(ee   d%ee   d>ee   fd)ZސdPd%ee   d>ee   fd*Z	 dPdWed%ee   d>ee   fd+Zd,ee   d>efd-Z	 dPdee   d>ee   fd.Z	 dTdee   d%ee   d>eee      fd/ZdXd0ZdXd1Z	 	 	 dYdee   d2ee   d%ee   d>eeee   f   fd3Zd2ed>efd4Zd5 Zd6 ZdPd7Z	 dPdYededZeeeef      dee   fd8ZdYed>ee   fd9ZdYed>efd:Z	 	 	 	 dZdYedZeeeeef         deeeef      d;ee   dee   d>eaeeeef   f   fd<Z	 	 	 	 dZdYededZeeeeef         deeeef      d;ee   dee   d>eee   ef   fd=Z	 	 	 d[dYededZeeeeef         deeeef      d;ee   f
d>Z	 	 	 d[dYededZeeeeef         deeeef      d;ee   f
d?Z	 	 	 d[dYededZeeeeef         deeeef      d;ee   d>ee   fd@Z	 	 	 	 dZdYedZeeeeef         deeeef      d;ee   dee   f
dAZ	 	 	 	 dZdYedZeeeeef         deeeef      d;ee   dee   f
dBZdee   dCee   d>ee   fdDZdee   d>ee   fdEZ	 dPdee   fdFZ	 dPdqeRdee   fdGZdqeRfdHZdI ZdJefdKZdL ZdM Zy(\  Routermodel_namesFcache_responsesi  default_cache_time_secondsNleastbusy_loggerlowesttpm_loggeroptional_callbacksINFOTr   simple-shuffle
model_listassistants_configsearch_toolsguardrail_list	redis_url
redis_host
redis_portredis_passwordredis_dbcache_kwargscaching_groups
client_ttlpolling_intervaldefault_prioritynum_retriesmax_fallbackstimeoutstream_timeoutdefault_litellm_paramsdefault_max_parallel_requestsset_verbosedebug_level)DEBUGr   default_fallbacks	fallbackscontext_window_fallbackscontent_policy_fallbacksmodel_group_aliasenable_pre_call_checksenable_tag_filteringtag_filtering_match_anyretry_afterretry_policymodel_group_retry_policyallowed_failsallowed_fails_policycooldown_timedisable_cooldownsrouting_strategy)r   
least-busyusage-based-routinglatency-based-routingcost-based-routingusage-based-routing-v2optional_pre_call_checksrouting_strategy_argsprovider_budget_configalerting_configrouter_general_settingsdeployment_affinity_ttl_secondsmodel_group_affinity_configignore_invalid_deploymentsreturnc0                    || _         |/| _        || _        || _        || _        || _        ddlm}0  |0       | _        dt        _
        | j                   du rQ|dk(  r$t        j                  t        j                         n(|dk(  r#t        j                  t        j                         |,xs
 t!               | _        || _        |xs g | _        |xs g | _        g | _        i | _        d}1d}2i }3|| _        ||p|nd}1|||3d	<   |||3d
<   |t1        |      |3d<   |||3d<   |	#t        j2                  d       t1        |	      |3d<   |3j5                  |       | j7                  |3      }2|
r7t        j8                   t        j:                  d%d|1i|3t        _        |
| _        t?        |2tA                     | _        tC        ||2      | _"        || _#        d| _$        || _%        g | _&        tO               | _(        i | _)        i | _*        i | _+        |xs i | _,        i | _-        i | _.        |G| j_                  |       | j`                  | _1        |D ]  }4d|4d   v sd| j,                  |4d   d   <   ! ng | _0        d| _2        |#|#| _3        nt        jf                  | _3        |%xs th        | _5        tm        | j8                  | jj                        | _7        |&| _8        tA               | _9        ||| _:        n;t        jt                  t        jt                  | _:        ntv        jx                  | _:        ||| _=        n;t        jz                  t        jz                  | _=        nt        j|                  | _=        |xs t        j~                  | _@        || _A        | | _B        |'| _C        |xs t        j                  }5| j                  |5       |5| _D        |t        j                  H|xs t        j                  }5| j                  | j                  j                  d|5i       n
d|5ig| _D        |xs t        j                  | _H        |xs t        j                  }6| j                  |6       |6| _I        t        t              | _L        t        t              | _M        t        t              | _N        g | _O        |xs i }t        j                  ||       | _Q        || _R        | j                  j                  d|       | j                  j                  dd       | j                  j                  di       j5                  d|i       i | _T        	 | j                  |'|)       d| _V        t        t        j                  t              r*t        j                  j                  | j                         n)t        j                  j                  | j                         t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _]        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _`        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _c        |)| _f        |*| _g        |-| _h        d| _i        t        j                  || j                        r|(|(j                  d       ndg}(d| _l        |!t        |!t              rt        d%i |!| _l        nt        |!t              r|!| _l        | j                  >t        j                  dj                  | j                  j                  d                    |"| _r        |.| _s        d| _t        |$t        |$t              rt        d%i |$| _t        nt        |$t              r|$| _t        | j                  >t        j                  d!j                  | j                  j                  d                    |+| _v        |(| j                  |(       | j                  rt        d" | j                  xs g D              s|| j                  g | _y        t        | j8                  | j                  d#d#d#| j                  $      }7| j                  j                  |7       t        j                  j                  |7       | j                  | j                          | j                          | j                          | j                          y)&a  
        Initialize the Router class with the given parameters for caching, reliability, and routing strategy.

        Args:
            model_list (Optional[list]): List of models to be used. Defaults to None.
            redis_url (Optional[str]): URL of the Redis server. Defaults to None.
            redis_host (Optional[str]): Hostname of the Redis server. Defaults to None.
            redis_port (Optional[int]): Port of the Redis server. Defaults to None.
            redis_password (Optional[str]): Password of the Redis server. Defaults to None.
            cache_responses (Optional[bool]): Flag to enable caching of responses. Defaults to False.
            cache_kwargs (dict): Additional kwargs to pass to RedisCache. Defaults to {}.
            caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None.
            client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600.
            polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms.
            default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None.
            num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2.
            timeout (Optional[float]): Timeout for requests. Defaults to None.
            default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}.
            set_verbose (bool): Flag to set verbose mode. Defaults to False.
            debug_level (Literal["DEBUG", "INFO"]): Debug level for logging. Defaults to "INFO".
            fallbacks (List): List of fallback options. Defaults to [].
            context_window_fallbacks (List): List of context window fallback options. Defaults to [].
            enable_pre_call_checks (boolean): Filter out deployments which are outside context window limits for a given prompt
            model_group_alias (Optional[dict]): Alias for model groups. Defaults to {}.
            retry_after (int): Minimum time to wait before retrying a failed request. Defaults to 0.
            allowed_fails (Optional[int]): Number of allowed fails before adding to cooldown. Defaults to None.
            cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1.
            routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle".
            routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}.
            alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None.
            provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None.
            deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600.
            ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
        Returns:
            Router: An instance of the litellm.Router class.

        Example Usage:
        ```python
        from litellm import Router
        model_list = [
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-1>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-2>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "openai-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "gpt-3.5-turbo",
                "api_key": <your-api-key>,
            },
        ]

        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
        ```
        r   )ServiceLoggingTr   r   localNredisurlhostportpasswordzwDeprecated 'redis_db' argument used. Please remove 'redis_db' from your config/database and use 'cache_kwargs' instead.dbtype)redis_cachein_memory_cache)r   r   modellitellm_params)cachedefault_cooldown_time)fallback_param*)params
router_objr   max_retriesmetadatar   r   r   )r   r   router_budget_limitingz+[32mRouter Custom Retry Policy Set:
{}[0mexclude_nonez3[32mRouter Custom Allowed Fails Policy Set:
{}[0mc              3   <   K   | ]  }t        |t                y wN)
isinstancerF   ).0cbs     r   	<genexpr>z"Router.__init__.<locals>.<genexpr>  s       8
4 r234   Fr   ttl_secondsenable_user_key_affinityenable_responses_api_affinityenable_session_id_affinityr   r   )r   r   r   r   r   r   litellm._service_loggerr   service_logger_objlitellmsuppress_debug_infor   setLevelloggingr   r   ra   r   r   r   r   deployment_namesdeployment_latency_mapr   strwarningupdate_create_redis_cacher   Cacher   r   r   rL   	schedulerr   default_deploymentr   provider_default_deployment_idsrz   pattern_routerteam_pattern_routersauto_routerscomplexity_routersr    model_id_to_deployment_index_map model_name_to_deployment_indicesset_model_listr   healthy_deployments_access_groups_cacher   r;   r   r:   cooldown_cacher   failed_callsr   openaiDEFAULT_MAX_RETRIESr   ROUTER_MAX_FALLBACKSrequest_timeoutr   r   r   r   r   validate_fallbacksr   appendr   r   r   inttotal_calls
fail_callssuccess_callsprevious_modelsChatchatr   
setdefaultdeployment_statsrouting_strategy_initaccess_groupsr   _async_success_callbacklistlogging_callback_manager"add_litellm_async_success_callbackdeployment_callback_on_successsuccess_callbackadd_litellm_success_callback#sync_deployment_callback_on_success_async_failure_callback"add_litellm_async_failure_callback$async_deployment_callback_on_failurefailure_callbackadd_litellm_failure_callbackdeployment_callback_on_failurer   r   r   router_budget_loggerr'   !should_init_router_budget_limiterr   dictr_   infoformat
model_dumpr   r   r   rT   r   add_optional_pre_call_checksanyr   rF   add_litellm_callback_initialize_alertinginitialize_assistants_endpointinitialize_router_endpointsapply_default_settings)8selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
cache_typer   cache_configm
_fallbacks_content_policy_fallbacksaffinity_callbacks8                                                           r   __init__zRouter.__init__   s   z '*D'&&<#$8!'>$:2@2B&*#t#f$%..w||<'%..w}}=#>'<'> 	$ "3(.B,2 	 ')#  	 ')$ Z%;
@V J$&/U#%'1V$%'*:V$)+9Z(#%-- N &)]T" -22<@K}}$ ' N: N N#2D #]_


 #-;
 !1"&-J*:<,02  	! 68AC # 	
 AC- GI-!
+-1__D$a 011PQD//2B0CG0LM  
  O EI!$!.D!(!6!6D*K.K+**D4F4F
 "3O 	 "*D  ,&22D%99D$!.D"".!(!6!6D!(!=!=D9'"9"9,& 0 3'"3"3
z:#(G,E,E,Q*Gg.G.GJ~~)%%sJ&78#&
"3!4 %H(H(H 	%
 %H(H(H 	" 	/HI(A%(3)
 (3(
 +6+
  	
 "8!=2LL(>4P	 '=###..y'B##..}a@##..z2>EE~.	
 ')	 	""-"7 	# 	
 "g55t<,,OO33 ,,OO33 g..5,,II88 )-(P(P'QG$g55t<,,OO99
 99/G+ g..5,,II33 )-(K(K'LG$%:"&<#/N,DH!AA!$:U:U
 (3(//0HI,D+E(37#,-$/$?,$?!L+6$0!  ,%**HOO))44$4G % 	%
 ( 	( CG!+.5,>,VAU,V)02DE,@)((4%**PWW11<<$<O :I#/--.FG
 ++C 8
..4"48
 5
 &&.*,' 7jj @@)..3+0,0,L,L! ##**+<=,,AABST+%%'++-((*##%r   c                      t        j                  t        j                        } | j                  | j
                  z   }d|v r|j                  d       |S )zS
        Returns a list of valid arguments for the Router.__init__ method.
        r6  )inspectgetfullargspecr   r=  args
kwonlyargsremove)arg_spec
valid_argss     r   get_valid_argszRouter.get_valid_args  sH    
 ))&//:]]X%8%88
Zf%r   c                 *    g }| j                  |       y)z;
        Apply the default settings to the router.
        N)r/  )r6  default_pre_call_checkss     r   r5  zRouter.apply_default_settings  s    
 :<))*ABr   c                 $   t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j
                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        | j                  B| j                  D ]2  }t         j                  j                  t         j                  |d       4 yy)z
        Pseudo-destructor to be invoked to clean up global data structures when router is no longer used.
        For now, unhook router's callbacks from all lists
        NF)require_self)r   r  #remove_callback_from_list_by_objectr  r   r#  r&  input_callbackservice_callback	callbacksr   )r6  callbacks     r   discardzRouter.discard  s3   
 	((LL++T	
 	((LL$$d	
 	((LL++T	
 	((LL$$d	
 	((LL""D	
 	((LL$$d	
 	((LLt	

 "". 3300TT%%xe U  4 /r   r8  c                     | j                  d      }|s2t        d      }|%t        |t              rt	        j
                  |      }|rt        di i | d|iS t        di | S )za
        Initializes either a RedisCache or RedisClusterCache based on the cache_config.
        startup_nodesREDIS_CLUSTER_NODESr   )getrv   r   r   jsonloadsr   r   )r8  rR  _env_cluster_nodess      r   r   zRouter._create_redis_cache  so     %((9!+,A!B!-*=OQT2U $

+= >$X'W,'W'WXX---r   r   c                 T    | j                   j                  || j                   _        yy)z
        Update the redis cache for the router, if none set.

        Allows proxy user to just do
        ```yaml
        litellm_settings:
            cache: true
        ```
        and caching to just work.
        N)r   r   )r6  r   s     r   _update_redis_cachezRouter._update_redis_cache  s%     ::!!)%*DJJ" *r   c                    t        j                  d|        dgt        D cg c]  }|j                   c}z   }|<t	        |t
              xr ||v }t	        |t              }|s|st        d| d| d      |t        j                  j                  k(  s|t        j                  k(  rt        | j                        | _
        t	        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _        t	        t        j                  t              r*t        j                   j#                  | j                         y y |t        j$                  j                  k(  s|t        j$                  k(  ret'        | j                  |      | _        t	        t        j                  t              r*t        j                   j#                  | j(                         y y |t        j*                  j                  k(  s|t        j*                  k(  ret-        | j                  |      | _        t	        t        j                  t              r*t        j                   j#                  | j.                         y y |t        j0                  j                  k(  s|t        j0                  k(  ret3        | j                  |      | _        t	        t        j                  t              r*t        j                   j#                  | j4                         y y |t        j6                  j                  k(  s|t        j6                  k(  ret9        | j                  i       | _        t	        t        j                  t              r*t        j                   j#                  | j:                         y y y c c}w )NzRouting strategy: r   zInvalid routing_strategy: 'z'. Valid options: z. Check 'router_settings.routing_strategy' in your config.yaml or the 'routing_strategy' parameter if using the Router SDK directly.)router_cache)r[  routing_args)r   r,  re   valuer   r   
ValueError
LEAST_BUSYr(   r   r   r   rL  r  r  rN  r  r1  USAGE_BASED_ROUTINGr+   r   USAGE_BASED_ROUTING_V2r,   lowesttpm_logger_v2LATENCY_BASEDr*   lowestlatency_logger
COST_BASEDr)   lowestcost_logger)r6  r   r   svalid_strategy_stringsis_valid_stringis_valid_enums          r   r  zRouter.routing_strategy_init  s     	""%78H7I#JK
 #3!36X1qww6X!X'+S1 ?$(>>  ''7IM"= 12B1C D&&<%= >\]   : : @ @@?#=#==$;$TD!'00$7&&--d.C.CD*.*?*?)@&'++T200EEdF[F[\ 3  C C I II?#F#FF$;!ZZ2%D! '++T200EEdF[F[\ 3  F F L LL?#I#II'A!ZZ2(D$ '++T200EEdF^F^_ 3  = = C CC?#@#@@(C!ZZ2)D% '++T200EEdF_F_` 3  : : @ @@?#=#==%=!ZZ&D" '++T200EEdF\F\] 3 K 7Ys   O c                 D   | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j
                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _	        y r   )
factory_functionr   acreate_assistantsadelete_assistantaget_assistantsacreate_threadaget_threada_add_messageaget_messagesarun_threadr6  s    r   r3  z%Router.initialize_assistants_endpoint\  s    "&"7"78R8R"S!%!6!6w7P7P!Q#44W5L5LM"33G4J4JK001D1DE!2273H3HI!2273H3HI001D1DEr   c                    | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j
                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _	        | j                  t        j                  d	      | _
        | j                  t        j                  d
      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                   d      | _        | j                  t        j"                  d      | _        | j                  t        j$                  d      | _        | j                  t        j&                  d      | _        | j                  t        j(                  d      | _        | j                  t        j*                  d      | _        | j                  t        j,                  d      | _        | j                  t        j.                  d      | _        | j                  t        j0                  d      | _        y)z+Helper to initialize core router endpoints.
moderation	call_typeanthropic_messagesagenerate_contentaadapter_generate_content
aresponsesafile_deleteafile_content	responsesaget_responsesacancel_responsesacompact_responsesadelete_responsesalist_input_items
_arealtime_aresponses_websocketacreate_fine_tuning_jobacancel_fine_tuning_jobalist_fine_tuning_jobsaretrieve_fine_tuning_jobalist_filesaimage_editallm_passthrough_routeN)rl  r   amoderationrz  aanthropic_messagesr{  r|  r}  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  
afile_listr  r  ru  s    r   _initialize_core_endpointsz!Router._initialize_core_endpointsg  sW   00< 1 
 $(#8#8&&2F $9 $
  #'"7"7&&2F #8 #
 "&!6!6%%1D "7 "
 *.)>)>--9T *? *
& //, 0 
 !11  N 2 
 "22!!_ 3 
 ..w/@/@K.X"33"".> 4 
 "&!6!6%%1D "7 "
 #'"7"7&&2F #8 #
 "&!6!6%%1D "7 "
 "&!6!6%%1D "7 "
 //, 0 
 &*%:%:))5L &; &
" (,'<'<++7P (= (
$ (,'<'<++7P (= (
$ '+&;&;**6N '< '
# *.)>)>--9T *? *
& //- 0 
  00= 1 
 '+&;&;**6N '< '
#r   c                 H   ddl m}m}m}m}m}m}m}m}m	}	m
}
m} | j                  |d      | _        | j                  |
d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |	d      | _        | j                  |d	      | _        | j                  |d
      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        y)z"Initialize vector store endpoints.r   )adeletealist	aretrieveasearchaupdatecreatedeleter  retrievesearchr   avector_store_searchrx  vector_store_searchvector_store_createavector_store_retrievevector_store_retrieveavector_store_listvector_store_listavector_store_updatevector_store_updateavector_store_deletevector_store_deleteN)litellm.vector_stores.mainr  r  r  r  r  r  r  r  r  r  r   rl  r  r  r  r  r  r  r  r  r  r  r  )r6  r  r  r  r  r  r  r  r  r  r  r   s               r   "_initialize_vector_store_endpointsz)Router._initialize_vector_store_endpoints  sv   	
 	
 	
 	
 %)$9$95 %: %
! $(#8#83 $9 $
  $(#8#83 $9 $
  '+&;&;!9 '< '
# &*%:%: 7 &; &
" #'"7"71 #8 #
 "&!6!6/ "7 "
 %)$9$95 %: %
! $(#8#83 $9 $
  %)$9$95 %: %
! $(#8#83 $9 $
 r   c                    ddl m} ddl m} ddl m} ddl m} ddl m} ddl m} ddl m} dd	l m} dd
l m	}	 ddl m
}
 ddl m} ddl m} | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |	d      | _        | j                  |d      | _        | j                  |
d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        y)z'Initialize vector store file endpoints.r   acreater  )r  )r  )aretrieve_content)r  r  r  )r  )r  )retrieve_content)r   avector_store_file_createrx  vector_store_file_createavector_store_file_listvector_store_file_listavector_store_file_retrievevector_store_file_retrieveavector_store_file_contentvector_store_file_contentavector_store_file_updatevector_store_file_updateavector_store_file_deletevector_store_file_deleteN)litellm.vector_store_files.mainr  r  r  r  r  r  r  r  r  r  r  r   rl  r  r  r  r  r  r  r  r  r  r  r  r  )r6  avector_store_file_create_fnavector_store_file_delete_fnavector_store_file_list_fnavector_store_file_retrieve_fnavector_store_file_content_fnavector_store_file_update_fnvector_store_file_create_fnvector_store_file_delete_fnvector_store_file_list_fnvector_store_file_retrieve_fnvector_store_file_content_fnvector_store_file_update_fns                r   '_initialize_vector_store_file_endpointsz.Router._initialize_vector_store_file_endpoints  s   	
	
 	X	
	
	
	
	
 	V	
	
	
 *.)>)>(4O *? *
& )-(=(='3M )> )
% (,'<'<&2K (= (
$ '+&;&;%1I '< '
# ,0+@+@*6S ,A ,
( +/*?*?)5Q +@ +
' +/*?*?)5Q +@ +
' *.)>)>(4O *? *
& *.)>)>(4O *? *
& )-(=(='3M )> )
% *.)>)>(4O *? *
& )-(=(='3M )> )
%r   c                     ddl m}m}m}m} | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        y)	z"Initialize Google GenAI endpoints.r   )r{  agenerate_content_streamgenerate_contentgenerate_content_streamr{  rx  r  r  r  N)litellm.google_genair{  r  r  r  rl  )r6  r{  r  r  r  s        r   "_initialize_google_genai_endpointsz)Router._initialize_google_genai_endpoints)  s    	
 	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 )-(=(=$0J )> )
% (,'<'<#/H (= (
$r   c                     ddl m}m} | j                  |d      | _        | j                  |d      | _        ddlm}m} | j                  |d      | _        | j                  |d      | _        y	)
z$Initialize OCR and search endpoints.r   )aocrocrr  rx  r  r  r  r  r  N)litellm.ocrr  r  rl  litellm.searchr  r  )r6  r  r  r  r  s        r    _initialize_ocr_search_endpointsz'Router._initialize_ocr_search_endpoints?  sd    )))$&)A	(((>2,,W	,J++Fh+Gr   c                    ddl m}m}m}m}m}m}m}m}m	}	m
}
m}m}m}m}m}m}m}m} | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |	d      | _	        | j'                  |d	      | _        | j'                  |d
      | _        | j'                  |
d      | _
        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        y)zInitialize video endpoints.r   )avideo_contentavideo_create_characteravideo_editavideo_extensionavideo_generationavideo_get_characteravideo_listavideo_remixavideo_statusvideo_contentvideo_create_character
video_editvideo_extensionvideo_generationvideo_get_character
video_listvideo_remixvideo_statusr  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  N)litellm.videosr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rl  )r6  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s                      r   _initialize_video_endpointsz"Router._initialize_video_endpointsK  s-   	
 	
 	
 	
 	
* "&!6!6)< "7 "
 !% 5 5(: !6 !
  000V//
l/S!22_ 3 
 !11N 2 
 #33&6 4 
 "22_ 3 
 !11N 2 
  000V'+'<'<#/H (= (
$ '+&;&;".F '< '
# %)$9$9 ,B %: %
! $(#8#8+@ $9 $
   000V//
l/S $ 5 5(: !6 !
  $44'8  5  
r   c           	      $   ddl m}m}m}m}m}m}m}m} ddl	m
}	 | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d	      | _        | j                  |d
      | _        | j                  |d      | _        | j                  |d      | _        |	j                         D ]#  \  }
}t        | |
| j                  ||
             % y)zInitialize container endpoints.r   )acreate_containeradelete_containeralist_containersaretrieve_containercreate_containerdelete_containerlist_containersretrieve_container)_generated_endpointsr  rx  r  r  r  r  r  r  r  N)litellm.containersr  r  r  r  r  r  r  r  #litellm.containers.endpoint_factoryr  rl  itemssetattr)r6  r  r  r  r  r  r  r  r  container_file_endpointsnamefuncs               r   _initialize_container_endpointsz&Router._initialize_container_endpoints  sK   		
 		
 		
	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 !% 5 5(: !6 !
  $44'8  5  
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
 "&!6!6)< "7 "
 !% 5 5(: !6 !

 388:JD$D$ 5 5dd 5 KL ;r   c                 4   | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j
                  d      | _        y)z*Initialize Anthropic Skills API endpoints.acreate_skillrx  alist_skills
aget_skilladelete_skillN)rl  r   r  r  r  r  ru  s    r   _initialize_skills_endpointsz#Router._initialize_skills_endpoints  s    !22!!_ 3 
 !11  N 2 
 //, 0 
 "22!!_ 3 
r   c                    ddl m} ddl m} ddl m} ddl m} ddl m} ddl m} ddl m} dd	l m} | j                  |d
      | _
        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        y)z-Initialize Google Interactions API endpoints.r   )acancelr  r  )aget)cancelr  r  rT  acreate_interactionrx  create_interactionaget_interactionget_interactionadelete_interactiondelete_interactionacancel_interactioncancel_interactionN)litellm.interactionsr  r  r  r  r	  r  r  rT  rl  r  r  r  r  r  r  r  r  )	r6  r  r  r  r  r  r  r  r  s	            r   "_initialize_interactions_endpointsz)Router._initialize_interactions_endpoints  s   GGGAEEE?#'#8#8+@ $9 $
  #'"7"7*> #8 #
 !% 5 5(: !6 !
  $44'8  5  
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
r   c                 $   | j                          | j                          | j                          | j                          | j	                          | j                          | j                          | j                          | j                          y)zvHelper to initialize specialized router endpoints (vector store, OCR, search, video, container, skills, interactions).N)	r  r  r  r  )_override_vector_store_methods_for_routerr  r  r  r  ru  s    r   !_initialize_specialized_endpointsz(Router._initialize_specialized_endpoints  sn    //1446//1--/668((*,,.))+//1r   c                 D    | j                          | j                          y r   )r  r  ru  s    r   r4  z"Router.initialize_router_endpoints  s    '')..0r   r   c           	          |y|D ]J  }t        |t              st        d| d      t        |      dk7  s1t        d| dt        |       d       y)z3
        Validate the fallbacks parameter.
        NzItem 'z' is not a dictionary.ry   zDictionary 'z%' must have exactly one key, but has z keys.)r   r+  r^  len)r6  r   fallback_dicts      r   r  zRouter.validate_fallbacks   sl     !+MmT2 6-8N!OPP=!Q& "=/1VWZ[hWiVjjpq 	 ,r   c                    |y d|v }d|v }d|v }|s|s|r| j                   g | _         d }| j                   D ]  }t        |t              s|} n |n|j                  xs ||_        |j                  xs ||_        |j
                  xs ||_        | j                  |_        | j                  r{| j                  |_        nit        | j                  | j                  |||| j                        }| j                   j                  |       t        j                  j                  |       d|v rzddlm | j                   g | _         t!        fd| j                   D              }|sA        }	| j                   j                  |	       t        j                  j                  |	       |D ]  }
d }|
d	v r
|
d
k(  rt#        | j                        }nM|
dk(  r-t%        | j                  | j&                  | j(                        }n|
dk(  rt+        | j                        }|v| j                   g | _         | j                   j                  |       t        j                  j                  |        y )Ndeployment_affinityresponses_api_deployment_checksession_affinityr   encrypted_content_affinityr   )EncryptedContentAffinityCheckc              3   6   K   | ]  }t        |        y wr   )r   )r   r   r!  s     r   r   z6Router.add_optional_pre_call_checks.<locals>.<genexpr>U  s!      %1B 2<=1s   )r  r  r  r   prompt_caching)r   r   )
dual_cacher   r   enforce_model_rate_limits)r$  )r   r   rF   r   r   r   r   r   r   r   r  r   r  r1  Elitellm.router_utils.pre_call_checks.encrypted_content_affinity_checkr!  r0  rH   r'   r   r   rG   )r6  r   r   r   r   existing_affinity_callbackr   r<  already_registeredec_callbackpre_call_check	_callbackr!  s               @r   r/  z#Router.add_optional_pre_call_checks  s    $+
 $9<T#T ,0HH 	& &8;S%S"$,)&&.*,'LP&--b"9:13. .
 *5.GG 0/ +C
 /LL 54 +H
 /II 21 +E
 88 +6 3388 /J %<** $ D D-E2O/I040P0P%! ''../@A00EEFWX
 (+CC &&.*,'!$ %11% " &;=''..{;00EEkR
 7N04I "  !118tzzJ	#;;0#zz+/+F+F#	
  #>>2djjI	 &&.*,'##**95,,AA)L5 7r   
deploymentc                 .   	 t        j                  |      }|d   }t        j                  r#t	        dd      }|j                  |      |d<   |S d|v r|d   dd dz   |d<   |S # t        $ r(}t        j                  dt        |              |d}~ww xY w)	z
        returns a copy of the deployment with the api key masked

        Only returns 2 characters of the api key and masks the rest with * (10 *).
        r      r   )visible_prefixvisible_suffixapi_keyNz
**********z+Error occurred while printing deployment - )
copydeepcopyr   redact_user_api_key_infor%   	mask_dict	Exceptionr   debugr   )r6  r,  _deployment_copyr   maskeres         r   print_deploymentzRouter.print_deployment}  s    	#}}Z8#34D#EN//,AaP5;5E5En5U !12 $# n,,:9,Ebq,IH,Ty)## 	!''=c!fXF G		s   AA# A# #	B,#BBr   messagesc                     	 t        j                  d| d       ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di |}|S # t
        $ r}|d}~ww xY w)	z
        Example usage:
        response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}]
        zrouter.completion(model=z,..)r   r<  original_functionr   kwargsNr   )r   r7  _completion_update_kwargs_before_fallbacksfunction_with_fallbacksr6  )r6  r   r<  r@  responser:  s         r   
completionzRouter.completion  s    
	!''*B5'(NO#F7O!)F:*.*:*:F&'00uV0L3t33=f=HO 	G	s   AA 	A+$A&&A+c           	      4   d }d }	 |j                         }||d<   | j                  |||j                  dd       |      }|d   j                         }|j                  dd       }|5t        j                  | j
                  ||f|d      }	|	j                          | j                  ||       |j                  dd        |d   }| j                  ||      }
|j                  d	d       }||
||
j                  k7  rd }n|
}| j                  |      s| j                  |
       i ||| j                  |d|}t        j                  di |}t!        j"                  d| d       t%        |t&              r.| j)                  |||      }|rt        j*                  d|d      t%        |t,              r| j/                  |||      S |S # t0        $ r@}t!        j"                  d| dt3        |       d       || j5                  ||       |d }~ww xY w)Nr   specific_deploymentr   r<  rG  request_kwargsr   silent_modelT)targetrA  r@  daemonr,  r@  r1  r,  r<  cachingclientzlitellm.completion(model=)[32m 200 OK[0mr   rD  r@  Response output was blocked. messager   llm_providermodel_responser<  initial_kwargs)[31m Exception [0mr   )r2  get_available_deploymentpop	threadingThread_silent_experiment_completionstart_update_kwargs_with_deployment_get_clientrT  r1  has_model_id routing_strategy_pre_call_checksr   r   rE  r   r,  r   rq   "_should_raise_content_policy_errorContentPolicyViolationErrorro   _completion_streaming_iteratorr6  r   (_set_deployment_num_retries_on_exception)r6  r   r<  r@  
model_namer,  #input_kwargs_for_streaming_fallbackr   rJ  threadpotential_model_clientdynamic_api_keymodel_clientinput_kwargsrD  _should_raiser:  s                    r   rA  zRouter._completion  s    

^	 39++-/;@/8 66!$*JJ/Dd$K%	 7 J ((89>>@N)--ndCL'
 #))==&1!	 //:f/UJJ~t,'0J%)%5%5%f &6 &" %jjD9O+*6#'='E'EE#5 $$U+555L $//&	
 L ))9L9H!&&+J<7NO
 (M2 $ G G(6 !H ! !!== >#%'  ($78::#+%#F ;   O 	!&&+J<7KCPQF8SZ[ %==aLG	s   GG G 	H;HHc                 L   ddl m}  ||      }|j                  d      }|!|j                  d      |u rt        |      |d<   d|vri |d<   |d   j	                  dd       d|d   d<   d|d	<   |j	                  d
d       |j	                  dd       |j	                  dd       |S )u  
        Prepare kwargs for a silent experiment by ensuring isolation from the primary call.

        Guarantee metadata isolation: safe_deep_copy falls back to the original
        reference when deepcopy fails (e.g. metadata contains UserAPIKeyAuth with
        parent_otel_span — an OTel Span that is not deepcopy-able). Force a shallow
        copy of the metadata dict so mutations (model_group, is_silent_experiment)
        never corrupt the main call's metadata.
        r   safe_deep_copyr   Nlitellm_parent_otel_spanTis_silent_experimentFstreamlitellm_call_idlitellm_logging_objstandard_logging_object)'litellm.litellm_core_utils.core_helpersrv  rT  r+  r_  )r6  r@  rv  silent_kwargsoriginal_metadatas        r   _get_silent_experiment_kwargsz$Router._get_silent_experiment_kwargs  s     	K&v. #JJz2)!!*-1BB(,->(?M*%]*(*M*%
 	j!%%&@$G<@j!"89 #(h 	+T2/63T: r   rJ  c           	      0    	 |j                  di       j                  dd      ryt        j                        t        j                  d          j
                  d
i |d   d<   t        j                         }t        j                  |       	  fd}|j                   |              |j                          y# |j                          w xY w# t        $ r.}t        j                  d d	t        |              Y d}~yd}~ww xY w)zE
        Run a silent experiment in the background (thread).
        r   rx  FN%Starting silent experiment for model model_groupc                  0  K    j                   dt        t        t                 d d {    t	        j
                         } t	        j                         }|| j                  |       | rt	        j                  | ddi d {    y y 7 `7 w)Nr   r<  return_exceptionsTr   )	acompletionr   r   rM   asyncio	all_taskscurrent_taskrP  gather)pendingcurrentr<  r6  r~  rJ  s     r   _run_silent_completionzDRouter._silent_experiment_completion.<locals>._run_silent_completionZ  s     *$** *!%d+;&<h!G (   &//1G%224G*0%nngNNNN  Os"   .BBABBBB#Silent experiment failed for model : r   )rT  r2  r3  r   r,  r  r  new_event_loopset_event_looprun_until_completecloser6  errorr   )r6  rJ  r<  r@  loopr  r:  r~  s   ```    @r   rb  z$Router._silent_experiment_completion>  s    +	zz*b)--.DeL}}X.H!&&7~F ?D>>HHM 8DM*%m4 ))+D""4(O ''(>(@A



 	!''5l^2c!fXN 	s5   "C A0C C	 8C 	CC 	D'$DDry  c                    K   y wr   r   r6  r   r<  ry  r@  s        r   r  zRouter.acompletions        	   c                    K   y wr   r   r  s        r   r  zRouter.acompletiony  r  r  c                    K   y wr   r   r  s        r   r  zRouter.acompletion  r  r  c                 &  K   	 ||d<   ||d<   ||d<   | j                   |d<   | j                  ||       |j                  d      xs | j                  }t	        j                         }| j                  |      }|r| j                  |||       d {   S |+t        |t              r | j                  di | d {   }n | j                  di | d {   }t	        j                         }	|	|z
  }
t        j                  | j                  j                  t        j                   |
d||	t#        |      	             |S 7 7 7 n# t$        $ r;}t        j                  t'        | |t)        j*                         |
             |d }~ww xY ww)Nr   r<  ry  r>  r?  priority)r   r<  r@  r  servicedurationry  
start_timeend_timeparent_otel_spanlitellm_router_instancerI  error_traceback_stroriginal_exceptionr   )_acompletionrB  rT  r   time_is_prompt_management_model_prompt_management_factoryr   r  schedule_acompletionasync_function_with_fallbacksr  create_taskr   async_service_success_hookrg   ROUTERr   r6  rE   	traceback
format_exc)r6  r   r<  ry  r@  request_priorityr  r  rD  r  	_durationr:  s               r   r  zRouter.acompletion  s    ,	#F7O!)F:%F8*.*;*;F&'00uV0L%zz*5N9N9NJ*.*J*J5*Q'*!<<%! =   
  +
;KS0Q!:!:!:!DV!DD!C!C!C!Mf!MMyy{H :-I''BB(//&+)%%Fv%N C 	 O- EM  		(,0#)(1(<(<(>'(	 G		sl   FBE
 EE
 F'E
 >E?E
 EA)E
 FE
 E
 E
 
	F6F		FFfallback_itemcomplete_response_object_usagec                     ddl m} t        t        t           t        | dd            }||gng }|)t        |d      r|j                  |j                  |       |j                  |      }t        | d|       y)zCMerge partial-stream usage with fallback-stream usage on the chunk.r   )BaseTokenUsageProcessorusageN)usage_objects)litellm.cost_calculatorr  r   r   rn   getattrhasattrr  r  combine_usage_objectsr  )r  r  r  r  r  combined_usages         r   _combine_fallback_usagezRouter._combine_fallback_usage  s     	DXe_gmWd&KL#(#4"*66@.44@  !?@0FF' G 
 	w7r   rZ  r[  c                 n    K   ddl m  G fddt              } fd} | |             S w)z
        Helper to iterate over a streaming response.

        Catches errors for fallbacks using the router's fallback system
        r   MidStreamFallbackErrorc                   4     e Zd Zdef fdZd Zd Z xZS )ERouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapperasync_generatorc                     t         |   |j                  j                  j                         || _        t        d      r j                  j                         | _        y y N)completion_streamr   custom_llm_providerlogging_obj_hidden_params)	superr=  r   r  r  _async_generatorr  r  r2  )r6  r  	__class__rZ  s     r   r=  zNRouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__init__  sd     &5(..(6(J(J . : :	 !  )8%>+;<*8*G*G*L*L*ND' =r   c                     | S r   r   ru  s    r   	__aiter__zORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__aiter__      r   c                 R   K   | j                   j                          d {   S 7 wr   )r  	__anext__ru  s    r   r  zORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__anext__  s!     !22<<>>>>s   '%')r   r   r   r   r=  r  r  __classcell__r  rZ  s   @r   FallbackStreamWrapperr    s    O O?r   r  c                   K   d } 	 2 3 d {   }| 7 
6 n# $ r}ddl m}  |j                        }t        t        t
           t        |dd             }	 t        t        j                  d            }j                  dj                        }j                  dj                        }j                  dj                        }	j                  d	<   |j                  s|j                  sd
<   ndddd|j                  ddgz   d
<   j                  |       j!                  |d|||	|d       d {  7  } t#        | d      rD| 2 3 d {  7  }
|
r.t%        |
t&              rt#        |
d      rj)                  |
|       |
 @6 nd  n+# t*        $ r}t-        j.                  d|        |d }~ww xY wY d }~nd }~ww xY wt1        j2                  d      5  t#        d      rG	 j5                          d {  7   n,# t6        $ r }t-        j8                  d|       Y d }~nd }~ww xY w| St#        | d      rG	 | j5                          d {  7   n,# t6        $ r }t-        j8                  d|       Y d }~nd }~ww xY wd d d        y # 1 sw Y   y xY w# t1        j2                  d      5  t#        d      rG	 j5                          d {  7   n,# t6        $ r }t-        j8                  d|       Y d }~nd }~ww xY w| St#        | d      rG	 | j5                          d {  7   n,# t6        $ r }t-        j8                  d|       Y d }~nd }~ww xY wd d d        w # 1 sw Y   w xY wxY ww)Nr   stream_chunk_builderchunksr  r   r   r   r   r>  r<  system  You are a helpful assistant. You are given a message and you need to respond to it. You are also given a generated content. You need to respond to the message in continuation of the generated content. Do not repeat the same content. Your response should be in continuation of this text: rolecontent	assistantTr  r  prefixr?  Fr   )r:  disable_fallbacksr   r   r   r  rA  r@  r  Fallback also failed: )shieldaclose7stream_with_fallbacks: error closing model_response: %s:stream_with_fallbacks: error closing fallback_response: %s)litellm.mainr  r  r   r   rn   r  r   rT  r   r   r   r  is_pre_first_chunkgenerated_contentrB  *async_function_with_fallbacks_common_utilsr  r   rl   r  r6  r   r  anyioCancelScoper  BaseExceptionr7  )fallback_responseitemr:  r  complete_response_objectr  r  r   r   r   r  fallback_errorr  r[  r<  rZ  r6  s               r   stream_with_fallbackszERouter._acompletion_streaming_iterator.<locals>.stream_with_fallbacks  s     $i"0 $J.) N)=+?)00,( 26UO4gtD2.D)"&sN,>,>w,G"HK0>0B0B#T^^1I @N?Q?Q2D4Q4Q@, @N?Q?Q2D4Q4Q@, ;?:K:KN#67++13F3F
 6>z25=(0 ,M
 )4+,+>+>*.
A 
6z2 88). 9  #MM.3&/5M5M(3!##1 N 	 	 	 & 0+>3D 	0 	0- -$.}>Q$R$+M7$C $ < <$13Q!" #0/ 4E #
  ))//00@A )()SN)h &&d3~x8"0"7"7"999, 177 Y ! 
 )4)8:"3":":"<<<, 177 \ !  433U&&d3~x8"0"7"7"999, 177 Y ! 
 )4)8:"3":":"<<<, 177 \ !  433s  N	   J1 G;GC*F%	E
F%F!E$"F&>F%$G%	G.GGGJ1 GJ1 N	3J%HHHJ%	I$H?:J%?IJ%I0)I,*I0/J%0	J9JJ%JJ%	N	%J.*N	1NM:K0)K,
*K0/M:0	L	9L	M:L	M:+M>M
?MM:	M.	M)	$M:)M.	.M:1	N:N?NN	litellm.exceptionsr  ro   )r6  rZ  r<  r[  r  r  r  s   ````  @r   _acompletion_streaming_iteratorz&Router._acompletion_streaming_iterator  s7      	>	?$7 	?(k	 k	Z %%:%<==s   .5c                 j    ddl m  G fddt              }| fd} | |             S )a  
        Sync equivalent of _acompletion_streaming_iterator.

        Wraps a sync streaming response so that MidStreamFallbackError
        (raised by CustomStreamWrapper.__next__) triggers the Router's
        fallback chain instead of surfacing directly to the caller.
        r   r  c                   4     e Zd Zdef fdZd Zd Z xZS )HRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrappersync_generatorc                     t         |   |j                  j                  j                         || _        t        d      r j                  j                         | _        y y r  )	r  r=  r   r  r  _sync_generatorr  r  r2  )r6  r  r  rZ  s     r   r=  zQRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapper.__init__r  sd     &4(..(6(J(J . : :	 !  (6$>+;<*8*G*G*L*L*ND' =r   c                     | S r   r   ru  s    r   __iter__zQRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapper.__iter__}  r  r   c                 ,    t        | j                        S r   )nextr   ru  s    r   __next__zQRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapper.__next__  s    D0011r   )r   r   r   r   r=  r  r  r  r  s   @r   SyncFallbackStreamWrapperr  q  s    	Oy 	O2r   r  c            	   3     K   d } 	 D ]  }|  	 t#        d      r	 j1                          | t#        | d      r	 | j1                          y y y # $ r}ddl m}  |j                        }t        t        t
           t        |dd             }	 t        t        j                  d            }j                  dj                        }j                  dj                        }j                  dj                        }	j                  d	<   |j                  s|j                  sd
<   ndddd|j                  ddgz   d
<   j                  |        j                   di |||	d} t#        | d      r<| D ]6  }
|
r.t%        |
t&              rt#        |
d      rj)                  |
|       |
 8 nd  n+# t*        $ r}t-        j.                  d|        |d }~ww xY wY d }~d }~ww xY w# t2        $ r!}t-        j4                  d|       Y d }~d }~ww xY w# t2        $ r }t-        j4                  d|       Y d }~y d }~ww xY w# t#        d      r>	 j1                          n,# t2        $ r }t-        j4                  d|       Y d }~nd }~ww xY w| Kt#        | d      r>	 | j1                          w # t2        $ r }t-        j4                  d|       Y d }~w d }~ww xY ww w xY ww)Nr   r  r  r  r   r   r   r   r>  r<  r  r  r  r  Tr  r?  )r   r   r   r  r  r  r  r  r   )r  r  r  r   r   rn   r  r   rT  r   r   r   rA  r  r  rB  rC  r  r   rl   r  r6  r   r  r  r  r7  )r  r  r:  r  r  r  r  r   r   r   r  r  	close_errr  r[  r<  rZ  router_selfs                r   r  zDRouter._completion_streaming_iterator.<locals>.stream_with_fallbacks  sY     $X*DJ +L >73&,,. %0W%w6)//1	60Y * B)=+?)00,( 26UO4gtD2.8)"&sN,>,>w,G"HK0>0B0B#[%:%:1I @N?Q?Q2#<<@, @N?Q?Q2#<<@, ;F:Q:QN#67++13F3F5=z25=(0 ,M
 )4+,+>+>*.
A 
6z2  ??). @  )L(K(K )()"+1I1I	)% 0*=->M -$.}>Q$R$+M7$C + C C$13Q!" #0/ .? #
  ))//00@A )(	)}B)N ) -33U%  ) -33X%  >73&,,.( -33U% 
 %0W%w6)//1( -33X% 60s  K5A I K5H  K5H- K5G=;G8D2GG8	G/G**G//G82I 8G==I  	H*	H%K5%H**K5-	I6IK5IK5K2'I87K28	J!JK2J!!K23KK2	K-K(#K2(K--K22K5r  )r6  rZ  r<  r[  r  r  r  r	  s    ```  @@r   rj  z%Router._completion_streaming_iteratorb  s:     	>	2(; 	2$ Z	 Z	x ))>)@AAr   c           	        K   	 |j                  di       j                  dd      ryt        j                  |      }t        j                  d|         | j
                  d
i |}||d   d<    | j                  d
|t        t        t           |      d| d{    y7 # t        $ r.}t        j                  d| d	t        |              Y d}~yd}~ww xY ww)z<
        Run a silent experiment in the background.
        r   rx  FNr  r  r  r  r  r   )rT  r2  r3  r   r,  r  r  r   r   rM   r6  r  r   )r6  rJ  r<  r@  r~  r:  s         r   _silent_experiment_acompletionz%Router._silent_experiment_acompletion  s     	zz*b)--.DeL}}X.H!&&7~F ?D>>HHM7CM*%m4 #$"" "d#34h?    
  	!''5l^2c!fXN 	sK   C"B# CA4B# B!B#  C!B# #	C,$CCCCc                 p  K   d}d}i }	 |j                         }||d<   t        |      }t        j                         }	| j                  |||j	                  dd      |       d{   }|}t        j                         }
|
|	z
  }t        j                  | j                  j                  t        j                  |d|	|
t        |                   | j                  ||       |d   j                         }|j	                  d	d      }|(t        j                   | j                  d ||d
|       | j                  ||       |j	                  d	d       |d   }| j                  ||      }| j                  |xx   dz  cc<   i ||| j                   |d|}|j	                  d	d       t#        j$                  d i |}|j'                  dd      }| j)                  ||d      }|]t+        |t
        j,                        rC|4 d{    	 | j/                  |||       d{    | d{   }ddd      d{    n&| j/                  |||       d{    | d{   }t+        t0              r.| j3                  |||      }|rt#        j4                  d|d      | j6                  |xx   dz  cc<   t9        j:                  d| d       | j                  |||       t+        |t<              r| j?                  |||       d{   S |S 7 7 7 7 7 # 1 d{  7  sw Y   xY w7 7 7 +# t"        j@                  $ rz}|j'                  di       j'                  dd      }|j'                  di       j'                  dd      }|xjB                  d| d| z  c_!        || jE                  ||       |d}~wtF        $ rY}t9        j:                  d| dtI        |       d       || jJ                  |xx   dz  cc<   || jE                  ||       |d}~ww xY ww)!z
        - Get an available deployment
        - call it with a semaphore over the call
        - semaphore specific to it's rpm
        - in the semaphore,  make a check against it's local rpm before running
        Nr   rG  rH  async_get_available_deploymentr  r,  r  r   rJ  )rJ  r<  rM  ry   rO  r{  max_parallel_requestsr,  r@  client_type)r,  r  r  rS  rT  rU  rV  zlitellm.acompletion(model=rR  )r,  rD  r  rY  r  r   z$

Deployment Info: request_timeout: z

timeout: r\  r]  r   )&r2  r   r  r  r_  r  r  r   r  rg   r  _track_deployment_metricsr  rd  _get_async_openai_model_clientr  r   r   r  rT  re  r   	Semaphore&async_routing_strategy_pre_call_checksrq   rh  ri  r  r   r,  ro   r  TimeoutrW  rk  r6  r   r  )r6  r   r<  r@  rl  r,  _timeout_debug_deployment_dictrm  r  r  r  r  r   rJ  rq  rr  	_responser  rpm_semaphorerD  rs  r:   deployment_request_timeout_paramdeployment_timeout_params                           r   r  zRouter._acompletion  s     

 	'[	28++-/;@/8@HJ#BB!$*JJ/Dd$K%	  C   J .8*yy{H :-I''BB(//&>)%%Fv%N C 	 **%8H +  ((89>>@N)--ndCL' ##7D77 %1!) ! //:f/UJJ~t,'0J>>% ? L Z(A-( $//&	
 L ^T2++;l;I4:JJ%t5K !,,%3 - M
 (Zw00. )= EE#-$/)9 F   
 &/H )== AA) +%5 B    "+? (M2 $ G G(6 !H ! !!== >#%'  z*a/*!&&,ZL8OP **%!!1 +  ($78!AA#+%#F B    OZ )

  / )=== +4  	/M/Q/Q "0c#T* - (F'I'I "(c)T" % IIABbAccn  pH  oI  J  JI%==aLG 		!&&,ZL8LSQRVHT[\ %
+q0+%==aLG		s   P6AM &L'F%M L"M L+)L%*	L+3L'4L+8M L)M  M !	M *M+B-M MM P6M P6M "M %L+'L+)M +L=1L42L=9M M M P3A5OP3AP..P33P6r@  metadata_variable_namec                    |j                  d| j                        |d<   |j                  dt        t	        j
                                      d}| j                  |      r|}|j                  |i       j                  ||d       y)zm
        Adds/updates to kwargs:
        - num_retries
        - litellm_trace_id
        - metadata
        r   litellm_trace_idNr   )r  r   )rT  r   r  r   r   uuid4_get_model_from_aliasr   )r6  r   r@  r  r   s        r   rB  z&Router._update_kwargs_before_fallbacks  s}     !'

=$:J:J K},c$**,.?@+/%%E%2 %0"5<<!8IJ	
r   	exceptionc                     t        |d      r|j                  y|j                  di       }|j                  d      }|	 t        |      |_        yy# t        t
        f$ r Y yw xY w)z
        Set num_retries from deployment litellm_params on the exception.

        This allows the retry logic in async_function_with_retries to use
        per-deployment retry settings instead of the global setting.
        r   Nr   )r  r   rT  r  r^  	TypeError)r6  r"  r,  r   dep_num_retriess        r   rk  z/Router._set_deployment_num_retries_on_exception  ss     9m,1F1F1R#(8"=(,,];&(+O(<	% ' 	* s    A A$#A$c                     | j                   j                         }|j                  di       xs i }|j                         D ]  \  }}|	|j	                  ||        |j	                  |i       j                  |       y)z
        Adds default litellm params to kwargs, if set.

        Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name
        r   N)r   r2  r_  r  r  r   )r6  r@  r  defaultsmetadata_defaultskeyr]  s          r   *_update_kwargs_with_default_litellm_paramsz1Router._update_kwargs_with_default_litellm_params  s|     ..335$LLR8>B #..*JC}c5) + 	0"5<<=NOr   function_namec                    |j                  di       j                         }|d   j                         }t        ||      }t        |      }|j                  |i       j                  d      }| j	                  ||      }	|j                  d      }
|	|d<   |
|d<   t        |t        di ||	      }| j                  |
       |S )z.
        Handle clientside credential
        
model_infor   )r   rI  r+  r  )r  r   idoriginal_model_idrl  r   r-  rN  r   )rT  r2  r6   r2   _generate_model_idrX   r[   upsert_deployment)r6  r,  r@  r+  r-  r   dynamic_litellm_paramsr  r  	_model_idr0  deployment_pydantic_objs               r   _handle_clientside_credentialz$Router._handle_clientside_credential  s      ^^L"5::<
#$45::<!;)&"
 "D'"
 jj!7<@@O++#4J , 
	 'NN40$
4*;
&'",")C,BC!#

 	. 	 	
 '&r   c                 L   | j                  di       xs i }t        |t              r|}n|j                  d      }|j                  d      xs g }|j                  d      xs g }|s|rt	        |      t	        |      z   }||d<   d|vr|j                  d      	|d   |d<   yyy)z
        Merge tools from deployment litellm_params with request kwargs.
        When both have tools, concatenate them (deployment tools first, then request tools).
        tool_choice: use request value if provided, else deployment's.
        r   Tr   toolstool_choiceN)rT  r   r+  r.  r  )r,  r@  dep_params_raw
dep_params	dep_tools	req_toolsmergeds          r   _merge_tools_from_deploymentz#Router._merge_tools_from_deployment	  s     $(8"=Cnd+'J'222EJNN7+1r	JJw'-2		)_tI6F$F7O&:>>-+H+T$.}$=F=! ,U&r   c                    | j                  ||       |j                  di       j                         }|d   d   }|d   j                  d      }|d   }t        |      rZ| j	                  |||      }|j
                  j                         }|j                  j                  }|j                  j                  }t        |	      }	|j                  |	i       j                  ||||d
       |j                  di       j                  d      }
|
rH||	   j                  d      xs g }t        |      }|
D ]  }||vs|j                  |        |||	   d<   |j                  di       j                  d      }|r:d| }||	   j                  d      xs g }||vr|j                  |       |||	   d<   ||d<   | j                  ||d         |d<   | j!                  ||	       y)a  
        3 jobs:
        - Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
        - Adds default litellm params to kwargs, if set.
        - Merges tools from deployment with request (proxy-configured tools + request tools).
        rM  r-  r   r   api_baserl  )rI  r,  r@  r+  r.  )r,  r-  rB  deployment_model_nametagslitellm_credential_namezCredential: r@  datar   )r@  r  N)r@  rT  r2  r7   r7  r-  r.  r   r   rB  r2   r  r   r  r  _get_timeoutr*  )r6  r,  r@  r+  r-  deployment_litellm_model_namedeployment_api_baserD  r6  r  deployment_tagsexisting_tagsmerged_tagstagcredential_namecredential_tags                   r   rd  z%Router._update_kwargs_with_deployment!	  s7    	))Z)O^^L"5::<
(23C(DW(M%()9:>>zJ *< 8#6:&*&H&H%fM 'I '# 1;;FFHJ,C,R,R,X,X)"9"H"H"Q"Q!C'"
 	0"5<<;(/)>		
 %..)92>BB6J"#9:>>vFL"M}-K&k)&&s+ ' 6AF)*62 %..)92>BB%
 +O+<=N"#9:>>vFL"M]2$$^45BF)*62)| --
+; < . 
y 	772H 	8 	
r   c                     | j                  ||d      }|j                  dd      }||||j                  k7  rd}|S |}|S )a  
        Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment

        The same OpenAI client is re-used to optimize latency / performance in production

        If dynamic api key is provided:
            Do not re-use the client. Pass model_client=None. The OpenAI/ AzureOpenAI client will be recreated in the handler for the llm provider
        asyncr  r1  N)re  rT  r1  )r6  r,  r@  ro  rp  rq  s         r   r  z%Router._get_async_openai_model_clientg	  sh     "&!1!1!&g "2 "

 !**Y5'&2#9#A#AAL  2Lr   rH  c                     |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      S )z=Helper to get stream timeout from kwargs or deployment paramsr   N)rT  r   r   )r6  r@  rH  s      r   _get_stream_timeoutzRouter._get_stream_timeout	  s_    
 JJ'. Gxx $G ""	G
 **../?F	
r   c                     |j                  dd      xsf |j                  dd      xsR |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      }|S )zAHelper to get non-stream timeout from kwargs or deployment paramsr   Nr  )rT  r   r   r6  r@  rH  r   s       r   _get_non_stream_timeoutzRouter._get_non_stream_timeout	  s    
 JJy$' 	@zz+T2	@xx4	@
 xx!4	@ ||	@ **..y$? 	 r   c                 ~    d}|j                  dd      r| j                  ||      }|| j                  ||      }|S )z6Helper to get timeout from kwargs or deployment paramsNry  FrG  )rT  rU  rX  rW  s       r   rI  zRouter._get_timeout	  sP    /3::h&..f4.HG?22D 3 G r   modelsc                    K   dt         dt        t           f fd}dt         dt        t           dt        f fd}t	        |t
              rQt        d |D              r?g }|D ]  }|j                   |d||d|        t        j                  |  d{   }|S t	        |t
              rt        d	 |D              rg }t        |      D ]'  \  }	}
|D ]  }|j                   |d||	|
d
|        ) t        j                  |  d{   }t        t        |            D cg c]  }g  }}|D ]A  }t	        |t              r||d      j                  |d          .|d   j                  |       C |S yy7 7 qc c}w w)a6  
        Async Batch Completion. Used for 2 scenarios:
        1. Batch Process 1 request to N models on litellm.Router. Pass messages as List[Dict[str, str]] to use this
        2. Batch Process N requests to M models on litellm.Router. Pass messages as List[List[Dict[str, str]]] to use this

        Example Request for 1 request to N models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    {"role": "user", "content": "is litellm becoming a better product ?"}
                ],
                max_tokens=15,
            )
        ```


        Example Request for N requests to M models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    [{"role": "user", "content": "is litellm becoming a better product ?"}],
                    [{"role": "user", "content": "who is this"}],
                ],
            )
        ```
        r   r<  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwzs
            Wrapper around self.async_completion that catches exceptions and returns them as a result
            r  Nr   r  r6  r   r<  r@  r:  r6  s       r   _async_completion_no_exceptionszARouter.abatch_completion.<locals>._async_completion_no_exceptions	  D     -T--WEHWPVWWWW 0   <$ "$ <$ 	949<9<idxc                    K   	  j                   d| |d| d{   |fS 7 # t        $ r}||fcY d}~S d}~ww xY wwr]  r^  )r   r<  rc  r@  r:  r6  s        r   *_async_completion_no_exceptions_return_idxzLRouter.abatch_completion.<locals>._async_completion_no_exceptions_return_idx	  sS     *$**TTVTT T  #vs4   A & $& A & 	=8=A =A c              3   <   K   | ]  }t        |t                y wr   )r   r+  r   r9  s     r   r   z+Router.abatch_completion.<locals>.<genexpr>	  s     -T8ajD.A8r   r  Nc              3   <   K   | ]  }t        |t                y wr   )r   r  rg  s     r   r   z+Router.abatch_completion.<locals>.<genexpr>	  s     /VX
1d0CXr   )r   rc  r<  ry   r   r   )r   r   rM   r  r   r  allr  r  r  	enumerateranger  tuple)r6  rZ  r<  r@  r`  re  _tasksr   rD  rc  rW  r  _final_responsess   `             r   abatch_completionzRouter.abatch_completion	  s    H				"&'7"8				+,	 	& h%#-T8-T*TF=gET\g`fgh   %^^V44HO$'C/VX/V,VF )( 3W#EMMB "'S7FL $ !4 &nnf55I<A#h-<P/Q<Pq<PO/Q%h.#HQK077D#A&--h7	 &
 #"# -W' 5 6/Qs8   BFE;A4FE=F&	E?/AF=F?Fc           	          K   dt         dt        t           f fd}g }|D ]  }|j                   |d||d|        t	        j
                  |  d{   }|S 7 w)a  
        Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router

        Use this for sending multiple requests to 1 model

        Args:
            model (List[str]): model group
            messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
            **kwargs: additional kwargs
        Usage:
            response = await self.abatch_completion_one_model_multiple_requests(
                model="gpt-3.5-turbo",
                messages=[
                    [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
                    [{"role": "user", "content": "hello good mornign"}],
                ]
            )
        r   r<  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwr]  r^  r_  s       r   r`  z]Router.abatch_completion_one_model_multiple_requests.<locals>._async_completion_no_exceptions
  ra  rb  r  Nr   )r   r   rM   r  r  r  )r6  r   r<  r@  r`  rm  message_requestrD  s   `       r   -abatch_completion_one_model_multiple_requestsz4Router.abatch_completion_one_model_multiple_requests
  sv     ,				"&'7"8		 'OMM/ /=C  ( !00 1s   AAAAc                    K   y wr   r   r  s        r   "abatch_completion_fastest_responsez)Router.abatch_completion_fastest_response6
  r  r  c                    K   y wr   r   r  s        r   rv  z)Router.abatch_completion_fastest_response>
  r  r  c                    K   |j                  d      D cg c]  }|j                          }}dt        dt        t        t        t        f      dt
        dt        dt        t        t        t        f   f
 fd}g dt        j                  ffd	}|D ]2  }t        j                   |d|||d
|      }	j                  |	       4 r_t        j                  t        j                          d{   \  }
|
D ]'  } ||       d{   }|d|j"                  d<   |c S  r_t        d      c c}w 7 F7 0w)z
        model - List of comma-separated model names. E.g. model="gpt-4, gpt-3.5-turbo"

        Returns fastest response from list of model names. OpenAI-compatible endpoint.
        ,r   r<  ry  r@  r   c                    K   	  j                   d| ||d| d{   }|S 7 # t        j                  $ r& t        j                  dj                  |               t        $ r}|cY d}~S d}~ww xY ww)zn
            Wrapper around self.acompletion that catches exceptions and returns them as a result
            r   r<  ry  Nz4Received 'task.cancel'. Cancelling call w/ model={}.r   )r  r  CancelledErrorr   r7  r-  r6  )r   r<  ry  r@  resultr:  r6  s         r   r`  zRRouter.abatch_completion_fastest_response.<locals>._async_completion_no_exceptionsT
  s~     	/t//hehW]haghh i)) %++JQQRWX  s>   A6' %' A6' A A3'A.(A3)A6.A33A6taskc                   K   	 |  d {   }t        |t        t        f      r@t        j                  d       D ]  }|j                           |	 j                  |        S 	 	 j                  |        y 7 n# t        $ r Y S w xY w# t        $ r Y /w xY w# t        $ r Y y w xY w# 	 j                  |        w # t        $ r Y w w xY wxY ww)Nz=Received successful response. Cancelling other LLM API calls.)	r   rq   ro   r   r7  r	  rC  KeyErrorr6  )r~  r}  tpending_taskss      r   check_responsezARouter.abatch_completion_fastest_response.<locals>.check_responseg
  s     #f}6I&JK)//W +
 +!!((. L!((. $         !((. s   CB
 A9AB
 A;$C%B( 'B 8C9B
 ;	BCBC
	BB( BB( 	B%"C$B%%C(C*B<;C<	CCCCCr{  )return_whenNT!fastest_response_batch_completionzAll tasks failedr   )splitstripr   r   r
   boolr   r   rq   ro   r6  r  Taskr  r  waitFIRST_COMPLETEDr  )r6  r   r<  ry  r@  r9  rZ  r`  r  r~  donecompleted_taskr}  r  s   `            @r   rv  z)Router.abatch_completion_fastest_responseF
  sV     &+[[%56%5!'')%56		"&tCH~"6	@D	PS	="5y@A	" 	w|| 	, E&&/ (6EKD
   &  (/7+B+B) #D- #'-n==%QUF))*MN!M #'	  *++C 7j# >s:   D?D6CD?4D;5D?D=D?D?+D?=D?r  c                    K   y wr   r   r6  r   r<  r  ry  r@  s         r   r  zRouter.schedule_acompletion
  r  r  c                    K   y wr   r   r  s         r   r  zRouter.schedule_acompletion
  r  r  c                 B  K   t        |      }t        t        j                               }t	        |||      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                   |       d {    t        j                         }
|
|	k  r|r]	  | j"                  d|||d| d {   }|j$                  j'                  di        |j$                  d   j)                  dd	i       |S | j
                  j/                  |j                  |j                         d {    t1        j2                  d|d      7 7 /7 7 7 # t*        $ r}t-        |d
|       |d }~ww xY w7 Hw)Nr  
request_idrl  requestFr   r  r/  rl  health_deploymentsr{  additional_headers%x-litellm-request-prioritization-usedTr  r  rl  %Request timed out while polling queuer
  rV  r   )r   r   r   r   rK   r   add_requestr  	monotonicr   r   _async_get_healthy_deploymentspollr  rl  r  sleepr  r  r  r   r6  r  remove_requestr   r  )r6  r   r<  r  ry  r@  r  _request_idr  r  	curr_timepoll_intervalmake_request_healthy_deploymentsrn  r  r:  s                    r   r  zRouter.schedule_acompletion
  s$     =VD$**,'"
 nn(((666 >>#dll2NN$	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 NN,	 (" "2$"2"2 #(6#EK# 	 ((334H"M(()=>EE<dC !  ..//??t 0    //?% S 	7' 3
  :x0
s   AHG1A,HG4<HG7H"G9#HHG= G;AG= 5HHH4H7H9H;G= =	HHHHr>  rA  .c                 \  K   t        |      }t        t        j                               }t	        |||      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                   |       d {    t        j                         }
|
|	k  r|ri	  ||i | d {   }t#        |j$                  t&              r<|j$                  j)                  di        |j$                  d   j+                  ddi       |S | j
                  j1                  |j                  |j                  
       d {    t3        j4                  d|d      7 7 ;7 7 7 # t,        $ r}t/        |d	|       |d }~ww xY w7 Iw)Nr  r  Fr  r  r  r  Tr  r  r  r
  rV  )r   r   r   r   rK   r   r  r  r  r   r   r  r  r  rl  r  r  r   r  r+  r  r   r6  r  r  r   r  )r6  r   r  r>  rA  r@  r  r  r  r  r  r  r  r  rn  r  r:  s                    r   _schedule_factoryzRouter._schedule_factory
  s     =VD$**,'"
 nn(((666 >>#dll2NN$	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 NN,	 (" 
"3T"DV"DD	i66=,,778LbQ,,-ABII@$G !  ..//??t 0    //?% Q 	7' 3
 E  :x0
s   AH,G=A,H,H <H,HH,"H#H,H,H
 HAH
 +5H, H*!H, H,H,H,H
 
	H'H""H''H,c                     | j                  |      }|t        |      dk7  ry|d   d   j                  dd       }|d|vry|j                  d      d   }|t        j
                  v S )Nrl  ry   Fr   r   r   /)get_model_listr  rT  r  r   )_known_custom_logger_compatible_callbacks)r6  r   r   litellm_modelsplit_litellm_models        r   r  z"Router._is_prompt_management_model*  s|    ((E(:
ZA!5"1&67;;GTJ C}$<+11#6q9"g&W&WWWr   c                   K   |j                  dd       }|$t        di dt               t               d|\  }}t	        t
        |      }| j                  |dddg|j                  dd             }| j                  ||	       |d
   j                         }|j                  dd       }t        |t              xr |j                  d      }|j                  d      xs |d
   j                  dd       }	|j                  d      xs |d
   j                  dd       }
|j                  dd       xs |d
   j                  dd       }|s,|	t        |	t              st        d|	 dt        |	             |
*t        |
t              st        d|
 dt        |
             |j!                  ||t#        |      |	|
|      \  }}}h d}|j%                         D ci c]  \  }}||vs|| }}}i |||}||d<   ||d<   ||d<   |	|d<   |
|d<   ||d<   | j'                  |      }|t)        |      dk(  r.|j                  d       t+        j,                  di | d {   S  | j.                  di | d {   S c c}}w 7 $7 w)Nr{  r  )r>  	rules_objr  userpromptr  rG  r   r<  rG  rM  r   r   zlitellm_agent/	prompt_idprompt_variablesprompt_labelz*Prompt ID is not set or not a string. Got=z, type=z2Prompt variables is set but not a dictionary. Got=)r@  )r   r<  non_default_paramsr  r  r  >   r  r  prompt_versionbitbucket_configdotprompt_configr  r<  r  r   r>  r   )rT  rs   rr   rw   r   LiteLLMLoggingr^  r_  rd  r2  r   r   
startswithr^  r   r+  get_chat_completion_promptru   r  r  r  r   r  r  )r6  r   r<  r@  litellm_logging_objectprompt_management_deploymentrH  r  is_litellm_agent_modelr  r  r  optional_paramsprompt_management_paramskvfiltered_data_model_lists                     r   r  z!Router._prompt_management_factory6  s=     "(,A4!H!)-; .)6!&"2"4 	.*"F "&n6L!M'+'D'D%(;< &

+@$ G (E (
$ 	++3F 	, 	
 ,,<=BBD$/ ",3"
 "9&&'78 	 JJ{+ !/K0

#k4
  	 "::
 
)*:;??
 	
 zz.$7 $;W<

#nd
# 	 &Z	3%?<YKwtT]N_`  '
;KT0RDEUDVV]^bcs^t]uv  #==@O-% > 
		
$
  "ZZ\
)TQQ6N-NAqD\ 	 
 @M?V??w%z(>$%'{%5!"!-~))U);#k"2a"7JJ*+ ,,6v6667T77A&AAA#
 7As7   GJJ*J/A4J#J$J=J
>J
Jr  c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   r  r>  r   r   r  r   )_image_generationrT  r   r  r   rC  r6  r6  r  r   r@  rD  r:  s         r   image_generationzRouter.image_generation  s    
	#F7O%F8*.*@*@F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	s   A.A1 1	B:A<<Bc           	         d}	 t        j                  d| d|        | j                  |dddg|j                  dd             }| j	                  ||	       |d
   j                         }| j                  ||	      }| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |d|}| j                  |xx   dz  cc<   t        j                  d| d       |S # t        $ rE}	t        j                  d| dt        |	       d       || j                   |xx   dz  cc<   |	d }	~	ww xY w)NrU  #Inside _image_generation()- model: 
; kwargs: r  r  r  rG  r  rM  r   ry   rN  r  rP  rQ  zlitellm.image_generation(model=rR  r\  r]  r   )r   r7  r^  r_  rd  r2  r  r  rg  r   r  r   r  r,  r6  r   r  )
r6  r  r   r@  rl  r,  rH  rq  rD  r:  s
             r   r  zRouter._image_generation  s   
*	!''5eWJvhO 66#)h?@$*JJ/Dd$K 7 J
 //:f/U./446D>>% ? L
 Z(A-( 11Z1H// $#33*	
 H z*a/*!&&1*=TU O 	!&&1*=QRUVWRXQYY`a %
+q0+G	s   C?D 	EA EEc           	      l  K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   r  r>  r   r?  r  r   )_aimage_generationrT  r   rB  r  r6  r  r  rE   r  r  r  s         r   aimage_generationzRouter.aimage_generation  s     	#F7O%F8*.*A*AF&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B4A A- $A+%A- *B4+A- -	B166B,,B11B4c           	      6  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nr  r  r  r  r  rG  rH  rM  r   r   ry   r  r  r  r  z litellm.aimage_generation(model=rR  r\  r]  r   )r   r7  r   r  r_  rd  r2  r  r  r   r  r   re  r   r  r  r  r  r,  r6  r   r  )r6  r  r   r@  rl  r  r,  rH  rq  rD  r  r:  s               r   r  zRouter._aimage_generation  s    
D	!''5eWJvhO  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L
 Z(A-(00 $#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&2:,>UV OqJ )
  . )=== *  	!&&2:,>RSVWXSYRZZab %
+q0+G	s   HAG F$B8G F'G F/-F).	F/7F+8F/<G F-G #G$	G -G.5G #H$G 'G )F/+F/-G /G5F86G=G G 	HA HHHfilec           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d{   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "whisper",
                "litellm_params": {
                    "model": "whisper-1",
                },
            },
        ])

        audio_file = open("speech.mp3", "rb")
        transcript = await client.atranscription(
        model="whisper",
        file=audio_file
        )

        ```
        r   r  r>  r?  Nr  r   )	_atranscriptionrB  r  r6  r  r  rE   r  r  )r6  r  r   r@  rD  r:  s         r   atranscriptionzRouter.atranscription1  s     .	#F7O!F6N*.*>*>F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   BAA AA BA 	B6BBBc           	      ,  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 G7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)Nz!Inside _atranscription()- model: r  r  r  r  rG  rH  rM  r   ry   )r  rP  rQ  r  r  r  zlitellm.atranscription(model=rR  r\  r]  r   )r   r7  r   r  r_  rd  r2  r  r  r   r  r   re  r   r  r  r  r  r,  r6  r   r  )r6  r  r   r@  rl  r  r,  rH  rq  rD  r  r:  s               r   r  zRouter._atranscription[  s    
B	!''3E7*VHM  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:f/U./446D>>% ? L
 Z(A-(--  #33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&/
|;RS OmF )
  . )=== *  	!&&/
|;OPSTUPVxW^_ %
+q0+G	s   HAG FB3G F"G F*(F$)	F*2F&3F*7G F(G F?	G (G)5G HG "G $F*&F*(G *F<0F31F<8G G 	HA HHHinputvoicec           	        K   	 ||d<   ||d<   | j                  |dddg|j                  dd      |       d{   }| j                  ||	       |d
   j                         }|d    | j                  j                         D ])  \  }}||vr|||<   |dk(  s||   j                  |       + | j                  ||d      }	|j                  dd      }
|
|	|
|	j                  k7  rd}n|	}t        j                  di i |d|i| d{   }|S 7 7 # t        $ r;}t        j                  t        | |t!        j"                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "tts",
                "litellm_params": {
                    "model": "tts-1",
                },
            },
        ])

        async with client.aspeech(
            model="tts",
            voice="alloy",
            input="the quick brown fox jumped over the lazy dogs",
            api_base=None,
            api_key=None,
            organization=None,
            project=None,
            max_retries=1,
            timeout=600,
            client=None,
            optional_params={},
        ) as response:
            response.stream_to_file(speech_file_path)

        ```
        r  r  r  r  r  rG  NrH  r?  r   r   r   rS  r  r1  rQ  r  r   )r  r_  rB  r2  r   r  r   re  rT  r1  r   aspeechr6  r  r  rE   r  r  )r6  r   r  r  r@  r,  rH  r  r  ro  rp  rq  rD  r:  s                 r   r  zRouter.aspeech  s    @4	#F7O#F7O#BB#)h?@$*JJ/Dd$K%	  C   J 00uV0L./446DM3399;1VO !F1I*_1I$$Q' < &*%5%5%f' &6 &" %jjD9O+*6#'='E'EE#5$__ l  H OM>  		(,0#)(1(<(<(>'(	 G		sR   E 6D DAD A4D DD E D D 	E"6EEE c           	      6  K   	 ||d<   t         |d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwNr   r  r>  r?  r  r   )
r  _arerankrB  r  r6  r  r  rE   r  r  r6  r   r@  rD  r:  s        r   arerankzRouter.arerank  s     	#F7O#F7O*.--F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   BAA 	A
A BA 	B6BBBc           	        K   d }	 t        j                  d| d|        | j                  ||j                  dd       |       d {   }| j	                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i || j                  |d	| d {   }| j                  |xx   dz  cc<   t        j                  d
| d       |S 7 7 8# t        $ rE}t        j                  d
| dt        |       d       || j                  |xx   dz  cc<   |d }~ww xY ww)NzInside _rerank()- model: r  rG  )r   rG  rI  rM  r   r   ry   )rP  rQ  zlitellm.arerank(model=rR  r\  r]  r   )r   r7  r  r_  rd  r2  r  r  r   r  r   r  r,  r6  r   r  )	r6  r   r@  rl  r,  rH  rq  rD  r:  s	            r   r  zRouter._arerank  s    
'	!''+E7*VHE  $BB$*JJ/Dd$K%  C   J
 //:f/U./446DgJ>>% ? L Z(A-($__ #33* 	 H z*a/*!&&(4KL O9  	!&&(4HQPWX %
+q0+G	sO   EAD D	BD D5D ED D 	EA EEEis_retryis_fallbackis_asyncc                    d|dg}	 ||d<   ||d<   |j                  d| j                        |d<   |j                  di       j                  d|i       | j	                  |||j                  dd       	      }|d
   j                         }	| j                  j                         D ])  \  }
}|
|vr|||
<   |
dk(  s||
   j                  |       + t        j                  di i |	|| j                  d|S # t        $ r}|d }~ww xY w)Nr  r  r   r  r   r   r  rG  r  r   )r  rP  r   )rT  r   r  r   r^  r_  r2  r   r  r   text_completionr   r6  )r6  r   r  r  r  r  r@  r<  r,  rH  r  r  r:  s                r   r  zRouter.text_completion7  s1    $78	#F7O%F8$*JJ}d>N>N$OF=!j"-44mU5KL 66!$*JJ/Dd$K 7 J ./446D3399;1VO !F1I*_1I$$Q' < **s-r-rSWSgSg-rkq-rss 	G	s   B2C9 ;=C9 9	D	DD	c           	        K   |j                  dd       9| j                  ||j                  d      | j                  ||f|       d {   S 	 ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 M7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr  )r   r  r>  rA  r@  r   r  r>  r?  r  r   )rT  r  r_  atext_completion_atext_completionrB  r  r6  r  r  rE   r  r  )	r6  r   r  r  r  r  r@  rD  r:  s	            r   r  zRouter.atext_completion\  s      ::j$'3//J/"&"7"7V_ 0   	#F7O%F8*.*@*@F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		sI   AC"	B
C"AB BB C"B 	C$6CCC"c           	      2  K   	 t        j                  d| d|        t        |      }| j                  |d|dg|j	                  dd       |       d {   }| j                  ||       |d   j                         }|d	   }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)N#Inside _atext_completion()- model: r  r  r  rG  rH  rM  r   r   ry   r  r  r  r  zlitellm.atext_completion(model=rR  r\  r]  r   )r   r7  r   r  r_  rd  r2  r  r  r   r  r   re  r   r  r  r  r  r,  r6  r   r  )r6  r   r  r@  r  r,  rH  rl  rq  rD  r  r:  s               r   r  zRouter._atext_completion  s    C	!''5eWJvhO  AH#BB#)f=>$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L Z(A-(// $#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&1*=TU OoH )
  . )=== *  	!&&1%8LSQRVHT[\  &!+&G	   HAG F"B8G F%G F-+F',	F-5F)6F-:G F+G !G"	G +G,5G !H"G %G 'F-)F-+G -F?3F64F?;G G 	HA HHH
adapter_idc           	        K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   r  r>  r   r   r  r  r   )_aadapter_completionrT  r   r  r   r  r6  r  r  rE   r  r  )	r6  r  r   r  r  r  r@  rD  r:  s	            r   aadapter_completionzRouter.aadapter_completion  s     	#F7O#-F< *.*C*CF&'$*JJ}d>N>N$OF=!j"-44mU5KL?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   CA0A= 4A;5A= :C;A= =	C6B<<CCc           	      2  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nz&Inside _aadapter_completion()- model: r  r  zdefault textr  rG  rH  rM  r   r   ry   )r  rP  rQ  r  r  r  z"litellm.aadapter_completion(model=rR  r\  r]  r   )r   r7  r   r  r_  rd  r2  r  r  r   r  r   re  r   r  r  r  r  r,  r6  r   r  )r6  r  r   r@  r  r,  rH  rl  rq  rD  r  r:  s               r   r  zRouter._aadapter_completion  s    C	!''8z&R  AH#BB#)nEF$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L Z(A-(22 ",#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&4ZL@WX OoH )
  . )=== *  	!&&4UG;OPSTUPVxW^_  &!+&G	r  c                 T   K   ddl m}  |j                  d| |d| d{   S 7 w)z
        Helper function to make a search API call through the router with load balancing and fallbacks.
        Reuses the router's retry/fallback infrastructure.
        r   SearchAPIRouter)router_instancer>  Nr   )&litellm.router_utils.search_api_routerr  async_search_with_fallbacks)r6  r>  r@  r  s       r   _asearch_with_fallbackszRouter._asearch_with_fallbacks*  s@     
 	K@_@@ 
 /
 
 
 	
 
s   (&(original_generic_functionc                 V   K   ddl m}  |j                  d| ||d| d{   S 7 w)z
        Helper function for search API calls - selects a search tool and calls the original function.
        Called by async_function_with_fallbacks for each retry attempt.
        r   r  )r  r   r  Nr   )r  r  "async_search_with_fallbacks_helper)r6  r   r  r@  r  s        r   _asearch_with_fallbacks_helperz%Router._asearch_with_fallbacks_helper7  sC      	KG_GG 
 &?
 	
 
 	
 
    )')guardrail_namec                    K   ||d<   ||d<   | j                   |d<   | j                  ||d       t        j                  d| d|         | j                  d	i | d{   }|S 7 w)
aw  
        Execute a guardrail with load balancing and fallbacks.

        Args:
            guardrail_name: Name of the guardrail to execute
            original_function: The guardrail's execution function (e.g., async_pre_call_hook)
            **kwargs: Additional arguments passed to the guardrail

        Returns:
            Result from the guardrail execution
        r   r  r>  litellm_metadatar   r@  r  z&Inside aguardrail() - guardrail_name: r  Nr   )_aguardrail_helperrB  r   r7  r  )r6  r  r>  r@  rD  s        r   
aguardrailzRouter.aguardrailG  s     " )w.?*+&*&=&="#,, #5 	- 	

 	##4^4DJvhW	
 <;;EfEE Fs   AA) A'!A)c                    K   |}| j                  |      }t        j                  d|j                  di       j                  d              ||d<    |di | d{   }|S 7 w)a}  
        Helper for aguardrail - selects a guardrail deployment and executes it.
        Called by async_function_with_fallbacks for each retry attempt.

        Args:
            model: The guardrail_name (named 'model' for fallback system compatibility)
            original_generic_function: The guardrail's execution function
            **kwargs: Additional arguments
        )r  zSelected guardrail deployment: r   	guardrailselected_guardrailNr   )get_available_guardrailr   r7  rT  )r6  r   r  r@  r  r	  rD  s          r   r  zRouter._aguardrail_helperf  s      !99) : 
 	##-.@.D.DEUWY.Z.^.^_j.k-lm	

 (:#$2<V<< =s   AA'A%A'rZ   c                     ddl m} | j                  D cg c]  }|j                  d      |k(  s| }}|st	        d|       t        |      dk(  r|d   S t        t         || ||            S c c}w )z
        Select a guardrail deployment using the router's load balancing strategy.

        Args:
            guardrail_name: Name of the guardrail to select

        Returns:
            Selected guardrail configuration dict
        r   r-   r  zNo guardrail found with name: ry   llm_router_instancer  r   )&litellm.router_strategy.simple_shuffler.   r   rT  r^  r  r   rZ   )r6  r  r.   gr  s        r   r
  zRouter.get_available_guardrail  s     	J **
*!aee4D.E.WA* 	 
 #=n=MNOO"#q(&q)) $($7$
 	

s
   A1A1c           	      f  K   	 ||d<   ||d<   | j                   |d<   | j                  ||d       t        j                  d| d|         | j                  d
i | d{   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |	             |d}~ww xY ww)
        Helper function to make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
        r   r  r>  r  r  z3Inside ageneric_api_call_with_fallbacks() - model: r  Nr  r   )(_ageneric_api_call_with_fallbacks_helperrB  r   r7  r  r6  r  r  rE   r  r  )r6  r   r>  r@  rD  r:  s         r   !_ageneric_api_call_with_fallbacksz(Router._ageneric_api_call_with_fallbacks  s     	#F7O2CF./*.*W*WF&'00FCU 1  "''EeWJW]V^_ @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B1AA* !A("A* 'B1(A* *	B.36B))B..B1rl  c                     d|v rT|d   rOddl m} 	  |||j                  d      |j                  d            \  }}}}|}|d   j	                  ||      |d<   |S # t        $ r |}Y 'w xY w)a  
        Add the deployment model to the endpoint for LLM passthrough route.

        e.g for bedrock invoke users can pass endpoint as /model/special-bedrock-model/invoke
          it should be actually sent as /model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke
        endpointr   )rt   r  rB  )r   r  rB  )r   rt   rT  r6  replace)r6  r@  r   rl  rt   stripped_model_namern  replacement_model_names           r   ;_add_deployment_model_to_endpoint_for_llm_passthrough_routezBRouter._add_deployment_model_to_endpoint_for_llm_passthrough_route  s     F:$6 1
4/?$(.

3H(I#ZZ
30,#Q1
 *=&
 "(
!3!;!;-"F:   4)3&4s   0A A)(A)c           	      ~  K   |j                  dd      }d}	 t        |      }	 | j                  |||j                  dd      |j                  dd             d{   }| j                  |||	       |d
   j                         }	|	d   }
| j                  |
xx   dz  cc<   | j                  |||
        |di i |	d| j                  i|}| j                  ||d      }|\t        |t        j                        rB|4 d{    	 | j                  ||       d{    | d{   }ddd      d{    n%| j                  ||       d{    | d{   }| j                  |
xx   dz  cc<   t!        j"                  d|
 d       |S 7 ?# t        $ r!}|r |dd|i| d{  7  cY d}~S |d}~ww xY w7 7 7 7 # 1 d{  7  sw Y   |xY w7 7 # t        $ rE}t!        j"                  d| dt%        |       d       || j&                  |xx   dz  cc<   |d}~ww xY ww)r  passthrough_on_no_deploymentFr  r<  NrG  )r   rI  r<  rG  r   rC  r   ry   )r@  r   rl  rP  r  r  r  z'ageneric_api_call_with_fallbacks(model=rR  r\  r]  r   )r_  r   r  rT  r6  rd  r2  r  r  r   re  r   r  r  r  r  r   r,  r   r  )r6  r   r  r@  r  r+  r  r,  r:  rH  rl  rD  r  s                r   r  z/Router._ageneric_api_call_with_fallbacks_helper  s     (.zz2PRW'X$;F	@H
#'#F#F#)#ZZ
D9(.

3H$(O	 $G $ 
 //%fM 0  ./446DgJZ(A-(LLUz M  1 t33 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&9*E\] Oy  /!:!Q!Q&!QQQQD )
  . )=== *  	!&&9%@TUXYZU[T\\cd  &!+&G	s  H=G, 8F FF !B'G, G	G, G$G%	G.G/G3G, >G?G, G(	G, $G*%5G, H=F 	G'G6F97G;G<G,  H=GGG, GGG, G%GG%!G, *G, ,	H:5A H55H::H=c           
         |j                   }t        d      }	 t        j                  d| d| d|        | j	                  |||       | j                  ||j                  dd      |j                  d	d      |
      }| j                  ||d       |d   j                         }|d   }| j                  |xx   dz  cc<   d|v r|d   r|d   j                  ||      |d<   ||d<   | j                  |       	 t        |d         \  }	}
}	}	 |di i ||
| j                  d|}| j                   |xx   dz  cc<   t        j"                  | d| d       |S # t        $ r d}
Y \w xY w# t        $ rG}t        j"                  | d| dt%        |       d       || j&                  |xx   dz  cc<   |d}~ww xY w)a  
        Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
        Args:
            model: The model to use
            original_function: The handler function to call (e.g., litellm.completion)
            **kwargs: Additional arguments to pass to the handler function
        Returns:
            The response from the handler function
        generic_api_callr.  z&Inside _generic_api_call() - handler: z	, model: r  r  r<  NrG  rH  rC  r   r   ry   r  rN  r  )r  rP  z(model=rR  r\  r]  r   )r   r2   r   r7  rB  r^  rT  r_  rd  r2  r  r  rg  rt   r6  r   r  r,  r   r  )r6  r   r>  r@  handler_namer  r,  rH  rl  rn  r  rD  r:  s                r    _generic_api_call_with_fallbacksz'Router._generic_api_call_with_fallbacks5  sC    )11!C,"
:	!''8iPUwV`ag`hi 00'= 1 
 66J5$*JJ/Dd$K%	 7 J //%fDV 0  ./446DgJZ(A-( V#z(:%+J%7%?%?z%Rz"(F7O 11Z1H+/?d7m/T,&1 ) +>#33 	H z*a/*!&&.
|3JK O!  +&*#+"  	!&&.w.B3q6('R  &!+&G	s>   C"E1 =E  AE1  E.+E1 -E..E1 1	G:AF<<Gc                     	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di |}|S # t        $ r}|d }~ww xY w)Nr   r  r>  r?  r   )
_embeddingrB  rC  r6  r6  r   r  r  r@  rD  r:  s          r   	embeddingzRouter.embedding  sj    	#F7O#F7O*.//F&'00uV0L3t33=f=HO 	G	s   ?A 	AAAc           	         d }	 t        j                  d| d|        | j                  |||j                  dd             }| j	                  ||       |d   j                         }|d   }| j                  ||d	      }|j                  d
d       }||||j                  k7  rd }	n|}	| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |	d|}
| j                  |xx   dz  cc<   t        j                  d| d       |
S # t         $ rE}t        j                  d| dt#        |       d       || j$                  |xx   dz  cc<   |d }~ww xY w)NzInside embedding()- model: r  rG  )r   r  rG  rM  r   r   syncr  r1  ry   rN  r  rP  rQ  zlitellm.embedding(model=rR  r\  r]  r   )r   r7  r^  r_  rd  r2  re  rT  r1  r  rg  r   r#  r   r  r,  r6  r   r  )r6  r  r   r@  rl  r,  rH  ro  rp  rq  rD  r:  s               r   r!  zRouter._embedding  s   
4	!''-eWJvhG 66$*JJ/Dd$K 7 J
 //:f/U./446DgJ%)%5%5%f& &6 &" %jjD9O+*6#'='E'EE#5Z(A-( 11Z1H(( "#33*	
 H z*a/*!&&*:,6MN O 	!&&*:,6J3q6(RYZ %
+q0+G	s   D+D0 0	E>9A E99E>c           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d }~ww xY wwr  )	_aembeddingrB  r  r6  r  r  rE   r  r  r"  s          r   
aembeddingzRouter.aembedding  s     	#F7O#F7O*.*:*:F&'00uV0L?T??I&IIHO J 		(,0#)(1(<(<(>'(	 G		r  c           	      .  K   d }	 t        j                  d| d|        t        |      }| j                  |||j	                  dd       |       d {   }| j                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d	|}	| j                  ||d
      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)NzInside _aembedding()- model: r  rG  )r   r  rG  rI  rM  r   r   ry   r&  r  r  r  zlitellm.aembedding(model=rR  r\  r]  r   )r   r7  r   r  r_  rd  r2  r  r  r   r)  r   re  r   r  r  r  r  r,  r6  r   r  )r6  r  r   r@  rl  r  r,  rH  rq  rD  r  r:  s               r   r(  zRouter._aembedding  s    
B	!''/wjI  AH#BB$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L
 Z(A-()) "#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&+J<7NO OmF )
  . )=== *  	!&&+J<7KCPQF8SZ[ %
+q0+G	s   HAG F B8G F#G F+)F%*	F+3F'4F+8G F)G G  	G )G*5G H G #G %F+'F+)G +F=1F42F=9G G 	HA HHHc           	      b  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   r>  r   r?  r  r   )_acreate_filerT  r   rB  r  r6  r  r  rE   r  r  r  s        r   acreate_filezRouter.acreate_file*  s     
	#F7O*.*<*<F&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B/AA( A& A( %B/&A( (	B,16B''B,,B/c                    K   	 ddl m} t        j                  d| d        t	               j                  |dddgj                  dd       	       d {   }d
t        dt        f fd}g }t        |t              r|j                   ||             n|D ]  }|j                   ||              t        j                  |  d {   }t        |      dk(  rt        d       |||      }	t        t        |d         }
|	|
j                   d<   |
S 7 7 M# t        $ rH}t        j"                  d| d dt%        |       d       | j&                  |xx   dz  cc<   |d }~ww xY ww)Nr   )add_model_file_id_mappingsr  r  r  files-api-fake-textr  rG  r   r<  rG  rI  r  r,  r   c           	      P  K   ddl m}  |      }j                  | |d       | d   j                         }|d   }j	                  | |      }j
                  |xx   dz  cc<   t        |d   	      \  }}}}t        t        t           j                  d
            }	t        t        t           j                  d            }
|
r|	st        d      t        |	      }|rt        |
|      }
|
|d<   d|v r|d   |j                  di       d<   t!        j"                  di i ||j$                  |d|}j'                  | |d      }|\t)        |t*        j,                        rB|4 d {    	 j/                  |        d {    | d {   }d d d       d {    n%j/                  |        d {    | d {   }j0                  |xx   dz  cc<   t3        j4                  d| d       |S 7 7 x7 p7 b# 1 d {  7  sw Y   LxY w7 ]7 Uw)Nr   ru  r-  rC  r   r   rM  ry   r  purposer  z2file and file_purpose are required for create_file)r3  )file_contentnew_model_namegcs_bucket_namer  r  rP  rQ  r  r  r  litellm.acreate_file(model=rR  r   )r}  rv  rd  r2  r  r  rt   r   r   rP   rT  rN   r6  r4   r3   r  r   r-  r   re  r   r  r  r  r  r   r,  )r,  rv  kwargs_copyrH  rl  rq  stripped_modelr  rn  r3  r  replace_model_in_jsonl_boolrD  r  r@  r  r6  s                 r   create_file_for_deploymentz8Router._acreate_file.<locals>.create_file_for_deploymentV  s    R,V433)&"0 4 
 ""2388:!']
#BB)&  C     ,1, =Mw-=9 3Q
 x(:;VZZ	=RSHY/F1CD7#L  /L#/+ /1%)'5D
 +/K'%- ./  **+=rB) #// /B#'#7#7".	
 & !% 0 0)& 7 !1 ! !,!7#4#42  -} #II'1DT J    *2>  -}} EE#-@P F    &.~H"":.!3.%**1*=TU  )  -
 $2  -}}}  .s   E/H&2H3H&6HH	HH	HH&(H)H&H"	H&H$7H&H	HH&HHHH&$H&No healthy deployments found.)r  r  model_file_id_mappingr8  , r\  r]  ry   )!litellm.router_utils.common_utilsr/  r   r7  r   async_get_healthy_deploymentsr_  r+  rO   r   r  r  r  r  r6  r   r  r"  r   r  )r6  r   r@  r/  r  r<  tasksr,  r  r>  returned_responser:  r  s   ` `         @r   r,  zRouter._acreate_fileB  s    
B	T!''5eWJvhO  AH(,(J(J#)6KLM$*JJ/Dd$K%!1 )K ) #V T V FV V p E-t478KLM"5JLL!;J!GH #6 &nne44I9~" ?@@$>$79%! !%%5y| D & ,,' %$i#P 5  	!++-eWBvh>RSVWXSYRZZab  &!+&G	sP   E:AD&  D"!A5D& D$A
D& !E:"D& $D& &	E7/AE22E77E:c           	      r  K   	 |)ddl m} | j                  |d      } |di | d{   S ddlm} t	        |      }| j                  |dddg|j                  d	d      |
       d{   }|d   j                         }|d   }	| j                  ||d       | j                  ||      }
| j                  |	xx   dz  cc<   t        |d         \  }}}} |di i ||| j                  |
d|}| j                  ||d      }|[t        |t        j                         rA|4 d{    | j#                  ||       d{    | d{   }ddd      d{    n%| j#                  ||       d{    | d{   }| j$                  |	xx   dz  cc<   t'        j(                  d|	 d       |S 7 7 V7 7 ~7 v7 h# 1 d{  7  sw Y   RxY w7 c7 [# t*        $ rE}t'        j,                  d| dt/        |       d       || j0                  |xx   dz  cc<   |d}~ww xY ww)z
        Create a vector store for a specific model.

        Args:
            model: Model name from router config
            **kwargs: Vector store creation parameters

        Returns:
            VectorStoreCreateResponse
        Nr   r  avector_store_createrx  r  zvector-store-api-fake-textr  rG  rH  r   r   rC  rM  ry   r  r7  r  r  r  z#litellm.avector_store_create(model=rR  r\  r]  r   )r  r  rl  litellm.vector_storesr   r  r_  r2  rd  r  r  rt   r   re  r   r  r  r  r  r   r,  r6  r"  r   r  )r6  r   r@  r  
factory_fnavector_store_create_sdkr  r,  rH  rl  rq  rn  r  rD  r  r:  s                   r   rE  zRouter.avector_store_create  s    O	}> "22'= 3 
 (1&111Q@H#BB#)6RST$*JJ/Dd$K%	  C   J ./446DgJ//%4 0   >>% ? L Z(A-( ,<$w-+P(A"Aq/ +>#33*	
 H !,,%3 - M (Zw00. )=EE#-@P F    &.~H	 )== AA)<L B    "*>z*a/*!&&5j\AXY O 2
T )  .	 )=== *  	!++5eW<PQTUVQWPXX_`  &!+&G	s   H7&G& F?G& H7=G& ,G-C G& -G.G& 1GG		GG	GG& "G#G& >G"?	G& G$	5G& >H7?G& G& G& G	GG& GGGG& $G& &	H4/A H//H44H7c                      y)aE  
        Override factory-generated vector store methods with router-aware implementations.
        This is called after _initialize_vector_store_endpoints() to ensure our custom
        methods that handle deployment selection and credential injection are used instead
        of the generic factory-generated ones.
        Nr   ru  s    r   r  z0Router._override_vector_store_methods_for_router,  s     	r   c           	      |  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   t        d      }| j	                  |||        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   r>  r   _acreate_batchr.  r  r  r   )rK  rT  r   r2   rB  r  r6  r  r  rE   r  r  r6  r   r@  r  rD  r:  s         r   acreate_batchzRouter.acreate_batch7  s     
	#F7O*.*=*=F&'$*JJ}d>N>N$OF=!%G.&" 00'= 1 
 @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   B<A(A5 ,A3-A5 2B<3A5 5	B9>6B44B99B<c                 b  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }|d   j                         }|d	   }| j                  ||d
       | j                  ||      }| j                  |xx   dz  cc<   t        |d	         \  }}	}}t        j                  di i ||	| j                  |d|}
| j                  ||d      }|\t        |t        j                         rB|4 d {    	 | j#                  ||       d {    |
 d {   }
d d d       d {    n%| j#                  ||       d {    |
 d {   }
| j$                  |xx   dz  cc<   t        j&                  d| d       |
S 7 a7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t(        $ rH}t        j*                  d| d| dt-        |       d       || j.                  |xx   dz  cc<   |d }~ww xY ww)Nz Inside _acreate_batch()- model: r  r  r0  r  rG  rH  r   r   rK  rC  rM  ry   r  r7  r  r  r  zlitellm.acreate_batch(model=rR  zlitellm._acreate_batch(model=r?  r\  r]  r   )r   r7  r   r  r_  r2  rd  r  r  rt   r   rM  r   re  r   r  r  r  r  r,  r6  r"  r   r  r6  r   r@  r  r,  rH  rl  rq  rn  r  rD  r  r:  s                r   rK  zRouter._acreate_batchV      
I	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K%	  C   J ./446DgJ//%fDT 0   >>% ? L Z(A-( ,<$w-+P(A"Aq,, +>#33*	
 H !,,%3 - M (Zw00. )= EE#-@P F    &.~H )== AA)<L B    "*>z*a/*!&&.zl:QR O{R )
  . )=== *  	!++/wb@TUXYZU[T\\cd  &!+&G	   H/AG F7CG $F:%G (G F<	G
F>GG G G 6G7	G  G5G 6H/7G :G <G>G G GG	GG G 	H,$AH''H,,H/c                    	K   	 t              }|3 j                  |dddgj                  dd      |       d{   }n j                         }|t	        d      g 	dt
        f	 fd	}|^t        |t              rNt        |      d
kD  r@t        j                  |D cg c]  } |t        t
        |             c}ddi d{   }n<|/t        |t              r |t        t
        |             d{   }nt	        d      |=t        |t              r|S t        |t              r|D ]  }t        |t              s|c S  	r	d
   t	        dj                  	            7 /c c}w 7 7 z# t        $ r;}t        j                  t!         t#        j$                         |             |d}~ww xY ww)z
        Iterate through all models in a model group to check for batch

        Future Improvement - cache the result.
        Nr  zretrieve-api-fake-textr  rG  r1  Router not yet initialized.rl  c                 2  K   	 ddl m} | d   j                  d      }| d   j                         }|j                  d      }|t	        d|        |st        |      \  }}}} |	      }j                  t        t        |       |d	       |j                  dd        |j                  dd        t        j                  d
i i |d|i| d {   S 7 # t        $ r0}dd l} |j                          
j                  |       Y d }~y d }~ww xY ww)Nr   ru  r   r   r  z2Model not found in litellm_params for deployment: r  aretrieve_batchrC  r   )r}  rv  rT  r2  r6  rt   rd  r   r+  r_  r   rV  r  	print_excr  )rl  rv  r   rH  r  rn  
new_kwargsr:  r  r@  receieved_exceptionsr6  s            r   try_retrieve_batchz2Router.aretrieve_batch.<locals>.try_retrieve_batch  sI    $ V&'78<<WEE%&67<<>D*.((3H*I'}'PQ[P\] 
 /7G"'84.1 "0!7J77#'j#9)&7 8 
 NN#8$?HH2D9!(!8!8 ""13F )"    !  $'I'')(//2 sA   DCC CC DC 	D$&D
DDDr   r  Tr=  z7Unable to find batch in any model. Received errors - {}r  )r   rA  r_  r  r6  rY   r   r  r  r  r  r   r+  rj   r-  r  rE   r  r  )
r6  r   r@  r  filtered_model_listrZ  resultsr}  r:  rY  s
   ` `      @r   rV  zRouter.aretrieve_batch  s    l	@H  <<'-:RST(.

3H$(O#)%5 =   $ '+&9&9&;#"* =>>#% % 5H % R $/2D9+,q0 ' &9%8E +40CU+KL%8!
 '+!  %0Z#T6 !3,.AB!    ?@@ "g|4"N.")%fl;#)M #*
 $*1-- IPP( sx2  		(,0#)(1(<(<(>'(	 G		s}   G:E< E0A%E< 'E3
E< E80E< ?E: #E< #G$%E< 
E< G#E< 3E< :E< <	G 6F;;G  Gc           	      |  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   t        d      }| j	                  |||        | j
                  d	i | d{   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d}~ww xY ww)
zZ
        Cancel a batch through the router with proper model-to-provider mapping.
        r   r>  r   _acancel_batchr.  r  Nr  r   )r^  rT  r   r2   rB  r  r6  r  r  rE   r  r  rL  s         r   acancel_batchzRouter.acancel_batch  s     	#F7O*.*=*=F&'$*JJ}d>N>N$OF=!%G.&" 00'= 1 
 @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		rN  c                 b  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }|d   j                         }|d	   }| j                  ||d
       | j                  ||      }| j                  |xx   dz  cc<   t        |d	         \  }}	}}t        j                  di i ||	| j                  |d|}
| j                  ||d      }|\t        |t        j                         rB|4 d {    	 | j#                  ||       d {    |
 d {   }
d d d       d {    n%| j#                  ||       d {    |
 d {   }
| j$                  |xx   dz  cc<   t        j&                  d| d       |
S 7 a7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t(        $ rH}t        j*                  d| d| dt-        |       d       || j.                  |xx   dz  cc<   |d }~ww xY ww)Nz Inside _acancel_batch()- model: r  r  zbatch-api-fake-textr  rG  rH  r   r   r^  rC  rM  ry   r  r7  r  r  r  zlitellm.acancel_batch(model=rR  zlitellm._acancel_batch(model=r?  r\  r]  r   )r   r7  r   r  r_  r2  rd  r  r  rt   r   r_  r   re  r   r  r  r  r  r,  r6  r"  r   r  rP  s                r   r^  zRouter._acancel_batch@  rQ  rR  c                   K   | j                  |      }|t        d      dt        ffd}t        j                  |D cg c]
  } ||       c}  d{   }dg dddd}|D ]g  }||d	   t        |d	      rt        |d	      |d	<   t        |d
      |d
<   |d   j                  |j                         t        |dd      du scd|d<   i |S c c}w 7 w)zQ
        Return all the batches across all deployments of a model group.
        r  NrT  r   c                 x   K   	 t        j                  di i | d    d {   S 7 # t        $ r Y y w xY ww)Nr   r   )r   alist_batchesr6  r?  s    r   rZ  z0Router.alist_batches.<locals>.try_retrieve_batch  sT     $22 ;/0;F;     s(   :+ )+ :+ 	7:7:r  F)objectrH  first_idlast_idhas_morere  rf  rH  rg  T)	r  r6  rY   r  r  r  r  extendrH  )r6  r   r@  r[  rZ  r\  final_resultsr}  s     `     r   rc  zRouter.alist_batches  s     #11U1C&9::	,? 	  5HI5HE '5HI
 

 
 F! ,49T07
0KM*-+269+Ei(f%,,V[[9 6:u5=04M*-  / J
s*   ?CCCCC-AC
Cr  c                   K   | j                  |j                  dd      |       |j                  d      ru| j                  |d         r`| j                  |d   |       d {   }|d   d   |d<   |d   j	                         }| j                  ||       |j                  |        |di | d {   S 7 V7 w)	Nr   rU  r?  r  )r   rI  r   rM  r   )rB  rT  r  r  r2  rd  r   )r6  r>  r  r@  r,  rH  s         r   )_pass_through_moderation_endpoint_factoryz0Router._pass_through_moderation_endpoint_factory  s      	,,**Wb) 	- 	
 ::g4#6#6&/#6#R#BBWo%  C   J ))9:7CF7O./446D//% 0  MM$&0000 1s%   A#C %B<&AC 7B>8C >C ry  )d
assistantsrw  rz  r}  r  r  r  r  r  r~  r  r  r  r  r  r  r  r  r  r  r  r{  r  r  r  r  rE  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  aupload_container_fileupload_container_filealist_container_fileslist_container_filesaretrieve_container_fileretrieve_container_fileadelete_container_filedelete_container_filer  r  r  r  r  r  r  r  r  r  r  r  c                 P    dv r&	 	 d
dt         t           dt         t           f fd}|S dv r&	 	 d
dt         t           dt         t           f fd}|S dv r%	 	 d
dt         t           dt         t           ffd}|S 	 	 d
dt         t           dt         t           f fd	}|S )z
        Creates appropriate wrapper functions for different API call types.

        Returns:
            - A synchronous function for synchronous call types
            - An asynchronous function for asynchronous call types
        )r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rQ  c                 ,     j                   ddi|S )Nr>  r   )r  r  rQ  r@  r>  r6  s      r   sync_wrapperz-Router.factory_function.<locals>.sync_wrapperb  s*    
 =t<< &7;A r   )r  r  r  r  c                 t    | r	d|vr| |d<   |j                  d      r j                  ddi|S  di |S )Nr  r   r>  r   )rT  r  rw  s      r   vector_store_sync_wrapperz:Router.factory_function.<locals>.vector_store_sync_wrappert  s\    
 '+@+N4GF01::g&@4@@ *;?E  )2622r   )r  r  r  r  r  r  c                      d| |d|S )Nr  rQ  r   r   )r  rQ  r@  r>  s      r   vector_store_file_sync_wrapperz?Router.factory_function.<locals>.vector_store_file_sync_wrapper  s'    
 ) (;!  r   c                   K   dk(  r j                   d| |d| d {   S dk(  r j                  ddi| d {   S dv r j                  ddi| d {   S dv r j                  d| d| d {   S dv r j                  ddi| d {   S d	v r j
                  d| d| d {   S d
k(  r j                  ddd| d {   S dv r j                  ddi| d {   S dv r j                  d| d| d {   S dv r j                  d| |d| d {   S dv r j                  d| d| d {   S y 7 O7 17 7 7 7 7 7 z7 [7 ;7 w)Nrl  )r>  r  rQ  rw  r>  r  )r  r  r  r  r  r  )r>  r  )rz  r}  r  r  r  r  r  r  r  r  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )	r  r  r  r  rm  ro  rq  rs   aretrieve_container_file_contentr  T)r>  r  )r  r  r  r  r  )r  rE  r  r  r  r  )r~  r  )r  r  r  r   ))_pass_through_assistants_endpoint_factoryrk  r   _init_vector_store_api_endpointsr  _init_containers_api_endpoints_init_responses_api_endpoints _init_interactions_api_endpoints)r  rQ  r@  ry  r>  r6  s      r   async_wrapperz.Router.factory_function.<locals>.async_wrapper  s    
 L(KTKK &7(;! 	   l*KTKK &7;A   339T99 &7      CTBB &7(;   
   > DTCC &7    
 
 AT@@ &7(;   
 66CTCC &715   
    @T?? &7      CTBB &7(;   
 ??CTCC &7(;! 	     
 CTBB &7(;   iHs   F
E1 F
E4F
#E7$ F
E:F
$E<% F
E>!F
'F (F
F F
(F)!F

F F
+F,F
4F
7F
:F
<F
>F
 F
F
F
F
F
NN)r   r   r   )r6  r>  ry  rx  rz  r}  r  s   ```    r   rl  zRouter.factory_function  s    d  
 
( 6:(,%-c]    
 
 6:(,3%-c]3 3 -, 
 
 6:(,	%-c]	 	 21 26$(C	!)#C	SMC	J r   c                    K   |r	d|vr||d<   |j                  d      r | j                  dd|i| d{   S  |di | d{   S 7 7 w)z
        Initialize the Vector Store API endpoints on the router.

        If a model is provided in kwargs, use model-based routing to get
        the deployment credentials. Otherwise, call the original function directly.
        r  r   r>  Nr   )rT  r  r6  r>  r  r@  s       r   r  z'Router._init_vector_store_api_endpoints!  st      #8#F,?F() ::g??? "3   '0000 1s!   4AAAAAAc                 D   K   |r	d|vr||d<    |di | d{   S 7 w)z
        Initialize the Containers API endpoints on the router.

        Container operations don't need model-based routing, so we call the
        original function directly with the custom_llm_provider.
        r  Nr   r   r  s       r   r  z%Router._init_containers_api_endpoints:  s2      #8#F,?F()&00000s     c                    K   ddl m} |j                  |j                  d            }|||d<    | j                  dd|i| d{   S 7 w)z
        Initialize the Responses API endpoints on the router.

        GET, DELETE, CANCEL Responses API Requests encode the model_id in the response_id, this function decodes the response_id and sets the model to the model_id.
        r   )ResponsesAPIRequestUtilsresponse_idNr   r>  r   )litellm.responses.utilsr  get_model_id_from_response_idrT  r  )r6  r>  r@  r  model_ids        r   r  z$Router._init_responses_api_endpointsJ  sh      	E+IIJJ}%
 &F7O;T;; 
/

 
 	
 
s   AAAAc                 V   K   |r	d|vr||d<   d|vrd|d<    |di | d{   S 7 w)z
        Initialize the Interactions API endpoints on the router.

        GET, DELETE, CANCEL Interactions API Requests don't need model-based routing,
        so we call the original function directly with the custom_llm_provider.
        r  geminiNr   r   r  s       r   r  z'Router._init_interactions_api_endpoints`  sE      #8#F,?F() .,4F()&00000r   rQ  c                    K   |E| j                   .| j                   d   }|j                  | j                   d          nt        d       |d||d| d{   S 7 w)z@Internal helper function to pass through the assistants endpointNr  r   z'custom_llm_provider' must be set. Either via:
 `Router(assistants_config={'custom_llm_provider': ..})` 
or
 `router.arun_thread(custom_llm_provider=..)`r|  r   )r   r   r6  )r6  r>  r  rQ  r@  s        r   r  z0Router._pass_through_assistants_endpoint_factorys  s      &%%1&*&<&<=R&S#d445EFG s  ' 
 3F
FL
 
 	
 
s   AAAAr:  r  r  c	                   K   t        j                  dt        j                                 |}	d}
|j	                  d      }d}|du s||| |	d|}d|vr| j
                  |d<   d|vrd	|d<   	 t        j                  d
       t        |      }|r)|j                  ||d       t        |i | d{   }|S t        |t        j                        r|@| j                  ||      }||	|j                  ||d       t        |i | d{   }|S dj                  |||      }t        j                  dj                  |             |xj                  dj                  |      z  c_        nt        |t        j                         r|@| j                  ||      }||	|j                  ||d       t        |i | d{   }|S dj                  |||      }t        j                  dj                  |             |xj                  dj                  |      z  c_        ||t        j                  d|        t#        |t%        t&        |            \  }
}|

|||   d   }
|
Dt        j                  d| d|        t)        |	d      r|	xj                  d| d| z  c_        |	|j                  |
|d       t        |i | d{   }|S t)        |	d      rW|	xj                  dj                  ||
      z  c_        t3        |      d	kD  r$|	xj                  dj                  |      z  c_        |	7 7 97 7 t# t*        $ ru}t-        |      }t        j.                  dj                  t'        |      t        j                         t1        | |       d{  7               t'        |      }Y d}~d}~ww xY ww)zD
        Common utilities for async_function_with_fallbacks
        	TracebackNr   rU  T)litellm_routerr  r   fallback_depthr   zTrying to fallback b/w models)r   )fallback_model_grouporiginal_model_group)r   r  zmodel={}. context_window_fallbacks={}. fallbacks={}.

Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContextWindowExceededError'. No context_window_fallback set. Defaulting                         to fallbacks, if available.{})msgz
{}zmodel={}. content_policy_fallback={}. fallbacks={}.

Set 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting                         to fallbacks, if available.{}zinside model fallbacks: r   z7No fallback model group found for original model_group=z. Fallbacks=rW  zlitellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}
{}

Debug Information:
Cooldown Deployments={}r  r  z<. Received Model Group={}
Available Model Group Fallbacks={}z
Error doing the fallback: {})r   r7  r  r  rT  r   r,  r@   r   rB   r   r   ContextWindowExceededError(_get_fallback_model_group_from_fallbacksr-  rW  ri  rA   r   r   r  r6  r   r  r=   r  )r6  r:  r  r   r   r   r  rA  r@  r  r  r  fallback_failure_exception_strrr  is_non_standard_fallback_formatrD  #context_window_fallback_model_grouperror_message#content_policy_fallback_model_groupgeneric_fallback_idxnew_exceptionr  s                         r   r  z1Router.async_function_with_fallbacks_common_utils  s     	##i	0D0D0F/G$HI#.4jj.A)+&$(<(DG #"4
 
 ,.,0,>,>L)</-.L)*L	@!&&'FG /R#/+ /##090D "4""" 
  !W??@+7 EE":$/ F  8 ;B00 ''4W4H &8&&&  H $O %m  %t  %t#%=y%M *..77=v)8 II}!==IAwBBC+7 EE":$/ F  8 ;B00 ''4W4H &8&&&  H $O %l  %s  %s#%=y%M *..77=v)8 II}!==I$)@%++.Fyk,RS -' $S+ 6(( (/4H4T+45I+J3+O('/)..QR]Q^^jktjuv 19=*228op{o|  }I  JS  IT  7U  U2,,##0D0D "4""" 
   %y1&&*i*p*p$+ & 12Q6"**4;;6* ! W2 H Z  	@@H!'' k  r  r&((*I04)9  	 .1-?*	@s   A-O0AM 5M6M ;O<AM M
M OB2M 
MM OD
M MM !A&OM 
M M M 	OAO-N0.OOOOc                   K   |j                  d      }|j                  dd      }|j                  d| j                        }|j                  d| j                        }|j                  d| j                        }|j                  dd      }	 | j                  |||||	       | | j                  |i |d|i d{   }	n | j                  |i | d{   }	t        j                  t        j                        rt        j                  d
|	        t        |	d      }	|	S 7 i7 P# t        $ r+}
| j                  |
|||||||       d{  7  cY d}
~
S d}
~
ww xY ww)z
        Try calling the function_with_retries
        If it fails after num_retries, fall back to another model group
        r   r  Fr   r   r   mock_timeoutN)r@  r  r   r   r   zAsync Response: r   )rD  attempted_fallbacks)rT  r_  r   r   r   _handle_mock_testing_fallbacksasync_function_with_retriesr   isEnabledForr   r   r7  r0   r6  r  )r6  rA  r@  r  r  r   r   r   r  rD  r:  s              r   r  z$Router.async_function_with_fallbacksH  s     &,ZZ%8,2JJ7JE,R$*JJ{DNN$K	39::&(E(E4
  4:::&(E(E4
  zz.$7 	//'#)A)A 0  '!A!A!A"#"2>"  "B!A!A4!R6!RR$11'--@%++.>xj,IJ7!$%H O S  
	HH!((	 	 	 	
	sg   B
E#1D, >D(?D, D*AD, 'E#(D, *D, ,	E 5EEEE E#E  E#c                    t        j                  |      }|j                  ,|j                  du rt        j                  |dd| d|       |j
                  ,|j
                  du rt        j                  |dd| d|       |j                  -|j                  du rt        j                  |dd| d|       yy)	a  
        Helper function to raise a litellm Error for mock testing purposes.

        Raises:
            litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params
            litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params
            litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params
        NTrU  #This is a mock exception for model=z#, to trigger a fallback. Fallbacks=)r   rX  rW  zF, to trigger a fallback.                     Context_Window_Fallbacks=zF, to trigger a fallback.                     Context_Policy_Fallbacks=)	r\   from_kwargsmock_testing_fallbacksr   InternalServerErrormock_testing_context_fallbacksr  %mock_testing_content_policy_fallbacksri  )r6  r@  r  r   r   r   mock_testing_paramss          r   r  z%Router._handle_mock_testing_fallbacks|  s     6AA&I66B#::dB--!=k]Jmnwmxy   >>J#BBdJ44!=k] K..F-GI   EEQ#IITQ55!=k] K..F-GI  R Rr   c                   K   t        j                  d       |j                  d      }|j                  d| j                        }t	        |      }|j                  d| j
                        }|j                  d| j                        }|j                  d| j                        }|j                  d      }	|j                  d      }
|j                  d	|j                  d
            xs i }d|v rFt        |d   t              r3| j                  |d         }||j                  dt        |      i       t        j                  d| d|
        d|d<   |
|d<   	 | j                  |	|        | j                  |g|i | d {   }t!        |dd       }|S 7 # t"        $ r}d }|}t%        |dd       }|t        |t&              r|}
	 | j)                  |j                  d      xs d|       d {  7  \  }}d}| j*                  |G|	xs$ |j                  d      xs |j                  d      }t-        |||| j*                        }||}
d}|s| j/                  ||||||       |
|d<   |
dkD  r| j1                  ||      }n t        j                  d|
        | j3                  ||
|
||      }t5        j6                  |       d {  7   t9        |
      D ]/  }	 |dz   |d<   |
|d<    | j                  |g|i | d {  7  }t;        j<                  |      r| d {  7  }t!        ||dz   |
      }|c cY d }~S # t"        $ r}|}| j1                  ||      }|
|z
  dz
  }|j                  d      }| | j)                  ||       d {  7  \  }}ng }|s(	 | j/                  ||||||       n# t"        $ r |w xY w| j3                  |||
||      }t5        j6                  |       d {  7   Y d }~*d }~ww xY w t?        |      t@        jB                  v r#tE        |d|
       ||dz   n|
}tE        |d|       |d }~ww xY ww)Nz#Inside async function with retries.r>  r   r   r   r   r   r   r  r   r  r  model_group_sizez/async function w/ retries: original_function - z, num_retries - r   attempted_retriesr   )r  r@  )rD  r  r   rU  r  Fr"  r  r   r   T)r  r  all_deploymentsr   regular_fallbacksr   )r@  r:  z#Retrying request with num_retries: )r:  remaining_retriesr   r  r  ry   )#r   r7  r_  r   r   r   r   r   rT  r   r   r  r   r  %_handle_mock_testing_rate_limit_error	make_callr1   r6  r  r  r  r   "_get_num_retries_from_retry_policyshould_retry_this_error	log_retry_time_to_sleep_before_retryr  r  rk  r!   is_async_callabler   r   LITELLM_EXCEPTION_TYPESr  )r6  rA  r@  r>  r   r  r   r   r   r  r   	_metadatar   rD  r:  current_attemptr  deployment_num_retriesr  _all_deployments_retry_policy_applies_model_group_for_retry_policy_retry_policy_retriesr   r  _modelrn  _timeoutactual_retries_attempteds                                r   r  z"Router.async_function_with_retries  sI    ##$IJ"JJ':;JJ{DNN;	<VD#)::&(E(E$
  $*::&(E(E$
  $*::&(E(E$
  &,ZZ%8jj/ !**%7J9OPVTV	I%*Y}5Ms*S,,	-8P,QJ%  "4c*o!FG##=>O=PP`al`mn	
 *+	%&  		
b	%66' 7  ,T^^,=OOOOH4!QDH O	 P
  X	%"O!"%,Qt%D"%1j&7 5 99jj)/R!1 :   $  %*!  ,0H0T
  V9==#?V6::gCV . )K0 =-E!%!2!2	)% )4"7K,0) ),,(<$4-E&/-E -  (3Im$ Qv9KL!''5k]C ::$"-'$8 0 ; K --,,,#(#5>25Dq5HI12/:Im,%3T^^4E%W%WPV%WWWH(::  *2>><!)*9A*=$/ H
 $O  ,2 *+& "^^6Q^?F(3o(E(I%,2JJw,?F) #'"E"E"(-= #F #  0 02, 1
$ 88&'4H0@9Q2;9Q 9   ) $"#G$  $??*;$/,@(8  @  H "--111Y,2' $6B &'7+J+JJ*M;G ,;+FOa'K ) *M;ST$$qX	%s   EQ!,F FF Q!F 
Q(AQ7G:8CQKQ+&MLM1L42M	QQQ!	PAP N#
!P/OPO	1PP
P
QPAQQQ!c                    K   |j                  d      } ||i |}t        j                  |      st        j                  |      r
| d{   }| j                  ||       d{   }|S 7 !7 w)z^
        Handler for making a call to the .completion()/.embeddings()/etc. functions.
        r   N)rD  r  )rT  r!   r  r?  isawaitableset_response_headers)r6  r>  rA  r@  r  rD  s         r   r  zRouter.make_callq  s}      jj)$d5f5..x8G<O<O=
 &~H22; 3 
 
  &
s$   A	A1A-A1&A/'A1/A1c                 @   |j                  dd      }| j                  |      }d}|:t        |      dk(  r,t        t        t
           |d   d   j                  d            }|:|du r5t        j                  d	|        t        j                  |d
d| d|      yy)z
        Helper function to raise a mock litellm.RateLimitError error for testing purposes.

        Raises:
            litellm.RateLimitError error when `mock_testing_rate_limit_error=True` passed in request params
        mock_testing_rate_limit_errorNr  ry   r   r   r   TzTlitellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=rU  r  z , to trigger a rate limit error.)r   rX  rW  r   )r_  r  r  r   r   r  rT  r   r,  r   RateLimitError)r6  r@  r  r  available_modelsr   s         r   r  z,Router._handle_mock_testing_rate_limit_error  s     9?

+T9
%  ..+.F%)'C0@,AQ,F/23CDHHWK
 *5-5!&&fgrfst ((!=k]Jjk'	  6 6r   r  r  r  r  c                 8   d}|t        |t              rt        |      }d}|t        |t              rt        |      }t        |t        j                        r||t        |t        j
                        r||t        |dd      }	|	t        j                  |	      s|	dvr|t        |t        j                        r|t        |t        j                        r|dk  r|t        |      dkD  r|t        |t        j                        r	 |dk  r||dk  r|y)au  
        1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
        2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None

        2. raise an exception for RateLimitError if
            - there are no fallbacks
            - there are no healthy deployments in the same model group
        r   Nstatus_code)i  i  ry   T)r   r  r  r   r  ri  r  _should_retryNotFoundErrorr
  r  AuthenticationError)
r6  r  r  r  r   r   r  _num_healthy_deployments_num_all_deploymentsr  s
             r   r  zRouter.should_retry_this_error  s*   " $% *z:Mt/T'*+>'?$ &:ot+L#&#7  ug@@A(4K ugAAB(4Ke]D9"7+@+@+M*,eW223KeV223(A-%1)*Q.eV778
 %) $q(Kr   c                 4    t        | j                  g|i |S )z~
        Sync wrapper for async_function_with_fallbacks

        Wrapped to reduce code duplication and prevent bugs.
        )r   r  )r6  rA  r@  s      r   rC  zRouter.function_with_fallbacks  s     "$"D"DVtVvVVr   c                 l    |yd}|D ])  }t        |j                               d   |k(  s#||   } |S  |S )a@  
        Returns the list of fallback models to use for a given model group

        If no fallback model group is found, returns None

        Example:
            fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}]
            model_group = "gpt-3.5-turbo"
            returns: ["gpt-4"]
        Nr   )r  keys)r6  r   r  r  r  s        r   r  z/Router._get_fallback_model_group_from_fallbacks  sQ     48DDIIK #{2'+K'8$##	  $#r   c                     | j                   y| j                   D ]C  }t        |t              sd|v s|d   }t        |t              s/t	        |      dkD  s>|d   c S  y)zX
        Returns the first model from the default_fallbacks list, if it exists.
        Nr   r   )r   r   r+  r  r  )r6  fallbackdefault_lists      r   _get_first_default_fallbackz"Router._get_first_default_fallback  s]     >>!H(D)cXo'}lD1c,6G!6K'?*	 '
 r   r  c                    |t        |      dk(  rn!|t        |t              rt        |      dkD  ryd}t        |d      r,t        |j                  d      r|j                  j
                  }t        |d      r|j                  }|%t        j                  |||| j                        }|S t        j                  ||| j                        }|S )	z
        Calculate back-off, then retry

        It should instantly retry only when:
            1. there are healthy deployments in the same model group
            2. there are fallbacks for the completion call
        Nry   r   rD  headerslitellm_response_headers)r  r   response_headersmin_timeout)r  r   r  )
r  r   r  r  rD  r  r  r   _calculate_retry_afterr   )r6  r:  r  r   r  r  r  r   s           r   r  z"Router._time_to_sleep_before_retry  s    " &3+?1+D+.5'(1,481j!gajj)&D zz11101 99'44"3'!1 ,,	G  44"3' ,,G r   c                   K   ddl m} 	 |j                  dd      }|t        d      |d   j                  d      y|d   d   j                  dd      }|d   d   j                  d	d      }|d   j                  d
i       xs i }	|	j                  dd      }
||
yt	        |
t
              rt        |
      }
| j                  |
      }|y| j                  ||      }|j                  dd      }|j                  dd      }|j                  j                  }|j                  j                  }|j                  dd      }|j                  dd      }t        | |
       ||	||||yt        |      }|j                  dd      }t               }|j                  d      }t         j"                  j$                  j'                  |
||      }g }|j)                   |||t*        j,                  j$                               t         j.                  j$                  j'                  |
||      }|j)                   ||dt*        j,                  j$                               | j0                  j3                  ||       d{    |S 7 # t4        $ r7}t7        j8                  dj'                  t        |                   Y d}~yd}~ww xY ww)zG
        Track remaining tpm/rpm quota for model in model_list
        r   )RedisPipelineIncrementOperationr|  Nzstandard_logging_object is Noner   r   r,  r  r-  r/  r  r,  received_model_nametpmrpmr  deployment_idtotal_tokens%H-%Mr/  current_minuter   )r)  increment_valuer   ry   )increment_listr  zOlitellm.router.Router::deployment_callback_on_success(): Exception occured - {})litellm.types.cachingr  rT  r^  r   r  r   get_deploymentget_router_model_infor   r  r  rJ   r   rw   strftimer`   TPMr]  r-  r  r   r   RPMr   async_increment_cache_pipeliner6  r   r7  )r6  r@  completion_responser  r  r  r|  deployment_namer  r-  r/  deployment_infodeployment_model_infor  r  tpm_litellm_paramsrpm_litellm_paramstpm_model_inforpm_model_infor  r  dtr  tpm_keypipeline_operationsrpm_keyr:  s                              r   r  z%Router.deployment_callback_on_successN  s'     	Jt	HN

)4I# '. !BCC&'++J7?"()9"::"F"J"J $# %%56zBFF!4 $$4599,KQr
^^D$/&"*C(RB #'"5"5r"5"B"*,0,F,F#2,7 -G -)
 *--eT:C)--eT:C *9)G)G)K)K&)8)G)G)K)K& &;%>%>ud%KN%:%>%>ud%KN B,0"$ K*2*2&.&.#DV#L &=&A&A.RS&T
 &'!#" *--33::. ;  NP# $**3#(4'OO11 *--33::. ;  $**3#()'OO11 jj??#6%5 @   
   	!''ahhF
 	sq   K3J KAJ K/J KB J .K/DJ JJ KJ 	K-K	K	KKc                    d}|d   j                  d      nc|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }|t	        | |      }|S y)z
        Tracks the number of successes for a deployment in the current minute (using in-memory cache)

        Returns:
        - key: str - The key used to increment the cache
        - None: if no key is found
        Nr   r   r  r-  r/  r  )rT  r   r  r   rJ   )	r6  r@  r  r  r  r/  r  r-  r)  s	            r   r"  z*Router.sync_deployment_callback_on_success  s     "#''
3; !12:>BB=RVWK 0155lBGM2Jd+B"bjB$W>C(, C Jr   c                    t        j                  d       	 |j                  dd      }t        |dd      }|j                  di       }|j                  di       }t        j
                  j                  j                  |      }	|j                  d	d      }
d}|	 t        j                  j                  |	
      }|
|
dk\  r|
}n||dk\  r|}n| j                  }t        |t              r3|j                  d      }|yt        | |       t        | ||||      }|S t        j                  d       y# t        $ r}|d}~ww xY w)a`  
        2 jobs:
        - Tracks the number of failures for a deployment in the current minute (using in-memory cache)
        - Puts the deployment in cooldown if it exceeds the allowed fails / minute

        Returns:
        - True if the deployment should be put in cooldown
        - False if the deployment should not be put in cooldown
        z1Router: Entering 'deployment_callback_on_failure'r"  Nr  rU  r   r-  )r  r   )r  r   r/  Fr  r  exception_statusr  r,  time_to_cooldownzWRouter: Exiting 'deployment_callback_on_failure' without cooldown. No model_info found.)r   r7  rT  r  r   litellm_core_utilsexception_mapping_utils_get_response_headersutils&_get_retry_after_from_exception_headerr   r   r+  rI   r?   r6  )r6  r@  r  r  r  r"  r  r   _model_infoexception_headersdeployment_cooldownheader_cooldown_time_to_cooldownr  r}  r:  s                   r   r(  z%Router.deployment_callback_on_failure  sp     	##$WX9	

;5I&y-D $ZZ(8"=N(,,\2>K ' : : R R h h#, !i !
 #1"4"4_d"K"O ,")--"V"V%6 #W # #.3F!3K$7! ,A1E$3!$($6$6!+t,/:t/D ( @,0"/ 3,0%5'0,%6 %++m  	G	s$   C&D3 >D3 D3 3	E<D>>Er  c                 (  K   |d   d   j                  dd      }|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }t	        |      }	t               }
|
j                  d      }t        j                  j                  j                  |||	      }| j                  j                  |d
|	t        j                  j                         d{    y7 w)z3
        Update RPM usage for a deployment
        r   r   r,  Nr  r-  r/  r  r  ry   )r)  r]  r  r   )rT  r   r  r   r   rw   r  r`   r  r]  r-  r   async_increment_cacher   r   )r6  r@  r  r  r  r  r  r-  r/  r  r  r  r  s                r   r%  z+Router.async_deployment_callback_on_failure?  s     !!12:>BB$
 -.z:>>}dS,-11,CIr
^^D$'"*C RB<VD

 "%%++22. 3 
 jj..-%%	 / 
 	
 	
s   DD
DD)r   r  c                     t        |      S )a  
        Helper to return what the "metadata" field should be called in the request data

        - New endpoints return `litellm_metadata`
        - Old endpoints return `metadata`

        Context:
        - LiteLLM used `metadata` as an internal field for storing metadata
        - OpenAI then started using this field for their metadata
        - LiteLLM is now moving to using `litellm_metadata` for our metadata
        )r    )r6  r@  s     r   '_get_metadata_variable_name_from_kwargsz.Router._get_metadata_variable_name_from_kwargsa  s     6f==r   c                    	 d|v rdnd}t        |      j                  t        |      d}|j                         D ]W  \  }}||ddfvr|||<   ||k(  st	        |t
              s*i ||<   ||   j                         D ]  \  }}|dk7  s|||   |<    Y t        | j                        dkD  r| j                  j                  d       | j                  j                  |       | j                  ||   d<   |S # t        $ r}|d	}~ww xY w)
z
        When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
        r  r   )exception_typeexception_stringr<  r>  r     r   N)r   r   r   r  r   r+  r  r  r_  r  r6  )	r6  r@  r:  _metadata_varprevious_modelr  r  
metadata_k
metadata_vs	            r   r  zRouter.log_retryq  s+   	&8F&B"
 
 #'q'"2"2$'FN 	 ]J8KLL()N1%-'Jq$,?46N=1282G2M2M2O.
J%)::<FN1-j9 3P  4''(1,$$((+  ''77;7K7KF=!"34M 	G	s+   AC7 C7 %#C7 	A-C7 7	D DDr  r  c                     |}| j                   j                  ||d      }|#d}| j                   j                  ||dd       |S |dz  }| j                   j                  ||d       |S )zf
        Update deployment rpm for that minute

        Returns:
        - int: request count
        T)r)  r  
local_onlyry   r   )r)  r]  r$  r   )r)  r]  r$  )r   	get_cache	set_cache)r6  r  r  r  request_counts        r   _update_usagezRouter._update_usage  s      

,,*:t - 
  MJJ  =Tr !   QMJJ  =T !  r   c                 n    | j                   y| j                   D ]  }t        |t              sd|v s y y)NFr   T)r   r   r+  )r6  r  s     r   _has_default_fallbackszRouter._has_default_fallbacks  s6    >>!H(D)(? ' r   rD  c                    |j                   r5t        |j                         dkD  r|j                   d   j                  dk7  ry|j                  d| j                        }|2d}|D ](  }t        |j                               d   |k(  s#||   } n |y| j                         ryt        j                  dj                  ||             y)z
        Determines if a content policy error should be raised.

        Only raised if a fallback is available.

        Else, original response is returned.
        r   content_filterFr   NTzyContent Policy Error occurred. No available fallbacks. Returning original response. model={}, content_policy_fallbacks={})choicesr  finish_reasonrT  r   r  r  r*  r   r7  r-  )r6  r   rD  r@  r   r  r  s          r   rh  z)Router._should_raise_content_policy_error  s     H$4$4 5 9"004DD#)::&(E(E$
 
 $/#' 0		$Q'50+/;( 1
 $/((*## H  O  O/	

 r   c                     g }	 | j                  |      \  }}t        |t              rg S 	 t	        | |      }g }|D ]  }|d   d   |v r|j                  |         ||fS # t        $ r Y Aw xY w)Nr  r  r-  r/  )#_common_checks_available_deploymentr   r+  r6  r>   r  )r6  r   r  r  rn  unhealthy_deploymentsr  r,  s           r   _get_healthy_deploymentszRouter._get_healthy_deployments  s    !#	"&"J"J #K #A *D1	 2
 !:$(;K!
 %'*J,'-1FF#**:6	 + #$444  		s   &A" "	A.-A.c                   K   g }	 | j                  |      \  }}t        |t              rg |fS 	 t	        | |       d{   }t        |      }g }|D ]  }|d   d   |vs|j                  |         ||fS # t        $ r Y Tw xY w7 Gw)z
        Returns Tuple of:
        - Tuple[List[Dict], List[Dict]]:
            1. healthy_deployments: list of healthy deployments
            2. all_deployments: list of all deployments
        r  r  Nr-  r/  )r0  r   r+  r6  r<   setr  )	r6  r   r  r  rn  r1  unhealthy_deployments_setr  r,  s	            r   r  z%Router._async_get_healthy_deployments  s      "$	"&"J"J #K #A *D1+++ 2
 'F$(;K'
 !
 %((=$>!$&*J,'-5NN#**:6 + #$444  		!
s9   B
(A9 B
 B B
"B
9	BB
BB
c                 r    t         j                  D ]$  }t        |t              s|j	                  |       & y)a  
        Mimics 'async_routing_strategy_pre_call_checks'

        Ensures consistent update rpm implementation for 'usage-based-routing-v2'

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        N)r   rN  r   r   r*  )r6  r,  r+  s      r   rg  z'Router.routing_strategy_pre_call_checks  s,     !**I)\2((4 +r   r  c           
      |  K   t         j                  D ].  }t        |t              s	 |j	                  ||       d{    0 y7 # t         j
                  $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          t!        | |j"                  ||d   d   | j$                         |d}~wt&        $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          |d}~ww xY ww)O  
        For usage-based-routing-v2, enables running rpm checks before the call is made, inside the semaphore.

        -> makes the calls concurrency-safe, when rpm limits are set for a deployment

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        Nr"  traceback_exceptionr  rK  rA  r-  r/  r
  )r   rN  r   r   async_pre_call_checkr  r  r  async_failure_handlerr  r  r  r`  ra  failure_handlerrc  r?   r  r   r6  )r6  r,  r  r  r+  r:  s         r   r  z-Router.async_routing_strategy_pre_call_checks%  s}    " !**I)\2(#88EUVVV + W-- ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'-04)*+,#-l#;D#A)-);); G  ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'GsF   $F<AAAF<AF9B<DF9"BF44F99F<rI  c           
        K   |}t         j                  D ]2  }t        |t              s	 |j	                  |||||       d{   }4 |S 7 # t
        $ r}	|t        j                  |j                  |	t        j                         t        j                                      t        j                  |j                  |	t        j                         f      j                          |	d}	~	ww xY ww)r8  r   r  r<  rI  r  Nr9  r;  )r   rN  r   r   async_filter_deploymentsr6  r  r  r=  r  r  r  r`  ra  r>  rc  )
r6  r   r  r<  r  rI  r  returned_healthy_deploymentsr+  r:  s
             r   !async_callback_filter_deploymentsz(Router.async_callback_filter_deploymentsb  s     ( (;$ **I)\2'@@"'0L%-+9-= A   1 +8 ,+1 ! ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'Gs;   &C1AAAC1A	C.BC))C..C1r   c                    |g}|j                         D ]  \  }}t        |t              r|j                  |       nOt        |t              r%|j                  t        j                  |             n|j                  t        |             t        |t              r|j                  |       t        |t              r%|j                  t        j                  |             |j                  t        |              dj                  |      }t        j                  |j                               }|j                         S )z
        Helper function to consistently generate the same id for a deployment

        - create a string from all the litellm params
        - hash
        - use hash as id
        rU  )r  r   r   r  r+  rU  dumpsjoinhashlibsha256encode	hexdigest)r6  r  r   partsr  r  
concat_strhash_objects           r   r2  zRouter._generate_model_id  s     "((*DAq!S!QAt$TZZ]+SV$!S!QAt$TZZ]+SV$ + WWU^
nnZ%6%6%89$$&&r   r  _model_name_litellm_paramsr  c           
      $   	 t        di |}t        di ||||d}t        j                  j	                         D ]0  }|j
                  j                  |      |j
                  |   ||<   2 |j                  j                  }|t        j                  ||i       |j
                  j                  }|j
                  j                  |j
                  j                  dz   |z   }t        j                  j	                         }	|j                         D 
ci c]  \  }
}|
|	vs|
| }}
}t        j                  ||i       | j                  |      dur3t        j                   d|j"                   d|j                  d	           y|j
                  j                  d
      xs g }|D ]  }	 t%        j&                  |        | j-                  |      }|j/                  d      }| j1                  ||j                  j                         |S c c}}
w # t$        j(                  $ r$}t+        d|j"                   d|d|       |d}~ww xY w# t2        $ r1}| j4                  rt        j6                  d| d       Y d}~y|d}~ww xY w)a^  
        Create a deployment object and add it to the model list

        If the deployment is not active for the current environment, it is ignored

        Returns:
        - Deployment: The deployment object
        - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
        r1  N
model_costr  rN  TzIgnoring deployment z% as it is not active for environment supported_environments	tag_regexz&Invalid regex in tag_regex for model 'z': u    — r   r   r  zError creating deployment: 1, ignoring and continuing with other deployments.r   )r[   rX   rh   model_fieldsr  r   rT  r-  r/  r   register_modelr   r  r  $deployment_is_active_for_environmentr   r   rl  recompiler  r^  _add_deploymentto_json _add_model_to_list_and_index_mapr6  r   r"  )r6  r  rN  rO  r  r   r,  fieldr  _custom_pricing_fieldsr  r  _shared_model_info
_tag_regexpatternexcr   r:  s                     r   _create_deploymentzRouter._create_deployment  s    P	-;-No-NN# !&-&	J 4@@EEG,,007C)3)B)B5)IK& H
 ",,//H#&& +  %3399K((<<H--AACG+U  &@%L%L%Q%Q%S"!,!2!2!4"!4AAW8W1!4  " ""!3 99Z9P &--*:+@+@*AAfgqg|g|  ~V  hW  gX  Y  $2266{CIrJ%JJw' & ---DJ&&D&9E11j&;&;&>&> 2  M"2 xx $@AVAV@WWZ"+U3%1   	..%//1!4ef 	si   AI B=I H%H*A!I $I 1HAI I I.III 	J%J
J

Jc                 t    |j                   j                  d      ry|j                   j                  d      ryy)z
        Check if the deployment is an auto-router deployment (semantic router).

        Returns True if the litellm_params model starts with "auto_router/"
        but NOT "auto_router/complexity_router" (which uses complexity routing).
        auto_router/complexity_routerFzauto_router/Tr   r  r6  r   s     r   _is_auto_router_deploymentz!Router._is_auto_router_deployment  s6     **+JK**>:r   c                    ddl m} |j                  j                  }|j                  j                  }||t        d      |j                  j                  }|t        d      |j                  j                  }|t        d       ||j                  |||||       }|j                  | j                  v rt        d|j                   d	      || j                  |j                  <   y)
z
        Initialize the auto-router deployment.

        This will initialize the auto-router and add it to the auto-routers dictionary.
        r   )r|   Nzzauto_router_config_path or auto_router_config is required for auto-router deployments. Please set it in the litellm_paramszfauto_router_default_model is required for auto-router deployments. Please set it in the litellm_paramszhauto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params)rl  auto_router_config_pathauto_router_configdefault_modelembedding_modelr  zAuto-router deployment 3 already exists. Please use a different model name.)
/litellm.router_strategy.auto_router.auto_routerr|   r   rl  rm  r^  auto_router_default_modelauto_router_embedding_modelrl  r  )r6  r,  r|   rl  rm  rn  ro  autor_routers           r   init_auto_router_deploymentz"Router.init_auto_router_deployment#  s!    	O %%== 	  -7,E,E,X,X"*/A/I M  %%?? 	  x  %%AA 	 "z  $.!,,$;1'+$($
   D$5$55)**?*?)@@st  4@*//0r   c                 <    |j                   j                  d      ryy)z
        Check if the deployment is a complexity-router deployment.

        Returns True if the litellm_params model starts with "auto_router/complexity_router"
        rg  TFrh  ri  s     r    _is_complexity_router_deploymentz'Router._is_complexity_router_deploymentR  s     **+JKr   c                    ddl m} |j                  j                  }|j                  j                  }|8|r6|j                  di       }|j                  d      xs |j                  d      }|t        d       ||j                  || |      }|j                  | j                  v rt        d	|j                   d
      || j                  |j                  <   y)z
        Initialize the complexity-router deployment.

        This will initialize the complexity-router and add it to the complexity-routers dictionary.
        r   r~   NtiersMEDIUMSIMPLEzcomplexity_router_default_model is required for complexity-router deployments, or configure tiers in complexity_router_config. Please set it in the litellm_params)rl  rn  r  complexity_router_configzComplexity-router deployment rp  )	;litellm.router_strategy.complexity_router.complexity_routerr   r   r|  complexity_router_default_modelrT  r^  rl  r  )r6  r,  r   r|  rn  ry  complexity_routers          r   !init_complexity_router_deploymentz(Router.init_complexity_router_deployment\  s    	
 %%>> 	! %%EE 	
  %=,00"=E!IIh/F599X3FM f 
 /?!,,'$(%=	/
   D$;$;;/
0E0E/FFyz  :K
 5 56r   c           	      P   |j                   d|j                   vs|j                   d   yt        d      }|t        d      |t        vrt        dt         d|       |j                   d   D ]"  }|t        vst        dt         d| d	|        ||j                   d   v ryy
)a  
        Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments

        Requires `LITELLM_ENVIRONMENT` to be set in .env. Valid values for environment:
            - development
            - staging
            - production

        Raises:
        - ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values
        - ValueError: If supported_environments is not set in model_info or not one of the valid values
        rS  TLITELLM_ENVIRONMENT)secret_namezPSet 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .envz#LITELLM_ENVIRONMENT must be one of z. but set as: z&supported_environments must be one of z for deployment: F)r-  r   r^  rR   )r6  r,  litellm_environment_envs       r   rY  z+Router.deployment_is_active_for_environment  s     !!)'z/D/DD$$%=>F,9NO&b  &@@56P5QQ_`s_tu  ))*BCD55 <=W<XXfgkfll}  I  ~J  K  D *"7"78P"QQr   c                 >   t        j                  |      }g | _        i | _        i | _        | j                          | j                          |D ]
  }|j                  d      }|j                  d      }t        |t              rI|j                         D ]6  \  }}t        |t              s|j                  d      s)t        |      ||<   8 |j                  di       }d|vr| j                  ||      }	|	|d<   |j                  dd       8t        |d   t               r%|d   D ]  }
|
|d<   | j#                  ||||        | j#                  ||||        t%        j&                  d| j)                                 |D ch c]  }|d   	 c}| _        y c c}w )	Nrl  r   os.environ/r-  r/  organization)r  rN  rO  r  z
Initialized Model List )r2  r3  r   r  r  "_invalidate_model_group_info_cache_invalidate_access_groups_cacher_  r   r+  r  r   r  rv   r2  rT  r  re  r   r7  get_model_namesr   )r6  r   original_model_listr   rN  rO  r  r  r  _idorgr9  s               r   r  zRouter.set_model_list  s   "mmJ702-02-//1,,. )E))L1K#ii(89O/40+113DAq!!S)all=.I-7]* 4 !&		, ;K ;&--k?K$'D!"">48D/J +>:C69ON3++(-$/(7$/	 ,  ; ''$) +$3 +	 ( 9 )F 	##'(<(<(>'?@	
 6@@ZAlOZ@@s   Fc                 ^   dd l }|j                  j                  }d}d|v r(|j                  d      d   }|t        j
                  v rd}|r	|}d }d }d }	nt	        j                  |j                  j                  |j                  j                  dd             \  }}}}	|t        j                  vr#t        j                  |      st        d|       | j                  j                  |j                  j                         |j                  j                  (t        |dd       t        |d      |j                  _        |j                  j                   (t        |d	d       t        |d	      |j                  _        d
|j"                  v r{| j$                  j'                  |j"                  |j)                  d             |j*                  j,                  r/| j.                  j                  |j*                  j,                         |j*                  j                  d      }
|j*                  j                  d      }|
Z|Xd
|v rT|
| j0                  vrt3               | j0                  |
<   | j0                  |
   j'                  ||j)                  d             |j                  j                  dg       xs g }|D ]i  }|j                  di       }dD ]P  }||v s||   j5                  d      s||   j7                  dd      }|j8                  j                  |d      ||<   R k |(| j;                  |||j                  j                         | j=                  |j                        r| j?                  |       | jA                  |j                        r| jC                  |       |S )Nr   Fr  Tr  r   r  zUnsupported provider - r  r  r   r   team_idteam_public_model_namedataSources
parameters)r  r)  r  rU  )r,  r  r   )r   rN  )"osr   r   r  r   r  rt   rT  provider_listr&   existsr6  r   r  r  r  r  rl  r   add_patternr]  r-  r/  r   r   rz   r  r  environ'_initialize_deployment_for_pass_throughrj  ru  rw  r  )r6  r,  r  r  is_prompt_management_modelr  r  r  rp  rB  _team_id_team_public_model_namedata_sourcesdata_sourcer   	param_keyenv_names                    r   r\  zRouter._add_deployment  s    #1177%*"-"/"5"5c":1"="g&W&WW-1*% #F"&"OH (( //55$.$=$=$A$A)4%# $7+@+@@,334GH"9:M9N OPP 	$$Z%>%>%D%DE
 %%))1
E40<,3J,FJ%%) %%))1
E40<,3J,FJ%%) *''' ++%%z'9'9t'9'L $$''44;;J<Q<Q<T<TU((,,Y7","7"7";";<T"U '3..t8886H6J))(3%%h/;;');););)N
 "0044]BGM2'K __\26F0	&6)+<+G+G+V%i088KH(*

x(DF9%	 1 ( *88%$7 //55 9  ***:S:S*T,,
,C
 00%44 1 
 22j2Ir   c                    |j                   j                  du rSddlm} |j                   j                  *t        j                  |j                   j                        }ni }|dk(  r|j                  d      xs |j                   j                  }|j                  d      xs |j                   j                  }|j                  d      xs |j                   j                  }||t        d	      |j                  |||
       nf|j                  d      xs |j                   j                  }	|j                  d      xs |j                   j                  }
|j                  ||	|
       	 y)a*  
        Optional: Initialize deployment for pass-through endpoints if `deployment.litellm_params.use_in_pass_through` is True

        Each provider uses diff .env vars for pass-through endpoints, this helper uses the deployment credentials to set the .env vars for pass-through endpoints
        Tr   )passthrough_endpoint_routerN	vertex_aivertex_projectvertex_locationvertex_credentialsz]vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints.)
project_idlocationr  rB  r1  )r  rB  r1  )r   use_in_pass_through>litellm.proxy.pass_through_endpoints.llm_passthrough_endpointsr  rF  r"   get_credential_valuesrT  r  r  r  r^  add_vertex_credentialsrB  r1  set_pass_through_credentials)r6  r,  r  r   r  credential_valuesr  r  r  rB  r1  s              r   r  z.Router._initialize_deployment_for_pass_throughf  s    $$88D@ ((@@L$6$L$L--EE%! %'!"k1%))*:; @!00?? 
 &))*;< A!00@@  
 &))*>? D!00CC #
 ")_-D$w  ,BB-,'9 C  &))*5 :!0099 
 &)))4 9!0088  ,HH(;%# I 
 r   c                 ^   |j                   j                  }|r| j                  |      ry|j                  d      }| j	                  |       |j                   j                  }|z|j                   j                  d      }t        j                  j                         D ]%  }|j                  j                  |      }|!|||<   ' t        j                  ||i       | j                  ||j                   j                         | j                  j                  |j                          |S )z
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added deployment
        - OR None (if deployment already exists)
        NTr   rN  rQ  rU  )r-  r/  rf  r]  r\  r.  rh   rW  r  r   rT  r   rX  r^  r   addrl  )r6  r,  _deployment_model_id_deploymentr5  _model_info_dictr_  field_values           r   add_deploymentzRouter.add_deployment  s#     *4477D$5$56J$K !((d(;
3 )),,	 %/%:%:%E%ESW%E%X3@@EEG(77;;EB*.9$U+ H ""y:J.KL 	--
(=(=(@(@ 	. 	
 	Z223r   r  removal_idxc                    | j                   j                         D ]  \  }}||kD  s|dz
  | j                   |<    || j                   v r| j                   |= t        | j                  j                               D ]y  \  }}||v r|j	                  |       g }|D ]-  }||kD  r|j                  |dz
         |j                  |       / t        |      dkD  r|| j                  |<   m| j                  |= { y)a"  
        Helper method to update deployment indices after a deployment has been removed from model_list.

        Parameters:
        - model_id: str - the id of the deployment that was removed
        - removal_idx: int - the index where the deployment was removed from model_list
        ry   r   N)r  r  r  r  rC  r  r  )r6  r  r  r  rc  rl  indicesupdated_indicess           r   (_update_deployment_indices_after_removalz/Router._update_deployment_indices_after_removal  s     #'"G"G"M"M"OM3[ GJQw55mD #P t<<<55h? $((M(M(S(S(U#VJg%{+ !O$#**373#**3/	  ?#a'DS55jA99*E# $Wr   c                    t        | j                        }| j                  j                  |       | j                          | j	                          ||| j
                  |<   n6|j                  di       j                  d      || j
                  |d   d   <   |j                  d      }|r<|| j                  vrg | j                  |<   | j                  |   j                  |       yy)a!  
        Helper method to add a model to the model_list and update both indices.

        Parameters:
        - model: dict - the model to add to the list
        - model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"]
        Nr-  r/  rl  )r  r   r  r  r  r  rT  r  )r6  r   r  rc  rl  s        r   r^  z'Router._add_model_to_list_and_index_map  s     $//"u%//1,,. >AD11(;YY|R(,,T2>ORD11%2Ed2KL YY|,
!F!FFDF55jA11*=DDSI r   c                 0   	 |j                   j                  xs d}| j                  |      }||j                  |j                  k(  ryd}|j                   j                  }| j                  }||v rU||   }|N| j
                  j                  |       | j                          | j                          | j                  ||       | j                  |       |S # t        $ r1}| j                  rt        j                  d| d       Y d}~y|d}~ww xY w)z
        Add or update deployment
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added/updated deployment
        rU  r  Nr  r  rN  zError upserting deployment: rV  )r-  r/  r  r   r  r   r_  r  r  r  r  r6  r   r   r7  )r6  r,  r  _deployment_on_routerr  r  deployment_fast_mappingr:  s           r   r3  zRouter.upsert_deployment  s/   (	#-#8#8#;#;#Ar :>:M:M- ;N ;! %0,,0E0T0TT .2 * 5 5 8 8*.*O*O' $;;"9-"HK".++K8??A<<>EE%2 F 
 :6 	..%++21#5fg 	s%   AC 
BC 	D$%DDDr/  c                    d}|| j                   v r| j                   |   }	 |P| j                  j                  |      }| j                          | j	                          | j                  ||       |S y# t        $ r Y yw xY w)z
        Parameters:
        - id: str - the id of the deployment to be deleted

        Returns:
        - The deleted deployment
        - OR None (if deleted deployment not found)
        Nr  )r  r   r_  r  r  r  r6  )r6  r/  deployment_idxr  s       r   delete_deploymentzRouter.delete_deployment@  s     666!BB2FN	)**>:779446==^ >   		s   AA4 4	B ?B c                     || j                   v rn| j                   |   }| j                  |   }t        |t              rt	        di |S t        |t              r|S t        dj                  t        |                  y)zl
        Returns -> Deployment or None

        Raise Exception -> if model found in invalid format
        zModel invalid format - {}Nr   )r  r   r   r+  rX   r6  r-  r   )r6  r  rc  r   s       r   r  zRouter.get_deployment\  su     t<<<77ACOOC(E%&!*E**E:. ; B B4; OPPr   c                     | j                  |      }|yt        di |j                  j                  d      j                  d      S )zE
        Returns -> dict of credentials for a given model id
        r  NTr   r   )r  rV   r   r.  )r6  r  r,  s      r   get_deployment_credentialsz!Router.get_deployment_credentialso  sU     ((((;
& 
''222E

*$*
'	(r   model_group_namec                    || j                   v rs| j                   |   }|rb| j                  |d      }t        |t              rt	        di |S t        |t              r|S t        dj                  t        |                  y)z
        Returns -> Deployment or None

        Raise Exception -> if model found in invalid format

        Optimized with O(1) index lookup instead of O(n) linear scan.
        r   zModel Name invalid - {}Nr   )r  r   r   r+  rX   r6  r-  r   )r6  r  r  r   s       r   "get_deployment_by_model_group_namez)Router.get_deployment_by_model_group_namez  s}     tDDD;;<LMG
3eT*%...z2 L#$=$D$DT%[$QRRr   c                    | j                  |      }|| j                  |      }|T| j                  j                  |      xs g }|r3|d   }t	        |t
              rt        di |}nt	        |t              r|}|yt        di |j                  j                  d      j                  d      }|j                  j                  {t        j                  |j                  j                        }|s-t        j                  d|j                  j                   d       |j                  |       |j!                  d	d       |j                  j"                  r|j                  j"                  |d
<   |S d|j                  j$                  v r-|j                  j$                  j'                  d      d   |d
<   |S d|d
<   |S )a  
        Get API credentials and provider info from a model name in model_list.
        Useful for passthrough endpoints (files, batches, etc.) that need credentials.

        This method tries to find a deployment by model_id first, and if not found,
        it tries to find by model_group_name (model_name).

        Args:
            model_id: Model ID or model name from model_list (e.g., "gpt-4o-litellm")

        Returns:
            Dictionary containing api_key, api_base, custom_llm_provider, etc.
            Returns None if model not found.

        Example:
            credentials = router.get_deployment_credentials_with_provider("gpt-4o-litellm")
            # Returns: {"api_key": "sk-...", "custom_llm_provider": "openai", ...}
        r  N)r  r   Tr   zCredential 'z' not found in credential_listrF  r  r  r
  r   )r  r  r   router   r+  rX   rV   r   r.  rF  r"   r  r   r   r   r_  r  r   r  )r6  r  r,  potential_wildcard_modelsdeployment_dictcredentialsr  s          r   (get_deployment_credentials_with_providerz/Router.get_deployment_credentials_with_provider  s   , ((((;
 @@!) A J
 (,(;(;(A(A((K(Qr%(";A">ot4!+!>o!>J<!0J . 
''222E

*$*
' 	
 $$<<H 2 H H))AA! %%--":#<#<#T#T"UUst 01OO5t< $$88 ))== %  J--3331;1J1J1P1P1V1V22K-.  2:K-.r   rX   r  c                      y r   r   r6  r,  r  r/  s       r   r  zRouter.get_router_model_info  s     	r   c                      y r   r   r  s       r   r  zRouter.get_router_model_info  s     	r   c           	         || j                  |      }||}|t        d      |j                  d      xs i j                  dd      }|%|j                  d      xs i j                  dd      }|}|j                  d      }t        |t              r|}ngt        |t
              rd|v rt	        di |}nGt        dt        |      j                   d	|j                  d      xs i j                  d
d             t        j                  |j                  |      \  }	}
}}|
dk(  r|t        j                  d|	 d       n|
dk7  r|	}d|v r| j                  j                  |      }|w|D ]r  }	 |j                  d      xs i j                  d
      |j                  d      xs i j                  d
      k(  r&|j                  d      xs i j                  d      } nt ||	}|j!                  dj#                  |
            sdj#                  |
|      }n|}t        j$                  |      }|j                  d      xs i }||j'                  t)        t*        |             |S # t        $ r Y w xY w)a  
        For a given model id, return the model info (max tokens, input cost, output cost, etc.).

        Augment litellm info with additional params set in `model_info`.

        For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.

        Returns
        - ModelInfo - If found -> typed dict with max tokens, input cost, etc.

        Raises:
        - ValueError -> If model is not mapped yet
        Nr  zDeployment not foundr-  
base_modelr   r   z.Deployment missing valid litellm_params. Got: z, deployment_id: r/  unknownr   r   azurez Could not identify azure model 'z'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-modelsr   z{}/z{}/{}r  r   )r  r^  rT  r   r[   r+  r   r   r   rt   r   r   r  r   r  r6  r  r-  get_model_infor   r   rk   )r6  r,  r  r/  r  r  r   litellm_params_datar   r  r  rn  potential_modelspotential_modelmodel_info_namer-  user_model_infos                    r   r  zRouter.get_router_model_info  s   & >--r-:K&(
344 !nn\28b==lDQ
$..)9:@bEEdJ  )nn-=>)>:0N+T2wBU7U+B.ABN01::; <"#->>,#?#E2"J"J4QZ"[!\^ 
 -4,D,D &&)-
)#Q ')j.@!''26(  ;Z  [ !G+Ee|#'#6#6#<#<=P#Q #/+;	! / 3 3L A GRLLTR *| < B!c$i ( %4$7$78H$I$OR"%#g, !& !& ( ,< = E-@ AB%nn-@%HO#O++/B
 %..6<"!d9o>?-  ) ! !s   4A-I,,	I98I9c                 \    || j                   v r| j                   |   }| j                  |   S y)a%  
        For a given model id, return the model info

        Returns
        - dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
        - None: could not find deployment in list

        Optimized with O(1) index lookup instead of O(n) linear scan.
        N)r  r   )r6  r/  rc  s      r   r  zRouter.get_model_infoP  s4     66677;C??3''r   c                 Z    | j                  |      }|y|d   }| j                  |      S )zT
        Return list of all models in the same model group as that model id
        )r/  Nrl  r  )r  r  )r6  r/  r-  rl  s       r   get_model_groupzRouter.get_model_group`  s>    
 ((B(/
-
""j"99r   c           	         ddl m} d}d}d}	 t        j                  j	                  |      }	 t        j                  |      }	 |I|j	                  dd      }|5t        j                  |      }||xs i } |t        t        |      |      }|7|5t        t         |t        t        |      j                         |            }|S ||}|S # t
        $ r Y w xY w# t
        $ r Y w xY w# t
        $ r Y hw xY w)z
        For a given model id, return the model info

        1. Check if model_id is in model info
        2. If not, check if litellm model name is in model info
        3. If not, return None
        r   )_update_dictionaryNr  r  )litellm.utilsr  r   rR  rT  r6  r  r   r+  rk   r2  )	r6  r  rl  r  r-  custom_model_infolitellm_model_name_model_infor  base_model_infos	            r   get_deployment_model_infoz Router.get_deployment_model_infol  s6    	5*.
,0=A%	 ' 2 2 6 6x @	,3,B,B,T)
	 ,.22<F
)&-&<&<:&NO&2,=,C),> 7--) (-J-V"<=BBD%J  +66JI  		
  		"  		s5   C C AC- 	CC	C*)C*-	C98C9user_facing_model_group_namec                 	   d}d}d}d}| j                  |      }|y|D ]'  }d}	d|v r|d   |k(  rd}	n!d|v r| j                  j                  |      d}	|	s9t        di |d   }
|
j                  }|j                  di       }|j                  di       }d}||j                  dd      }||j                  dd      }||j                  dd      }d}||j                  d	d      }||j                  d	d      }||j                  d	d      }	 |j                  d
d      }|| j                  ||
j                        }nd}d\  }}	 t        j                  |
j                  |
j                        \  }}}}|Tt        j$                  ||      }|g }|j                  di       }|j                  dd      }t'        |ddddd|||d
      }|t)        di ||gd|}nG||j*                  vr|j*                  j-                  |       |j                  dd      -|d   (|j.                  |d   |j.                  kD  r
|d   |_        |j                  dd      -|d   (|j0                  |d   |j0                  kD  r
|d   |_        |j                  dd      (|j2                  |d   |j2                  kD  r
|d   |_        |j                  dd      (|j4                  |d   |j4                  kD  r
|d   |_        |j                  dd      	 |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_         |j                  dd      |d   
|d   |_!        |j                  dd      ||j                  d      }|j                  d	d      ||j                  d	      }|	|d}||z  }||d}||z  }* ||||_"        |||_#        |||_        |S # t        $ r d}Y 0w xY w# t        j                  j                  $ r8}t        j                  dj!                  t#        |                   Y d}~Nd}~ww xY w) z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info
        Nr  Frl  Tr   r-  r  r  r/  )r  rl  )rU  rU  r  z.litellm.router.py::get_model_group_info() - {}moder  r   )
r)  
max_tokensmax_input_tokensmax_output_tokensinput_cost_per_tokenoutput_cost_per_tokenlitellm_providerr  supported_openai_paramssupports_system_messages)r  	providersr  r  r  r  "supports_parallel_function_callingsupports_visionsupports_function_callingsupports_web_searchsupports_url_contextsupports_reasoningr  r   )$r  r   r  r[   #configurable_clientside_auth_paramsrT  r  r   r6  r   rt   r  
exceptionsBadRequestErrorr   r  r-  r   get_supported_openai_paramsModelMapInfor]   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )r6  r  r  model_group_info	total_tpm	total_rpmr  r   r   is_matchr   model_litellm_paramsmodel_info_dict_deployment_tpm_deployment_rpmr  r-  r  rX  rn  r:  r  db_model_infor  s                           r   _set_model_group_infozRouter._set_model_group_info  s<    6:#'	#'	SW+((K(@
EH%%*=*L%''--k:F+Fe4D.EFN BB 0
 $)99-=r#B #iib9O .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B	"*..tT:'!%!?!?!)n6J6J "@ "J "&J
 +1'M<4;4L4L(..(6(J(J51|Q !*1*M*M'\+' +2.0+ !&		, ;$((8)##%)&*)**+%1,C-1
  '#1 $'C&2^ %$   '7'A'AA$..55lCNN#5t<H"#56B(99A%&89*;;< 9CCU8V$5NN#6=I"#67C(::B%&9:*<<= :DDW9X$6>>"8$?K$99A!"89&;;< =G.=$9 >>"94@L$::B!"9:&<<= >H/>$: NN#GN "#GHDPJN$GNN#4d;G"#45=7;$4NN#>EQ"#>?4GAE$>NN#8$?K"#89TA;?$8NN#94@L"#9:dB<@$9 NN#7>J"#78D@:>$7NN#<dCO"#<=I?I1@$< >>%.:?V&0nnU&;O>>%.:?V&0nnU&;O*$ !I_,	*$ !I_,	K  L '$'0 $$'0 $ 3>7 !D  A  "!
" %%55 %++DKKCPQFS s*   4Q*0Q<*Q98Q9<S-SSc                     || j                   v rT| j                   |   }t        |t              r|}nt        |t              r|d   du ry|d   }ny| j	                  ||      S | j	                  ||      S )z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info or hidden model group
        hiddenTNr   )r  r  )r   r   r   r+  r  )r6  r  r  _router_model_groups       r   get_model_group_infozRouter.get_model_group_info  s     $000))+6D$$&*#D$'>T)*.w-'--/-8 .   ))#+ * 
 	
r   c                   K   t               }|j                  d      }g }g }| j                  |      }|y|D ]  }|j                  di       j                  d      }|d   j                  d      }	||	=|j	                  t
        j                  j                  j                  ||	|	             |j	                  t
        j                  j                  j                  ||	|	              ||z   }
| j                  j                  |

       d{   }|y|dt        |       }|t        |      d }d}|!|D ]  }t        |t              s|d}||z  } d}|!|D ]  }t        |t              s|d}||z  } ||fS 7 qw)z
        Returns current tpm/rpm usage for model group

        Parameters:
        - model_group: str - the received model name from the user (can be a wildcard route).

        Returns:
        - usage: Tuple[tpm, rpm]
        r  r  Nr  r-  r/  r   r   )r/  r   r  )r  r   )rw   r  r  rT  r  r`   r  r]  r-  r  r   async_batch_get_cacher  r   r  )r6  r  r  r  tpm_keysrpm_keysr   r   r/  r  combined_tpm_rpm_keyscombined_tpm_rpm_valuestpm_usage_listrpm_usage_list	tpm_usager  	rpm_usages                    r   get_model_group_usagezRouter.get_model_group_usage  s     
 ! ((K(@
E %		, ; ? ? EB+01A+B+F+F,M z]2OO##))00'#1 1  OO##))00'#1 1   * !)8 3(,

(H(H& )I )
 #
 #*)@3x=)Q)@X)Q $(	%#a% ($%	NI	 $ $(	%#a% ($%	NI	 $
 )##3#
s   DFF<F$F5F)maxsizec                 $    | j                  |      S )z
        Cached version of get_model_group_info, uses @lru_cache wrapper

        This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request
        )r  )r6  r  s     r   _cached_get_model_group_infoz#Router._cached_get_model_group_info  s     ((55r   c                 <  K   | j                  |      }||j                  |j                  }nd }||j                  |j                  }nd }||i S | j                  |       d {   \  }}i }|||xs dz
  |d<   ||d<   |||xs dz
  |d<   ||d<   |S 7 1w)Nr   x-ratelimit-remaining-tokenszx-ratelimit-limit-tokensx-ratelimit-remaining-requestszx-ratelimit-limit-requests)r  r  r  r  )r6  r  r  	tpm_limit	rpm_limitcurrent_tpmcurrent_rpmreturned_dicts           r   get_remaining_model_group_usagez&Router.get_remaining_model_group_usage  s     <<[I',<,@,@,L(,,II',<,@,@,L(,,II!2I)-)C)CK)P#P [ <E q=M89 9BM45 >G q?M:; ;DM67 $Qs   A&B(B)2Bc                 ~  K   t        |t              rt        |d      rt        |j                  t              r|j                  j                  di        ||j                  d   d<   |j                  d   }d|vr?d|vr;|9| j                  |       d{   }|j                         D ]  \  }}|	|||<    |S 7 &w)a  
        Add the most accurate rate limit headers for a given model response.

        ## TODO: add model group rate limit headers
        # - if healthy_deployments > 1, return model group rate limit headers
        # - else return the model's rate limit headers
        r  r  zx-litellm-model-groupr  r  N)r   r   r  r  r+  r  r#  r  )r6  rD  r  r  remaining_usageheaderr]  s          r   r  zRouter.set_response_headers  s      x+"23822D9##../CRH  ##$89' "*!8!89M!N /6HH4<NN+(,(L(L) # &5%:%:%<MFE(5:*62 &= #s   BB=B;B=2
B=c                     | j                   j                          t        |      D ]T  \  }}|j                  d      }|s|| j                   vrg | j                   |<   | j                   |   j	                  |       V y)z
        Build model_name -> deployment indices mapping for O(1) lookups.

        This index allows us to find all deployments for a given model_name in O(1) time
        instead of O(n) linear scan through the entire model_list.
        rl  N)r  clearrj  rT  r  )r6  r   rc  r   rl  s        r   _build_model_name_indexzRouter._build_model_name_indexC  sq     	--335#J/JC<0JT%J%JJHJD99*E55jAHHM 0r   c                 v   g | _         | j                          | j                          t        |      D ]  \  }}|j	                  di       }|j	                  d      }|G|j	                  dd      }|j	                  di       }| j                  ||      }d|vri |d<   ||d   d<   | j                  ||        y)z
        Build model index from model list to enable O(1) lookups immediately.
        This is called during initialization to avoid the race condition where
        requests arrive before model_id_to_deployment_index_map is populated.
        r-  r/  Nrl  rU  r   rU  )r   r  r  rj  rT  r2  r^  )r6  r   rn  r   r-  r  rl  r   s           r   '_build_model_id_to_deployment_index_mapz.Router._build_model_id_to_deployment_index_mapS  s     //1,,.!*-HAu<4J!~~d+H "YY|R8
!&+;R!@22:~Nu,*,E,',4l#D)111Q .r   exclude_team_modelsc                    g }|p|| j                   v r`| j                   |   }|D ]L  }| j                  |   }d|v sd|d   v s|r|d   j                  d      r6|j                  |d   d          N |S | j                  j                         D ]U  }| j                  |   }| j                  |   }d|v s&d|d   v s.|r|d   j                  d      rE|j                  |       W |S )z
        if 'model_name' is none, returns all.

        Returns list of model id's.

        Optimized with O(1) or O(k) index lookup when model_name provided,
        instead of O(n) linear scan.
        r-  r/  r  )r  r   rT  r  r  r  )r6  rl  r,  idsr  rc  r   r  s           r   get_model_idszRouter.get_model_idsn  s    !TBBB??
K"C OOC0E#u,|9L1L.53F3J3J93U$

5#6t#<= #" 
 !AAFFH;;HE,5(TU<5H-H*u\/B/F/Fy/Q JJx( I 
r   candidate_idc                     || j                   v S )a  
        O(1) membership check for a deployment ID without allocating large lists.

        Note: Call sites may pass a variable named `model` when it actually
        contains a deployment ID. This helper expects the deployment ID string.

        Uses the existing `model_id_to_deployment_index_map` which is kept
        in sync by `_build_model_id_to_deployment_index_map` and model-list
        mutation helpers.
        )r  )r6  r0  s     r   rf  zRouter.has_model_id  s     tDDDDr   c                 p   |sy|| j                   v s| j                  |      r|S | j                  d      }|sy|D ]x  }|j                  di       }|j                  d      }||k(  xs2 |xr |j	                  d|       xs |xr |j	                  d|       }|sb|j                  d      }|sv|c S  y)a  
        Resolve model_name from model_id.

        This method attempts to find the correct model_name to use with the router
        so that litellm_params can be automatically injected from the model config.

        Strategy:
        1. First, check if model_id directly matches a model_name or deployment ID
        2. If not, search through router's model_list to find a match by litellm_params.model
        3. Return the model_name if found, None otherwise

        Args:
            model_id: The model_id extracted from decoded video_id
                     (could be model_name or litellm_params.model value)

        Returns:
            model_name if found, None otherwise. If None, the request will fall through
            to normal flow using environment variables.
        Nr  r   r   r  :rl  )r   rf  r  rT  endswith)r6  r  
all_modelsr,  r   actual_modelmatchesrl  s           r    resolve_model_name_from_model_idz'Router.resolve_model_name_from_model_id  s    ,  t'''4+<+<X+FO ((D(9
$J'^^,<bAN)--g6L
 ( L J\%:%:Qxj>%JL J\%:%:Qxj>%J  '^^L9
%% %$ r   team_model_namer  c                     | j                  ||      }|sy|D ]9  }|j                  di       j                  d      |k(  s(|j                  d      c S  y)z
        Map a team model name to a team-specific model name.

        Returns:
        - deployment id: str - the deployment id of the team-specific model
        - None: if no team-specific model name is found
        rl  r  Nr-  r  rl  )r  rT  )r6  r9  r  rZ  r   s        r   map_team_modelzRouter.map_team_model  s[     $$$QEyyr*..y9WDyy.. 
 r   c                 |    |/|d   j                  d      |k(  r||d   j                  d      k(  ry|	|d   |k(  ryy)zU
        Get the team-specific model name if team_id matches the deployment.
        r-  r  r  Trl  Fr
  )r6  rl  r   r  s       r   should_include_deploymentz Router.should_include_deployment  sV     l#''	2g=eL1556NOO#l(;z(Ir   model_aliasc                    g }|| j                   v rv| j                   |   }|D ]`  }| j                  |   }| j                  |||      s'|'|j                         }||d<   |j	                  |       P|j	                  |       b |S |lt        | j                        D ]T  \  }}| j                  |||      s|'|j                         }||d<   |j	                  |       D|j	                  |       V |S )z
        Return all deployments of a model name

        Used for accurate 'get_model_list'.

        if team_id specified, only return team-specific models

        Optimized with O(1) index lookup instead of O(n) linear scan.
        )rl  r   r  rl  )r  r   r>  r2  r  rj  )	r6  rl  r?  r  returned_modelsr  rc  r   alias_models	            r   _get_all_deploymentszRouter._get_all_deployments  s    68 >>>;;JGG ,11) 2  #. ',jjl4?L1'..{;'..u5 : !   (8
U11) 2  #.&+jjl4?L1'..{;'..u5 9 r   c                    | j                         xs g }g }|D ]m  }|j                  d      }| j                  |      r(| j                  ||      }|s;|j	                  |       M|j	                  |j                  dd             o |S )u  
        Returns all possible model names for the router, including models defined via model_group_alias.

        If a team_id is provided, only deployments configured with that team_id (i.e. team‐specific models)
        will yield their team public name.
        r-  )r,  r  rl  rU  )r  rT  _is_team_specific_model_get_team_specific_modelr  )r6  r  deploymentsr   r,  r-  r9  s          r   r  zRouter.get_model_names,   s     ))+1r%J#5J++J7"&"?"?)7 #@ # #&&7"":>>,#CD & r   c                 ~    |j                  d      xs i }|y||j                  d      k(  r|j                  d      S y)a  
        Get the team-specific model name if team_id matches the deployment.

        Args:
            deployment: DeploymentTypedDict - The model deployment
            team_id: Optional[str] - If passed, will return router models set with a `team_id` matching the passed `team_id`.

        Returns:
            str: The `team_public_model_name` if team_id matches
            None: If team_id doesn't match or no team info exists
        r-  Nr  r  r
  )r6  r,  r  r-  s       r   rF  zRouter._get_team_specific_modelC   sD     &0^^L%A%GR
jnnY//>>":;;r   r-  c                 >    t        |xr |j                  d            S )z
        Check if model info contains team-specific configuration.

        Args:
            model_info: Model information dictionary

        Returns:
            bool: True if model has team-specific configuration
        r  )r  rT  )r6  r-  s     r   rE  zRouter._is_team_specific_modelX   s     J<:>>)#<==r   c                 l   g }|#|| j                   vr|S || j                   |   fg}n#t        | j                   j                               }|D ]d  \  }}t        |t              r|}n*t        |t
              rt        di |}|d   du r<|d   }nC|j                  | j                  ||             f |S )z
        Helper function to get model list from model alias.

        Used by `.get_model_list` to get model list from model alias.
        r
  Tr   )rl  r?  r   )	r   r  r  r   r   r+  rb   rh  rC  )r6  rl  rA  alias_itemsr?  model_value_router_model_name_model_values           r   get_model_list_from_model_aliasz&Router.get_model_list_from_model_aliasd   s     68!!7!77&&&(>(>z(JKLKt55;;=>K(3$K+s+*5"K.8G;G)T1)5g)>&""))1{ *  )4$ r   c                    g }|"|j                  | j                  ||             |j                  | j                  |             t        |      dk(  r| j                  j                  |      xs g }|A|| j                  v r3| j                  |   j                  |      xs g }|j                  |       |*|(|D ]#  }t        di |}||d<   |j                  |       % ||| j                  z  }|S )z
        Includes router model_group_alias'es as well

        if team_id specified, returns matching team-specific models
        r;  r  r   rl  r   )
rh  rC  rO  r  r   r  r   rY   r  r   )r6  rl  r  rA  r  #potential_team_only_wildcard_modelsr9  deployment_typed_dicts           r   r  zRouter.get_model_list   s     68!""))Z)Q 	00J0G	
 1$(,(;(;(A(A*(M(SQS% "w$2K2K'K--g6<<ZHNB 4 *001TU%*C*O2A,?,D!,D):D),7#**+@A 3
 t.Or   c                 8    | j                   j                          y)zInvalidate the cached model group info.

        Call this whenever self.model_list is modified to ensure the cache is rebuilt.
        N)r  cache_clearru  s    r   r  z)Router._invalidate_model_group_info_cache   s    
 	))557r   c                     d| _         y)zInvalidate the cached access groups.

        Call this whenever self.model_list is modified to ensure the cache is rebuilt.
        N)r  ru  s    r   r  z&Router._invalidate_access_groups_cache   s    
 %)!r   model_access_groupc                    |du xr
 |du xr |du }|r| j                   | j                   S ddlm}  |t              }| j	                  ||      }|rq|D ]l  }|j                  d      }	|	s|	j                  dg       xs g D ]=  }
| |
|k(  s|d   }||
   j                  |       %|d   }||
   j                  |       ? n |rt        |      | _         | j                   S |S )a  
        If model_name is provided, only return access groups for that model.

        Parameters:
        - model_name: Optional[str] - the received model name from the user (can be a wildcard route). If set, will only return access groups for that model.
        - model_access_group: Optional[str] - the received model access group from the user. If set, will only return models for that access group.
        - team_id: Optional[str] - the team id, to resolve team-specific models
        Nr   r   r;  r-  r  rl  )r  collectionsr   r  r  rT  r  r+  )r6  rl  rV  r  
_use_cacher   r  r   r9  r  groups              r   get_model_access_groupszRouter.get_model_access_groups   s
     $Q#5#=Q'T/ 	
 $33?,,,+#D)((J(P
eeL1!,"!E!K!K-9$(::-.|_
 -e 4 ; ;J G)*<J)%077
C "L   (,](;D%,,,r   c                     | j                  |      }t        |      dk(  ry|j                  |g       }|D ]   }| j                  j	                  |        y y)zG
        Return True if model access group is a wildcard route
        )rV  r   Fr  T)r[  r  rT  r   r  )r6  rV  r  rZ  r   s        r   )_is_model_access_group_for_wildcard_routez0Router._is_model_access_group_for_wildcard_route   sm     441 5 
 }"""#5r:E""(((7C 
 r   c                     t        |       }i }g d}|D ]K  }||v r||   ||<   |dk(  s| j                  dk(  s%| j                  j                  j	                         ||<   M |S )a  
        Get router settings method, returns a dictionary of the settings and their values.
        For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after
        )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )varsr   rd  r\  rU  )r6  	_all_vars_settings_to_returnvars_to_includevars        r   get_settingszRouter.get_settings	!  s}    
 J	 
  #Ci+4S>#C(..))-DD+/+D+D+Q+Q+V+V+X#C( # #"r   c                    g d}g d}| j                         }|D ]  }||v rg||v rt        ||         }t        | ||       '|dk(  r1|d   ||   k7  r&| j                  ||   |j	                  di              t        | |||          nt        j                  dj                  |              t        j                  d| j                                 y)	z-
        Update the router settings.
        )r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   zSetting {} is not allowedzUpdated Router settings: N)rd  r  r  r  rT  r   r7  r-  )r6  r@  _allowed_settings_int_settings_existing_router_settingsrc  _casted_values          r   update_settingszRouter.update_settings*!  s    


 %)$5$5$7!C''-'$'s$4MD#}5 1156HIVTW[X22-3C[28** 73 3  D#vc{3%++,G,N,Ns,ST' ( 	##&?@Q@Q@S?T$UVr   c                 |   |d   d   }t        |      }|dk(  rhdj                  |      }| j                  j                  |d|      }|5t	        j
                  | |       | j                  j                  |d|      }|S |dk(  r]|j                  d	      du r%| d
}| j                  j                  |d|      }|S | d}| j                  j                  |d|      }|S |j                  d	      du r$| d}| j                  j                  ||      }|S | d}| j                  j                  ||      }|S )a  
        Returns the appropriate client based on the given deployment, kwargs, and client_type.

        Parameters:
            deployment (dict): The deployment dictionary containing the clients.
            kwargs (dict): The keyword arguments passed to the function.
            client_type (str): The type of client to return.

        Returns:
            The appropriate client based on the given client_type and kwargs.
        r-  r/  r  z{}_max_parallel_requests_clientTr)  r$  r  )r  r   rS  ry  _stream_async_client_async_client_stream_client)r)  r  _client)r   r-  r   r%  r5    set_max_parallel_requests_clientrT  )r6  r,  r@  r  r  r  	cache_keyrQ  s           r   re  zRouter._get_client]!  s    l+D1+LV+T119@@JIZZ))$AQ * F ~%FF,0
 --!dEU .  MG#zz(#t+'j(<=	--!dEU .  'j6	--!dEU .  zz(#t+'j7	--!4D .  'j0	--!4D .  r   c           	      f	   t        j                  d|        t        |      }t               }	 t	        j
                  |      }d}	d}
d}t        |      }t               }|j                  d      }| d| }| j                  j                  |d	|
      xs i }t        |      D ]  \  }}|j!                  di       }|j!                  di       }d}	 |j!                  dd      }||j!                  dd      }| j#                  ||      }|xs |j!                  dd      }t%        |t&              rZ|j!                  dd      Ht%        |d   t(              r5||d   kD  r-|j+                  |       d	}	|
dj                  ||d   |      z  }
|j!                  dd      }| j                  j                  |d	|
      xs d}t%        |t&              r| j.                  dk7  rv|j!                  |d      ||<   t1        |||         }t%        |t&              rB|j!                  dd      0t%        |d   t(              r|d   |k  r|j+                  |       d	}|M|j!                  d      <|j!                  d      }|)t3        t5        d!i ||      s|j+                  |       |t        j6                  du s|xs |}t	        j8                  |t5        d!i |      \  }}}}t	        j:                  ||      }|]t        j<                  j?                  |      } dg}!| jA                         D ]8  \  }"}#|"|vs|"|!v st        j                  d|"        |j+                  |       :  tC        |      tC        |      k(  r<	 |d	u rtE        |      |	d	u r't	        jF                  dj                  |
      |d       tC        |      dkD  r!t        |      D $%cg c]  \  }$}%|$|vr|% }}$}%tC        |      dkD  rt        j<                  jI                  |      }|S # t        $ r9}t        j                  dj                  t        |                   |cY d}~S d}~ww xY w# t        $ r8}t        j,                  dj                  t        |                   Y d}~d}~ww xY wc c}%}$w )"a  
        Filter out model in model group, if:

        - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
        - filter models above rpm limits
        - if region given, filter out models not in that region / unknown region
        - [TODO] function call and model doesn't support function calling
        z2Starting Pre-call checks for deployments in model=)r<  zllitellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {}NFrU  r  z:rpm:Trl  r   r-  r  r  r   r  z%Model={}, Max Input Tokens={}, Got={}zAn error occurs - {}r/  r   r   r  allowed_model_region)r   rt  r  r  )passed_paramsresponse_formatz1INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=r  z~litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.
{}rV  r   )%r   r7  r  r4  r   token_counterr6  r  r-  r   r   rw   r  r   r%  rj  rT  r  r   r+  r  r  r"  r   maxrx   r[   drop_paramsrt   r  r  get_non_default_paramsr  r  rd   r  _get_order_filtered_deployments)&r6  r   r  r<  rI  _returned_deploymentsinvalid_model_indicesinput_tokensr:  _context_window_error_potential_error_str_rate_limit_errorr  r  r  r  model_group_cacherc  r,  rO  r  _deployment_modelr  r-  r  current_request_cache_localcurrent_requestrt  _dep_model_for_paramsr  rn  r  r  special_paramsr  r  ids&                                         r   _pre_call_checkszRouter._pre_call_checks!  sp     	##@H	
 !%%8 9 #	)"00(CL !&!!<^L W-G5 01JJ  ?O !   	 	  ))>?OC(nn-=rBO$..r:K !%W(__\4@
%!0!4!4\4!HJ!77)u 8 
 %/$T/2E2Egt2T! z40"'94@L #:.@#A3G(:6H+II-11#604-,CJJ 1 *+= > ,, ! #tR0H 

$$ TDT %   	 ( ,d3))-EE.?.C.CHa.P!(+"%/1B81L#
 5'++E48D #?5#93?+E2oE-11#6,0)  *"&&'=>J'5'9'9:P'Q$'3,'5'H'H-A .11#6  )g.A.AU.J(9(BU% ,,/#1#DO#D)' +2*M*M/(;+'
 +2 *1)M)M&4 *N *& '8%8N 2 8 8 :1$;;^@S177"STUSV W 255c: !;a  @p $%-B)CC !D(/  '$.88 ^  e  e,  !#  $%) &&;<%<DAq11 < " % $%)$+MM$Q$Q%%! %$c  	)!''~  F  FF
 )(	)r  W%//0F0M0McRSf0UVVWX%s<   P$ B8Q)"R-$	Q&-.Q!Q&!Q&)	R*2-R%%R*c                 t    || j                   vry| j                   |   }t        |t              r|}|S |d   }|S )z
        Get the model from the alias.

        Returns:
        - str, the litellm model name
        - None, if model is not in model group alias
        Nr   )r   r   r   )r6  r   _items      r   r!  zRouter._get_model_from_aliasb"  sK     ...&&u-eS!E  'NEr   c                 X    | j                   D cg c]  }|d   d   |k(  s| c}S c c}w )z6
        Get the deployment by litellm model.
        r   r   )r   )r6  r   r9  s      r    _get_deployment_by_litellm_modelz'Router._get_deployment_by_litellm_modelu"  s2      ??T?aa0@.A'.Je.S?TTTs   ''rG  c                    d}|N|j                  d      xs i }|j                  d      xs i }|j                  d      xs |j                  d      }|du r|| j                  |      fS | j                  |      rM| j                  |      }	|	*|	j                  j
                  }
|
|	j                  d      fS t        d	| d
      | j                  |      }||}|| j                  vr| j                  j                  |      }|r||fS |3|| j                  v r%| j                  |   j                  |      }|r||fS | j                  F| j                  j                         }| j                  d   j                         |d<   ||d   d<   ||fS | j                  |      }t!        |      dk(  r| j                  |      }t#        j$                  t&        j(                        rt#        j*                  d|        t!        |      dk(  r| j-                         rB| j/                         }|r0t#        j0                  d| d| d       |}| j                  |      }t!        |      dk(  rU| j3                  |      d| dj5                  |      }nd| dj5                  |      }t7        j8                  ||d      t6        j:                  r%|t6        j:                  v rt6        j:                  |   }||fS )aF  
        Common checks for 'get_available_deployment' across sync + async call.

        If 'healthy_deployments' returned is None, this means the user chose a specific deployment

        Returns
        - str, the litellm model name
        - List, if multiple models chosen
        - Dict, if specific model chosen
        Nr   r  user_api_key_team_idTr  r  r   zBLiteLLM Router: Trying to call specific deployment, but Model ID :z does not exist in Model ID mapr   r   r  r   zinitial list of deployments: zModel 'z7' not found. Attempting to use default fallback model 'z'.zYou passed in model=z+. There is no 'model_name' with this stringz1. There are no healthy deployments for this modelrU  rV  )rT  r  rf  r  r   r   r.  r^  r!  r   r   get_deployments_by_patternr   r   r2  rC  r  r   r  r   r   r7  r*  r  r,  r  r-  r   r  model_alias_map)r6  r   r<  r  rG  rI  request_team_idr   r  r,  deployment_model_model_from_aliaspattern_deploymentsupdated_deploymentr  fallback_modelrW  s                    r   r0  z*Router._common_checks_available_deployment{"  s   & *.%%))*5;H-112DEK&ll& >!%%&<=  $&$??e?LLLu%,,e,<J%#-#<#<#B#B ')>)>D)>)QQQTUZT[[z{  !66U6C(%E((("&"5"5"P"P #Q # #111  +#t'@'@@&*&?&?#',, -  $
 ' "555 &&2%)%<%<%A%A%C"7;7N7N$8$& ##34 AF"#34W=000 #7757I"#q("&"G"Ge"G"T --gmm<!''/0C/DE "#q(**,!%!A!A!C!)..!%(_`n_ooqr +E*.*C*Cu*C*U' &'1,&&%&8@ 4UG;fgnnG !5UG;lmttG --#!#  ""u0G0G'G++E )))r   c           	        K   | j                  |||||      \  }}t        ||      }t        j                  t        j
                        rt        j                  d|        t        ||      }t        j                  t        j
                        rt        j                  d|        t        |t              r|S t        | |       d{   }t        j                  t        j
                        rt        j                  d|        | j                  ||      }| j                  |||t        t        t           |      nd||	       d{   }| j                   r,|*| j#                  |t        t        t$           |      ||
      }t'        | |||| j)                  |             d{   }t+        |      dk(  rt-        | ||       d{   }	|	|S 7 7 7 17 w)z
        Get the healthy deployments for a model.

        Returns:
        - List[Dict], if multiple models chosen
        *OR*
        - Dict, if specific model chosen
        r   r<  r  rG  rI  )r  rI  z'healthy_deployments after team filter: z-healthy_deployments after web search filter: r  Ncooldown deployments: r  cooldown_deploymentsr@  r   r  r<  rI  )r  r   rI  r  r  r   r  r   r  )r0  r8   r   r  r   r   r7  r9   r   r+  r<   _filter_cooldown_deploymentsrC  r   r   rM   r   r  r
   r/   r  r  rD   )
r6  r   rI  r<  r  rG  r  r  r  r"  s
             r   rA  z$Router.async_get_healthy_deployments"  s#    $ &*%M%M 3) &N &
"" 7 3)

 !--gmm<!''9:M9NO < 3)

 !--gmm<!''?@S?TU )40&&%D$(;K&
  
 !--gmm<!''*@AU@V(WX"?? 3!5 @ 

 %)$J$J 3:B:NT*+X6TX)- %K %
 
 &&8+?"&"7"7$(d5H$I!-	 #8 # %< $) 3#'#O#O$%
 
 "#q(A(,!1 I
 O""] 

$
sJ   CG(GBG(G"AG(2G$3#G(G&	G("G($G(&G(c           
        K   | j                   dk7  rR| j                   dk7  rC| j                   dk7  r4| j                   dk7  r%| j                   dk7  r| j                  |||||      S 	 t        |      }| j                  |||||       d{   }||j                  }|j
                  }| j                  ||||||	       d{   }t        |t              r|S |j                  d
      rt        |      dk(  r|d   S t        j                         }	| j                   dk(  r4| j                  (| j                  j                  ||||       d{   }
n| j                   dk(  r4| j                  (| j                  j                  ||||       d{   }
n| j                   dk(  r5| j                  )| j                  j                  |||||       d{   }
n`| j                   dk(  rt!        | ||      S | j                   dk(  r2| j"                  &| j"                  j                  ||       d{   }
nd}
|
t%        | ||       d{   }|t'        j(                  d| d| j+                  |
       d|        t        j                         }||	z
  }t-        j.                  | j0                  j3                  t4        j6                  |d||	|             |
S 7 V7  7 7 W7 7 7 # t8        $ r}t;        j<                         }|j|j                  dd      }|Vt?        j@                  |jB                  ||f      jE                          t-        j.                  |jG                  ||             |d}~ww xY ww)z
        Async implementation of 'get_available_deployments'.

        Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps).
        r   r   r   r   r   r  r   rI  r<  r  rG  Nr   rI  r<  r  rG  r  "_encrypted_content_affinity_pinnedry   r   r  r  r<  r  r  r  r<  r  rI  r  r  r  r  $get_available_deployment for model: , Selected deployment:  for model: 2<routing_strategy>.async_get_available_deploymentsr  r  ry  r  r  r  r{  r;  )$r   r^  r   async_pre_routing_hookr   r<  rA  r   r+  rT  r  r  rb  async_get_available_deploymentsrf  rd  r.   r   rD   r   r,  r;  r  r  r   r  rg   r  r6  r  r  r`  ra  r>  rc  r=  )r6  r   rI  r<  r  rG  r  pre_routing_hook_responser  r  r,  r"  r  r  r:  r:  r  s                    r   r  z%Router.async_get_available_deploymentX#  s2     !!%==%%)99%%)==%%)@@%%500!$7- 1  F	@P /3.I.I-!$7 /J / )% )41774== )-(J(J-!$7!1 )K ) # -t4** ""#GH+,1*1--J%%)AA,,8 22RR$),?!)#	 S    %%)==**6 00PP$),?!)#	 Q    %%)@@--9 33SS$),?!)#'5 T    &&*::%(,(;  %%5))5 //OO$),? P    "
!"E,0%5# 	
  !&&6ug=TUYUjUjkuUvTw  xD  EJ  DK  L yy{H :-I''BB(//&R%5)% C 	 ])#0&0  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G!	s   A"N%$K2 	K
8K2 K"K2 N#K2 <N=AK2 K%AK2 K(AK2 K+!K2 7N8;K2 3K.4K2 K0BK2 NK2 "K2 %K2 (K2 +K2 .K2 0K2 2	N;BM==NNc           
        K   	 t        |      }| j                  |||||       d{   }||j                  }|j                  }| j	                  ||||||       d{   }t        |t              r`|j                  di       }	|	j                  d      r|S t        j                  d|j                  di       j                  d       d	|d
      | j                  |      }
t        |
      dk(  rt        j                  d| d|d
      t        j                         }| j                  dk(  r4| j                  (| j                  j!                  ||
||       d{   }n| j                  dk(  r5| j"                  )| j"                  j!                  ||
|||       d{   }n`| j                  dk(  rt%        | |
|      S | j                  dk(  r2| j&                  &| j&                  j!                  ||
       d{   }nd}|t)        | ||       d{   }|t+        j,                  d| d| j/                  |              t        j                         }||z
  }t1        j2                  | j4                  j7                  t8        j:                  |d|||             |S 7 7 P7 Q7 7 7 # t<        $ r}t?        j@                         }|j|j                  dd      }|VtC        jD                  |jF                  ||f      jI                          t1        j2                  |jK                  ||             |d}~ww xY ww)z
        Async version of get_available_deployment_for_pass_through

        Only returns deployments configured with use_in_pass_through=True
        r  Nr  r   r  Deployment r-  r/  C does not support pass-through endpoint (use_in_pass_through=False)rU  rV  r  r   Model z has no deployments configured with use_in_pass_through=True. Please add use_in_pass_through: true to the deployment configurationr   r  r   r  r   r  r   r  r  z7async_get_available_deployment_for_pass_through model: , selected deployment: r  r  r{  r;  )&r   r  r   r<  rA  r   r+  rT  r   r   _filter_pass_through_deploymentsr  r  perf_counterr   rb  r  rd  r.   r   rD   r   r,  r;  r  r  r   r  rg   r  r6  r  r  r`  ra  r>  rc  r=  )r6  r   rI  r<  r  rG  r  r  r  r   pass_through_deploymentsr  r,  r"  r  r  r:  r:  r  s                      r   /async_get_available_deployment_for_pass_throughz6Router.async_get_available_deployment_for_pass_through#  s    E	@P /3.I.I-!$7 /J / )% )41774== )-(J(J-!$7!1 )K ) # -t4!4!8!89I2!N!%%&;<..!11"-.A.E.ElTV.W.[.[\`.a-b  cf  !g#%'  (,'L'L$7 (M ($ +,1--$UG  ,n  o!#  **,J%%)AA,,8 22RR$),D!)#	 S    %%)@@--9 33SS$),D!)#'5 T    &&*::%(,(@  %%5))5 //OO$),D P    "
!"E,0%5# 	
  !&&I%Pghlh}h}  I  iJ  hK  L ((*H :-I''BB(//&R%5)% C 	 i)#P&2  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G	s   M$J> J.8J> !J1"8J> MCJ> #J4$AJ> 'J7(!J> 	M
;J> J:J>  J<!BJ> -M.J> 1J> 4J> 7J> :J> <J> >	MBM		MMc                    K   || j                   v r+| j                   |   j                  |||||       d{   S || j                  v r+| j                  |   j                  |||||       d{   S y7 >7 w)z
        This hook is called before the routing decision is made.

        Used for the litellm auto-router to modify the request before the routing decision is made.
        r  N)r  r  r  )r6  r   rI  r<  r  rG  s         r   r  zRouter.async_pre_routing_hook$  s       D%%%**51HH-!$7 I    D+++007NN-!$7 O    )s!   5A:A68A:0A81A:8A:c                    | j                  ||||      \  }}t        |t              r|S t        |      }t	        | |      }| j                  ||      }| j                  r|| j                  ||||      }t        |      dk(  rU| j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      | j                  d
k(  r*| j                  | j                  j                  ||      }n| j                  dk(  rt!        | ||      S | j                  dk(  r+| j"                  | j"                  j                  |||      }nx| j                  dk(  r,| j$                   | j$                  j                  ||||      }n=| j                  dk(  r,| j&                   | j&                  j                  ||||      }nd}|nt)        j*                  d| d       | j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      t)        j*                  d| d| j-                  |       d|        |S )zB
        Returns the deployment based on routing strategy
        r   r<  r  rG  r  r  Nr  r   r  	model_idsr  r   r   r   cooldown_listr   r  r   r  r   r  r  rI  r   r  r   r  z, No deployment availabler  r  )r0  r   r+  r   r>   r  r   r  r  r/  r  get_min_cooldownrc   r   r   get_available_deploymentsr.   rd  r   rb  r   r,  r;  )r6  r   r<  r  rG  rI  r  r  r  r  _cooldown_time_cooldown_listr,  s                r   r^  zRouter.get_available_deployment$  s8    &*%M%M 3	 &N &
"" )40&&+L,
  9$(;K 
 #?? 3!5 @ 
 &&8+?"&"7"7$7!-	 #8 # "#q(**e*<I!00AA#6F B N 7(,?ON ','+'B'B,	    L0T5J5J5V..HH!7J I J ""&66 "$($7  !!%<<))522LL!$7- M J !!%::%%1..HH!$7!	 I J !!%==((411KK!$7!	 L J J!&&6ug=VW **e*<I!00AA#6F B N 7(,?ON ','+'B'B,	  	""25'9PQUQfQfgqQrPss  AF  @G  H	
 r   c                 4   | j                  ||||      \  }}t        |t              r`|j                  di       }|j                  d      r|S t	        j
                  d|j                  di       j                  d       d|d	      | j                  |
      }t        |      dk(  rt	        j
                  d| d|d	      t        |      }	t        | |	      }
| j                  ||
      }| j                  r|| j                  ||||      }t        |      dk(  rU| j                  |      }| j                  j                  ||	      }t        | |	      }t!        ||| j                  |      | j"                  dk(  r*| j$                  | j$                  j'                  ||      }n| j"                  dk(  rt)        | ||      S | j"                  dk(  r+| j*                  | j*                  j'                  |||      }nx| j"                  dk(  r,| j,                   | j,                  j'                  ||||      }n=| j"                  dk(  r,| j.                   | j.                  j'                  ||||      }nd}|nt1        j2                  d| d       | j                  |      }| j                  j                  ||	      }t        | |	      }t!        ||| j                  |      t1        j2                  d| d | j5                  |              |S )!a  
        Returns deployments available for pass-through endpoints (based on load balancing strategy)

        Similar to get_available_deployment, but only returns deployments with use_in_pass_through=True

        Args:
            model: Model name
            messages: Optional list of messages
            input: Optional input data
            specific_deployment: Whether to find a specific deployment
            request_kwargs: Optional request parameters

        Returns:
            Dict: Selected deployment configuration

        Raises:
            BadRequestError: If no deployment is configured with use_in_pass_through=True
            RouterRateLimitError: If no pass-through deployments are available
        r  r   r  r  r-  r/  r  rU  rV  r  r   r  z has no deployment configured with use_in_pass_through=True. Please add use_in_pass_through: true in the deployment configurationr  r  Nr  r  r  r  r   r  r   r  r   r  r   r  r   z1get_available_deployment_for_pass_through model: z, no available deploymentsr  )r0  r   r+  rT  r   r  r  r  r   r>   r  r   r  r/  r  r  rc   r   r   r  r.   rd  r   rb  r   r,  r;  )r6  r   r<  r  rG  rI  r  r   r  r  r  r  r  r  r,  s                  r   )get_available_deployment_for_pass_throughz0Router.get_available_deployment_for_pass_through3%  s   8 &*%M%M 3	 &N &
"" )400445ErJN!!"78** --)*=*A*A,PR*S*W*WX\*])^  _b  c!#  $(#H#H 3 $I $
  '(A-))   (i  j  ,M,
  9$(;K 
 $(#D#D 8!5 $E $
  &&8+?'+'<'<$<!-	 (= ($ '(A-**e*<I!00AA#6F B N 7(,?ON ','+'B'B,	    L0T5J5J5V..HH!7O I J ""&66!$($<  !!%<<))522LL!$<- M J !!%::%%1..HH!$<!	 I J !!%==((411KK!$<!	 L J J!&&CE7Jde **e*<I!00AA#6F B N 7(,?ON ','+'B'B,	  	""?wF]^b^s^st~^  ^A  B	
 r   r  c                     t        j                  t        j                        rt        j                  d|        t        |      }|D cg c]  }|d   d   |vr| c}S c c}w )a  
        Filters out the deployments currently cooling down from the list of healthy deployments

        Args:
            healthy_deployments: List of healthy deployments
            cooldown_deployments: List of model_ids cooling down. cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]

        Returns:
            List of healthy deployments
        r  r-  r/  )r   r  r   r   r7  r4  )r6  r  r  cooldown_setr,  s        r   r  z#Router._filter_cooldown_deployments%  sr     !--gmm<!''*@AU@V(WX/0 2
1
,'-\A 1
 	
 
s   A!c                     t        j                  dt        |       d       |D cg c]&  }|j                  di       j                  dd      r|( }}t        j                  dt        |       d       |S c c}w )a  
        Filter out deployments configured with use_in_pass_through=True

        Args:
            healthy_deployments: List of healthy deployments

        Returns:
            List[Dict]: Only includes a list of deployments that support pass-through
        z%Filter pass-through deployments from z healthy deploymentsr   r  FzFound z& deployments with pass-through enabled)r   r7  r  rT  )r6  r  r,  r  s       r   r  z'Router._filter_pass_through_deployments%  s     	##3C8K4L3MMab	
 2$
1
~~.3778MuU 1 	! $
 	##S1233YZ	
 ('$
s   +A8c                     	 |j                  di       j                  dd      }||| j                  ||       yyy# t        $ r+}t        j                  dt        |              Y d}~yd}~ww xY w)z7
        Tracks successful requests rpm usage.
        r-  r/  Nz$Error in _track_deployment_metrics: )rT  r(  r6  r   r  r   )r6  r,  r  rD  r  r:  s         r   r  z Router._track_deployment_metrics	&  s    		Y!~~lB7;;D$GH'&& "2 (    	Y!''*NsSTvh(WXX	Ys   8= 	A1!A,,A1c                 H    t        ||| j                  | j                        S )Nr  )r  r   r   )r6  r"  r  s      r   rC   z(Router.get_num_retries_from_retry_policy&  s)     2#%)%B%B**	
 	
r   c                    | j                   }|yt        |t        j                        r|j                  |j                  S t        |t        j
                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S yy)a  
        BadRequestErrorRetries: Optional[int] = None
        AuthenticationErrorRetries: Optional[int] = None
        TimeoutErrorRetries: Optional[int] = None
        RateLimitErrorRetries: Optional[int] = None
        ContentPolicyViolationErrorRetries: Optional[int] = None
        N)r   r   r   r  AuthenticationErrorAllowedFailsr  TimeoutErrorAllowedFailsr  RateLimitErrorAllowedFailsri  'ContentPolicyViolationErrorAllowedFailsr  BadRequestErrorAllowedFails)r6  r"  r   s      r   get_allowed_fails_from_policyz$Router.get_allowed_fails_from_policy$&  s     >B=V=V' y'"="=>$DDP'GGGy'//2$==I'@@@y'"8"89$??K'BBBy'"E"EF$LLX'OOOy'"9"9:$@@L'CCC M ;r   c                 J   ddl m} | j                  y | j                  } ||j                  dg|j                        }|| _        t        j                  j                  |       t        j                  j                  |j                         t        j                  d       y )Nr   )SlackAlertingslack)alerting_thresholdalertingdefault_webhook_urlz2[94m
Initialized Alerting for litellm.Router[0m
)1litellm.integrations.SlackAlerting.slack_alertingr  r   r  webhook_urlslack_alerting_loggerr   r  r1  r!  !response_taking_too_long_callbackr   r,  )r6  r  router_alerting_config_slack_alerting_loggers       r   r2  zRouter._initialize_alertingL&  s    S'151E1E!.5HHY 6 B B"
 &<"((==>TU((EE"DD	
 	""H	
r   CustomRoutingStrategyc                 `    t        | d|j                         t        | d|j                         y)a  
        Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router

        Use this to set your custom routing strategy

        Args:
            CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase
        r^  r  N)r  r^  r  )r6  r  s     r   set_custom_routing_strategyz"Router.set_custom_routing_strategyd&  s4     	&!::	

 	,!@@	
r   c                 N    d t         _        | j                  j                          y r   )r   r   flush_cacheru  s    r   r  zRouter.flush_cache{&  s    

 r   c                     g t         _        g t         _        g t         _        g t         _        d | _        | j                          y r   )r   r   r  r&  r#  r   r  ru  s    r   resetzRouter.reset&  s6    #% *,'#% *,' r   )F)r   r   )FFF)T)rl  r  )NNNN)NNNNN)NF)r   N)NNN)NNFN)NNF(   r   r   r   r4  r   __annotations__r   r   r  r   r  tenacityr   r(   r   r+   r   r   r   r   r	   r   ra   rY   r
   r   rU   rf   rZ   r+  rl  floatr   rb   r_   rT   r^   ri   rS   r=  staticmethodrF  r5  rP  r   r   r   rY  re   r  r3  r  r  r  r  r  r  r  r  r  r  r4  r  r/  r;  rq   ro   rE  rA  r  rb  r   rM   r  rl   rn   r  r  rj  r  r  rB  r6  rk  r*  rX   r7  r@  rd  r  rU  rX  rI  rp  rt  rv  r  r   r  r  r  r  r  r  r  rN   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  rp   r#  r!  r)  r(  rO   r-  r,  rE  r  rj   rM  rK  rV  r_  r^  rc  rk  rl  r  r  r  r  r   r  r  r#   wrapr  r  r  r  r  r  rC  r  r  r  r  r"  r(  r%  r  r  r{   r(  r*  rh  r2  r  rg  r  r  rC  r2  re  r[   rj  ru  rw  r  rY  r  r  r\  r  r  r  r^  r3  r  r  r  r  r  r  r  r  r  rk   r  r]   r  r  r  r   r   r  r#  r  r)  r+  r/  rf  r8  r<  r>  rC  r  rF  rE  rO  r  r  r  r[  r]  rd  rj  re  r  r!  r  r0  rA  r  r  r}   r  r^  r  r  r  r  rC   r  r2  rW   r  r  r  r   r   r   r   r      sU    uK&+OXd^+&11H:>h67>:>h67>MQeL(C,G&H!IJQ ;?<@=A#'$($((,"&*/ ,0*.%) #'*. 7;!06 )+)+ ',%*(,     ,0 DH&(DH48 "#/3FJ+0kY&$*+T$sCx.-AAB
Y& $$78Y& t$789Y& !&8!9:Y& C=Y& SMY& SMY& !Y&  3-!Y&" "$#Y&$ %Y&& !K
'Y&, -Y&0 #5/1Y&2 #3-3Y&6 c]7Y&8  
9Y&> %?Y&@ !AY&B !)!
CY&H (0}IY&J KY&L _-MY&N $I
OY&T UY&V #'WY&X #'YY&Z $eC!::;;<
[Y&` !%aY&b #cY&d "&eY&f gY&h +t#$
iY&n #'#
oY&t  
uY&z '
{Y&@  
AY&F $D>GY&H "&
IY&X #++@"AYY&Z  $[Y&\ !))@ A]Y&^ ".1_Y&` "*!"
aY&f *-gY&h &.d3S	>.B%CiY&j %)kY&l 
mY&v DI   D .38n.	z,,	-. ."+ +M %os&: ;MTXM^	FD
P0
dF
P
,
HB
H+MZ
"
H21$ mM(01F(GmM^4 2$(c3h$8	}11	2&cc$(c3h$8c	}11	2cJ. .`11+/91j $()9$:DKDM	 
 UZ$()9$:DKEN	 
 kp$()9$:DI'RV-Y`afYgJgDh	"M1	2  	33 '(3 	3j 8*8(08 
8 8(N>+N> tCH~&N> 	N>
 
N>`B+B tCH~&B 	B
 
BB+/9@ii$(c3h$8i	}12	3i^ 1;	

 
 !)	

 

*"04	, EOPP4<SMP	P, NR''(,'=Ec]'	'@ > >t > > >. (,	D
D
 D
  }	D

 
D
L t 4

"&
	%s
#	$
"&	%s
#	$$	4 	t 	uczAR8S 	Z#S	Z# T#s(^,d4S#X3G.HHIZ#x++$(.>)?$@+^ $(c3h$8BI$-	  SX$(c3h$8BI%.	  	M,M, tCH~&M, 	M,f di$()9$:FISZ[`Sa	 
 $()9$:FISZ[_S`	  @@ '(@ 	@D?? ? $	?
 CHo? S#X?B
X 
X 
XaBaB '(aB S#X	aBFs 3 , ,C ,\c # *Fs F3 FP( (3 (TD) DC DLT3 Ts T3 Tl3 *)C )^ $)&+#(## # 4.	#
 d^# 4.#R $)&+#(## # 4.	#
 d^# 4.#JDS D# DT $)&+#(  4.	
 d^ 4.:DS D DL
x 


5=
  $> $,:!
!
 
!
F-5>38n-0>A	c3h>OO5=ObJJ-5J` $)	 S$Y 4.	 
"6c4i 0 6 6x $(	 S$Y 4.	 
2DuS$Y'7 D DN 
	0GG 
	GT^S$Y^@	 
	>NN 
	Nd  $v}v 
	vp   
	 DNN 
	N`//l .21#1 &c]1D QB#B c"e
BN
 .21#1 &c]18 .21#1 &c]1 
#
2 .21#1 &c]1, .2(,	
#
 &c]
 %	
,}!}! $D>}! D>	}!
 #+4.}! #+4.}! c]}! }! }!~ V[[]1 1l &*$(3737-- c]- D>	-
 #+4.- #+4.-^ V[[]C% C%J $ :>!!)1#!L /3*.3737,0FF &d^F "$	F
 #+4.F #+4.F $D>FPW &*$S$s)^,-$ c]$ 
$s)		$2Xc] $ /3*.// / 	/
 &d^/ "$/ 
sEz	/f@D! 
#!FJ 
JX 
+3C= 
D>>	/	0> # # #t #J 4<TN	6 $$$1$;?$	$L5c 5Xd^ 5.55,4TN5	tDz4:%	&5<54 5( 15	;; #4.; n-	;F *.041,1, "$Z1, 4 012	1,
 #4.1, !1, n-1,f'c '4 '>`` ` 	`
 ` 
*	`D D -@j -@^~ RV +KJ +KZ&z &d &P/A /Ah*  B:$:;>:GJ:x' '8L 'R$F$F*-$F	$FN 6:JJ%-c]J	J81J 18J;O 1fC HZ,@ 8s x
/C &	(3 	(8D> 	( #	*	0KK	$sCx.	!KZ 
 	$,- ! 	
 
  58>A	  !	`U4#567` !` SM	`
 
`D $  
:# 
:(4. 
:66),6	)	6pf f >Af 	.	!f P
 
8P 
>G$G$	x}hsm+	,G$R 1266	.	!6 36 cSVh B ;?""*23-"	"HN$ N4 N R$ R8 MR""3-"EI"	c"HE E E4 4	#4lc C HSM & FJ&*5=c]	& &*!%	33 c]3 #	3
 
!	"3jx} S	 0 IM-8@	#*
>(4. 
>T 
> +/$"3-$	!	"$N JN("3-(9A#(	$*+	,(T8) %),0!%	.SM. %SM. #	.
 
c49n	.`"%	.#B1Wf4v *.M%M% "M% tCH~&	M%
 !M%^3 8C= &Uc Ud U 48,0.3)-x*x* 4S#X/0x* c4i()	x*
 &d^x* !x* 
sE$*%%	&x*| 48,0.3+/a#a# a# 4S#X/0	a#
 c4i()a# &d^a# #4.a# 
tDz4	 a#N 48,0.3aa a 4S#X/0	a
 c4i()a &d^aN 48,0.3RR R 4S#X/0	R
 c4i()R &d^Rp 48,0.3%% % 4S#X/0	%
 c4i()% &d^% 
(	)%T 48,0.3)-{{ 4S#X/0{ c4i()	{
 &d^{ !{@ 48,0.3)-aa 4S#X/0a c4i()	a
 &d^a !aF
#':
EI#Y
	d
.(#':(	d(: FJY,4TNY$ BF
"
19#
&Dy &DP
0
%>
.!r   r   )r  r2  enumrG  r?  rU  r   rZ  r`  r  r  rX  r   	functoolsr   typingr   r   r   r	   r
   r   r   r   r   r   r   r   r  httpxr
  r   pydanticr   typing_extensionsr   r   litellm.litellm_core_utils2litellm.litellm_core_utils.exception_mapping_utilsr   litellm._loggingr   litellm._uuidr   litellm.caching.cachingr   r   r   r   litellm.constantsr   "litellm.integrations.custom_loggerr   #litellm.litellm_core_utils.asyncifyr   r}  r   r    ,litellm.litellm_core_utils.coroutine_checkerr!   .litellm.litellm_core_utils.credential_accessorr"   %litellm.litellm_core_utils.dd_tracingr#   *litellm.litellm_core_utils.litellm_loggingr$   r  0litellm.litellm_core_utils.sensitive_data_maskerr%   $litellm.llms.openai_like.json_loaderr&   &litellm.router_strategy.budget_limiterr'   "litellm.router_strategy.least_busyr(   #litellm.router_strategy.lowest_costr)   &litellm.router_strategy.lowest_latencyr*   &litellm.router_strategy.lowest_tpm_rpmr+   )litellm.router_strategy.lowest_tpm_rpm_v2r,   r  r.   )litellm.router_strategy.tag_based_routingr/   /litellm.router_utils.add_retry_fallback_headersr0   r1    litellm.router_utils.batch_utilsr2   r3   r4   /litellm.router_utils.client_initalization_utilsr5   2litellm.router_utils.clientside_credential_handlerr6   r7   r@  r8   r9   #litellm.router_utils.cooldown_cacher:   &litellm.router_utils.cooldown_handlersr;   r<   r=   r>   r?   ,litellm.router_utils.fallback_event_handlersr@   rA   rB   *litellm.router_utils.get_retry_from_policyrC   r  !litellm.router_utils.handle_errorrD   rE   >litellm.router_utils.pre_call_checks.deployment_affinity_checkrF   ;litellm.router_utils.pre_call_checks.model_rate_limit_checkrG   Dlitellm.router_utils.pre_call_checks.prompt_caching_deployment_checkrH   >litellm.router_utils.router_callbacks.track_deployment_metricsrI   rJ   litellm.schedulerrK   rL   litellm.types.llms.openairM   rN   rO   rP   litellm.types.routerrQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   litellm.types.servicesrg   litellm.types.utilsrh   ri   rj   rk   r  rl   rm   rn   r  ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   &router_utils.pattern_match_deploymentsrz   opentelemetry.tracer{   _Spanrq  r|   r}   r}  r   Enumr   r   r   r   r   <module>r     s          	    #          &  ! 9 " 2   9 ; B K M 8 P P E G F H N J P A M 
 R >  
 2      0 0 
 * 9 
   G1 DDJ $)) sV sVr   