idZddlZddlmZmZmZddlmZmZmZm Z m Z m Z ddl Z ddl mZddlmZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZdd lmZm Z m!Z!ddl"m#Z#ddl"m#Z$ddl"m%Z%m&Z&dZ'GddZ(GddeZ)y)a Provider budget limiting Use this if you want to set $ budget limits for each provider. Note: This is a filter, like tag-routing. Meaning it will accept healthy deployments and then filter out deployments that have exceeded their budget limit. This means you can use this with weighted-pick, lowest-latency, simple-shuffle, routing etc Example: ``` openai: budget_limit: 0.000000000001 time_period: 1d anthropic: budget_limit: 100 time_period: 7d ``` N)datetime timedeltatimezone)AnyDictListOptionalTupleUnion)verbose_router_logger) DualCache)RedisPipelineIncrementOperation) CustomLoggerSpan)duration_in_seconds)_get_tags_from_request_kwargs)%_get_prometheus_logger_from_callbacks)AllMessageValues)DeploymentTypedDictLiteLLM_Params RouterErrors) BudgetConfig)GenericBudgetConfigTypeStandardLoggingPayloadceZdZdZdZdeeeffdZdedefdZ dedefdZ dede fd Z dded edefd Z d ZdZdZdZdefdZdeeeffdZdeeeffdZy )_LiteLLMParamsDictViewz Lightweight attribute view over `litellm_params` dict. This avoids pydantic construction in request hot-path while preserving attribute-style access used by `litellm.get_llm_provider(...)`. _paramsparamsc||_yNr)selfr s w/Users/manta/Documents/Projects/TheRoad-I1/.venv/lib/python3.12/site-packages/litellm/router_strategy/budget_limiter.py__init__z_LiteLLMParamsDictView.__init__6s  keyreturnc8|jj|Sr"rgetr#r's r$ __getattr__z"_LiteLLMParamsDictView.__getattr__9||$$r&c8|jj|Sr"r*r,s r$ __getitem__z"_LiteLLMParamsDictView.__getitem__<r.r&c||jvSr"rr,s r$ __contains__z#_LiteLLMParamsDictView.__contains__?sdll""r&Ndefaultc:|jj||Sr"r*)r#r'r3s r$r+z_LiteLLMParamsDictView.getBs||W--r&c6|jjSr")rkeysr#s r$r6z_LiteLLMParamsDictView.keysEs||  ""r&c6|jjSr")rvaluesr7s r$r9z_LiteLLMParamsDictView.valuesHs||""$$r&c6|jjSr")ritemsr7s r$r;z_LiteLLMParamsDictView.itemsKs||!!##r&c,t|jSr")iterrr7s r$__iter__z_LiteLLMParamsDictView.__iter__NDLL!!r&c,t|jSr")lenrr7s r$__len__z_LiteLLMParamsDictView.__len__Qs4<<  r&c,t|jSr"dictrr7s r$rEz_LiteLLMParamsDictView.dictTr?r&c,t|jSr"rDr7s r$ model_dumpz!_LiteLLMParamsDictView.model_dumpWr?r&r")__name__ __module__ __qualname____doc__ __slots__rstrrr%r-r0boolr2r+r6r9r;r>intrBrErGr&r$rr,sItCH~%s%s%%s%s%###.s.S.C.#%$"!!"d38n""DcN"r&rc8eZdZ d6dedeedeeeeee e e fffdZ d7de dedeee d eed eed eef d Zd ee e e fdee e e fde e efde e efdeee de e efdee d eee e e fe ffdZ d6dee e e fd ee d eee e e efe e efeee ffdZde deded efdZde de dededed ef dZde dedefdZdZdede de defd Zd!Zd"Zd#Zd$e d eefd%Zd&e d eefd'Zd(e d eefd)Z d*e d ee fd+Z!d&e d,ed-efd.Z"d&e d eefd/Z#d&e d ee fd0Z$d&e defd1Z%e& d6deedeeeeee e e fffd2Z'd3Z( d6deeeeee e e fffd4Z)d5Z*y)8RouterBudgetLimitingN dual_cacheprovider_budget_config model_listcp||_g|_tj|j ||_d|_d|_|j|j||jttjtr tjj!|yy)N)rU)rSredis_increment_operation_queueasyncio create_task(periodic_sync_in_memory_spend_with_redisrTdeployment_budget_configtag_budget_config_init_provider_budgets_init_deployment_budgets_init_tag_budgets isinstancelitellm callbackslistlogging_callback_manageradd_litellm_callback)r#rSrTrUs r$r%zRouterBudgetLimiting.__init__\s%VX,DIIKL # #LP%DH ##% %%%<   g'' .  , , A A$ G /r&modelhealthy_deploymentsmessagesrequest_kwargsparent_otel_spanr(c :Kt|tr|g}t|dk(r|Sg}|j||d{\}}} } t|dkDr|jj ||d{} | xsdgt|z} i} t |D]\}}t| |xsd| |<|j||| | | |t|\}}t|dk(r(ttjjd||S|S77w) z Filter out deployments that have exceeded their provider budget limit. Example: if deployment = openai/gpt-3.5-turbo and openai spend > openai budget limit then skip this deployment r)rgriN)r6rjri)rgprovider_configsdeployment_configsdeployment_providers spend_mappotential_deployments request_tags: )r`rErA0_async_get_cache_keys_for_router_budget_limitingrSasync_batch_get_cache enumeratefloat$_filter_out_deployments_above_budgetr ValueErrorr+no_deployments_with_provider_budget_routingvalue)r#rfrgrhrirjrr cache_keysrnrorp_current_spendscurrent_spendsrqidxr'deployment_above_budget_infos r$async_filter_deploymentsz-RouterBudgetLimiting.async_filter_deploymentstsx& )4 0#6"7  " #q (& &,.GG 3)H       z?Q $(OO$I$I!1%J%O$3#Msec*o6MN+-I%j1S!&~c':'Ac!B #2 99$7!1#5%9#&;:#1:  %,()Q. #OOUUVVXYuXvw) (& &Q s"=  F model_namelitellm_paramsrf model_infoiddeployment_spend:z+Exceeded budget for deployment model_name: z, litellm_params.model: , model_id: tag_spend:zExceeded budget for tag='z ', tag_spend=z, tag_budget_limit=)rwrTrA _get_llm_provider_for_deployment max_budgetr+budget_duration+_track_provider_remaining_budget_prometheusr[r debugr\_get_budget_config_for_tagappend)r#rrrgrnrorprqrsrr deploymentis_within_budgetrconfig current_spend debug_msg _model_name_litellm_params_litellm_model_namemodel_id_tag_tag_budget_config _tag_spends r$ryz9RouterBudgetLimiting._filter_out_deployments_above_budgetsP,-/$()<=OC# **1223C8H#DDZPH//-h7F((0 $-MM)(1V5K5K4LMs%MDD!)+%+%6%6E ((]f>O>O-O&CH:RP]^bcictctbu$v 49+R8HH4+0( ,,1A(nn\: ",..1A"B"Hb&5&9&9'&B#%>>,;??E11/9F$-MM+H:Qv7M7M6NOQT%M((]f>O>O-O&QR]Q^^vxKwLLXYaXbbderdsswx~xNxNwO%P -33I>49+R8HH4+0( %%*:(D)-)H)H)N&)%.]](a0B0R0R/ST& /99 *.@.K.K K*CD6WaVbbuwIwTwTvU)VI177 B8ykD%&BBBr&cKg}i}i}g}|D]5}|jri|j|}|j||E|j|} | 2| j&| ||<|jd|d| j|j r\|j dij d} | 9|j| } | &| || <|jd| d| j|jst|} | D]7} |j| } | s|jd| d| j98||||fSw)aB Returns list of cache keys to fetch from router cache for budget limiting and provider and deployment configs Returns: Tuple[List[str], Dict[str, GenericBudgetInfo], Dict[str, GenericBudgetInfo], List[Optional[str]]]: - List of cache keys to fetch from router cache for budget limiting - Dict of provider budget configs `provider_configs` - Dict of deployment budget configs `deployment_configs` - List of resolved providers aligned by deployment index `deployment_providers` rrrrrrmr) rTrr_get_budget_config_for_providerrr[r+!_get_budget_config_for_deploymentr\rr)r#rgrir}rnrorprr budget_configrrsrrs r$ruzERouterBudgetLimiting._async_get_cache_keys_for_router_budget_limitings(!# 9;;=46-J**@@L$++H5'$($H$H$RM%1)99E5B(2"))-hZq9V9V8WX ,,%>>,;??E'$($J$J8$TM$07D*84"))/z=;X;X:YZ%%<#1  )D)-)H)H)N&)"))(a0B0R0R/ST)?.L      sC8E ;$E  ,E start_time_key current_time ttl_secondscK|jj|d{}|(|jj|||d{|St|S797w)z Checks if the key = `provider_budget_start_time:{provider}` exists in cache. If it does, return the value. If it does not, set the key to `current_time` and return the value. Nr'r|ttl)rSasync_get_cacheasync_set_cacherx)r#rrr budget_starts r$_get_or_set_budget_start_timez2RouterBudgetLimiting._get_or_set_budget_start_time_sl"__<<^LL  //11",K2   \"" M s!AA'A A AA spend_key response_costcK|jj|||d{|jj|||d{|S7,7w)a Handle start of new budget window by resetting spend and start time Enters this when: - The budget does not exist in cache, so we need to set it - The budget window has expired, so we need to reset everything Does 2 things: - stores key: `provider_spend:{provider}:1d`, value: response_cost - stores key: `provider_budget_start_time:{provider}`, value: current_time. This stores the start time of the new budget window rN)rSr)r#rrrrrs r$_handle_new_budget_windowz.RouterBudgetLimiting._handle_new_budget_windowpsm(oo--K.   oo--l .      s!"AA%A A AArcK|jjj|||d{t|||}|jj |y7.w)a Increment spend within existing budget window Runs once the budget start time exists in Redis Cache (on the 2nd and subsequent requests to the same provider) - Increments the spend in memory cache (so spend instantly updated in memory) - Queues the increment operation to Redis Pipeline (using batched pipeline to optimize performance. Using Redis for multi instance environment of LiteLLM) rN)r'increment_valuer)rSin_memory_cacheasync_incrementrrWr)r#rrr increment_ops r$"_increment_spend_in_current_windowz7RouterBudgetLimiting._increment_spend_in_current_windowshoo--==>   7) ,,33LA s,AA/Ac&Ktjd|jdd}| td|jdd}t |jdd}|jd ijd d}| td |j |} | r4d |d | j } d|} |j| | | |d{|j|} | r4d|d | j } d|}|j| | ||d{t|}t|dkDrP|D]J}|j|}|sd|d |j }d|}|j||||d{Lyy77p7 w)z)Original method now uses helper functionsz/in RouterBudgetLimiting.async_log_success_eventstandard_logging_objectNz$standard_logging_payload is requiredrrrrrcustom_llm_providerzcustom_llm_provider is requiredrrprovider_budget_start_time:)rrrrrzdeployment_budget_start_time:rztag_budget_start_time:) r rr+rzrMrr_increment_spend_for_keyrrrAr)r#kwargs response_obj start_timeend_timestandard_logging_payloadrrrrrrr[deployment_spend_keydeployment_start_time_keyrsrr_tag_spend_key_tag_start_time_keys r$async_log_success_eventz,RouterBudgetLimiting.async_log_success_events##$UVEKZZ %tF   $ +CD D7;;OQO 488RHI#)::.>#C#G#G !4$   &>? ?<<=PQ ""5!6a 8U8U7VW  ;;N:OPN//+#-+ 0  $(#I#I(#S #%6xjBZBjBjAk#l *Gz(R %//6.8+ 0  5V< | q $%)%D%DT%J"%$TF!,>,N,N+OP#-C4&*I'77&8"0':&3 8% !)   s=CFF AFF 5F/FF F FFrcHK|jytjtjj }t |j}|j|||d{}||j|||||d{}nk||z |kDr4tjd|j|||||d{}n/|||z z }t|} |j||| d{tjd|d|y777T7&w)N)rrr)rrrrrz,Budget window expired - resetting everything)rrrzIncremented spend for z by ) rrnowrutc timestamprrrr rrOr) r#rrrrrrrremaining_timettl_for_increments r$rz-RouterBudgetLimiting._increment_spend_for_keysY  ( ( 0 ||HLL1;;= )-*G*GH !??)%#@   !%!?!?#-)+' "@"L\)[ 8 ! ' '(V W!%!?!?#-)+' "@"L)L<,GHN #N 3 99#=FW:   ##$YKtM? C E  sHA+D"-D.D" D;D" D /D"9D :!D"D"D" D"c<K |jd{tjtd{;7&7#t$rM}t j dt|tjtd{7Yd}~Vd}~wwxYww)z Handler that triggers sync_in_memory_spend_with_redis every DEFAULT_REDIS_SYNC_INTERVAL seconds Required for multi-instance environment usage of provider budgets NzError in periodic sync task: ) _sync_in_memory_spend_with_redisrXsleepDEFAULT_REDIS_SYNC_INTERVAL Exceptionr errorrMr#es r$rZz=RouterBudgetLimiting.periodic_sync_in_memory_spend_with_rediss  ;;===mm/= %++.KCPQF8,TUmm/ sWBA? AAABAA B =B B  BBBBcK |jjsytjd|jt |jdkDrCt j|jjj|jg|_y#t$r+}tjdt|Yd}~yd}~wwxYww)a How this works: - async_log_success_event collects all provider spend increments in `redis_increment_operation_queue` - This function pushes all increments to Redis in a batched pipeline to optimize performance Only runs if Redis is initialized Nz.Pushing Redis Increment Pipeline for queue: %sr)increment_list*Error syncing in-memory cache with Redis: ) rS redis_cacher rrWrArXrYasync_increment_pipelinerrrMrs r$#_push_in_memory_increments_to_redisz8RouterBudgetLimiting._push_in_memory_increments_to_redis#s ??.. ! ' '@44 47781<##OO//HH'+'K'KI 46D 0  ! ' '>r&rcT|jy|jj|dSr")r\r+)r#rs r$rz/RouterBudgetLimiting._get_budget_config_for_tags*  ! ! )%%))#t44r&rc |jdxsi}t|tr|jxsd}|}n>t|tr!|jdxsd}t |}n d}t i}t jt||\}}}}|S#t$rtjd|YywxYw)Nrrrf)rfrz+Error getting LLM provider for deployment: ) r+r`rrfrErraget_llm_providerrMrr r)r#rdeployment_litellm_paramsrfprovider_resolution_params_rs r$rz5RouterBudgetLimiting._get_llm_provider_for_deployments (27G(H(NB %3^D177=22K*5t<155g>D"-C-.*-CB-G*+2+C+C%j9, (A"Aq#"   ! ' '=j\J   sBB!CCrrcFt}|r|j|||yy)z Optional helper - emit provider remaining budget metric to Prometheus This is helpful for debugging and monitoring provider budget limits. rN)rtrack_provider_remaining_budget)r#rrrprometheus_loggers r$rz@RouterBudgetLimiting._track_provider_remaining_budget_prometheuss2BC   = =!) >  r&cJK|j|}|yd|d|j}|jjr.|jjj |d{}n#|jj |d{}| t |SdS777w)ab GET the current spend for a provider from cache used for GET /provider/budgets endpoint in spend_management_endpoints.py Args: provider (str): The provider to get spend for (e.g., "openai", "anthropic") Returns: Optional[float]: The current spend for the provider, or None if not found Nrrrl)rrrSrrrx)r#rrrrs r$_get_current_provider_spendz0RouterBudgetLimiting._get_current_provider_spends<O##L1"&&'89E !r&c|j|jjD]\}}|td|d|jt|ts8t |j d|j d|j|<t j|j||j|tjd|jyy)Nz$No budget config found for provider z, provider_budget_config: r time_period)rr)rrz#Initalized Provider budget config: ) rTr;rzr`GenericBudgetInfor+rXrYrr r)r#rrs r$r]z+RouterBudgetLimiting._init_provider_budgets.s  & & 2$($?$?$E$E$G &>$>xjHbcgc~c~bA"&*;<O & < 8 >BK#-I)--l;K.223DE  ! ' '6{mCVWgVhhtu~tA '$0)!2 0!,"00846D1;I--i8+!. ##4T5R5R4S T r&ctjyddlm}m}|durt d|j |ji|_tjjD]P\}}t|tr tdi|}t|j|j}||j|<Rtjd|jy)Nr)CommonProxyErrors premium_userTz,Tag budgets are an Enterprise only feature, rzInitialized Tag Budget Config: rP)rar\litellm.proxy.proxy_serverrr rznot_premium_userr;r`rErrrrr r)r#rr rr_generic_budget_configs r$r_z&RouterBudgetLimiting._init_tag_budgetsjs  $ $ , N t #>?P?a?a>bc   ! ! )%'D "(/(A(A(G(G(I $D$,d3%1%G4F%G"%6.>>/::& ",BD " "4 ()J ##-d.D.D-E F r&r")NN)+rHrIrJr r rEr rrrrMrr%rrrrrxr ryrurOrrrrrrZrrrrrrrrrr staticmethodrr]r^r_rPr&r$rRrR[s#  HH!)H $*+T$sCx.-AA B H:*.+/ I'I'"I'4 012 I' ! I' #4. I' dI'VYC#DcN3YC"$sCx.1YCs$556 YC !&7!78 YC #8C=1 YCU #YC3iYC tDcN#S( )YC|*.C !$sCx.1C !C   S  S# #$ S# #$ Xc]   C J#!#16#EH# #"      8BB-2B9<B.9v1 (1 1  1  1 f$"J#J(5/J4YY #Y$,=:   ( $*+T$sCx.-AA B @:  !  $*+T$sCx.-AA B ! F r&rR)*rKrXrrrtypingrrrr r r ralitellm._loggingr litellm.caching.cachingr litellm.caching.redis_cacher"litellm.integrations.custom_loggerrr*litellm.litellm_core_utils.duration_parserr)litellm.router_strategy.tag_based_routingr'litellm.router_utils.cooldown_callbacksrlitellm.types.llms.openairlitellm.types.routerrrrlitellm.types.utilsrrrrrrrRrPr&r$rsh(22::2-GAJS7RR,AO,","^g  <g  r&