
    ia                         d dl Z d dlmZmZ d dlmZmZmZmZmZm	Z	 d dl
Z
d dl
mZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ erd d
lmZ e	eef   ZneZ G d de      Z G d de      Zy)    N)datetime	timedelta)TYPE_CHECKINGAnyDictListOptionalUnion)ModelResponsetoken_counterverbose_logger)	DualCache)CustomLogger)safe_divide_seconds)!_get_parent_otel_span_from_kwargs)LiteLLMPydanticObjectBase)Spanc                   8    e Zd ZU dZeed<   dZeed<   dZeed<   y)RoutingArgsi  ttlr   lowest_latency_buffer
   max_latency_list_sizeN)	__name__
__module____qualname__r   float__annotations__r   r   int     w/Users/manta/Documents/Projects/TheRoad-I1/.venv/lib/python3.12/site-packages/litellm/router_strategy/lowest_latency.pyr   r      s"    C#$5$!#3#r!   r   c                   Z   e Zd ZU dZeed<   dZeed<   dZeed<   i fde	de
fdZd	 Zd
 Zd Z	 	 	 	 ddededeeeeef         deeeef      dee   dee   fdZ	 	 	 ddededeeeeef         deeeef      dee   f
dZ	 	 	 ddededeeeeef         deeeef      dee   f
dZy)LowestLatencyLoggingHandlerF	test_flagr   logged_successlogged_failurerouter_cacherouting_argsc                 2    || _         t        di || _        y )Nr    )r(   r   r)   )selfr(   r)   s      r"   __init__z$LowestLatencyLoggingHandler.__init__"   s    ('7,7r!   c                 ,   	 	 | j                  |      }|d   j                  |      y |d   |   j                  dd       }|d   j                  d      xs i j                  dd       }||y t        |t              rt	        |      }	 | d}t        j                         j                  d      }	t        j                         j                  d      }
t        j                         j                  d      }|	 d	|
 d	| }||z
  }d }|j                  d
d       |d
   du r|j                  d|      |z
  }|}d }d}t        |t              rt        |dd       }||j                  }|j                  }t        |t              r|j                         }n|}t        ||      }|t        |      }n|}|/t        |t              r|j                         }n|}t        ||      }t!        |      }| j"                  j%                  ||      xs i }||vri ||<   t'        ||   j                  dg             | j(                  j*                  k  r%||   j-                  dg       j/                  |       n,||   d   d | j(                  j*                  dz
   |gz   ||   d<   |t'        ||   j                  dg             | j(                  j*                  k  r%||   j-                  dg       j/                  |       n,||   d   d | j(                  j*                  dz
   |gz   ||   d<   |||   vri ||   |<   ||   |   j                  dd      |z   ||   |   d<   ||   |   j                  dd      dz   ||   |   d<   | j"                  j1                  ||| j(                  j2                         | j4                  r| xj6                  dz  c_        y y # t8        $ r7}t;        j<                  dj?                  t	        |                   Y d }~y d }~ww xY w)Nlitellm_paramsmodel_group
model_infoid_map%Y-%m-%d%H%M-streamTcompletion_start_timer   usagekeyparent_otel_spanlatency   time_to_first_tokentpmrpmr;   valuer   `litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}) _select_metadata_fieldget
isinstancer   strr   nowstrftimer   getattrcompletion_tokenstotal_tokensr   total_secondsr   r   r   r(   	get_cachelenr)   r   
setdefaultappend	set_cacher   r%   r&   	Exceptionr   	exceptionformat)r+   kwargsresponse_obj
start_timeend_timemetadata_fieldr/   r1   latency_keycurrent_datecurrent_hourcurrent_minuteprecise_minuteresponse_ms!time_to_first_token_response_timefinal_valuer?   rM   _usagerL   response_secondsttft_secondsr<   request_count_dictes                            r"   log_success_eventz-LowestLatencyLoggingHandler.log_success_event&   s   W	 "88@N&'++N;C$%56~FJJ!4 -.22<@FBKKDRVW&"*C(RB
	 "-T2'||~66zB'||~66t<!)!8!8!>$0><..AQ!R&3481::h-9fX>NRV>V 

#:HE
R 6 8C7;# lM:$\7DAF),2,D,D)'-':': &k9=/:/H/H/J,/:,&9,.?' '2*/*<K*:K<H)*KYW$E$S$S$U !- 0Q2E ,.?3/ $EV#L %%//':J 0   	 # //-/&r* *2.229bAB''==> 'r*55iDKKKX8J28N!9C))??!C9EHS}9U&r*95
 '2.r2667LbQR++AAB +2.9912 &!45 /r23HIId//EEI 0 +2.1 ");B)??=?&r*>: 'r*>:>>uaH<W #2&~6u= 'r*>:>>uaH1L #2&~6u= !!++#+=4CTCTCXCX , 
 >>''1,' " 	$$ryyF
 	s$   &O AO .M#O 	P-PPc                   K   	 | j                  |      }|j                  dd      }t        |t        j                        rm|d   j                  |      y|d   |   j                  dd      }|d   j                  d      xs i j                  dd      }||yt        |t
              rt        |      }	 | d}	| j                  j                  |	       d{   xs i }
||
vri |
|<   t        |
|   j                  d	g             | j                  j                  k  r%|
|   j                  d	g       j                  d
       n,|
|   d	   d| j                  j                  dz
   d
gz   |
|   d	<   | j                  j                  |	|
| j                  j                         d{    yy7 7 # t         $ r7}t#        j$                  dj'                  t        |                   Y d}~yd}~ww xY ww)zR
        Check if Timeout Error, if timeout set deployment latency -> 100
        rU   Nr.   r/   r0   r1   r2   r;   r=   g     @@r>   rB   rD   )rE   rF   rG   litellmTimeoutr   rH   r(   async_get_cacherP   r)   r   rQ   rR   async_set_cacher   rT   r   rU   rV   )r+   rW   rX   rY   rZ   r[   
_exceptionr/   r1   r\   rg   rh   s               r"   async_log_failure_eventz3LowestLatencyLoggingHandler.async_log_failure_event   s.    A	!88@NK6J*goo6*+//?G"()9":>"J"N"N%t#K !!1266|DJOOdB #*bj#B, W
	 &1M"6K"//??K?PPVTV ' !3313*2. .r266y"EF++AAB +2.99)RHOOPVW<Nr<R%=GD--CCaG=ILR8=T*2.y9 ++;;'0 --11 <    1 Q"  	$$ryyF
 	sn   G7AF4 G7AF4 G7A F4 F0CF4 )F2*F4 .G70F4 2F4 4	G4=-G/*G7/G44G7c                 `  K   	 	 | j                  |      }|d   j                  |      y |d   |   j                  dd       }|d   j                  d      xs i j                  dd       }||y t        |t              rt	        |      }	 | d}t        j                         j                  d      }	t        j                         j                  d      }
t        j                         j                  d      }|	 d	|
 d	| }||z
  }d }|j                  d
d       |d
   du r|j                  d|      |z
  }|}d}d }t        |t              rt        |dd       }||j                  }|j                  }t        |t              r|j                         }n|}t        ||      }|t        |      }n|}|/t        |t              r|j                         }n|}t        ||      }t!        |      }| j"                  j%                  ||d       d {   xs i }||vri ||<   t'        ||   j                  dg             | j(                  j*                  k  r%||   j-                  dg       j/                  |       n,||   d   d | j(                  j*                  dz
   |gz   ||   d<   |t'        ||   j                  dg             | j(                  j*                  k  r%||   j-                  dg       j/                  |       n,||   d   d | j(                  j*                  dz
   |gz   ||   d<   |||   vri ||   |<   ||   |   j                  dd      |z   ||   |   d<   ||   |   j                  dd      dz   ||   |   d<   | j"                  j1                  ||| j(                  j2                         d {    | j4                  r| xj6                  dz  c_        y y 7 7 *# t8        $ r7}t;        j<                  dj?                  t	        |                   Y d }~y d }~ww xY ww)Nr.   r/   r0   r1   r2   r3   r4   r5   r6   r7   Tr8   r   r9   )r;   r<   
local_onlyr=   r>   r?   r@   rA   rB   z\litellm.router_strategy.lowest_latency.py::async_log_success_event(): Exception occured - {}) rE   rF   rG   r   rH   r   rI   rJ   r   rK   rL   rM   r   rN   r   r   r   r(   rn   rP   r)   r   rQ   rR   ro   r   r%   r&   rT   r   rU   rV   )r+   rW   rX   rY   rZ   r[   r/   r1   r\   r]   r^   r_   r`   ra   rb   rc   rM   r?   rd   rL   re   rf   r<   rg   rh   s                            r"   async_log_success_eventz3LowestLatencyLoggingHandler.async_log_success_event	  s    X	 "88@N&'++N;C$%56~FJJ!4 -.22<@FBKKDRVW&"*C(RB

 "-T2'||~66zB'||~66t<!)!8!8!>$0><..AQ!R&3481::h-9fX>NRV>V 

#:HE
R 6 8C 7;#lM:$\7DAF),2,D,D)'-':': &k9=/:/H/H/J,/:,&9,.?' '2*/*<K*5K<H)*KYW$E$S$S$U !- 0Q2E ,.?3/ $EV#L ++;;')9#' <   
  # //-/&r* *2.229bAB''==> 'r*55iDKKKX8J28N!9C))??!C9EHS}9U&r*95
 '2.r2667LbQR++AAB +2.9912 &!45 /r23HIId//EEI 0 +2.1 ");B)??=?&r*>: 'r*>:>>uaH<W #2&~6u= 'r*>:>>uaH1L #2&~6u= ''77#+=4CTCTCXCX 8   
 >>''1,' "uj  	$$nuuF
 	sk   P.&O+ P.AO+ /P.0F"O+ O&F+O+ >O)?%O+ $P.&O+ )O+ +	P+4-P&!P.&P++P.Nr/   healthy_deploymentsmessagesinputrequest_kwargsrg   c                    i }t        d      }t        j                         j                  d      }	t        j                         j                  d      }
t        j                         j                  d      }|	 d|
 d| }d}|y|}|D ]   }|d   d   |vsd	d
g|d
d
di||d   d   <   " 	 t	        ||      }|j                         }t        j                  t        |      t        |            }t        |      }g }|j                         D ]2  \  }}d}|D ]  }||d   d   k(  s|} |!|j                  dd      xsS |j                  di       j                  dd      xs/ |j                  di       j                  dd      xs t        d      }|j                  dd      xsS |j                  di       j                  dd      xs/ |j                  di       j                  dd      xs t        d      }|j                  d	g       }|j                  dg       }|j                  |i       j                  dd
      }|j                  |i       j                  dd
      }d}|duxr- |j                  dd      duxr |d   du xr t        |      d
kD  }|r,|D ]  } t        | t               s|| z  } |t        |      z  }n+|D ]  } t        | t               s|| z  } |t        |      z  }|j                  di       j                  dd      }!|!|||!<   ||z   |kD  s|dz   |kD  r |j                  ||f       5 t        |      d
k(  ryt        |d       }"|"d
   d   }| j                   j"                  |z  }#|"D $cg c]  }$|$d   ||#z   k  s|$ }%}$t        j$                  |%      }&|&d
   }| j'                  |      }'||'|v r|||'   d<   |S # t
        $ r d
}Y -w xY wc c}$w )z>Common logic for both sync and async get_available_deploymentsinfr3   r4   r5   r6   Nr0   r1   r=   r   )r@   rA   )rv   textr@   r.   rA   r?   g        r7   Tapi_base r>   c                     | d   S )Nr>   r    )xs    r"   <lambda>zHLowestLatencyLoggingHandler._get_available_deployments.<locals>.<lambda>   s    1r!   rk   _latency_per_deployment)r   r   rI   rJ   r   rT   itemsrandomsamplelistrP   dictrF   rG   rR   sortedr)   r   choicerE   )(r+   r/   ru   rv   rw   rx   rg   r   lowest_latencyr]   r^   r_   r`   
deploymentall_deploymentsdinput_tokens_items_all_deploymentspotential_deploymentsitemitem_map_deploymentm_deployment_tpm_deployment_rpmitem_latencyitem_ttft_latencyitem_rpmitem_tpmtotaluse_ttft_call_latency_deployment_api_basesorted_deploymentsbufferr   valid_deploymentsrandom_valid_deploymentr[   s(                                           r"   _get_available_deploymentsz6LowestLatencyLoggingHandler._get_available_deployments  s    #%u||~..z:||~..t4!006(><..9IJ
%,$At$O;s"Aa$8:, 56 %	((GL
 !&&(!==fs6{C/0 !#-335ND(K(1\?400"#K ) " t,  ??#3R8<<UDI ??<488E  <	  t,  ??#3R8<<UDI ??<488E  <	  $<<	26L (-BB G||NB7;;E1EH||NB7;;E1EH Ed* /"&&x6dB/"8,4/ )*Q.	  %6M!-7. &7  %s+<'==%1M!-7. &2  %s<'88 $/??3CR#H#L#LB$  $/@L'(<= <'/9a</1%,,k<-HIG 6J $%* $$9~N ,A.q1 ""88>I *
)!QqT^f5L-LA) 	 

 #)--0A"B,Q/
44^D%.N*J ( >*) Q  	L	x
s   .O O,O,O)(O)c                    K   | d}t        |      }| j                  j                  ||       d {   xs i }| j                  ||||||      S 7 w)Nr2   r:   )r   r(   rn   r   	r+   r/   ru   rv   rw   rx   r\   r<   rg   s	            r"   async_get_available_deploymentsz;LowestLatencyLoggingHandler.async_get_available_deployments6  s      %T*+L,
 ##332B 4    	 	 ..
 	
s   1AAAc                     | d}t        |      }| j                  j                  ||      xs i }| j                  ||||||      S )z>
        Returns a deployment with the lowest latency
        r2   r:   )r   r(   rO   r   r   s	            r"   get_available_deploymentsz5LowestLatencyLoggingHandler.get_available_deploymentsT  su     %T*+L,
 ''2B (   	 	 ..
 	
r!   )NNNN)NNN)r   r   r   r%   boolr   r&   r   r'   r   r   r,   ri   rq   rt   rH   r   r	   r   r   r
   r   r   r   r    r!   r"   r$   r$      s   ItNCNCEG 8Y 8d 8ZxEN[B 48,0)--1NN "N 4S#X/0	N
 c4i()N !N %TNNh 48,0)-

 "
 4S#X/0	

 c4i()
 !
D 48,0)-

 "
 4S#X/0	

 c4i()
 !
r!   r$   )r   r   r   typingr   r   r   r   r	   r
   rl   r   r   r   litellm.caching.cachingr   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   r   litellm.types.utilsr   opentelemetry.tracer   _Spanr   r$   r    r!   r"   <module>r      s_     ( B B  @ @ - ; G U 91DD$+ $V	
, V	
r!   