
    j,j             	       ^   d dl Z d dlZd dlmZmZ d dlmZmZmZmZm	Z	m
Z
mZmZmZmZ d dlZd dlmZmZ d dlmZ d dlmZmZ d dl d dlmZ d d	lmZ d d
lmZ erd dlmZ neZ G d de      Z 	 dde
e!   de"dee!   de#fdZ$de#de	e!e!f   fdZ%de
e!   de!de&fdZ'de
e!   de	e!e!f   fdZ(y)    N)datetime	timedelta)
TYPE_CHECKINGAny	AwaitableCallableDictListLiteralOptionalTuplecast)print_verboseverbose_loggerCustomLogger)LiteLLM_TeamTableUserAPIKeyAuth)*_sanitize_prometheus_label_nameStandardLoggingPayload)!get_end_user_id_for_cost_tracking)AsyncIOSchedulerc                      e Zd Zd Zdeeee   f   fdZdedefdZ	dede
e   fdZdedee   de
e   fd	Zdedeeee   f   fd
Zdedee   fdZdeddfdZdedee   dee   ddfdZdededdfdZdefdZdeeee   f   ddfdZdedefdZd Zdedee   fdZd Zdede
e   de
e   de
e   d e
e   d!e
e   d"e
e   d#e
e   d$efd%Zd!e
e   d"e
e   de
e   de
e   d&e d'e!fd(Z"de
e   de
e   de
e   d e
e   d!e
e   d"e
e   d#e
e   d'e!d$efd)Z#de
e   de
e   d*e d+e fd,Z$d*e d e
e   de
e   de
e   d!e
e   d"e
e   d$efd-Z%d. Z&	 dud/e d0e'd1e(d2e
e   fd3Z)d4e d1e(fd5Z*d6e fd7Z+	 dvd6e d$ed8e!fd9Z,e-d:e'defd;       Z.d<ed*e d0e'fd=Z/d<ed*e d0e'fd>Z0d?e1d@edAe
e   dBe
e   dCef
dDZ2d@edAedBedCefdEZ3d@edAe
e   dBe
e   dCefdFZ4d@edAe
e   dBe
e   dCefdGZ5d@edAedBedCedHef
dIZ6dJefdKZ7dLedMe!dNe!fdOZ8dPe
e!   dMe
e!   de!fdQZ9dRe:dSe;e<ee=   e
e1   f      f   dTe:ee=   ge;d   f   dUe>dV   fdWZ?dX Z@dY ZAdZ ZBd[ ZCd\eeDee(f      fd]ZEd^eeF   fd_ZGd!e
e   d"e
e   d`e!dae!d'e!f
dbZHdceddedMe
e!   dPe
e!   d'e!deFfdeZIdfeFfdgZJd1e(fdhZKde
e   de
e   d'e!die!dje
e!   f
dkZLdededie!dje
e!   d'e!de(fdlZMdmeNde!fdnZOdoe=dpe=de
e!   fdqZPe-dreQfds       ZRe-dt        ZSy)wPrometheusLoggerc                 
   	 ddl m}m}m} | j	                         | _        | j                  |      | _        | j                  |      | _        | j                  |      | _	        | j                  dd| j                  d            | _        | j                  dd| j                  d            | _        | j                  dd	| j                  d      t        
      | _        | j                  dd| j                  d      t        
      | _        | j                  dd| j                  d      t        
      | _        | j                  dd| j                  d            | _        | j                  dd| j                  d            | _        | j                  dd| j                  d            | _        | j                  dd| j                  d            | _        | j                  dd| j                  d            | _        | j                  dd| j                  d            | _        | j                  dd| j                  d            | _        | j                  dd| j                  d            | _        | j                  d d!| j                  d             | _        | j                  d"d#| j                  d"            | _        | j                  d$d%g d&      | _        | j                  d'd(g d&      | _        | j                  d)d*| j                  d+            | _        | j                  d,d-| j                  d.            | _        | j                  d/d0| j                  d/      t        
      | _        | j                  d1d2d3g      | _         | j                  d4d5| j                  d4            | _!        | j                  d6d7| j                  d6            | _"        | j                  d8d9| j                  d8            | _#        | j                  d:d;| j                  d:            | _$        | j                  d<d=| j                  d<            | _%        | j                  d>d?| j                  d>            | _&        | j                  d@dA| j                  d@            | _'        | j                  dBdC| j                  dB            | _(        | j                  dDdEdFg      | _)        | j                  dGdHg dI      | _*        | j                  dJdK| j                  dJ            | _+        y # tX        $ r}t[        dLt]        |              |d }~ww xY w)MNr   )CounterGauge	Histogram$litellm_proxy_failed_requests_metriczjTotal number of failed responses from proxy - the client did not get a success response from litellm proxy)namedocumentation
labelnames#litellm_proxy_total_requests_metriczXTotal number of requests made to the proxy server - track number of client side requests$litellm_request_total_latency_metricz0Total latency (seconds) for a request to LiteLLM)r%   bucketslitellm_llm_api_latency_metricz1Total latency (seconds) for a models LLM API call*litellm_llm_api_time_to_first_token_metricz-Time to first token for a models LLM API calllitellm_spend_metriczTotal spend on LLM requests)r%   litellm_total_tokens_metricz7Total number of input + output tokens from LLM requestslitellm_input_tokens_metricz.Total number of input tokens from LLM requestslitellm_output_tokens_metricz/Total number of output tokens from LLM requests$litellm_remaining_team_budget_metriczRemaining budget for teamlitellm_team_max_budget_metriczMaximum budget set for team*litellm_team_budget_remaining_hours_metricz*Remaining days for team budget to be reset'litellm_remaining_api_key_budget_metriczRemaining budget for api key!litellm_api_key_max_budget_metriczMaximum budget set for api key-litellm_api_key_budget_remaining_hours_metricz.Remaining hours for api key budget to be reset,litellm_remaining_api_key_requests_for_modelzLRemaining Requests API Key can make for model (model based rpm limit on key))hashed_api_keyapi_key_aliasmodel*litellm_remaining_api_key_tokens_for_modelzJRemaining Tokens API Key can make for model (model based tpm limit on key)litellm_remaining_requestszWLLM Deployment Analytics - remaining requests for model, returned from LLM API Provider!litellm_remaining_requests_metriclitellm_remaining_tokensz:remaining tokens for model, returned from LLM API Providerlitellm_remaining_tokens_metriclitellm_overhead_latency_metricz;Latency overhead (milliseconds) added by LiteLLM processing(litellm_provider_remaining_budget_metriczHRemaining budget for provider - used when you set provider budget limitsapi_providerlitellm_deployment_statezlLLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outagelitellm_deployment_cooled_downzLLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down$litellm_deployment_success_responseszOLLM Deployment Analytics - Total number of successful LLM API calls via litellm$litellm_deployment_failure_responseszLLM Deployment Analytics - Total number of failed LLM API calls for a specific LLM deploymeny. exception_status is the status of the exception from the llm api!litellm_deployment_total_requestszXLLM Deployment Analytics - Total number of LLM API calls via litellm - success + failure+litellm_deployment_latency_per_output_tokenz3LLM Deployment Analytics - Latency per output token'litellm_deployment_successful_fallbackszfLLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model#litellm_deployment_failed_fallbackszbLLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model(litellm_callback_logging_failures_metriczTTotal number of failures when emitting logs to callbacks (e.g. s3_v2, langfuse, etc)callback_name&litellm_llm_api_failed_requests_metricz5deprecated - use litellm_proxy_failed_requests_metric)end_userr6   r7   r8   team
team_aliasuserlitellm_requests_metriczdeprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, userz(Got exception on init prometheus client )/prometheus_clientr   r    r!   _parse_prometheus_configlabel_filters_create_metric_factory_counter_factory_gauge_factory_histogram_factoryget_labels_for_metricr"   r&   LATENCY_BUCKETSr'   r)   r*   r+   litellm_tokens_metricr-   r.   r/   r0   r1   r2   r3   r4   r5   r9   r;   r=   r>   r?   rA   rB   rC   rD   rE   rF   rG   rH   rI   rK   rP   	Exceptionr   str)selfkwargsr   r    r!   es         x/Users/manta/Documents/Projects/TheRoad-I1/backend/.venv/lib/python3.12/site-packages/litellm/integrations/prometheus.py__init__zPrometheusLogger.__init__%   s   U	CC "&!>!>!@D %)$?$?$HD!"&"="=e"DD&*&A&A)&LD#8<8M8M; K55: 9N 9D5 8<7L7L:x559 8M 8D4 9=8O8O6B55: ( 9P 9D5 372I2I0C556VW'	 3J 3D/ ?C>U>U<?  55@ ( ?V ?D;" )-(=(=&-556LM )> )D% *.)>)>-I556ST *? *D& 04/D/D-@556ST 0E 0D, 150E0E.A556TU 1F 1D- 9=8K8K6+55: 9L 9D5 372E2E0-556VW 3F 3D/ ?C>Q>Q<<55@ ?R ?D; <@;N;N9.55= <O <D8 6:5H5H30557 6I 6D2 BFATAT?@55C BU BD> AE@S@S>^G AT AD= ?C>Q>Q<\G ?R ?D; 6:5H5H,i557 6I 6D2 483F3F*L555 4G 4D0 483J3J1M555 ( 4K 4D0 =A<O<O:Z*+ =P =D9 -1,?,?*~556PQ -@ -D) 372G2G0 ]556VW	 3H 3D/ 9=8M8M;o55: 9N 9D5 9=8M8M; @55: 9N 9D5 6:5J5J8x557 6K 6D2 @D?V?VBS55A @W @D< <@;P;P9x**+TU<D8 8<7L7L5t**+PQ8D4 =A<Q<Q?t+, =R =D9 ;?:O:O=U ;P ;D7 ,0+@+@. a556OP ,A ,D(  	DSVHMNG	s   T0T3 3	U<UUreturnc                 .   ddl }ddlm} |j                  }|si S t	        j
                  d|        g }t               | _        |D ]S  }t        |t              r	 |di |}n|}|j                  |       | j                  j                  |j                         U | j                  |      }|j                  r:| j                  |       ddj!                  |j"                        z   }t%        |      | j'                  |      }	| j)                  |	       |	S )zNParse prometheus metrics configuration for label filtering and enabled metricsr   N)PrometheusMetricsConfigzprometheus config: z!Configuration validation failed:

 )litellm%litellm.types.integrations.prometheusrd   prometheus_metrics_configr   debugsetenabled_metrics
isinstancedictappendupdatemetrics_validate_all_configurations
has_errors_pretty_print_validation_errorsjoinall_error_messages
ValueError_build_label_filters_pretty_print_prometheus_config)
r]   rg   rd   configparsed_configsgroup_configparsed_configvalidation_resultserror_messagerS   s
             r`   rR   z)PrometheusLogger._parse_prometheus_config@  s   Q22 I26(;< "u"L,- 7 G, G ,!!-0  ''(=(=> # ">>~N((001CD@499"55D M ]++ 11.A 	,,];    r{   c                    g }g }|D ]u  }|j                   D ]d  }| j                  |      }|r|j                  |       (|j                  s5| j	                  ||j                        }|sT|j                  |       f w t        ||      S )z>Validate all metric configurations and return collected errors)metric_errorslabel_errors)rq   _validate_single_metric_namero   include_labels_validate_single_metric_labelsValidationResults)r]   r{   r   r   rz   metric_namemetric_errorlabel_errors           r`   rr   z-PrometheusLogger._validate_all_configurationsl  s    $F%~~#@@M!((6 (("&"E"E#V%:%:#K #$++K8  . %  !}<XXr   r   c                 l    ddl m} |t         |t                    vrt	        | |t                    S y)zValidate a single metric namer   )get_args)r   valid_metricsN)typingr   rk   DEFINED_PROMETHEUS_METRICSMetricValidationError)r]   r   r   s      r`   r   z-PrometheusLogger._validate_single_metric_name  s8     	$c(+E"FGG('&'AB  r   labelsc                     ddl m} t        j                   |t        |            }|D cg c]	  }||vs| }}|rt        |||      S yc c}w )z#Validate labels for a single metricr   )r   r   invalid_labelsvalid_labelsN)r   r   PrometheusMetricLabels
get_labelsr   LabelValidationError)r]   r   r   r   r   labelr   s          r`   r   z/PrometheusLogger._validate_single_metric_labels  se     	  .88+[9

 .4QVEuL7P%VQ''-) 
  Rs
   	AAc                     i }|D ]A  }|j                   D ]0  }|j                  s| j                  |      "|j                  ||<   2 C |S )z1Build label filters from validated configurations)rq   r   r   )r]   r{   rS   rz   r   s        r`   rx   z%PrometheusLogger._build_label_filters  sQ    $F%~~((88EM5;5J5Jk2	  . % r   c                     | j                  ||      }|rG| j                  |j                  |j                  |j                         t        |j                        y)z
        Ensure that all the configured labels are valid for the metric

        Raises ValueError if the metric labels are invalid and pretty prints the error
        r   T)r   "_pretty_print_invalid_labels_errorr   r   r   rw   message)r]   r   r   r   s       r`   "_validate_configured_metric_labelsz3PrometheusLogger._validate_configured_metric_labels  s]     99+vN33'33*99(55 4 
 [0011r   r~   Nc           	         	 ddl m} ddlm} ddlm} ddlm}  |       } |dd      }|j                  d	       |j                   ||d
             |j                  r|j                  D cg c]  }|j                   }	}|j                  d   j                  }
 |ddj                  |	       d      }|j                   ||d
              |ddddd      }|j                  ddd       t        |
      D ]  }|j                  |        |j                  |       |j                   r|j                   D ]  } |d|j                   ddj                  |j"                         d      }|j                   ||d
              |d|j                   ddddd      }|j                  ddd       t        |j$                        D ]  }|j                  |        |j                  |        |j                  d	       yc c}w # t&        $ rc |j                  D ]!  }t)        j*                  |j,                         # |j                   D ]!  }t)        j*                  |j,                         # Y yw xY w)z-Pretty print all validation errors using richr   ConsolePanelTableTextu(   🚨🚨 Configuration Validation Errorsbold redstylere   redborder_stylezInvalid Metric Names: ,    📊 Valid Metric NamesT
bold greenleftgreentitleshow_headerheader_styletitle_justifyr   Available Metricscyanr   no_wrapzInvalid Labels for '': u   🏷️ Valid Labels for ''Valid LabelsN)rich.consoler   
rich.panelr   
rich.tabler   	rich.textr   printr   r   r   ru   
add_columnsortedadd_rowr   r   r   ImportErrorr   errorr   )r]   r~   r   r   r   r   consoler   r_   invalid_metricsr   metrics_error_textmetrics_tablemetricr   labels_error_textlabels_tabler   r   r   s                       r`   rt   z0PrometheusLogger._pretty_print_validation_errors  s}   L	:,((&iG C:VE MM$MM%E:; "//+=+K+K#+KaAMM+K   # !3 @ @!-  &*,TYY-G,HI$&" e$6UKL %3 $!-"(!(! (('vt )  %]3F!))&1 4 m, "../<<E(,.u/@/@.ATYYuOcOcEdDef()% MM%(9"NO#( :5;L;L:MQO$(%1&,%,$L !++N&RV+W!'(:(:!;$,,U3 "< MM,/' =* MM$i#l  	: 2 @ @$$\%9%9: !A1>>$$[%8%89  ?		:s&   A-H /HFH H A)I<;I<r   r   c                    	 ddl m} ddlm} ddlm} ddlm}  |       } |d| ddj                  |       d	d
      }	 |ddddd      }
|
j                  ddd       t        |      D ]  }|
j                  |        |j                  d       |j                   ||	d             |j                  |
       |j                  d       y# t        $ r* t        j                  d| d| dt        |              Y yw xY w)z8Pretty print error message for invalid labels using richr   r   r   r   r   u%   🚨🚨 Invalid Labels for Metric: 'z'
Invalid labels: r   z'
Please specify only valid labels belowr   r   u$   🏷️ Valid Labels for this MetricTr   r   r   r   r   r   r   re   r   r   zInvalid labels for metric 'r   z. Valid labels: N)r   r   r   r   r   r   r   r   ru   r   r   r   r   r   r   r   )r]   r   r   r   r   r   r   r   r   r   r   r   s               r`   r   z3PrometheusLogger._pretty_print_invalid_labels_error  s   %	,((&iG 7}DWX\XaXabpXqWr  s[  \ E !< )$$L ##N&$#O-$$U+ . MM$MM%E:;MM,'MM$ 	  -k]#n=MM]^deq^r]st	s   CC 0DDinvalid_metric_namer   c           	         	 ddl m} ddlm} ddlm} ddlm}  |       } |d| dd	      } |d
dddd      }	|	j                  ddd       t        |      D ]  }
|	j                  |
        |j                  d       |j                   ||d             |j                  |	       |j                  d       y# t        $ r' t        j                  d| dt        |              Y yw xY w)z=Pretty print error message for invalid metric name using richr   r   r   r   r   u   🚨🚨 Invalid Metric Name: 'z1'
Please specify one of the allowed metrics belowr   r   r   Tr   r   r   r   r   r   r   re   r   r   zInvalid metric name: z. Valid metrics: N)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r]   r   r   r   r   r   r   r   r   r   r   s              r`   "_pretty_print_invalid_metric_errorz3PrometheusLogger._pretty_print_invalid_metric_errorE  s    %	,((&iG 12E1FFxy E "/ )$$M $$%8PT$U /%%f- 0 MM$MM%E:;MM-(MM$ 	  '(;'<<MfUbNcMde	s   B;B> >-C.-C.c                     | j                  |      }|r<| j                  |j                  |j                         t	        |j
                        y)z]
        Raises ValueError if the metric name is invalid and pretty prints the error
        )r   r   N)r   r   r   r   rw   r   )r]   r   r   s      r`   _valid_metric_namez#PrometheusLogger._valid_metric_namet  sQ     11+>33$)$5$5UEXEX 4  U]]++	 r   rS   c                    	 ddl m} ddlm} ddlm} ddlm}  |       } |dd      } |d	d
dd      }|j                  ddd
       t        | d      r8| j                  r,t        | j                        D ]  }	|j                  |	        n|j                  d        |dd
dd      }
|
j                  ddd
       |
j                  dd       |rIt        |j                               D ],  \  }}|rdj                  |      nd}|
j                  ||       . n|
j                  dd       |j                  d       |j                   ||d             |j                  |       |j                  |
       |j                  d       y!# t         $ rT t#        j$                  dt        | d      rt        | j                        nd        t#        j$                  d |        Y y!w xY w)"z>Pretty print the processed prometheus configuration using richr   r   r   r   r   z"Prometheus Configuration Processedz	bold bluer   u   📊 Enabled MetricsTzbold magentar   )r   r   r   r   zMetric Namer   r   rl   z8[yellow]All metrics enabled (no filter applied)[/yellow]u   🏷️  Label Filtersr   zAllowed Labelsyellowr   z[dim]No labels specified[/dim]z+[yellow]No label filtering applied[/yellow]z*[dim]All default labels will be used[/dim]re   bluer   zEnabled metrics: zAll metricszLabel filters: N)r   r   r   r   r   r   r   r   r   hasattrrl   r   r   itemsru   r   r   r   info)r]   rS   r   r   r   r   r   r   r   r   r   r   r   
labels_strs                 r`   ry   z0PrometheusLogger._pretty_print_prometheus_config  s   @	C,((&iG =[QE ", +$	M $$]&$$Ot./D4H4H$T%9%9:F!))&1 ; %%N
 !. )$	L ##M#N##$4H#E+1-2E2E2G+H'K " 		&)= 
 !((jA ,I $$A@ MM$MM%F;<MM-(MM,'MM$ 	C#GDRcDdF4+?+?$@jw#xy /- AB	Cs   FF AG+*G+c                 R    t        | d      sy| j                  sy|| j                  v S )z3Check if a metric is enabled based on configurationrl   T)r   rl   )r]   r   s     r`   _is_metric_enabledz#PrometheusLogger._is_metric_enabled  s1     t./ ##d2222r   c                       fd}|S )zMCreate a factory function that returns either a real metric or a no-op metricc                  |    | r| d   n|j                  dd      }j                  |      r | i |S t               S )Nr   r#    )getr   
NoOpMetric)argsr^   r   metric_classr]   s      r`   factoryz8PrometheusLogger._create_metric_factory.<locals>.factory  sA    %)$q'vzz&"/EK&&{3#T4V44!|#r   rf   )r]   r   r   s   `` r`   rT   z'PrometheusLogger._create_metric_factory  s    	$ r   c                     t         j                  |      }|| j                  vr|S | j                  |   }|D cg c]	  }||v s| }}|S c c}w )zE
        Get the labels for a metric, filtered if configured
        )r   r   rS   )r]   r   default_labelsconfigured_labelsr   filtered_labelss         r`   rX   z&PrometheusLogger.get_labels_for_metric  sp     0::;G d000!! !..{;  .
-e:K1KE~ 	 
 	
s   	AAc                   K   ddl m} t        j                  d|        |j	                  d      }|t        |t              st        d|       |j	                  dd      }|j	                  di       xs i }|j	                  d	i       }	t        |d
      }
|d	   d   }|d	   d   }|d	   d   }|d	   d   }|d	   d   }|d   }|d   }|d   }|d	   j	                  d      }|d	   j	                  d      }i |r|ni |r|ni }|t        |t              r|d   }ng }t        d| d| d| d|
 d| 
       t        d@i d|
d|d|d|d    d |d    d!|d"|d#|d$|d	   d%   d&d'd|d(|d)|d*|d*   d+|d+   d,|d-   d.d d/d d0t        |1      d2|d	   j	                  d3      }|/t        |t              r|j                  d4      rdd5lm}  ||      }| j!                  |
||||||||6	       | j#                  ||
|||||||7	       | j%                  ||||||8       d {    | j'                  ||||	9       | j)                  |||||||:       | j+                  |||||       |d;   d<u rHt-        | j/                  d=>      |?      } | j0                  j2                  d@i |j5                          y y 7 w)ANr   r   z@prometheus Logging - Enters success logging function for kwargs standard_logging_objectz)standard_logging_object is required, got=r8   r   litellm_paramsmetadata
prometheusservice_typeuser_api_key_user_iduser_api_key_hashuser_api_key_aliasuser_api_key_team_iduser_api_key_team_aliascompletion_tokenstotal_tokensresponse_costrequester_metadatauser_api_key_auth_metadatarequest_tagsz'inside track_prometheus_metrics, model z, response_cost z, tokens_used z, end_user_id z, user_api_key rL   r6   r7   requested_modelmodel_grouprM   rN   rO   
user_emailuser_api_key_user_emailstatus_code200litellm_model_nametagsmodel_idapi_baser@   custom_llm_providerexception_statusexception_classcustom_metadata_labelsr   routeuser_api_key_request_routezsk-)
hash_token)	end_user_iduser_api_keyr   r8   user_api_teamuser_api_team_aliasuser_idr   enum_values)	standard_logging_payloadr  r  r   r8   r  r  r  r  )r  r  r  r   r   r   )r  r   r^   r   )r^   r8   r  r   r  r  r  streamTr&   r   supported_enum_labelsr  rf   )litellm.types.utilsr   r   rj   r   rm   rn   rw   r   r   UserAPIKeyLabelValuesget_custom_labels_from_metadatar\   
startswithlitellm.proxy.utilsr  ._increment_top_level_request_and_spend_metrics_increment_token_metrics#_increment_remaining_budget_metrics#_set_virtual_key_rate_limit_metrics_set_latency_metrics"set_llm_deployment_success_metricsprometheus_label_factoryrX   r&   r   inc)r]   r^   response_obj
start_timeend_timer   r  r8   r   	_metadatar  r  r  r   r  r  output_tokenstokens_usedr   _requester_metadatar   combined_metadata_tagsr  r  _labelss                             r`   async_log_success_eventz(PrometheusLogger.async_log_success_event  s    >NvhW	

 FLZZ%F
  $+:$d4
 ;<T;UV  

7B'$4b9?R"&&z26	7
 +:67MN/
;<OP5jABVW0<=ST6zB%
 11DE.~>0A.Fz.R.V.V /
 6N6

#*
+ 	#-
&9"r-
-G)R-
 $/J$d5
 -^<EE5eW<L]O[ijuiv  wE  FQ  ER  Ra  bn  ao  p	
 , 
 
'
 -
 5]C	

 1?
 
 +
 
 0
;<UV
 
 
  %
 
 .j9
 .j9
  22GH!
" "#
$ !%
& $C*$'
, +:6::,-
8 $<-''.6%l3L 	;;#%1' 3'# 	< 
	
 	%% &>#%1' 3# 	& 	
  66' 3%1)' 7 
 	
 	
 	00%1	 	1 	
 	!!%1' 3 $ 	" 	
 	//J+}	

 %X.$6.&*&@&@ E 'A ' (	G <D44;;FgFJJL 7K	
s   H-K/K0BKr  r  r  r   r8   r  r  r  r  c
                 X   t        j                  d       |t        |t              r|d   }
t	        | j                  d      |	      }t	        | j                  d      |	      } | j                  j                  di |j                  |d          t	        | j                  d      |	      } | j                  j                  di |j                  |d	          t	        | j                  d
      |	      } | j                  j                  di |j                  |d          y )Nz2prometheus Logging - Enters token metrics functionr   r&   r  r  r,   r   r-   prompt_tokensr.   r   rf   )r   rj   rm   rn   r%  rX   rZ   r   r&  r-   r.   )r]   r  r  r  r   r8   r  r  r  r  r/  r0  s               r`   r   z)PrometheusLogger._increment_token_metrics  sX    	QR $/J$d5
 -^<E*"&"<"<A #= # $	
 +"&"<"<9 #= # $	
 	*""))4G488$^4	
 +"&"<"<9 #= # $	
 	0((//:':>>$_5	
 +"&"<"<: #= # $	
 	1))00;7;??$%89	
r   r   r   c                   K   |j                  di       j                  dd       }|j                  di       j                  dd       }|j                  di       j                  dd       }	|j                  di       j                  dd       }
| j                  ||||
|	       d {    | j                  |||||       d {    y 7 #7 w)Nr   user_api_key_team_spenduser_api_key_team_max_budgetuser_api_key_spenduser_api_key_max_budget)r  r   r   key_max_budget	key_spend)r  r  
team_spendteam_max_budgetr   )r   -_set_api_key_budget_metrics_after_api_request*_set_team_budget_metrics_after_api_request)r]   r  r  r  r   r   r   _team_spend_team_max_budget_api_key_spend_api_key_max_budgets              r`   r!  z4PrometheusLogger._increment_remaining_budget_metrics  s      %((R8<<%t
 *--j"=AA*D
 (++J;?? $
 -00R@DD%t
 @@%1'.$ A 
 	
 	
 ==' 3",' > 
 	
 	
	
	
s$   B"C$C%CC
C
Cc
                 "   t        | j                  d      |	      }
 | j                  j                  di |
j	                          t        | j                  d      |	      }
 | j
                  j                  di |
j	                  |       y )NrP   r  r  r+   rf   )r%  rX   rP   r   r&  r+   )r]   r  r  r   r8   r  r  r  r   r  r0  s              r`   r  z?PrometheusLogger._increment_top_level_request_and_spend_metrics   s     +"&"<"<5 #= # $	
 	,$$++6g6::<*"&"<"<2 #= # $	
 	)!!((37377Fr   r^   r   c                    ddl m}  ||      }d| }d| }|j                  |t        j                        xs t        j                  }	|j                  |t        j                        xs t        j                  }
| j
                  j                  |||      j                  |	       | j                  j                  |||      j                  |
       y )Nr   )#get_model_group_from_litellm_kwargszlitellm-key-remaining-requests-zlitellm-key-remaining-tokens-)	)litellm.proxy.common_utils.callback_utilsrE  r   sysmaxsizer5   r   rk   r9   )r]   r  r   r^   r   rE  r    remaining_requests_variable_nameremaining_tokens_variable_nameremaining_requestsremaining_tokenss              r`   r"  z4PrometheusLogger._set_virtual_key_rate_limit_metrics  s    	
 :&A-k]; 	) ,I)V& LL93;;GV3;; 	 LL7ET 	 	99@@,k	

# 
!77>>,k	

#
r   c                    |j                  d      xs t        j                         }|j                  d      }	|j                  dd       }
|j                  dd       }| j                  |
|      }|C|j                  dd      du r/| j                  j                  |||||      j                  |       nt        j                  d	       | j                  |
|      }|Ht        | j                  d
      |      } | j                  j
                  di |j                  |       | j                  |	|      }|It        | j                  d      |      } | j                  j
                  di |j                  |       y y )Nr)  r(  api_call_start_timecompletion_start_time)r(  r)  r  FTzUTime to first token metric not emitted, stream option in model_parameters is not Truer)   r  r  r'   rf   )r   r   now_safe_duration_secondsr*   r   observer   rj   r%  rX   r)   r'   )r]   r^   r8   r  r   r  r  r  r)  r(  rN  rO  time_to_first_token_secondsapi_call_total_time_secondsr0  total_time_secondss                   r`   r#  z%PrometheusLogger._set_latency_metrics@  s    $ZZ
3Ex||~)/L)A
$jj)>E &

+BD I&*&A&A** 'B '
#
 (3

8U+t3;;BB"# g12  g '+&A&A* 'B '
# '2.&*&@&@ @ 'A ' (	G 7D//66AAII+
 "88! 9 
 ).&*&@&@ F 'A ' (	G =D55<<GwGOO" *r   c           	      R  K   ddl m} t        j                  d|        |j	                  dd      }|j	                  di       }|j	                  di       xs i }t        |d	      }	|d
   d   }
|d
   d   }|d
   d   }|d
   d   }|d
   d   }|j	                  dd        	 | j                  j                  |	||||||
      j                          | j                  |       y # t        $ r7}t        j                  dj                  t        |                   Y d }~y d }~ww xY ww)Nr   r   z@prometheus Logging - Enters failure logging function for kwargs r8   r   r   r   r   r   r   r   r   r   r   r   	exception0prometheus Layer Error(): Exception occured - {})r  r   r   rj   r   r   rK   r   r&  "set_llm_deployment_failure_metricsr[   rW  formatr\   )r]   r^   r'  r(  r)  r   r8   r  r   r  r  r  r   r  r  r_   s                   r`   async_log_failure_eventz(PrometheusLogger.async_log_failure_event  sW    >NvhW	

 

7B';A::%r<
   $4b9?R7
 +:67MN/
;<OP5jABVW0<=ST6zB%
 	

;%	77>>"# ce33F; 	  	$$BII#a&Q 	s1   B D'#A C$ #D'$	D$--DD'D$$D'request_dataoriginal_exceptionuser_api_key_dicttraceback_strc                 `  K   ddl m} 	 |j                  ||j                  di             }t	        |j
                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  dd      t        t        |dd            t        t        |dd            | j                  |      ||j                  	      }t!        | j#                  d
      |      } | j$                  j&                  di |j)                          t!        | j#                  d      |      } | j*                  j&                  di |j)                          y# t,        $ r7}	t/        j0                  dj3                  t        |	                   Y d}	~	yd}	~	ww xY ww)ar  
        Track client side failures

        Proxy level tracking - failed client side requests

        labelnames=[
                    "end_user",
                    "hashed_api_key",
                    "api_key_alias",
                    REQUESTED_MODEL,
                    "team",
                    "team_alias",
                ] + EXCEPTION_LABELS,
        r   StandardLoggingPayloadSetupproxy_server_requestr   rc  r8   r   r  N)rL   rO   r   r6   r7   rM   rN   r   r  r  r	  r  r  r"   r  r  r&   rX  rf   )*litellm.litellm_core_utils.litellm_loggingrb  _get_request_tagsr   r  r  r  r   api_key	key_aliasteam_idrN   r\   getattr_get_exception_class_namerequest_router%  rX   r"   r   r&  r&   r[   r   rW  rZ  )
r]   r\  r]  r^  r_  rb  r/  r  r0  r_   s
             r`   async_post_call_failure_hookz-PrometheusLogger.async_post_call_failure_hook  s    *	
(	/AA+%1%5%56Lb%Q B E 0*66&..,77088/99&..,77 , 0 0" =(:M4 PQ!$W-?PT%U!V $ > >?Q R'55K /&*&@&@ F 'A ' (	G =D55<<GwGKKM.&*&@&@ E 'A ' (	G <D44;;FgFJJL 	$$BII#a&Q 		s/   F.E E+ *F.+	F+4-F&!F.&F++F.datac                 \  K   	 ddl m} t        |j                  |j                  |j
                  |j                  dd      |j                  |j                  |j                  |j                  d|j                  |j                  ||j                  di                   }t        | j                  d	
      |      } | j                  j                   di |j#                          y# t$        $ r7}t'        j(                  dj+                  t-        |                   Y d}~yd}~ww xY ww)zp
        Proxy level tracking - triggered when the proxy responds with a success response to the client
        r   ra  r8   r   r  rc  rd  )rL   r6   r7   r   rM   rN   rO   r   r  r  r  r&   r  r  rX  Nrf   )re  rb  r  r  rg  rh  r   ri  rN   r  r   rl  rf  r%  rX   r&   r   r&  r[   r   rW  rZ  r\   )r]   rn  r^  responserb  r  r0  r_   s           r`   async_post_call_success_hookz-PrometheusLogger.async_post_call_success_hook  s    !	 0*66088/99 $" 5&..,77&..,77!'550BB#')-2H")M C K  /&*&@&@ E 'A ' (	G <D44;;FgFJJL 	$$BII#a&Q 		s/   D,C$C) (D,)	D)2-D$D,$D))D,request_kwargsc                    	 t        j                  d       |j                  di       }|j                  di       xs i }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  d	d      }|j                  d
d      }	t        ||||	|rt	        t        |dd            nd|r| j                  |      nd||d   d   |d   d   |d   d   |d   d   |j                  dg             }
	 | j                  |xs d|||	xs d       |Gt        | j                  d      |
      } | j                  j                  di |j                          t        | j                  d      |
      } | j                  j                  di |j                          y# t        $ r7}t        j                  dj                  t	        |                   Y d}~yd}~ww xY w)a  
        Sets Failure metrics when an LLM API call fails

        - mark the deployment as partial outage
        - increment deployment failure responses metric
        - increment deployment total requests metric

        Args:
            request_kwargs: dict

        (setting remaining tokens requests metricr   r   r8   Nr   r  r  rW  r  r  r   r   r   r   r   r   )r  r  r  r@   r  r	  r   r6   r7   rM   rN   r  r   r  r  r  r@   rD   r  r  rE   zLPrometheus Error: set_llm_deployment_failure_metrics. Exception occured - {}rf   )r   rj   r   r  r\   rj  rk  set_deployment_partial_outager%  rX   rD   r   r&  rE   r[   rZ  )r]   rr  r  _litellm_paramsr  r   r  r  rW  llm_providerr  r0  r_   s                r`   rY  z3PrometheusLogger.set_llm_deployment_failure_metrics  sZ   J	  !KL?M?Q?Q)2@$ -001A2FL"O!/!3!3GT!B266}dKK/33JEH/33JEH&**;=I*../DdKL 0#5!!)DMC	=$?@SW BKD229=PT +7
C'  7zB( .j9:PQ3J?- .11."E-K2 ..#5#;!!)/R	 /  $2*.*D*D$J +E + !,	 A99@@K7KOOQ.&*&@&@ C 'A ' (	G :D2299DGDHHJ 	  ^eeF 	s   GG 	H-G??Hr+  c                    	 t        j                  d       |j                  d      }|y |d   }|j                  di       xs i }|j                  di       }	|j                  dd       }
|j                  dd       }|	j                  d      xs i }|j                  d	d       }d }d }|d
   d   x}r$|j                  dd       }|j                  dd       }|d
   j                  d      x}rKt        | j	                  d      |      } | j
                  j                  di |j                  |dz         |rI	 t        | j	                  d      |      } | j                  j                  di |j                  |       |rHt        | j	                  d      |      } | j                  j                  di |j                  |       	 | j                  |
xs d|xs d|xs d|xs d       t        | j	                  d      |      } | j                  j                  di |j                          t        | j	                  d      |      } | j                  j                  di |j                          ||z
  }d }|j                  dd       |d   du r|j                  d|      |z
  }|xs |}|j                         }d }|T|dkD  rN||z  }t        | j	                  d      |      } | j                   j                  di |j                  |       y y y # t"        $ r7}t        j$                  dj'                  t)        |                   Y d }~y d }~ww xY w) Nrt  r   r  r   r   r8   r  
model_infoidhidden_paramsadditional_headersx_ratelimit_remaining_requestsx_ratelimit_remaining_tokenslitellm_overhead_time_msr>   r  r  i  r;   r=   r   ru  rC   rE   r  TrO  r   rF   zLPrometheus Error: set_llm_deployment_success_metrics. Exception occured - {}rf   )r   rj   r   r%  rX   r>   r   rR  r;   rk   r=   set_deployment_healthyrC   r&  rE   total_secondsrF   r[   rW  rZ  r\   )r]   rr  r(  r)  r  r+  r  r  rw  r*  r  rx  _model_infor  rK  rL  r}  r  r0  response_ms!time_to_first_token_response_time_latency_latency_secondslatency_per_tokenr_   s                            r`   r$  z3PrometheusLogger.set_llm_deployment_success_metricsq  s&   I	  !KL""#<= % (//
;H,001A2FL"O'++J;I!/!3!3GT!B*../DdKL#--5;K"tT2H04.2%=o%N$& !  &8%;%;4d&" $6#9#92D$  ,D,c,-.' . 3*.*D*D$E +E + !,	 <44;;FgFNN,t3 " 3*.*D*D$G +E + !,	 >66==HHLL&  2*.*D*D$E +E + !,	 <44;;FgFJJ$ ''#5#;!R!R)/R	 (  /&*&@&@ F 'A ' (	G =D55<<GwGKKM.&*&@&@ C 'A ' (	G :D2299DGDHHJ &.
%:KEI- ""8T2>"8,4 #&&'>IJV 2 #D"R{H'557 !%(]Q->$4}$D!2*.*D*D$Q +E + !,	 H@@GG '+, .?(  	$$^eeF
 	s   (L# K5L# #	M#,-MM#rW  c                     d}t        | d      rt        | d      xs d}t        |      dk\  r|d   j                         |dd  z   dz   }|| j                  j
                  z  }|S )Nr   rx     r   .)r   rj  lenupper	__class____name__)rW  exception_class_names     r`   rk  z*PrometheusLogger._get_exception_class_name  s{    !9n-#*9n#E#K  #$)$Q'--/2Fqr2JJSP ! 		 3 3 < <<##r   original_model_groupc                   K   ddl m}m} t        j                  d||       |j                  di       }|j                  |      }|j                  d      }t        t        t           |j                  d      xs g       }	t        |||d   |d	   |d
   |d   t        t        |dd            | j                  |      |		      }
t        | j                  d      |
      } | j                  j                   di |j#                          yw)zF

        Logs a successful LLM fallback event on prometheus

        r   StandardLoggingMetadatarb  zLPrometheus: log_success_fallback_event, original_model_group: %s, kwargs: %sr   r  r8   r  r   r   r   r   r  N	r   fallback_modelr6   r7   rM   rN   r  r	  r  rG   r  r  rf   )re  r  rb  r   rj   r   get_standard_logging_metadatar   r
   r\   r  rj  rk  r%  rX   rG   r   r&  )r]   r  r^   r]  r  rb  r*  standard_metadata
_new_modelr/  r  r0  s               r`   log_success_fallback_eventz+PrometheusLogger.log_success_fallback_event  s#    	

 	Z 	

 JJz2.	'EE" F  	
 ZZ(
T#Y

6 2 8b9+0%,-@A+,@A"#9:()BC );]D!QR ::;MN

 +"&"<"<E #= # $	
 	<44;;FgFJJL   D
Dc                   K   ddl m}m} t        j                  d||       |j                  d      }|j                  di       }t        t        t           |j                  d      xs g       }|j                  |      }	t        |||	d   |	d	   |	d
   |	d   t        t        |dd            | j                  |      |	      }
t        | j                  d      |
      } | j                  j                   di |j#                          yw)z@
        Logs a failed LLM fallback event on prometheus
        r   r  zLPrometheus: log_failure_fallback_event, original_model_group: %s, kwargs: %sr8   r   r  r  r   r   r   r   r  Nr  rH   r  r  rf   )re  r  rb  r   rj   r   r   r
   r\   r  r  rj  rk  r%  rX   rH   r   r&  )r]   r  r^   r]  r  rb  r  r*  r/  r  r  r0  s               r`   log_failure_fallback_eventz+PrometheusLogger.log_failure_fallback_eventC  s#    	

 	Z 	

 ZZ(
JJz2.	T#Y

6 2 8b9'EE" F  	 ,0%,-@A+,@A"#9:()BC );]D!QR ::;MN

 +"&"<"<A #= # $	
 	80077B'BFFHr  stater  r  r  r@   c           	          t        | j                  d      t        ||||            } | j                  j                  di |j                  |       y)z+
        Set the deployment state.
        rA   r  ru  r  Nrf   )r%  rX   r  rA   r   rk   )r]   r  r  r  r  r@   r0  s          r`   set_litellm_deployment_statez-PrometheusLogger.set_litellm_deployment_statep  s`     +"&"<"<6 #= # .#5!!)		

 	-%%,,7w7;;EBr   c                 .    | j                  d||||       y )Nr   r  r]   r  r  r  r@   s        r`   r  z'PrometheusLogger.set_deployment_healthy       	))!8X|	
r   c                 .    | j                  d||||       y )Nr  r  r  s        r`   rv  z.PrometheusLogger.set_deployment_partial_outage  r  r   c                 .    | j                  d||||       y )N   r  r  s        r`   set_deployment_complete_outagez/PrometheusLogger.set_deployment_complete_outage  r  r   r  c                 ^    | j                   j                  |||||      j                          y)zn
        increment metric when litellm.Router / load balancing logic places a deployment in cool down
        N)rB   r   r&  )r]   r  r  r  r@   r  s         r`    increment_deployment_cooled_downz1PrometheusLogger.increment_deployment_cooled_down  s+     	++22(LBR	

#%r   rJ   c                 X    | j                   j                  |      j                          y)za
        Increment metric when logging to a callback fails (e.g., s3_v2, langfuse, etc.)
        )rJ   N)rI   r   r&  )r]   rJ   s     r`   "increment_callback_logging_failurez3PrometheusLogger.increment_callback_logging_failure  s'     	55<<' 	= 	

#%r   providerspendbudget_limitc                 z    | j                   j                  |      j                  | j                  ||             y)z?
        Track provider remaining budget in Prometheus
        
max_budgetr  N)r?   r   rk   _safe_get_remaining_budget)r]   r  r  r  s       r`   track_provider_remaining_budgetz0PrometheusLogger.track_provider_remaining_budget  s<     	55<<XFJJ++' , 	
r   r  c                 .    |t        d      S ||S ||z
  S )Ninf)float)r]   r  r  s      r`   r  z+PrometheusLogger._safe_get_remaining_budget  s*     <=E!!r   data_fetch_function.set_metrics_function	data_type)teamskeysc           	        K   ddl m} |y	 d}d} |||       d{   \  }}|t        |      }||z   dz
  |z  }	 ||       d{    t        d|	dz         D ]'  } |||       d{   \  }}
 ||       d{    ) y7 i7 @7 7 # t        $ r.}t        j                  d| d	t        |              Y d}~yd}~ww xY ww)
ab  
        Generic method to initialize budget metrics for teams or API keys.

        Args:
            data_fetch_function: Function to fetch data with pagination.
            set_metrics_function: Function to set metrics for the fetched data.
            data_type: String representing the type of data ("teams" or "keys") for logging purposes.
        r   prisma_clientNr  2   )	page_sizepager  zError initializing z budget metrics: )litellm.proxy.proxy_serverr  r  ranger[   r   rW  r\   )r]   r  r  r  r  r  r  rn  total_counttotal_pages_r_   s               r`   _initialize_budget_metricsz+PrometheusLogger._initialize_budget_metrics  s      	= 	DI&9#$' !D+ "!$i '2Q69DK 't,,, aq1 3id SSa*4000 2! - T0 	$$%i[0A#a&J 	st   
C
B B*B 	B

#B -B.B  BB C
B 
B B B 	C$C=C
CC
c           	        K   ddl m ddlm t	        j
                  d       ydt        dt        dt        t        t           t        t           f   ffd	}| j                  || j                  d
       d{    y7 w)zY
        Initialize team budget metrics by reusing the generic pagination logic.
        r   )get_paginated_teamsr  NzDPrometheus: skipping team metrics initialization, DB not initializedr  r  rb   c                 ^   K    | |       d {   \  }}|t        |      }||fS 7 w)N)r  r  r  )r  )r  r  r  r  r  r  s       r`   fetch_teamszEPrometheusLogger._initialize_team_budget_metrics.<locals>.fetch_teams  sE      (;+yt( "E; "!%j+%%"s   -+-r  r  r  r  )1litellm.proxy.management_endpoints.team_endpointsr  r  r  r   rj   intr   r
   r   r   r  _set_team_list_budget_metrics)r]   r  r  r  s     @@r`   _initialize_team_budget_metricsz0PrometheusLogger._initialize_team_budget_metrics	  s     	
 	=   V 	&	&"%	&4)*HSM9:	& -- +!%!C!C . 
 	
 	
s   A:B>B?Bc           
      >  K   ddl m} ddlm ddlm ddlm t        j                  d       ydt        d	t        d
t        t        |t        t        f      t        t           f   ffd}| j!                  || j"                  d       d{    y7 w)z\
        Initialize API key budget metrics by reusing the generic pagination logic.
        r   )Union)UI_SESSION_TOKEN_TEAM_ID)_list_key_helperr  NzCPrometheus: skipping key metrics initialization, DB not initializedr  r  rb   c                    K    || d d d d dd 
       d {   }|j                  dg       }|j                  d      }|t        |      }||fS 7 8w)NT)
r  r  sizer  ri  rh  key_hashexclude_team_idreturn_full_objectorganization_idr  r  )r   r  )r  r  key_list_responser  r  r  r  r  s        r`   
fetch_keyszGPrometheusLogger._initialize_api_key_budget_metrics.<locals>.fetch_keys:  sz      '7+ 8#' $' ! %((4D+//>K"!$i$$!!s   AA9Ar  r  )r   r  litellm.constantsr  ;litellm.proxy.management_endpoints.key_management_endpointsr  r  r  r   rj   r  r   r
   r\   r   r   r  _set_key_list_budget_metrics)r]   r  r  r  r  r  s      @@@r`   "_initialize_api_key_budget_metricsz3PrometheusLogger._initialize_api_key_budget_metrics(  s      	!>	
 	=   U 	%	%"%	%4c>123Xc]BC	%* -- *!%!B!B . 
 	
 	
s   BBBBc                   K   ddl m} ddlm} |j                  j
                  }|r[|j                  rO|j                  |       d{   r4	 | j                          d{    |j                  |       d{    yy| j                          d{    y7 R7 ;7 ## |j                  |       d{  7   w xY w7 +w)a  
        Handler for initializing remaining budget metrics for all teams to avoid metric discrepancies.

        Runs when prometheus logger starts up.

        - If redis cache is available, we use the pod lock manager to acquire a lock and initialize the metrics.
            - Ensures only one pod emits the metrics at a time.
        - If redis cache is not available, we initialize the metrics directly.
        r   )'PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME)proxy_logging_obj)
cronjob_idN)
r  r  r  r  db_spend_update_writerpod_lock_managerredis_cacheacquire_lock$_initialize_remaining_budget_metricsrelease_lock)r]   r  r  r  s       r`   #initialize_remaining_budget_metricsz4PrometheusLogger.initialize_remaining_budget_metricsU  s      	N@,CCTT  0 < <%22B 3   CCEEE*77#J 8    ;;=== F*77#J 8   
 >sf   ACB	CB! !B"B! &C;B<CCCB! C!B>7B:8B>>Cc                    K   t        j                  d       | j                          d{    | j                          d{    y7 7 w)z[
        Helper to initialize remaining budget metrics for all teams and API keys.
        z%Emitting key, team budget metrics....N)r   rj   r  r  )r]   s    r`   r  z5PrometheusLogger._initialize_remaining_budget_metricss  sD      	DE2244455777 	57s!   )AA	AAAAr  c                 ^   K   |D ]$  }t        |t              s| j                  |       & yw)z8Helper function to set budget metrics for a list of keysN)rm   r   _set_key_budget_metrics)r]   r  keys      r`   r  z-PrometheusLogger._set_key_list_budget_metrics{  s*      C#~.,,S1 s   --r  c                 <   K   |D ]  }| j                  |        yw)z9Helper function to set budget metrics for a list of teamsN)_set_team_budget_metrics)r]   r  rM   s      r`   r  z.PrometheusLogger._set_team_list_budget_metrics  s     D))$/ s   r;  r<  c                 |   K   |r4| j                  ||xs d|||       d{   }| j                  |       yy7 w)z
        Set team budget metrics after an LLM API request

        - Assemble a LiteLLM_TeamTable object
            - looks up team info from db if not available in metadata
        - Set team budget metrics
        r   )ri  rN   r  r  r   N)_assemble_team_objectr  )r]   r  r  r;  r<  r   team_objects          r`   r>  z;PrometheusLogger._set_team_budget_metrics_after_api_request  sS       $ : :%.4" *+ !; ! K ))+6     <:<ri  rN   c                   K   ddl m} ddlm}m} |xs d|z   }	t        |||	|      }
	  ||||       d{   }|r|j                  |
_
        |
S 7 # t        $ r-}t        j                  dt        |              |
cY d}~S d}~ww xY ww)z
        Assemble a LiteLLM_TeamTable object

        for fields not available in metadata, we fetch from db
        Fields not available in metadata:
        - `budget_reset_at`
        r   )get_team_objectr  user_api_key_cache)ri  rN   r  r  )ri  r  r  Nz4[Non-Blocking] Prometheus: Error getting team info: )litellm.proxy.auth.auth_checksr  r  r  r  r   r[   r   rj   r\   budget_reset_at)r]   ri  rN   r  r  r   r  r  r  _total_team_spendr  	team_infor_   s                r`   r  z&PrometheusLogger._assemble_team_object  s      	CP"Za=8'!#!	

	-+#5 I *3*C*CK'
  	  Fs1vhO 		sC   'BA AA BA 	B
"B?B
 BB

BrM   c                    t        |j                  |j                  xs d      }t        | j	                  d      |      } | j
                  j                  di |j                  | j                  |j                  |j                               |j                  Rt        | j	                  d      |      } | j                  j                  di |j                  |j                         |j                  ct        | j	                  d	      |      } | j                  j                  di |j                  | j                  |j                  
             yy)z
        Set team budget metrics for a single team

        - Remaining Budget
        - Max Budget
        - Budget Reset At
        r   )rM   rN   r/   r  r  r  Nr0   r1   r  rf   )r  ri  rN   r%  rX   r/   r   rk   r  r  r  r0   r  r1   %_get_remaining_hours_for_budget_reset)r]   rM   r  r0  s       r`   r  z)PrometheusLogger._set_team_budget_metrics  se    ,,"

 +"&"<"<B #= # $	
 	91188C7CGG++??jj , 	
 ??&.&*&@&@ @ 'A ' (	G 7D//66AAEEdooV+.&*&@&@ L 'A ' (	G CD;;BBMWMQQ::$($8$8 ;  ,r   c                    t        |j                  |j                  xs d      }t        | j	                  d      |      } | j
                  j                  d
i |j                  | j                  |j                  |j                               |j                  Rt        | j	                  d      |      } | j                  j                  d
i |j                  |j                         |j                  F | j                  j                  d
i |j                  | j                  |j                  	             yy)z{
        Set virtual key budget metrics

        - Remaining Budget
        - Max Budget
        - Budget Reset At
        r   )r6   r7   r2   r  r  r  Nr3   r   rf   )r  tokenrh  r%  rX   r2   r   rk   r  r  r  r3   r  r4   r  )r]   r^  r  r0  s       r`   r  z(PrometheusLogger._set_key_budget_metrics  sM    ,,22+55;
 +"&"<"<E #= # $	
 	<44;;FgFJJ++,77'-- , 	
 ''3.&*&@&@ C 'A ' (	G :D2299DGDHH!,, ,,8ED>>EEPPTT::$5$E$E ;  9r   r9  r:  c                 |   K   |r4| j                  ||xs d|||       d {   }| j                  |       y y 7 w)Nr   )r  r   r9  r:  r   )_assemble_key_objectr  )r]   r  r   r   r9  r:  r^  s          r`   r=  z>PrometheusLogger._set_api_key_budget_metrics_after_api_request*  sV      &*&?&?)#5#;-#+ '@ ' ! (():; !r  c                 H  K   ddl m} ddlm}m} |xs d|z   }	t        ||||	      }
	 |
j                  r0 ||
j                  ||       d{   }|r|j                  |
_        |
S 7 # t        $ r,}t        j                  dt        |              Y d}~|
S d}~ww xY ww)z2
        Assemble a UserAPIKeyAuth object
        r   )get_key_objectr  )r  rh  r  r  )hashed_tokenr  r  Nz3[Non-Blocking] Prometheus: Error getting key info: )r  r  r  r  r  r   r  r  r[   r   rj   r\   )r]   r  r   r9  r:  r   r  r  r  _total_key_spendr^  
key_objectr_   s                r`   r  z%PrometheusLogger._assemble_key_object<  s      	BP%Nm;*(%"	
	 &&#1!2!8!8"/'9$ 

 8B8R8R%5 !   	  Ec!fXN  ! 	s@   'B"$A* A(A* &B"(A* *	B3!BB"BB"r  c                 h    |t        j                  |j                        z
  j                         dz  S )z6
        Get remaining hours for budget reset
        i  )r   rP  tzinfor  )r]   r  s     r`   r  z6PrometheusLogger._get_remaining_hours_for_budget_reseta  s.    
 hll?+A+ABB
-/D! 	!r   r(  r)  c                 j    t        |t              r#t        |t              r||z
  j                         S y)z
        Compute the duration in seconds between two objects.

        Returns the duration as a float if both start and end are instances of datetime,
        otherwise returns None.
        N)rm   r   r  )r]   r(  r)  s      r`   rQ  z'PrometheusLogger._safe_duration_secondsi  s.     j(+
8X0Nz)88::r   	schedulerc                 V   ddl m} ddlm} t        j
                  j                  t              }t        j                  dt        |             t        |      dkD  rLt        t        |d         }t        j                  d|z         | j                  |j                  d|dd	
       yy)z
        Initialize budget metrics as a cron job. This job runs every `PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES` minutes.

        It emits the current remaining budget metrics for all Keys and Teams.
        r   )2PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTESr   )callback_typezfound %s prometheus loggerszNInitializing remaining budget metrics as a cron job executing every %s minutesintervalprometheus_budget_metrics_jobT)minutesr{  replace_existingN)r  r  "litellm.integrations.custom_loggerr   rg   logging_callback_managerget_custom_loggers_for_typer   r   rj   r  r   add_jobr  )r  r  r   prometheus_loggersprometheus_loggers        r`   "initialize_budget_metrics_cron_jobz3PrometheusLogger.initialize_budget_metrics_cron_jobx  s     	YC ,,HH. I  	 	:C@R<ST!"Q& $%57I!7L M  `DE !EEJ2!%   'r   c                      ddl m}  ddlm} ddlm} dt        j                  v r)ddl m}m	}  |       }|j                  |        | |      }n |        }|j                  d|       |j                  d       y	)
a  
        Mount the Prometheus metrics endpoint with optional authentication.

        Args:
            require_auth (bool, optional): Whether to require authentication for the metrics endpoint.
                                        Defaults to False.
        r   )make_asgi_app)verbose_proxy_logger)appPROMETHEUS_MULTIPROC_DIR)CollectorRegistrymultiprocessz/metricsz;Starting Prometheus Metrics on /metrics (no authentication)N)rQ   r  litellm._loggingr  r  r   osenvironr"  r#  MultiProcessCollectormountrj   )r  r  r   r"  r#  registrymetrics_apps          r`   _mount_metrics_endpointz(PrometheusLogger._mount_metrics_endpoint  sf     	492 &3I(*H..x8'1K'/K 			*k*""I	
r   N)g      ?)Tr  
__module____qualname__ra   r	   r\   r
   rR   r   rr   r   r   r   r   r   rx   r   rt   r   tupler   r   ry   boolr   rT   r   rX   r1  r   r  r   rn   r  r!  r  r"  r#  r[  r[   r   rm  rq  rY  r$  staticmethodrk  r  r  r  r  r  rv  r  r  r  r  r  r   r   r   r   r   r  r  r  r  r  r  r  r   r  r>  r  r  r  r=  r  r   r  rQ  r   r  r+  rf   r   r`   r   r   #   s   Yv*$sDI~*> *XY4 YDU Y.	'	((,S		&	',4 Dd3i<P c 4PS9 *P:"3P:	P:d))04S	)IMc)	)V)#&)7<)	)^	,c 	,DC!#tCy.1DC	DCL
3c 
3d 
35	c.hMT8
"88
 c]8
 sm	8

 %SM8
 }8
  }8
 &c]8
 #8
 +8
t$
}$
 &c]$
 sm	$

 %SM$
 $
 $
LGc]G smG %SM	G
 }G  }G &c]G #G G +G<  sm   %SM   	  
   D@@ }@ sm	@
 %SM@  }@ &c]@ +@D)` (,AA &A *	A
  }AF''-;'RV V|  #RR
 +R Rh $Y $3 $ $,M$',M15,MKT,M\+I$'+I15+IKT+IZCC  C 3-	C
 3-C C2	
	
 	
 		

 	
	
	
 3-	
 3-		

 	
	
	
 3-	
 3-		

 	
  	
  		

$)
9>
	""5/	"2:5/	"		"+%c9U49hsm;S5T+U&UV+ 'S	{IdO'CD+ ?+	+Z
>+
Z><82sN23420>O9P 0
7}7 &c]7 	7
 7 74(( ( 	(
 UO( ( 
(T11f) )V<sm< %SM< 	<
 < E?<$#!#!  #! 	#!
 E?#! #! 
#!J!X !RW !  
%	 6F  > 
 
r   r   r  r  tagrb   c                    |j                         }|j                         D ci c]  \  }}|| v r|| }}}t        j                  j                  |v rt        d|j                  id      |d<   |j                  7|j                  j                         D ]  \  }}t        |      }|| v s|||<    |j                  7t        |j                        }	|	j                         D ]  \  }}|| v s|||<    | D ]  }||vsd||<    |S c c}}w )z
    Returns a dictionary of label + values for prometheus.

    Ensures end_user param is not sent to prometheus if it is not supported.
    user_api_key_end_user_idr   )r   r   rL   N)
model_dumpr   UserAPIKeyLabelNamesEND_USERvaluer   rL   r
  r   r  get_custom_labels_from_tags)
r  r  r2  	enum_dictr   r8  r   r  sanitized_keycustom_tag_labelss
             r`   r%  r%    s?    &&(I
 &OO--LE5)) 	u-   $$**o=&G68L8LM%'

#
 ))5%<<BBDJC;C@M 5516.	 E #78H8HI+113JC++',$ 4 ''%)OE" ' =s   D	r   c                    t         j                  }|t        |      dk(  ri S i }|D ]  }|}|j                  d      r|j	                  ddd      n|}|j                  d      }| }|D ]+  }t        |t              r|j                  |d      }nd}|+ n |qt        |t              s|||j	                  dd      <    |S )z)
    Get custom labels from metadata
    Nr   z	metadata.r   r  r  r  )
rg   !custom_prometheus_metadata_labelsr  r  replacesplitrm   rn   r   r\   )r   r  resultr  original_key
keys_partsr8  parts           r`   r  r    s     44D|s4yA~	F141Lckk+r1-RUYYs^
D%&		$-}  E3!75:F<''S12# & Mr   r  configured_tagc                 t    ddl ddlm}  |       }|j                  |      t	        fd| D              S )a  
    Check if any of the request tags matches a wildcard configured pattern

    Args:
        tags: List[str] - The request tags
        configured_tag: str - The configured tag

    Returns:
        bool - True if any of the request tags matches the configured tag, False otherwise

    e.g.
    tags = ["User-Agent: curl/7.68.0", "User-Agent: python-requests/2.28.1", "prod"]
    configured_tag = "User-Agent: curl/*"
    _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True

    configured_tag = "User-Agent: python-requests/*"
    _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True

    configured_tag = "gm"
    _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # False
    r   N)PatternMatchRouterc              3   D   K   | ]  }j                  |         yw))patternstringN)match).0r2  reregex_patterns     r`   	<genexpr>z;_tag_matches_wildcard_configured_pattern.<locals>.<genexpr>$	  s     Kdsrxxcx:ds    )rM  .litellm.router_utils.pattern_match_deploymentsrG  _pattern_to_regexany)r  rE  rG  pattern_routerrM  rN  s       @@r`   (_tag_matches_wildcard_configured_patternrT  	  s4    0 Q')N"44^DMKdKKKr   c                     ddl m} t        j                  }|t	        |      dk(  ri S i }|D ]3  } |d|       }|| v rd||<   d|v rt        | |      rd||<   /d||<   5 |S )a  
    Get custom labels from tags based on admin configuration.

    Supports both exact matches and wildcard patterns:
    - Exact match: "prod" matches "prod" exactly
    - Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0"

    Reuses PatternMatchRouter for wildcard pattern matching.

    Returns dict of label_name: "true" if the tag matches the configured tag, "false" otherwise

    {
        "tag_User-Agent_curl": "true",
        "tag_User-Agent_python_requests": "false",
        "tag_Environment_prod": "true",
        "tag_Environment_dev": "false",
        "tag_Service_api_gateway_v2": "true",
        "tag_Service_web_app_v1": "false",
    }
    r   r   tag_truer   )r  rE  false)rh   r   rg   custom_prometheus_tagsr  rT  )r  r   configured_tagsrA  rE  
label_names         r`   r9  r9  '	  s    , V44O#o"6!";	F)4tN;K5LM
 T!!'F: . %Mn&
 "(F: %z! *$ Mr   r,  ))r%  rG  r   r   r   r   r   r   r   r	   r
   r   r   r   r   rg   r$  r   r   r  r   litellm.proxy._typesr   r   rh   r   r  r   litellm.utilsr   apscheduler.schedulers.asyncior   r   r\   r  rn   r%  r  r0  rT  r9  rf   r   r`   <module>r_     s    
 
 (    : ; B 3 Q 6 ;?Q"
| Q"
nD ,9,&, 
#, 
	,^d tCH~ @L
s)L%(L	LB0d3i 0DcN 0r   