
    ju              -       <	   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
mZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmZ e
rd dlmZ neZej0                  j3                   ej4                                dZ ej8                  dd	      Zed	k(  r e        d d
lmZ dZ  G d de      Z!de"fdZ# G d d      Z$ ejJ                          ejL                  dddd       ejL                  dddd       ejL                  dedd       ejL                  ddd       ejL                  d d!d"       ejL                  d#d$dd%       ejL                  d&dd'       ejL                  d(dd%       ejL                  d)dd*       ejL                  d+d,e'd-.       ejL                  d/d0d,e'd1d23       ejL                  d4d0d,e'd5d63       ejL                  d7d0d,e'd89       ejL                  d:de(d;<       ejL                  d=de)d><       ejL                  d?de)d@<       ejL                  dAd,dBC       ejL                  dDd,dEC       ejL                  dFdGddH       ejL                  dIde(dJ<       ejL                  dKd,e'dL<       ejL                  dMde"dN<       ejL                  dOdPd0d,e'dQ9       ejL                  dRd,dST       ejL                  dUd,dVT       ejL                  dWd0d,dXY       ejL                  dZd0d,d[Y       ejL                  d\d]e)d^<       ejL                  d_d0d,d`Y       ejL                  dad0d,dbY       ejL                  dcde"dddef       ejL                  dgde"dhdif       ejL                  djde"dk<       ejL                  dld,d0dmn       ejL                  dod,d0dpn       ejL                  dqd,d0drn       ejL                  dsde)dtduf       ejL                  dvde)dwdxf      dye'fdz                                                                                                                                                                                                                                                                                 Z*e+d{k(  r e*        yy)|    N)TYPE_CHECKINGAnyOptionalUnion)load_dotenv)!DEFAULT_NUM_WORKERS_LITELLM_PROXY)get_secret_bool)FastAPIzlitellm.secretsLITELLM_MODEDEV)Enumc                       e Zd ZdZdZy)LiteLLMDatabaseConnectionPool
   <   N)__name__
__module____qualname__database_connection_pool_limit database_connection_pool_timeout     p/Users/manta/Documents/Projects/TheRoad-I1/backend/.venv/lib/python3.12/site-packages/litellm/proxy/proxy_cli.pyr   r   #   s    %'"')$r   r   returnc                 d   ddl m} |j                  d|         |j                  d|        t        j                  |       }t        j                  |j
                        }|j                  |       t        j                  |d      }t        j                  |j                  |            }|S )Nr   )verbose_proxy_loggerzurl: zparams: T)doseq)query)
litellm._loggingr   debugurlparseparse_qsr   update	urlencode
urlunparse_replace)urlparamsr   
parsed_urlparsed_queryencoded_querymodified_urls          r   append_query_paramsr-   (   s    5se}-&23""3'J$$Z%5%56L&&|4@M&&z':':':'OPLr   c                   ,   e Zd Zed        Zed        Zededededee	ef   fd       Z
e	 	 ddeded	ee   d
ee   def
d       Ze	 ddedededededee   fd       Ze	 ddededededededee   fd       Zed        Zed        Zed        Zy)ProxyInitializationHelpersc                  t    t         j                  j                  d      } t        j                  d|  d       y )Nlitellmz
LiteLLM: Current Version = 
)	importlibmetadataversionclickecho)pkg_versions    r   _echo_litellm_versionz0ProxyInitializationHelpers._echo_litellm_version6   s.    ((00;

2;-rBCr   c                     t        d       t        j                  d|  d| d      }t        t        j                  |j                         d             y )Nz)
LiteLLM: Health Testing models in confighttp://:z/health)r'      )indent)printhttpxgetjsondumps)hostportresponses      r   _run_health_checkz,ProxyInitializationHelpers._run_health_check;   sA    :;9974&$w!?@djj34r   rD   rE   modeltestc                 Z   |xs d}t        j                  d|        dd l}d|  d| }t        |t              r|}nt        d      |j                  d|      }|j                  j                  j                  |d	d
dgd      }t        j                  d|        t        d|        |j                  j                  j                  |d	d
dgd      }	|	D ]  }
t        j                  d|
         t        d       |j                  j                  |d
      }t        |       y )Nzgpt-3.5-turbozE
LiteLLM: Making a test ChatCompletions request to your proxy. Model=r   r;   r<   zInvalid test valuez
My API Key)api_keybase_urluserz*this is a test request, write a short poem)rolecontent   )rH   messages
max_tokensz
LiteLLM: response from proxy zN
 LiteLLM: Making a test ChatCompletions + streaming r equest to proxy. Model=T)rH   rQ   streamz'LiteLLM: streaming response from proxy z$
 making completion request to proxy)rH   prompt)r6   r7   openai
isinstancestr
ValueErrorOpenAIchatcompletionscreater?   )rD   rE   rH   rI   request_modelrU   api_baseclientrF   stream_responsechunkcompletion_responses               r   _run_test_chat_completionz4ProxyInitializationHelpers._run_test_chat_completionA   s^    0

TUbTcd	
 	TF!D6*dC H122|hG;;**11 #K  2 	
 	

4XJ?@]^k]lm	
 !++1188 #K  9 	
 %EJJ@HI %56$0077(T 8 
 	!"r   N
log_configkeepalive_timeoutr   c                     ddl }d| |d}|t        d|        ||d<   n|j                  rt        d       d|d<   |||d<   |S )	z8
        Get the arguments for `uvicorn` worker
        r   Nzlitellm.proxy.proxy_server:app)apprD   rE   zUsing log_config: rd   z,Using json logs. Setting log_config to None.timeout_keep_alive)r1   r?   	json_logs)rD   rE   rd   re   r1   uvicorn_argss         r   _get_default_unvicorn_init_argsz:ProxyInitializationHelpers._get_default_unvicorn_init_argsw   sn     	 4

 !&zl34)3L&@A)-L&(1BL-.r   rg   ssl_certfile_pathssl_keyfile_pathciphersc                     ddl }ddlm} ddlm} t        d| d| d        |       }	| d| g|	_        |+|)t        d| d	| d
       ||	_        ||	_        |||	_	        |j                   || |	             y)z5
        Initialize litellm with `hypercorn`
        r   N)serve)Config)[1;32mLiteLLM Proxy: Starting server on r<   z using Hypercorn[0m
/[1;32mLiteLLM Proxy: Using SSL with certfile:  and keyfile: [0m
)asynciohypercorn.asynciorp   hypercorn.configrq   r?   bindcertfilekeyfilern   run)
rg   rD   rE   rl   rm   rn   rv   rp   rq   configs
             r   _init_hypercorn_serverz1ProxyInitializationHelpers._init_hypercorn_server   s     	++:4&$G`a	
 q'((-=-IDEVDWWefvew  xA  B 0FO-FN"!( 	E#v&'r   num_workersmax_requests_before_restartc           	      R   t         j                  dk(  rnddl} G fddj                  j                  j
                        }t        d|  d d| d	       |  d |d
ddddd}	|||	d<   ||t        d| d| d       ||	d<   ||	d<    |||	      j                          y)z-
        Run litellm with `gunicorn`
        ntr   Nc                   0     e Zd Zd fd	Zd Zd Z xZS )NProxyInitializationHelpers._run_gunicorn_server.<locals>.StandaloneApplicationc                     |xs i | _         || _        t        |           d d}|dz   }t	                t	        d       t	        d| d       t	        d       t	        d d       y )	Nz curl --location 'http://0.0.0.0:z/chat/completions' \ai  
                --header 'Content-Type: application/json' \
                --data ' {
                "model": "gpt-3.5-turbo",
                "messages": [
                    {
                    "role": "user",
                    "content": "what llm are you"
                    }
                ]
                }'
                

                z[1;34mLiteLLM: Test your local proxy with: "litellm --test" This runs an openai.ChatCompletion request to your proxy [In a new terminal tab][0m
z8[1;34mLiteLLM: Curl Command Test for your local proxy
 z [0m
z;[1;34mDocs: https://docs.litellm.ai/docs/simple_proxy[0m
z5[1;34mSee all Router/Swagger docs on http://0.0.0.0:)optionsapplicationsuper__init__r?   )selfrg   r   _endpoint_strcurl_command	__class__rE   s        r   r   zWProxyInitializationHelpers._run_gunicorn_server.<locals>.StandaloneApplication.__init__   s    &}"#&  " 7tf<QR  "    p RS_R``jk X NtfT^_r   c                 Z   | j                   H| j                  j                         D ci c]"  \  }}|| j                   j                  v r|||$ }}}ni }|j                         D ]<  \  }}| j                   | j                   j	                  |j                         |       > y c c}}w N)cfgr   itemssettingssetlower)r   keyvaluer}   s       r   load_configzZProxyInitializationHelpers._run_gunicorn_server.<locals>.StandaloneApplication.load_config   s    88' +/,,*<*<*>*>JC$(("3"338I U
*>    F"(,,.JCxx+SYY[%8 #1s   'B'c                     | j                   S r   )r   )r   s    r   loadzSProxyInitializationHelpers._run_gunicorn_server.<locals>.StandaloneApplication.load   s    '''r   r   )r   r   r   r   r   r   __classcell__)r   rE   s   @r   StandaloneApplicationr      s    $L9(r   r   rr   r<   z with z workers[0m
zuvicorn.workers.UvicornWorkerT-iX  z+%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s)ry   workersworker_classpreload	accesslogtimeoutaccess_log_formatmax_requestsrs   rt   ru   rz   r{   )rg   r   )osnamegunicorn.app.baserg   baseBaseApplicationr?   r|   )
rD   rE   rg   r   rl   rm   r   gunicornr   gunicorn_optionss
    `        r   _run_gunicorn_serverz/ProxyInitializationHelpers._run_gunicorn_server   s     77d?$7	(HLL$5$5$E$E 7	(r 	:4&$vk]Zkl	
 fAdV$";!N
 '2/J^,(-=-IDEVDWWefvew  xA  B ,=Z(*:Y'#/?@DDFr   c                      	 ddg} t        t        j                  d      5 }t        j                  | ||       d d d        y # 1 sw Y   y xY w# t
        $ r}t        d| d       Y d }~y d }~ww xY w)Nollamarp   w)stdoutstderrzh
                LiteLLM Warning: proxy started with `ollama` model
`ollama serve` failed with Exceptionz-. 
Ensure you run `ollama serve`
            )openr   devnull
subprocessPopen	Exceptionr?   )commandr   es      r   _run_ollama_servez,ProxyInitializationHelpers._run_ollama_serve  sr    
	)Gbjj#&'  I '&& 	iijhk l 	s3   A AA AA A 	A0A++A0c                     dd l }|j                  |j                  |j                        5 }|j                  d| f      dk(  cd d d        S # 1 sw Y   y xY w)Nr   	localhost)socketAF_INETSOCK_STREAM
connect_ex)rE   r   ss      r   _is_port_in_usez*ProxyInitializationHelpers._is_port_in_use)  sA    ]]6>>6+=+=>!<<d 349 ?>>s   AAc                  *    t         j                  dv ryy)zBHelper function to determine the event loop type based on platform)win32cygwincliNuvloop)sysplatformr   r   r   _get_loop_typez)ProxyInitializationHelpers._get_loop_type0  s     <<55r   )NNr   )r   r   r   staticmethodr9   rG   rW   intr   boolrc   r   dictrk   r
   r~   r   r   r   r   r   r   r   r/   r/   5   s   D D 5 5
 3#3#3# 3# D#I	3# 3#j  %)+/	 SM $C=	
 
 4  "& ( ( (  ( 	 (
  ( # (  (D  6:cGcGcG cG 	cG
 cG cG &.c]cG cGJ   : :  r   r/   z--hostz0.0.0.0z!Host for the server to listen on.HOST)defaulthelpenvvarz--port  zPort to bind the server to.PORTz--num_workerszNumber of uvicorn / gunicorn workers to spin up. By default, it equals the number of logical CPUs in the system, or 4 workers if that cannot be determined.NUM_WORKERSz
--api_basezAPI base URL.)r   r   z--api_versionz2024-07-01-previewz$For azure - pass in the api version.z--modelz-mz)The model name to pass to litellm expectsz--aliaszThe alias for the model - use this to give a litellm model name (e.g. "huggingface/codellama/CodeLlama-7b-Instruct-hf") a more user-friendly name ("codellama")z	--add_keyz	--headerszheaders for the API callz--saveTzSave the model-specific config)is_flagtyper   z--debugFzTo debug the inputDEBUG)r   r   r   r   r   z--detailed_debugzTo view detailed debug logsDETAILED_DEBUGz--use_queuez)To use celery workers for async endpoints)r   r   r   r   z--temperaturezSet temperature for the model)r   r   r   z--max_tokenszSet max tokens for the modelz--request_timeoutz+Set timeout in seconds for completion callsz--drop_paramszDrop any unmapped params)r   r   z--add_function_to_promptz5If function passed but unsupported, pass it as promptz--configz-cz]Path to the proxy configuration file (e.g. config.yaml). Usage `litellm --config config.yaml`z--max_budgetz`Set max budget for API calls - works for hosted models like OpenAI, TogetherAI, Anthropic, etc.`z--telemetryzZHelps us know if people are using this feature. Turn this off by doing `--telemetry False`z--log_configz&Path to the logging configuration filez	--versionz-vzPrint LiteLLM versionz--healthz:Make a chat/completions request to all llms in config.yaml)
flag_valuer   z--testz4proxy chat completions url to make a test request toz--test_asyncz9Calls async endpoints /queue/requests and /queue/response)r   r   r   z--iam_token_db_authz!Connects to RDS DB with IAM tokenz--num_requestsr   z-Number of requests to hit async endpoint withz--run_gunicornzTStarts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)z--run_hypercornz@Starts proxy via hypercorn, instead of uvicorn (supports HTTP/2)z--ssl_keyfile_pathz^Path to the SSL keyfile. Use this when you want to provide SSL certificate when starting proxySSL_KEYFILE_PATH)r   r   r   r   z--ssl_certfile_pathz_Path to the SSL certfile. Use this when you want to provide SSL certificate when starting proxySSL_CERTFILE_PATHz	--ciphersz!Ciphers to use for the SSL setup.z--use_prisma_db_pushzHUse prisma db push instead of prisma migrate for database schema updates)r   r   r   z--localzfor local debuggingz--skip_server_startupzASkip starting the server after setup (useful for migrations only)z--keepalive_timeoutzSSet the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter)KEEPALIVE_TIMEOUTz--max_requests_before_restartz]Restart worker after this many requests (uvicorn: limit_max_requests, gunicorn: max_requests)MAX_REQUESTS_BEFORE_RESTARTuse_prisma_db_pushc&           
         t               }&|rddlm}'m}(m})m}* n	 ddlm}'m}(m})m}* |du rt        j                          y |rd|v r|t        j                          |du rt        j                  | |       y |du rt        j                  | |||       y |rt        j                  |      } |*dZi d	|d
|d|d|d|	d|
d|d|d|d|d|d|d|d|d|d|d| 	 dd l},d}-d}.i }/|st%        d      rddlm}0 t+        j,                  d      }1t+        j,                  d       }2t+        j,                  d!      }3t+        j,                  d"      }4t+        j,                  d#      }5 |0|1|2|3$      }6d%|3 d&|6 d'|1 d&|2 d(|4 
}7|5r|7d)|5 z  }7|7t*        j.                  d*<   d+t*        j.                  d<   dd,lm}8 t+        j,                  d-d       Jt+        j,                  d-      d+k(  r2 |8       }9|9j5                         D ]  \  }:};|;t*        j.                  |:<    |	 	 dd l}< |(       }=|<j9                  |=j;                  |/            }>|>j=                  d0d       }?|?&d1|?v r"|?d1   du rdd l}@d|@_         |@jC                          |>j=                  d2i       }/|/i }/|/j=                  d3d       }A|Add l}@ |'dZi A|@_"        |/r%|/j=                  d4d       }B|=jG                  |B|5       |/j=                  d6d       }C|C7t+        j,                  d*      "dd7l$m%}D  |D       }C|CrCt*        j.                  d*<   |/j=                  d8tL        jN                  jP                        }-|/j=                  d9tL        jR                  jP                        }.CrCjU                  d:      rzt+        jV                         }EtX        jZ                  j]                  dt*        jZ                  j_                  d;             dd l}@dd<lm0}F  |FCd =      }Ct+        jb                  |E       C#te        Ct              rCt*        j.                  d*<   |7t+        j,                  d*      "dd7l$m%}D  |D       }C|CrCt*        j.                  d*<   |4tL        jN                  jP                  }-tL        jR                  jP                  }.t+        j,                  d*d       t+        j,                  d>d       	 dd?l3m4}G t+        j,                  d*d       .|-|.d@}H Gd*d =      }Ctk        |C|H      }I|It*        j.                  d*<   t+        j,                  d>d       9|-|.d@}Ht+        j,                  d>      }Ctk        |C|H      }I|It*        j.                  d><   tm        j8                  dAgdB       d}JJrEddDl8m9}K ddEl:m;}Lm<}M  |M|/j=                  dF            dCu r
 Kd G       nLj{                  |" H       nt}        dI       |dJk(  r+t        j                  |      rt        j                  dKdL      }dd l}@|
du r@j                          ddMlCm}) |#rt}        dN       y t        j                  | ||!|$O      }N|%|%NdP<   |dCu rV|dCu rR||t}        dQ| dR| dS       |NdT<   ||NdU<   t        j                         }O|OrONdV<    |,j8                  dZi NdW|i y |du rt        j                  | ||)||||%X       y |du rt        j                  |)| |||| Y       y y # t        $ r}+t        d|+ d      d }+~+wt        $ r&}+dt        |+      v r|+ddlm}'m}(m})m}* Y d }+~+d }+~+ww xY w# t"        $ r t        d      w xY w# t"        $ r t        d.      w xY w# tn        $ r dC}JY w xY w)[Nr   )KeyManagementSettingsProxyConfigrg   save_worker_config   zMissing dependency z$. Run `pip install 'litellm[proxy]'`zlitellm[proxy]Tr   rH   aliasr^   api_versionr    detailed_debugtemperaturerR   request_timeout
max_budget	telemetrydrop_paramsadd_function_to_promptheaderssaver}   	use_queuezLuvicorn, gunicorn needs to be imported. Run - `pip install 'litellm[proxy]'`d   r   IAM_TOKEN_DB_AUTH)generate_iam_auth_tokenDATABASE_HOSTDATABASE_PORTDATABASE_USERDATABASE_NAMEDATABASE_SCHEMA)db_hostdb_portdb_userzpostgresql://r<   @/z?schema=DATABASE_URLTrue)decrypt_env_varUSE_AWS_KMSz?yaml needs to be imported. Run - `pip install 'litellm[proxy]'`)config_file_pathlitellm_settingsri   general_settingskey_management_settingskey_management_system)r  r   database_url)$construct_database_url_from_env_varsr   r   zos.environ/z../..)get_secret_str)default_value
DIRECT_URL)
get_secret)connection_limitpool_timeoutprisma)capture_outputF)check_prisma_schema_diff)PrismaManagershould_update_prisma_schemadisable_prisma_schema_update)db_url)use_migratezYUnable to connect to DB. DATABASE_URL found in environment, but prisma package not found.r   i   i   )rg   z>LiteLLM: Setup complete. Skipping server startup as requested.)rD   rE   rd   re   limit_max_requestsrs   rt   ru   ssl_keyfilessl_certfileloopr   )rD   rE   rg   r   rl   rm   r   )rg   rD   rE   rl   rm   rn   r   )Hlocalsproxy_serverr   r   rg   r   ModuleNotFoundErrorImportErrorrW   r/   r9   r   rG   rc   rB   loadsuvicornr   r	    litellm.proxy.auth.rds_iam_tokenr   r   getenvenviron*litellm.secret_managers.aws_secret_managerr   r   rv   r|   
get_configrA   r1   ri   _turn_on_json_key_management_settingsinitialize_secret_managerlitellm.proxy.utilsr  r   r   r   r   
startswithgetcwdr   pathinsertabspathr  chdirrV   litellm.secret_managers.mainr
  r-   r   FileNotFoundError litellm.proxy.db.check_migrationr  litellm.proxy.db.prisma_clientr  r  setup_databaser?   r   randomrandint_turn_on_debuglitellm.proxy.proxy_serverrk   r   r   r~   )PrD   rE   r^   r   rH   r   add_keyr   r   r    r   r   rR   r   r   r   r}   r   r   rI   localr   
test_asynciam_token_db_authnum_requestsr   healthr5   run_gunicornrun_hypercornrm   rl   rn   rd   r   skip_server_startupre   r   argsr   r   rg   r   r   r  db_connection_pool_limitdb_connection_timeoutr  r   r   r   r   db_name	db_schematoken_db_urlr   new_env_varkvrv   proxy_config_configr  r1   r  r  r  r  original_dirr  r
  r(   r,   is_prisma_runnabler  r  r  rj   	loop_typesP                                                                                   r   
run_serverrO  8  s`   L 8D	
 	
	 , $"88:U"x'7"446~"44T4@t|"<<T4PTUjj)G 	
	
	
 	
 $		

 	
 *	
 $	
 "	
 ,	
 "	
  	
 $	
 $:	
 	
 	
  !	
"  #	
&	 $'  " 0C DPii0Gii0Gii0Gii0G		"34I+'E
 &gYaway'!G9UGXi[11)0BJJ~&.4BJJ*+ 	O IImT*6		-(F2 *+K#))+1 !

1 , 
 '=Lkk,"9"96"9"RSG  '{{+=tD ,#33$[1T9$(!%%'&{{+=rB'#% &6&:&:)4'# '23H 4-40  (8(<(<+T)% 66*?RX 7  ,//EL#		.(A(ITCE1=BJJ~.'7';';0-LLRR($ %5$8$82-NNTT%!  7 7 F!yy{rwww/ 2-l$O&'J|S,I-9

>* >bii7?P?AL-9

>* >-LLRR % .NNTT "
 IInd+7yyt,8+C99^T2> -E(=F $.nD#QL#6|V#LL1=BJJ~.99\40< -E(=F $&99\#:L#6|V#LL/;BJJ|,z$?%)" "U 0(,,-KL 
 -D9!00AS=S0To 4<6FFtL>>$.DT!""$ 	3 P 1QQ!/	 R 
 '21LL-.5 ]e%; ,1A1MHIZH[[ijzi{  |E  F /?]+/@^,2AACI'0V$GKK # T!&;;'"3!1,G <  d"&=="3!1 >  #c	 # 	%%aS(LM   	3q6)  	^  	^ 	l  !U n % +%*"+sN   [5 %] "] B3]2 5	\?>\\?\::\?]]/2^ ^__main__),r3   rB   r   r3  r   r   urllib.parseparser!   typingr   r   r   r   r6   r@   dotenvr   litellm.constantsr   r.  r	   fastapir
   r*  appendr)  config_filenamer   litellm_modeenumr   r   r   rW   r-   r/   r   optionr   floatr   rO  r   r   r   r   <module>r]     s     	   
  6 6    ? 8G 			 #ryy/5M 	*D *

 
@ @F i&IRX h+HQWX- 
g	 lD? 	/
 tT(S  
k
 $O k4.HIh46VW		 		& 		4 T4S Ds1O 		6	 ot2LM	@
 	h	 		k	 		e	 		1	 		  	E
 	?
 	D	 	,	 		8	 	_	 	K	 		i 		j 		,	 	S	 iu;PQ	L	 		^ #		h(qF Gq R N X J @ Y |qh zL r   