
    j                         d dl Z d dlZd dlmZmZmZmZmZ d dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ  ej>                  e       Z! G d d      Z"y)    N)OptionalListCallableDictLiteral)conversation_thread_metric)JsonListStringPublicTraceThread   )evaluation_resulthelpers   )evaluation_tasks_executor)types)score_result   )trace)threads_clientc                      e Zd Zdej                  dee   dededdf
dZ	 ddee   d	ee   d
e	e
j                     deegef   deegef   dedej                  fdZded	ee   d
e	e
j                     deegef   deegef   dedej$                  fdZ ej*                  d      de	eed   ef      d
e	e
j                     de	ej2                     fd       Zy)ThreadsEvaluationEngineclientproject_namenumber_of_workersverbosereturnNc                 J    || _         || _        || _        || _        || _        y )N)_client_project_name_number_of_workers_verbose_threads_client)selfr   r   r   r   s        /Users/manta/Documents/Projects/TheRoad-I1/backend/.venv/lib/python3.12/site-packages/opik/evaluation/threads/evaluation_engine.py__init__z ThreadsEvaluationEngine.__init__   s*     )"3%    filter_stringeval_project_namemetricstrace_input_transformtrace_output_transformmax_traces_per_threadc                    t        |      dk(  rt        d      | j                  j                  | j                  |      }t        |      dk(  rt        j                  d|       |D cg c]  }|j                  dk(  s| }	}t        |	      dk(  rt        j                  d| d      t        |	      t        |      k  ra|D cg c]  }|j                  dk(  s|j                    }
}|	D cg c]  }|j                   }}t        j                  d	|
 d
| d| d       |	D cg c](  }t        j                  | j                  ||||||      * c}}t        j                  || j                   | j"                        }t%        j&                  || j                  | j                         t)        j*                  |      S c c}w c c}w c c}w c c}w )Nr   zNo metrics provided)r   r&   z%No threads found with filter_string: inactivez,No closed threads found with filter_string: z'. Only closed threads can be evaluated.activezSome threads are active: z with filter_string: z). Only closed threads will be evaluated: .)threadr'   r(   r)   r*   r+   )workersr   )r   r   )results)len
ValueErrorr!   search_threadsr   
exceptionsEvaluationErrorstatusidLOGGERwarning	functoolspartialevaluate_threadr   executer   r    r   log_feedback_scoresr   ThreadsEvaluationResult)r"   r&   r'   r(   r)   r*   r+   threadsr0   inactive_threadsactive_threads_idsinactive_threads_idsevaluation_tasksr2   s                 r#   evaluate_threadsz(ThreadsEvaluationEngine.evaluate_threads$   s    w<1233&&55++' 6 
 w<1,,7G  29XvFMMZ<WFX A%,,>}oMtu  !"S\1(/"(/f6==H3L		  " =M#M<L&FII<L #MNN+,>+??TUbTc  dM  Nb  Mc  cd  e  +
 + $$"3&;'=&; +
 	 ,33d&=&=t}}
 	##$"4"4T=Q=Q	
 !88IIK Y" $N
s$   +G GGG1G)-Gr0   c                    t        j                  ||||| j                  | j                  j                        j                         }|d   }t        |      dk(  rDt        j                  d|j                   d       t        j                  |j                  g       S || j                  }t        j                  ||ddd	|
      }	t        j                  |	| j                  j                        5  | j!                  ||      }
|
D cg c]  }|j"                   }}t        j$                  d|i       d d d        t        j                  |j                  
      S c c}w # 1 sw Y   /xY w)N)r0   r)   r*   max_resultsr   r   
discussionr   zThread 'z2' has no conversation traces. Skipping evaluation.)	thread_idscores)conversationr(   evaluation_task
evaluation)inputname
created_byr   )
trace_datar   evaluation_results)output)r   load_conversation_threadr   r   opik_client
model_dumpr3   r:   r;   r9   r   ThreadEvaluationResultr   	TraceDataopik_contexttrace_context_evaluate_conversation__dict__update_current_trace)r"   r0   r'   r(   r)   r*   r+   conversation_dictrM   rS   r2   resultoutputss                r#   r>   z'ThreadsEvaluationEngine.evaluate_thread`   sa    $<<"7#9-++<<++
 *, 	 )6|!NN699+%WX %;; ))B  $ $ 2 2 __#/GD"#*	

 ''!<<++
 11,HG 6==W6vWG=--6JG5TU
 !77ii
 	
 >
 
s   4E&E!E&!E&&E/metrics_calculation)rQ   rM   )rolecontentc                    g }|D ]  }	 t         j                  d|j                         |j                  |      }t         j                  d|j                         t	        |t
              r|j                  |       n|j                  |        |S # t        $ rg}t         j                  d|j                  d       |j                  t        j                  |j                  dt        |      d             Y d }~d }~ww xY w)NzMetric %s score startedzMetric %s score endedzCFailed to compute metric %s. Score result will be marked as failed.T)exc_infog        )rQ   valuereasonscoring_failed)r:   debugrQ   score
isinstancelistextendappend	Exceptionerrorr   ScoreResultstr)r"   rM   r(   score_resultsmetricra   es          r#   r]   z.ThreadsEvaluationEngine._evaluate_conversation   s     9;F6Dl34fkkBfd+!((0!((0 2   YKK!  
 $$ ,,#[[!"1v'+	 s   BB	DAC<<D)i  )__name__
__module____qualname__r   ThreadsClientr   rt   intr$   r   r   ConversationThreadMetricr   r	   r   rA   rG   r
   rY   r>   opiktrackr   r   r   rs   r]    r%   r#   r   r      s   &,,& sm& 	&
 & 
&* &*:J}:J $C=:J 0IIJ	:J
  ()=(>(CD:J !)*>)?)D E:J  #:J 
	2	2:Jx3
3
 $C=3
 0IIJ	3

  ()=(>(CD3
 !)*>)?)D E3
  #3
 
	1	13
j TZZ*+4(9 :C ?@A 0IIJ 
l&&	'	 ,r%   r   )#r<   loggingtypingr   r   r   r   r   r~   opik.exceptionsr6   opik.opik_contextr[   $opik.evaluation.metrics.conversationr   opik.rest_apir	   r
    r   r   enginer   r   engine_typesr(   r   api_objectsr   api_objects.threadsr   	getLoggerrx   r:   r   r   r%   r#   <module>r      sP      : :  $ ( K ; ( . * "   1			8	$` `r%   