
    j                         d dl Z d dlmZmZmZmZmZ d dlmZ  ed      Z	 e j                  e      Zde	defdZdedefdZ	 	 dd	ee	   d
ee   dee   deee	      fdZy)    N)ListOptionalTypeVarSequenceAnyTitemreturnc                 L    t        j                  |       }t        |      }|dz  S )Ni   )jsonable_encoderencode_get_json_size)r	   encoded_for_jsonsizes      /Users/manta/Documents/Projects/TheRoad-I1/backend/.venv/lib/python3.12/site-packages/opik/message_processing/batching/sequence_splitter.py_get_expected_payload_size_MBr   
   s)    '..t4*+D;    objc                 .   	 t        | t              rt        | j                  d            dz   S t        | t        t
        f      rt        t        |             S t        | t        d            ryt        | t              rad}t        | j                               }| j                         D ].  \  }}||v st        |      }t        |      }|||z   dz   dz   z  }0 |dz
  S t        | t              rd}| D ]  }|t        |      dz   z  } |dz
  S t        | t              rt        t        |             S t        j                  dt        |              t        t        |             S # t         $ r% t        j                  dd	       t        d
      cY S w xY w)a'  
    Compute the size of the resulting JSON without actually doing the JSON
    encoding, which is CPU and memory consuming. This assumes that we only
    receive basic Python objects, strings, booleans, numbers, list and dicts
    and that the object does not contain any cyclic reference.
    zutf-8   N      z5Unexpected object seen during JSON size estimation %rzFailed to compute object size.T)exc_infoinf)
isinstancestrlenr   intfloattypedictsetkeysitemsr   listboolLOGGERdebug	Exception)r   r   allowed_keyskeyvalueencoded_keyencoded_valuer	   s           r   r   r      sf   !c3szz'*+a//c5\*s3x= T$Z(T"Dsxxz?L!iik
U,&"0"5K$25$9MK-7!;a??D	 *
 !8OT"Dt,q00 !8OT"s3x= LLGc s3x=  5EU|s;   ,E& )E& E& /AE& 5*E&  .E& #E& 32E& &+FFr$   max_payload_size_MB
max_lengthc                    |	|J d       |t        |       }|t        d      }g }g }d}| D ]p  }|dn
t        |      }||k\  r|j                  |g       *t        |      |k(  }||z   |kD  }	|s|	r|j                  |       |g}|}[|j                  |       ||z  }r t        |      dkD  r|j                  |       |S )Nz1At least one limitation must be set for splittingr   g        r   )r   r   r   append)
r$   r/   r0   batchescurrent_batchcurrent_batch_size_MBr	   item_size_MBbatch_is_already_full+batch_will_exceed_memory_limit_after_addings
             r   split_into_batchesr9   ;   s	   
  +1G ;H Z
"#ElGM#&&.C4QRV4W 	 ..NND6" #M 2j @!L03FF 	4 !$ONN=)!FM$0!  &!\1!) , =A}%Nr   )NN)loggingtypingr   r   r   r   r   opik.jsonable_encoderr   r   	getLogger__name__r'   r   r   r   r   r9    r   r   <module>r@      s     9 9 0CL			8	$   e  ( ( (Z ,0 $,A;,!%, , 
$q']	,r   