
    i,                        d Z ddlmZ ddlZddlZddlZddlmZ ddlmZ ddl	m
Z
mZ ddZdddZdd	Zdd
ZddZedk(  r ej$                   e              yy)u\  G4.2 canary — token-count delta measurement (system.md, tiktoken cl100k_base).

Computes baseline and candidate token counts for system.md and emits the delta.
Used to verify Rule A/C/E lift achieves the expected token reduction
(target threshold: candidate - baseline <= -850).

Exit criteria (analyzer):
  exit_criteria_pass == True  iff  token_delta <= -850

Usage (config mode):
    python scripts/canary/g4_2_token_count.py \
        --config canary_config.json \
        --baseline-version 16.202605041200 \
        --candidate-version 17.202605042018 \
        --output results/token_count.json
    )annotationsN)datetime)Path)AnyDictc                 L    t        j                         j                         dz   S )NZ)r   utcnow	isoformat     "scripts/canary/g4_2_token_count.pyiso8601_nowr      s    ??&&(3..r   c                j    t        d|  t        j                         t        j                  |       y )Nz[g4_2_token_count] ERROR: file)printsysstderrexit)msgcodes     r   _failr   !   s#    	&se
,3::>HHTNr   c           	        | j                   rt        | j                         }|j                         st        d|        	 |j	                  dd      5 }t        j                  |      }d d d        j                  d      |j                  d      |j                  d      |j                  d	      |j                  d
      |j                  d      |j                  d      dS 	 | j                  rt        j                  | j                        nd }| j                  rt        j                  | j                        nd }| j                  | j                  | j                  | j                  | j                   dS # 1 sw Y   xY w# t        $ r}t        d|        Y d }~4d }~ww xY w# t        $ r}t        d|        Y d }~d }~ww xY w)Nz--config file not found: rutf-8encodingz--config parse error: pidscene_index_listshot_index_list_per_scenemodel_routingprompt_source_modecard_commit_hashchain_bg_card_snapshot_hashr   r    r!   r"   r#   r$   r%   z%failed to parse list/dict JSON args: )configr   existsr   openjsonload	Exceptiongetr    loadsr!   r   r"   r#   r$   r%   )argscfg_pathfpcfgexcr    r!   s          r   load_pinningr4   &   s   {{$ -hZ89	2sW5iim 6
 775> #(: ;),1L)M WW_5"%''*>"? #(: ;+.773P+Q
 	
	=151F1FDJJt,,-D 	
 -- JJt55637 	" xx,%>++"55 11'+'G'G / 65 	2*3%011	2&  =5cU;<<=sI   F	 E<*F	 &AF. <FF	 		F+F&&F+.	G7G

Gc                    g d}|D cg c]  }| j                  |      dv s| }}|rt        d| d       y y c c}w )Nr&   )N z#pinning missing required field(s): zF. Provide via --config <json> or individual flags. NO silent defaults.)r-   r   )pinningrequiredkmissings       r   validate_pinningr;   N   sW    H
 #C(Qgkk!n
&Bq(GC1' ;S T	
  Ds   ;;c                    t        j                  d      } | j                  dt        d        | j                  dt        dd       | j                  d	t        dd
       | j                  dt        dd       | j                  dt        dd       | j                  dt        d        | j                  dt        d        | j                  dt        d        | j                  dt        d        | j                  dt        d g d       | j                  dt        d        | j                  dt        d        | j                  dt        d       | j                         }t        |      }t        |       	 dd l}j                  d      }t        |j                        }||j                  z  d z  }||j                  z  d z  }|j!                         st        d!|        |j!                         st        d"|        |j#                  d#$      }|j#                  d#$      }	t%        |j'                  |            }
t%        |j'                  |	            }||
z
  }||j(                  k  }t+               ||j                  |j                  |
|||j(                  |d%d&d'id(}|j,                  d)k(  rHt/        j0                  |t2        j4                  d*d+d,       t2        j4                  j7                  d-       nht        |j,                        }|j8                  j;                  dd.       |j=                  d/d#$      5 }t/        j0                  ||d*d+d,       d d d        t?        d0|j                   d1|
 d2|j                   d1| d3| d4|j(                   d5| t2        j@                  6       |rdS d7S # t        $ r t        d       Y Rw xY w# 1 sw Y   xxY w)8NzAG4.2 canary: token-count delta (system.md, tiktoken cl100k_base).)descriptionz--config)typedefaultz--baseline-versionTz=Baseline prompt version directory name (e.g. 16.202605041200))r>   r8   helpz--candidate-versionz>Candidate prompt version directory name (e.g. 17.202605042018)z--prompt-base-pathzprompts/_base/scene_detailz)Base directory containing version subdirs)r>   r?   r@   z--exit-criteria-thresholdiz0token_delta upper bound for pass (default: -850)z--pidz--scene-index-listz--shot-index-list-per-scenez--model-routingz--prompt-source-mode)Nr   db)r>   r?   choicesz--card-commit-hashz--chain-bg-card-snapshot-hashz--output)r>   r8   r   ztiktoken not installed. Install via `pip install tiktoken` (or use the backend venv: backend/.venv/bin/pip install tiktoken). NO silent fallback.cl100k_basez	system.mdzbaseline system.md not found: zcandidate system.md not found: r   r   ztiktoken cl100k_basetoken_countzDscripts/canary/g4_2_token_count.py (system.md, tiktoken cl100k_base))	timestampr7   baseline_prompt_versioncandidate_prompt_versionbaseline_token_countcandidate_token_counttoken_deltaexit_criteria_thresholdexit_criteria_passmeasurement_toolmeasurement_scripts-F   )ensure_asciiindent	sort_keys
)parentsexist_okwz[g4_2_token_count]
  baseline (z): z tokens
  candidate (z tokens
  token_delta: z
  threshold: z
  exit_criteria_pass: r      )!argparseArgumentParseradd_argumentstrint
parse_argsr4   r;   tiktokenImportErrorr   get_encodingr   prompt_base_pathbaseline_versioncandidate_versionr(   	read_textlenencoderK   r   outputr*   dumpr   stdoutwriteparentmkdirr)   r   r   )parserr/   r7   r_   encoderbase_dirbaseline_pathcandidate_pathbaseline_textcandidate_textrH   rI   rJ   	exit_passoutout_pathr1   s                    r   mainrx   \   s   $$WF 
d;
,3\  ^
-C$]  _
,3 <H  J 3#tO  Q c48
,3E
5CN
)TB
.S$ 4  6
,3E
7c4P

t<D4 GW
 ##M2GD))*Ht444{BM 6 66DN!.}o>?  "//?@A!++W+=M#--w-?Nw~~m<=~ >?'*>>Kt;;;I !]#'#8#8$($:$: 4!6"#'#?#?'2a 
C  {{c		#szza4P

$dT:]]3]1RIIc2E!tL 2 
,,-S1E0F G../s3H2I J% '445 6!!*	- ZZ 1 q A  
"	

` 21s   =N< O<OOO __main__)returnr\   )rX   )r   r\   r   r]   rz   None)r/   zargparse.Namespacerz   Dict[str, Any])r7   r|   rz   r{   )rz   r]   )__doc__
__future__r   rY   r*   r   r   pathlibr   typingr   r   r   r   r4   r;   rx   __name__r   r   r   r   <module>r      s\     #   
   /
%P
_!D zCHHTV r   