+
    i,                        R t ^ RIHt ^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RI	H
t
Ht R R ltRR R	 lltR
 R ltR R ltR R lt]R8X  d   ]P$                  ! ]! 4       4       R# R# )u\  G4.2 canary — token-count delta measurement (system.md, tiktoken cl100k_base).

Computes baseline and candidate token counts for system.md and emits the delta.
Used to verify Rule A/C/E lift achieves the expected token reduction
(target threshold: candidate - baseline <= -850).

Exit criteria (analyzer):
  exit_criteria_pass == True  iff  token_delta <= -850

Usage (config mode):
    python scripts/canary/g4_2_token_count.py \
        --config canary_config.json \
        --baseline-version 16.202605041200 \
        --candidate-version 17.202605042018 \
        --output results/token_count.json
)annotationsN)datetime)Path)AnyDictc                   V ^8  d   QhRR/# )   returnstr )formats   ""scripts/canary/g4_2_token_count.py__annotate__r      s     / /S /    c                 V    \         P                  ! 4       P                  4       R ,           # )Z)r   utcnow	isoformatr   r   r   iso8601_nowr      s    ??&&(3..r   c               $    V ^8  d   QhRRRRRR/# )r   msgr
   codeintr	   Noner   )r   s   "r   r   r   !   s!      s # d r   c                n    \        R V  2\        P                  R7       \        P                  ! V4       R# )z[g4_2_token_count] ERROR: fileN)printsysstderrexit)r   r   s   &&r   _failr!   !   s#    	&se
,3::>HHTNr   c                    V ^8  d   QhRRRR/# )r   argszargparse.Namespacer	   Dict[str, Any]r   )r   s   "r   r   r   &   s     % %) %n %r   c                   V P                   '       d   \        V P                   4      pVP                  4       '       g   \        R V 24        VP	                  RRR7      ;_uu_ 4       p\
        P                  ! V4      pRRR4       RXP                  R4      RTP                  R4      RTP                  R4      R	TP                  R	4      R
TP                  R
4      RTP                  R4      RTP                  R4      /#  V P                  '       d!   \
        P                  ! V P                  4      MRpV P                  '       d!   \
        P                  ! V P                  4      MRpRV P                  RXRXR	V P                  R
V P                  RV P                  RV P                   /#   + '       g   i     EL5; i  \         d   p\        RT 24        Rp?ELWRp?ii ; i  \         d   p\        RT 24        Rp?LRp?ii ; i)z--config file not found: rutf-8encodingNz--config parse error: pidscene_index_listshot_index_list_per_scenemodel_routingprompt_source_modecard_commit_hashchain_bg_card_snapshot_hashz%failed to parse list/dict JSON args: )configr   existsr!   openjsonload	Exceptiongetr+   loadsr,   r*   r-   r.   r/   r0   )r#   cfg_pathfpcfgexcr+   r,   s   &      r   load_pinningr=   &   s   {{{$  -hZ89	2sW55iim 6
 3775>(: ;'1L)MSWW_5 #''*>"?(: ;)3773P+Q
 	
	=151F1F1FDJJt,,-D 	
 --- JJt55637 	" 	txx,#%>++d55D11%t'G'G / 655 	2*3%011	2&  =5cU;<<=sZ   F? (F+?F? G& 3G& "G& +F<	6F? <F? ?G#
GG#&H	1HH	c                    V ^8  d   QhRRRR/# )r   pinningr$   r	   r   r   )r   s   "r   r   r   N   s     
 
n 
 
r   c                    . ROpV Uu. uF  q P                  V4      R9   g   K  VNK  	  ppV'       d   \        RV R24       R# R# u upi )r*   Nz#pinning missing required field(s): zF. Provide via --config <json> or individual flags. NO silent defaults.)r*   r+   r,   r-   r.   r/   r0   )N )r7   r!   )r?   requiredkmissings   &   r   validate_pinningrE   N   sW    H
 #C(Qkk!n
&Bqq(GC1' ;S T	
  Ds
   AAc                   V ^8  d   QhRR/# )r   r	   r   r   )r   s   "r   r   r   \   s     _! _!c _!r   c                 J   \         P                  ! R R7      p V P                  R\        RR7       V P                  R\        RRR7       V P                  R	\        RR
R7       V P                  R\        RRR7       V P                  R\        R=RR7       V P                  R\        RR7       V P                  R\        RR7       V P                  R\        RR7       V P                  R\        RR7       V P                  R\        R. R>OR7       V P                  R\        RR7       V P                  R\        RR7       V P                  R\        RR7       V P                  4       p\        V4      p\        V4        ^ RIpXP                  R4      p\        VP                  4      pWQP                  ,          R,          pWQP                  ,          R,          pVP!                  4       '       g   \        RV 24       VP!                  4       '       g   \        RV 24       VP#                  R R!7      pVP#                  R R!7      p	\%        VP'                  V4      4      p
\%        VP'                  V	4      4      pW,
          pWP(                  8*  pR"\+        4       R#VR$VP                  R%VP                  R&V
R'VR(VR)VP(                  R*VR+R,R-R.R///pVP,                  R08X  dJ   \.        P0                  ! V\2        P4                  R1^RR27       \2        P4                  P7                  R34       Mq\        VP,                  4      pVP8                  P;                  RRR47       VP=                  R5R R!7      ;_uu_ 4       p\.        P0                  ! VVR1^RR27       RRR4       \?        R6VP                   R7V
 R8VP                   R7V R9V R:VP(                   R;V 2\2        P@                  R<7       V'       d   ^ # ^#   \         d    \        R4        ELi ; i  + '       g   i     L; i)?zAG4.2 canary: token-count delta (system.md, tiktoken cl100k_base).)descriptionz--configN)typedefaultz--baseline-versionTz=Baseline prompt version directory name (e.g. 16.202605041200))rI   rB   helpz--candidate-versionz>Candidate prompt version directory name (e.g. 17.202605042018)z--prompt-base-pathzprompts/_base/scene_detailz)Base directory containing version subdirs)rI   rJ   rK   z--exit-criteria-thresholdz0token_delta upper bound for pass (default: -850)z--pidz--scene-index-listz--shot-index-list-per-scenez--model-routingz--prompt-source-mode)rI   rJ   choicesz--card-commit-hashz--chain-bg-card-snapshot-hashz--output)rI   rB   ztiktoken not installed. Install via `pip install tiktoken` (or use the backend venv: backend/.venv/bin/pip install tiktoken). NO silent fallback.cl100k_basez	system.mdzbaseline system.md not found: zcandidate system.md not found: r'   r(   	timestampr?   baseline_prompt_versioncandidate_prompt_versionbaseline_token_countcandidate_token_counttoken_deltaexit_criteria_thresholdexit_criteria_passmeasurement_toolztiktoken cl100k_basemeasurement_scriptstoken_countzDscripts/canary/g4_2_token_count.py (system.md, tiktoken cl100k_base)-F)ensure_asciiindent	sort_keys
)parentsexist_okwz[g4_2_token_count]
  baseline (z): z tokens
  candidate (z tokens
  token_delta: z
  threshold: z
  exit_criteria_pass: r   i)Nr   db)!argparseArgumentParseradd_argumentr
   r   
parse_argsr=   rE   tiktokenImportErrorr!   get_encodingr   prompt_base_pathbaseline_versioncandidate_versionr2   	read_textlenencoderT   r   outputr4   dumpr   stdoutwriteparentmkdirr3   r   r   )parserr#   r?   rf   encoderbase_dirbaseline_pathcandidate_pathbaseline_textcandidate_textrQ   rR   rS   	exit_passoutout_pathr:   s                    r   mainr   \   s   $$WF 
d;
,3\  ^
-C$]  _
,3 <H  J 3#tO  Q c48
,3E
5CN
)TB
.S$ 4  6
,3E
7c4P

t<D4 GW
 ##M2GD))*H444{BM 6 66DN!!.}o>?  ""//?@A!++W+=M#--w-?Nw~~m<=~ >?'>K;;;I 	[]7!4#8#8"D$:$: 4!6{!4#?#?i2a 
C  {{c		#szza4P

$dT:]]3]11RIIc2E!tL 2 
,,-S1E0F G../s3H2I J% '445 6!!*	- ZZ 1 q A  
"	

` 21s   >O5 7P5PPP"	__main__)   )__doc__
__future__r   rb   r4   r   r   pathlibr   typingr   r   r   r!   r=   rE   r   __name__r    r   r   r   <module>r      sZ     #   
   /
%P
_!D zHHTV r   