
    zkiD9                        d Z ddlmZ ddlZddlZddlZddlmZ ddlmZ  e	       Z
ddZddZddZdd	Z	 	 	 	 	 	 	 	 dd
Zedk(  r ej"                   e              yy)u8  G4.5a canary — token-count delta (system.md, tiktoken cl100k_base, Task 4.5).

Computes baseline (v19) and candidate (v20) token counts for
`prompts/_base/scene_detail/<version>/system.md` and emits the signed delta.

Override RO-1 binding gate (signed convention):
  token_delta = candidate_token_count - baseline_token_count
  pass condition: token_delta <= -1000  (system-only, RO-1 binding)
which is mathematically equivalent to baseline - candidate >= 1000.

`system_plus_inject_delta <= -800` = secondary report-only metric (no gate).

PR4-B4 carry — this script is **CP-free**: it reads prompt files directly
via filesystem and does NOT import the render-strategy snapshot hash
helper. The `render_strategy_card_snapshot_hash` echoed in pinning is
operator-supplied and verified by analyzer cross-check against the 4 metric
scripts. Trap #6 — token gate signed: candidate < baseline ⇒ delta is
negative.

Trap #1 silent-fallback ban — applies to pinning loading; tiktoken is hard
dependency (no fallback).

Exit criteria (analyzer):
  exit_criteria_pass == True  iff  token_delta <= -1000 (default threshold)

Usage (config mode):
    python scripts/canary/g4_5a_token_count.py \
        --baseline-version 19.202605050814 \
        --candidate-version 20.202605051240 \
        --output results/g4_5a_token_count.json
    )annotationsN)datetime)Pathc                 L    t        j                         j                         dz   S )NZ)r   utcnow	isoformat     #scripts/canary/g4_5a_token_count.pyiso8601_nowr   7   s    ??&&(3..r   c                    t        | dd      rt        | j                        }|j                         s3t	        d| t
        j                         t        j                  d       	 |j                  dd      5 }t        j                  |      ddd       dfd
} |d       |d       |d       |d       |d       |d       |d      dS 	 t        | dd      rt        j                  | j                        nd}t        | dd      rt        j                  | j                        nd}t        | dd      t        | dd      t        | dd      t        | dd      t        | dd      dS # 1 sw Y   xY w# t        $ r>}t	        d	| t
        j                         t        j                  d       Y d}~&d}~ww xY w# t        $ r=}t	        d| t
        j                         t        j                  d       Y d}~d}~ww xY w)u   Load pinning dict from --config OR individual flags.

    Trap #1 — `_DEFAULT_SENTINEL` is the only allowed default; absent keys
    round-trip as None and `_validate_pinning()` rejects them.
    configNz/[g4_5a_token_count] ERROR: --config not found: file   rutf-8encodingz1[g4_5a_token_count] ERROR: --config parse error: c                H    j                  | t              }|t        u rd S |S )N)get_DEFAULT_SENTINEL)keyvalcfgs     r   _cfg_or_nonez-_load_pinning_from_args.<locals>._cfg_or_noneS   s'    ''#01C"334<<r   pidscene_index_listshot_index_list_per_scenemodel_routingprompt_source_modecard_commit_hash"render_strategy_card_snapshot_hash)r   r   r    r!   r"   r#   r$   z6[g4_5a_token_count] ERROR: failed to parse JSON args: )r   str)getattrr   r   existsprintsysstderrexitopenjsonload	Exceptionloadsr   r    )argscfg_pathfpexcr   r   r    r   s          @r   _load_pinning_from_argsr5   ;   s    tXt$$ A(LZZ HHQK	sW5iim 6	=
  & ,-? @)5+* */:"./C"D ,-? @2>43
 	
 t/6 JJt,,- 	 t8$? JJt556 	" tUD),%> =%d,@$G#D*<dC.56/

 
W 65 	CC5IZZ HHQKK	F  DSEJ	
 	sI   (E= ;E1E= AG 1E:6E= =	G3F??G	H3HHc                   |j                   dk(  rHt        j                  | t        j                  ddd       t        j                  j                  d       yt        |j                         }|j                  j                  dd       |j                  dd	
      5 }t        j                  | |ddd       ddd       y# 1 sw Y   yxY w)z2Write JSON output to args.output ('-' for stdout).-F   T)ensure_asciiindent	sort_keys
)parentsexist_okwr   r   N)
outputr-   dumpr)   stdoutwriter   parentmkdirr,   )outr1   out_pathr3   s       r   _emit_outputrH      s    {{c		#szza4P

$dT:]]3]1RIIc2E!tL 211s   CC
c                 p
   t        j                  d      } | j                  dt        d d       | j                  dt        dd	       | j                  d
t        dd	       | j                  dt        dd       | j                  dt        dd       | j                  dt        dd       | j                  dt        d d       | j                  dt        d d       | j                  dt        d        | j                  dt        d        | j                  dt        d        | j                  dt        d        | j                  dt        d g d        | j                  d!t        d        | j                  d"t        d        | j                  d#t        d$       | j                         }t        |      }g }	 d%d l}|j                  d*      }t!        |j"                        }||j$                  z  d+z  }	||j&                  z  d+z  }
|	j)                         s|j                  d,|	        |
j)                         s|j                  d-|
        |r?t        |||      }t        ||       |D ]   }t        d'| t        j                  (       " y)	 |	j+                  d./      }|
j+                  d./      }	 t/        |j1                  |            }t/        |j1                  |            }||z
  }||j2                  k  }|j4                  9|j6                  -|j4                  }|j6                  }||z
  }||j8                  k  }nd }d }d }d }i d4t;               d5|d6|j$                  d7|j&                  d8|d9|d:|d;|j2                  d<|d=|d>|d?|d@|j8                  dA|dBdCdD|dEdFdGi}t        ||       t        dH|j$                   dI| dJ|j&                   dI| dK| dL|j2                   dM| dN| dOt        j                  (       |rd%S d)S # t        $ rM d&}|j                  |       t        |||      }t        ||       t        d'| t        j                  (       Y y)w xY w# t,        $ rU}|j                  d0|        t        |||      }t        ||       t        d1| t        j                  (       Y d }~y)d }~ww xY w# t,        $ rU}|j                  d2|        t        |||      }t        ||       t        d3| t        j                  (       Y d }~y)d }~ww xY w)PNzbG4.5a canary: token-count delta (system.md, tiktoken cl100k_base, PR4-B4 CP-free, Trap #6 signed).)descriptionz--configz{Optional JSON config file. If provided, overrides individual --baseline-version / --candidate-version / --pid / etc. flags.)typedefaulthelpz--baseline-versionTz=Baseline prompt version directory name (e.g. 19.202605050814))rK   requiredrM   z--candidate-versionz=Candidate prompt version directory name (e.g. 20.<timestamp>)z--prompt-base-pathzprompts/_base/scene_detailz)Base directory containing version subdirsz--exit-criteria-thresholdiz?token_delta upper bound for pass (default: -1000, RO-1 binding)z(--system-plus-inject-secondary-thresholdizGsystem+inject combined delta secondary threshold (report-only, no gate)z--baseline-system-plus-injectzVOptional baseline system+inject token count (operator-supplied, secondary report-only)z--candidate-system-plus-injectzWOptional candidate system+inject token count (operator-supplied, secondary report-only)z--pid)rK   rL   z--scene-index-listz--shot-index-list-per-scenez--model-routingz--prompt-source-mode)Nr   db)rK   rL   choicesz--card-commit-hashz$--render-strategy-card-snapshot-hashz--output)rK   rN   r   zatiktoken not installed. Install via `pip install tiktoken` (or backend venv). NO silent fallback.z[g4_5a_token_count] ERROR: r   r   cl100k_basez	system.mdzbaseline system.md not found: zcandidate system.md not found: r   r   zprompt file read failed: z4[g4_5a_token_count] ERROR: prompt file read failed: ztiktoken encode failed: z3[g4_5a_token_count] ERROR: tiktoken encode failed: 	timestamppinningbaseline_versioncandidate_versionbaseline_token_countcandidate_token_counttoken_deltaexit_criteria_thresholdexit_criteria_passsystem_plus_inject_baselinesystem_plus_inject_candidatesystem_plus_inject_delta&system_plus_inject_secondary_threshold!system_plus_inject_secondary_passmeasurement_tooltiktoken cl100k_basemeasurement_failuresmeasurement_scriptstoken_countzvscripts/canary/g4_5a_token_count.py (system.md, tiktoken cl100k_base, PR4-B4 CP-free, Trap #6 signed, RO-1 -1000 gate)z [g4_5a_token_count]
  baseline (z): z tokens
  candidate (z tokens
  token_delta: z.  (signed: candidate - baseline)
  threshold: z<  (pass iff token_delta <= threshold)
  exit_criteria_pass: z
  system_plus_inject_delta: z (secondary report-only))argparseArgumentParseradd_argumentr%   int
parse_argsr5   tiktokenImportErrorappend_failure_outputrH   r(   r)   r*   get_encodingr   prompt_base_pathrT   rU   r'   	read_textr/   lenencoderY   baseline_system_plus_injectcandidate_system_plus_injectr^   r   )parserr1   rS   rb   rj   msgrF   encoderbase_dirbaseline_pathcandidate_pathbaseline_textcandidate_textr4   rV   rW   rX   	exit_passspi_baselinespi_candidate	spi_deltaspi_secondary_passs                         r   mainr      sr   $$/F M   L	   L	   ,8	   #-   2%   '9   (9   c48
,3E
5CN
)TB
$	   ,3E
.S$   
t<D%d+G
 ') ##M2GD))*Ht444{BM 6 66DN !##,]O<	
   "##-n-=>	
 dG-ABS$'C/u5CJJG (%///A'1171C"7>>-#@A #GNN>$B C (*>>Kt;;;I 	((4--97799!L0	DDD 	 	![]7 	D11 	T33	
 	 4 	 !6 	{ 	"4#?#? 	i 	&| 	' 	#I 	177  	,-?!" 	2#$ 	 4%& 	3 
'C8 d	,,-S
  !../s 
! "% '445 6!!* ,''0k 2"
	# ZZ 1 q   " 	
 	##C(dG-ABS$+C51

CL  ##&?u$EFdG-ABS$B3%H	
   ##&>se$DEdG-ABS$A#G	
 sE    P $Q6 =4S AQ32Q36	S?ASS	T5 AT00T5c                    i dt               d|d| j                  d| j                  dd dd dd d| j                  d	d
dd dd dd d| j                  dd ddd|dddiS )NrR   rS   rT   rU   rV   rW   rX   rY   rZ   Fr[   r\   r]   r^   r_   r`   ra   rb   rc   rd   zUscripts/canary/g4_5a_token_count.py (system.md, tiktoken cl100k_base, PR4-B4 CP-free))r   rT   rU   rY   r^   )r1   rS   rb   s      r   rm   rm   q  s    
[]7 	D11 	T33	
 	 	  	t 	"4#?#? 	e 	&t 	' 	#D 	177  	,T!" 	2#$ 	 4%& 	D 
' r   __main__)returnr%   )r1   argparse.Namespacer   dict)rF   r   r1   r   r   None)r   rh   )r1   r   rS   r   rb   listr   r   )__doc__
__future__r   re   r-   r)   r   pathlibr   objectr   r   r5   rH   r   rm   __name__r+   r
   r   r   <module>r      s   > #   
   H /DN	M`!F
  
	B zCHHTV r   