#!/usr/bin/env python3 # -*- coding: utf-8 -*- """G4.6 canary — shot_validator character_ids presence ratio. shot_validator manifest 의 모든 shot 에 대해 `character_ids` non-empty 비율을 측정. v3 baseline 은 필드 자체가 없어 ratio=0.0. v4 candidate 는 entity map 매핑이 성공한 shot 에서 ratio>0. Scope: shot_validator manifest.json 의 data.scenes[].shots[]. Generic logic — 시나리오 의존 어휘 0. character_ids 의 의미만 검증. Exit criteria (analyzer, OUT-OF-BAND): candidate.metrics.character_ids_present_ratio >= baseline.metrics.character_ids_present_ratio (no degradation) Usage: python scripts/canary/g4_6_shot_validator_character_ids_present.py \\ --cp-root data/projects//checkpoints/episodes//shot_validator \\ --prompt-version 4. \\ --role candidate \\ --output results/g4_6_shot_validator_candidate.json """ from __future__ import annotations import argparse import json import sys from datetime import datetime, timezone from pathlib import Path def iso8601_now() -> str: return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") def main() -> int: parser = argparse.ArgumentParser( description="G4.6 canary: shot_validator character_ids presence ratio." ) parser.add_argument("--cp-root", type=str, required=True, help="shot_validator checkpoint dir (containing manifest.json)") parser.add_argument("--prompt-version", type=str, required=True) parser.add_argument("--role", type=str, default="candidate", choices=["baseline", "candidate"]) parser.add_argument("--output", type=str, required=True) args = parser.parse_args() cp_root = Path(args.cp_root) manifest_path = cp_root / "manifest.json" measurement_failures: list[str] = [] if not manifest_path.exists(): measurement_failures.append(f"manifest not found: {manifest_path}") return _emit_and_exit(args, measurement_failures, scenes_count=0, shots_count=0, with_chars_count=0, failed_count=0) try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError) as exc: measurement_failures.append(f"manifest parse error: {exc}") return _emit_and_exit(args, measurement_failures, scenes_count=0, shots_count=0, with_chars_count=0, failed_count=0) # strict sentinel reads — silent default 차단 (G4.6 Codex I4). data = manifest.get("data") if not isinstance(data, dict): measurement_failures.append("manifest.data missing or not dict") return _emit_and_exit(args, measurement_failures, scenes_count=0, shots_count=0, with_chars_count=0, failed_count=0) scenes = data.get("scenes") if not isinstance(scenes, list): measurement_failures.append("data.scenes missing or not list") return _emit_and_exit(args, measurement_failures, scenes_count=0, shots_count=0, with_chars_count=0, failed_count=0) shots_count = 0 with_chars_count = 0 failed_scenes_count = 0 for sc in scenes: if not isinstance(sc, dict): measurement_failures.append("scene entry not dict") continue si = sc.get("scene_index") if sc.get("validator_status") == "failed": failed_scenes_count += 1 # G4.6 failed 마킹된 씬은 ratio 분모/분자 모두 제외 (LLM 매핑이 실행되지 않음) continue shots = sc.get("shots") if not isinstance(shots, list): measurement_failures.append( f"scene {si}: shots missing or not list" ) continue for sh in shots: if not isinstance(sh, dict): measurement_failures.append( f"scene {si}: shot entry not dict" ) continue shots_count += 1 char_ids = sh.get("character_ids") if isinstance(char_ids, list) and len(char_ids) > 0: with_chars_count += 1 # candidate role 의 shots_count == 0 → measurement_failure (manifest 비어있거나 # 모든 씬이 failed). degradation block 의미 없는 sample. if args.role == "candidate" and shots_count == 0: measurement_failures.append( "candidate role with shots_count=0 — manifest empty or all scenes failed" ) return _emit_and_exit(args, measurement_failures, scenes_count=len(scenes), shots_count=shots_count, with_chars_count=with_chars_count, failed_count=failed_scenes_count) def _emit_and_exit(args: argparse.Namespace, measurement_failures: list[str], scenes_count: int, shots_count: int, with_chars_count: int, failed_count: int) -> int: ratio = (with_chars_count / shots_count) if shots_count > 0 else 0.0 out = { "timestamp": iso8601_now(), "prompt_version": args.prompt_version, "role": args.role, "metrics": { "scenes_total": scenes_count, "scenes_failed": failed_count, "shots_evaluated": shots_count, "shots_with_character_ids": with_chars_count, "character_ids_present_ratio": ratio, }, "measurement_failures": measurement_failures, "measurement_scripts": { "character_ids_present": ( "scripts/canary/g4_6_shot_validator_character_ids_present.py " "(shot_validator manifest, character_ids non-empty ratio)" ) }, } if args.output == "-": json.dump(out, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True) sys.stdout.write("\n") else: out_path = Path(args.output) out_path.parent.mkdir(parents=True, exist_ok=True) with out_path.open("w", encoding="utf-8") as fp: json.dump(out, fp, ensure_ascii=False, indent=2, sort_keys=True) print( f"[g4_6_shot_validator_character_ids_present] role={args.role} " f"prompt_version={args.prompt_version}\n" f" scenes_total={scenes_count} scenes_failed={failed_count}\n" f" shots_evaluated={shots_count} shots_with_character_ids={with_chars_count}\n" f" ratio={ratio:.4f}\n" f" measurement_failures: {len(measurement_failures)}\n" f"Exit criteria (analyzer): candidate.ratio >= baseline.ratio (degradation block)", file=sys.stderr, ) if measurement_failures: return 1 return 0 if __name__ == "__main__": sys.exit(main())