#!/usr/bin/env python3
"""Image-first gallery for the retained Astra/Muse/Flash judge experiment.

Offline renderer: no model calls, no DB writes, no production edits.
Reverse-order images and raw prose keep the same display labels; aggregate
winners alone are converted back to original image identities.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import sys
from pathlib import Path
from urllib.parse import quote

BACKEND = Path(__file__).resolve().parents[2]
ROOT = BACKEND.parent
sys.path.insert(0, str(BACKEND))
sys.path.insert(0, str(Path(__file__).resolve().parent))

MODELS = [
    ("gpt-6-astra", "GPT-6 Astra", "xhigh · 9월 5일 새 호출"),
    ("meta/muse-spark-1.3-contributor", "Muse Spark 1.3", "Contributor · 9월 3일 기록"),
    ("google/gemini-3.8-flash", "Gemini 3.8 Flash", "9월 3일 기록"),
    ("gemini-3.1-pro-preview", "Gemini 3.1 Pro", "high · 9월 5일 정·역 새 호출"),
    ("x-ai/grok-4.6", "Grok 4.6", "high · 9월 5일 정·역 새 호출"),
]


def asset(path: Path) -> dict:
    path = path.resolve()
    return {"url": "/" + quote(path.relative_to(ROOT).as_posix(), safe="/"),
            "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}


def merge_runs(out: Path, baseline: Path, run_dirs: list[Path]) -> Path:
    """Offline merge; keep historical files and verify every retained input part."""
    from astra_judge_pilot import compare, digest, save

    if out.resolve() in {baseline.resolve(), *(p.resolve() for p in run_dirs)}:
        raise ValueError("Merged output must be separate from source runs")
    manifest = json.loads((baseline / "input_manifest.json").read_text())
    results = json.loads((baseline / "results.json").read_text())
    prior_summary = json.loads((baseline / "summary.json").read_text())
    expected_parts = {k: v["parts"] for k, v in manifest["inputs"].items()}
    sources, fresh, actual_calls, costs = [], {}, 0, 0.0
    for run in run_dirs:
        current = json.loads((run / "input_manifest.json").read_text())
        if (current["archived_contract_sha256"] != manifest["archived_contract_sha256"]
                or {k: v["parts"] for k, v in current["inputs"].items()} != expected_parts
                or current["shots"] != manifest["shots"]):
            raise ValueError(f"Comparison input drift in {run}")
        model = current["model"]
        if model in fresh:
            raise ValueError(f"Duplicate fresh model: {model}")
        per_tag = json.loads((run / "results.json").read_text())[model]
        slots = [per_tag[t][o] for t in manifest["shots"] for o in ("forward", "reverse")]
        for slot in slots:
            if slot["reasoning_effort"] != current["effort"]:
                raise ValueError(f"Effort drift in {run}")
        actual_calls += sum(s.get("http_attempt_count", 0) for s in slots)
        costs += sum((s.get("usage") or {}).get("cost", 0) or 0 for s in slots)
        fresh[model] = current["effort"]
        results[model] = per_tag
        sources.append({"directory": str(run), "model": model, "effort": current["effort"],
                        "results_sha256": digest((run / "results.json").read_bytes())})
    records = json.loads(Path(manifest["records"]).read_text())
    summary = compare(results, records, manifest["shots"])
    summary.update(fresh_http_attempts=actual_calls, fresh_reported_cost_usd=round(costs, 6),
                   previous_models=prior_summary["models"], fresh_sources=sources)
    manifest.update(model="multi-model-comparison", fresh_sources=sources,
                    gallery_models=[(m, name, f"{fresh[m]} · 이번 병렬 실행" if m in fresh else note)
                                    for m, name, note in MODELS if m in results],
                    gallery_note="Flash high · Muse high · Grok 4.6 high를 새로 병렬 실행했습니다. "
                    "Astra xhigh와 Gemini Pro high는 앞선 결과를 보존해 비교합니다. "
                    "모두 출력 상한 16,000 · 같은 이미지/참조 SHA와 보관 프롬프트/스키마. "
                    "세 새 모델은 OpenRouter JSON Schema 모드이며 이전 Muse·Flash는 JSON Object 모드/8,000이므로 "
                    "차이를 effort만의 효과로 단정할 수 없습니다. "
                    "정·역순 일치는 정확도가 아닌 안정성이고 하드위반은 모델의 주장입니다. "
                    "병렬 호출 시간에는 제공사 큐·캐시 영향이 포함됩니다.")
    save(out / "input_manifest.json", manifest)
    save(out / "results.json", results)
    save(out / "summary.json", summary)
    for name in ("judge_sys.txt", "judge_schema.json", "headers.json"):
        (out / name).write_bytes((baseline / name).read_bytes())
    return build(out)


def single_output_quality(slot: dict) -> dict:
    """Post-run integrity audit, not another judge or a change to source results.

    JSON Schema permits NUL and other C0 controls inside strings. Preserve the
    raw response but don't interpret its score as a valid visual assessment.
    Apply this exact mechanical rule to every model, after the paid experiment.
    """
    def controls(value):
        if isinstance(value, str):
            return sum(ord(c) < 32 and c not in "\n\r\t" for c in value)
        if isinstance(value, dict):
            return sum(controls(v) for v in value.values())
        if isinstance(value, list):
            return sum(controls(v) for v in value)
        return 0

    count = controls(slot.get("raw", {}))
    return {"valid": bool(slot.get("ok")) and count == 0,
            "control_characters": count,
            "rule": "post_run_c0_controls_except_tab_cr_lf_v1"}


def build_single(out: Path, baseline: Path, run_dirs: list[Path]) -> Path:
    """One-image repeatability table and a gallery; paired runs are read-only."""
    import statistics
    from astra_judge_pilot import digest, save
    from build_muse_compare_gallery import _shot_texts
    from muse_judge_pilot import _resolve_ref_path

    if out.resolve() in {baseline.resolve(), *(p.resolve() for p in run_dirs)}:
        raise ValueError("Single gallery must not replace source results")
    paired_manifest = json.loads((baseline / "input_manifest.json").read_text())
    paired_summary = json.loads((baseline / "summary.json").read_text())
    results, manifests, sources = {}, {}, []
    for run in run_dirs:
        manifest = json.loads((run / "input_manifest.json").read_text())
        if manifest["evaluation"] != "single_candidate" or manifest["repeats"] != 2:
            raise ValueError("This table requires two independent single-image repetitions")
        model = manifest["model"]
        if model in results:
            raise ValueError(f"Duplicate model {model}")
        if manifest["shots"] != paired_manifest["shots"]:
            raise ValueError("Single and paired shot populations differ")
        results[model] = json.loads((run / "results.json").read_text())[model]
        for orders in results[model].values():
            for slot in orders.values():
                slot["schema_ok"] = bool(slot.get("ok"))
                slot["output_quality"] = single_output_quality(slot)
                if slot["schema_ok"] and not slot["output_quality"]["valid"]:
                    slot["ok"] = False
                    slot["error"] = ("본문 손상: JSON 형식은 통과했지만 허용하지 않은 제어문자 "
                                     f"{slot['output_quality']['control_characters']}개. "
                                     "원문 보존 · 유효 점수 집계 제외 · 재호출 없음")
        manifests[model] = manifest
        sources.append({"directory": str(run), "model": model,
                        "results_sha256": digest((run / "results.json").read_bytes())})
    first = next(iter(manifests.values()))
    for manifest in manifests.values():
        if (manifest["system_sha256"] != first["system_sha256"] or
                manifest["schema_sha256"] != first["schema_sha256"] or
                {k: v["parts"] for k, v in manifest["inputs"].items()} !=
                {k: v["parts"] for k, v in first["inputs"].items()}):
            raise ValueError("Single model inputs differ")
    models = [(m, name, manifests[m]["effort"]) for m, name, _ in MODELS if m in results]
    tags = first["shots"]
    summary = {"models": {}, "images": [], "shots": [],
               "note": "No ground truth. Independent score rankings are derived, not a model choosing between two images. Ties remain ties.",
               "integrity_audit": "Post-run: exclude JSON-valid responses with C0 controls other than TAB/CR/LF. Applied to all models; raw scores/responses retained; no retries."}
    winner = lambda a, b: "A" if a > b else "B" if b > a else "tie"
    for model, name, effort in models:
        slots = [s for t in tags for s in results[model][t].values()]
        deltas, flag_agreement, pair_stable, pairs_complete = [], 0, 0, 0
        tie_runs, matched, comparable = 0, 0, 0
        for tag in tags:
            orders = results[model][tag]
            for candidate in ("A", "B"):
                samples = [orders[f"{candidate}_r{rep}"] for rep in (1, 2)]
                scores = [s.get("raw", {}).get("score") if s.get("ok") else None for s in samples]
                row = {"model": model, "tag": tag, "candidate": candidate, "scores": scores,
                       "returned_scores": [s.get("raw", {}).get("score") for s in samples],
                       "fails": [s.get("raw", {}).get("candidate_fails") if s.get("ok") else None for s in samples]}
                if all(s.get("ok") for s in samples):
                    row.update(mean_score=statistics.mean(scores), score_delta=abs(scores[0]-scores[1]))
                    deltas.append(row["score_delta"])
                    flag_agreement += row["fails"][0] == row["fails"][1]
                summary["images"].append(row)
            rankings, score_pairs = [], []
            for rep in (1, 2):
                a, b = [orders[f"{c}_r{rep}"] for c in ("A", "B")]
                if a.get("ok") and b.get("ok"):
                    av, bv = a["raw"]["score"], b["raw"]["score"]
                    rankings.append(winner(av, bv)); score_pairs.append([av, bv])
                    tie_runs += av == bv
                else:
                    rankings.append(None); score_pairs.append(None)
            mean_winner = None
            if all(r is not None for r in rankings):
                pairs_complete += 1
                pair_stable += rankings[0] == rankings[1]
                mean_winner = winner(sum(s[0] for s in score_pairs), sum(s[1] for s in score_pairs))
            old = next((s for s in paired_summary["shots"] if s["tag"] == tag and s["model"] == model), None)
            if old and mean_winner is not None:
                comparable += 1
                matched += mean_winner == old["combined"]
            summary["shots"].append({"model": model, "tag": tag, "score_rankings": rankings,
                "score_pairs": score_pairs, "mean_score_winner": mean_winner,
                "paired_combined": old["combined"] if old else None,
                "paired_orders": [old["forward"], old["reverse"]] if old else None})
        summary["models"][model] = {"name": name, "effort": effort, "calls": len(slots),
            "ok": sum(bool(s.get("ok")) for s in slots), "errors": sum(not s.get("ok") for s in slots),
            "schema_ok": sum(s["schema_ok"] for s in slots),
            "corrupt_outputs": sum(s["output_quality"]["control_characters"] > 0 for s in slots),
            "http_attempts": sum(s.get("http_attempt_count", 0) for s in slots),
            "repeat_complete_images": len(deltas), "same_score_images": sum(d == 0 for d in deltas),
            "mean_absolute_score_delta": round(statistics.mean(deltas), 3) if deltas else None,
            "same_failure_flag_images": flag_agreement,
            "score_rank_stable_shots": pair_stable, "score_rank_complete_shots": pairs_complete,
            "derived_tie_runs": tie_runs, "matches_paired_mean_rank": matched,
            "comparable_paired_shots": comparable,
            "median_seconds": round(statistics.median(s["elapsed_seconds"] for s in slots), 3),
            "reasoning_tokens": sum(((s.get("usage") or {}).get("completion_tokens_details") or {}).get("reasoning_tokens", 0) for s in slots),
            "hard_violation_mentions": sum(len(s["raw"]["readings"]["hard_violations"]) for s in slots if s.get("ok"))}
    records_path = Path(first["records"])
    records = json.loads(records_path.read_text())
    texts = _shot_texts(first["episode_id"])
    shots = []
    for tag in tags:
        images = {c: asset(records_path.parent / f"{tag}_{c.lower()}.png") for c in ("A", "B")}
        refs = [{**asset(_resolve_ref_path(r)), "label": r.get("label", "참조")} for r in records[tag].get("refs", [])]
        for candidate in ("A", "B"):
            hashes = [p["sha256"] for p in first["inputs"][f"{tag}_{candidate}"]["parts"] if p["type"] == "image_url"]
            if hashes != [r["sha256"] for r in refs] + [images[candidate]["sha256"]]:
                raise ValueError("Gallery images differ from evaluated images")
        judged = {m: {key: {k: v for k, v in slot.items() if k in (
            "raw", "ok", "error", "usage", "elapsed_seconds", "candidate", "repeat", "schema_ok", "output_quality")}
            for key, slot in results[m][tag].items()} for m, _, _ in models}
        shots.append({"tag": tag, "description": texts.get(tag, ""), "images": images,
                      "refs": refs, "prompt": records[tag]["prompt"], "judgments": judged})
    manifest = {**first, "model": "all-five-single", "models": models, "sources": sources,
                "paired_baseline": str(baseline)}
    save(out / "input_manifest.json", manifest); save(out / "results.json", results); save(out / "summary.json", summary)
    for filename in ("judge_sys.txt", "judge_schema.json", "prompt.diff"):
        (out / filename).write_bytes((run_dirs[0] / filename).read_bytes())
    data = json.dumps({"models": models, "shots": shots, "summary": summary}, ensure_ascii=False).replace("<", "\\u003c")
    style = TEMPLATE.split("<style>", 1)[1].split("</style>", 1)[0]
    page = out / "index.html"
    page.write_text(SINGLE_TEMPLATE.replace("__STYLE__", style).replace("__DATA__", data))
    return page


def build(out: Path) -> Path:
    from build_muse_compare_gallery import _shot_texts
    from muse_judge_pilot import _resolve_ref_path

    manifest = json.loads((out / "input_manifest.json").read_text())
    results = json.loads((out / "results.json").read_text())
    models = [entry for entry in manifest.get("gallery_models", MODELS) if entry[0] in results]
    if manifest.get("provider") == "openrouter" and not manifest.get("gallery_models"):
        models = [(m, name, f"{manifest['effort']} · 이번 새 호출" if m == manifest["model"] else note)
                  for m, name, note in models]
        manifest.setdefault("gallery_note",
            f"이번 새 호출: {manifest['model']} · effort={manifest['effort']} · "
            f"출력 상한 {manifest['max_completion_tokens']:,}. 나머지는 이전 응답입니다. "
            "같은 보관 이미지/프롬프트/스키마 · 정역순 일치는 정확도가 아닙니다.")
    summary = json.loads((out / "summary.json").read_text())
    records_path = Path(manifest["records"])
    records = json.loads(records_path.read_text())
    texts = _shot_texts(manifest["episode_id"])
    shots = []
    for tag in manifest["shots"]:
        record = records[tag]
        inputs = manifest["inputs"][f"{tag}_forward"]["parts"]
        image_hashes = [p["sha256"] for p in inputs if p["type"] == "image_url"]
        images = {label: asset(records_path.parent / f"{tag}_{label.lower()}.png")
                  for label in ("A", "B")}
        if [images[x]["sha256"] for x in ("A", "B")] != image_hashes[-2:]:
            raise ValueError(f"{tag}: gallery candidate bytes differ from judged bytes")
        references = []
        for ref in record.get("refs", []):
            entry = {**asset(_resolve_ref_path(ref)), "label": ref.get("label", "참조")}
            references.append(entry)
        if [r["sha256"] for r in references] != image_hashes[:-2]:
            raise ValueError(f"{tag}: gallery reference bytes differ from judged bytes")
        judgments = {model: results.get(model, {}).get(tag, {}) for model, _, _ in models}
        # Keep verdict output and usage only, not duplicate full API envelopes.
        judgments = {m: {o: {k: v for k, v in s.items() if k in {
            "ok", "raw", "normalized", "elapsed_seconds", "usage", "error",
            "display_to_canonical"}} for o, s in orders.items()}
            for m, orders in judgments.items()}
        shots.append({"tag": tag, "description": texts.get(tag, ""),
                      "prompt": record["prompt"], "images": images,
                      "refs": references, "judgments": judgments,
                      "production_selected": record.get("selected"),
                      "production": record.get("cross_model_order", {}).get("slots", [])})
    data = {"models": models, "shots": shots, "summary": summary}
    encoded = json.dumps(data, ensure_ascii=False).replace("<", "\\u003c")
    page = out / "index.html"
    template = TEMPLATE.replace("세 모델 정·역 결과", "모델별 정·역 결과")
    if manifest.get("gallery_note"):
        begin = template.index('<div class="note">')
        end = template.index('</div>', begin) + len('</div>')
        import html
        template = template[:begin] + '<div class="note">' + html.escape(manifest["gallery_note"]) + '</div>' + template[end:]
    page.write_text(template.replace("__DATA__", encoded), encoding="utf-8")
    return page


TEMPLATE = r'''<!doctype html>
<html lang="ko"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
<title>VLM 모델별 thinking · 정역순 이미지 판정 비교</title>
<style>
:root{color-scheme:dark;--bg:#111318;--panel:#1a1e26;--line:#353c48;--muted:#a5afc0;--accent:#a2d8cb}
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:#edf1f6;font:15px/1.65 -apple-system,BlinkMacSystemFont,'Apple SD Gothic Neo',sans-serif}
main{max-width:1536px;margin:auto;padding:30px 28px 70px}h1{font-size:32px;line-height:1.25;margin:6px 0 12px;letter-spacing:-1px}h2{font-size:23px;margin:0}h3{margin:0;font-size:18px}p{margin:8px 0}a{color:var(--accent)}.eyebrow{font-size:12px;letter-spacing:2px;color:var(--accent)}.muted{color:var(--muted);font-size:13px}
.note{border-left:3px solid var(--accent);padding:10px 15px;margin:18px 0;background:#182322}.stats,.judges{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}.stat,.judge{border:1px solid var(--line);border-radius:12px;background:var(--panel);padding:18px}.stat:first-child,.judge:first-child{border-color:#568c7f}.metric{font-size:28px;line-height:1.3;margin-top:8px}.stat small{font-size:13px;font-weight:400;color:var(--muted)}
nav{display:flex;gap:9px;flex-wrap:wrap;margin:18px 0}button,select{font:inherit;background:#242b36;color:inherit;border:1px solid var(--line);border-radius:7px;padding:7px 14px;cursor:pointer}button[aria-pressed=true]{background:var(--accent);border-color:var(--accent);color:#13231f;font-weight:650}.toolbar{position:sticky;top:0;z-index:3;background:#111318f5;padding:12px 0;border-bottom:1px solid var(--line);display:flex;align-items:center;gap:14px;flex-wrap:wrap}.toolbar label{margin-left:auto}.shot{padding-top:24px}.description{font-size:17px;margin:10px 0 18px}.images{display:grid;grid-template-columns:1fr 1fr;gap:16px}.image{margin:0;min-width:0}.image a{display:block;background:#090b0d;border:1px solid var(--line);border-radius:10px;overflow:hidden}.image img{display:block;width:100%;height:440px;object-fit:contain}.image figcaption{padding:8px 0;font-weight:650}.hash{font:11px ui-monospace,monospace;color:var(--muted);margin-left:8px}.judges{margin-top:18px;align-items:start}.pill{display:inline-block;padding:2px 9px;border-radius:5px;background:#2c433e;color:#c2f3e6;font-size:13px}.winner{display:flex;justify-content:space-between;gap:12px;margin:12px 0}.candidate{border-top:1px solid var(--line);padding-top:12px;margin-top:12px}.candidate strong{font-size:14px}.candidate p{font-size:14px}.violation{color:#ffc2a2;background:#352a25;padding:8px 10px;border-radius:6px}.nohv{color:var(--muted);font-size:13px}details{margin-top:14px;border-top:1px solid var(--line);padding-top:10px}summary{cursor:pointer;color:var(--accent)}dt{font-weight:650;margin-top:8px;font-size:13px}dd{margin:3px 0 10px;font-size:13px;color:#cbd3df}pre{white-space:pre-wrap;overflow-wrap:anywhere;font:13px/1.7 ui-monospace,monospace;max-height:480px;overflow:auto}.refs{display:flex;gap:14px;flex-wrap:wrap;padding-top:12px}.refs figure{margin:0;max-width:220px}.refs img{width:220px;height:150px;object-fit:contain;background:#080a0d;border-radius:7px}.refs figcaption{font-size:12px;overflow-wrap:anywhere}.summary-table{width:100%;border-collapse:collapse;margin-top:18px;font-size:13px}.summary-table td,.summary-table th{border-bottom:1px solid var(--line);padding:10px;text-align:left}.summary-table tr{cursor:pointer}.summary-table tr:hover{background:#232a34}.ribbon{display:flex;gap:12px;flex-wrap:wrap;margin:10px 0;font-size:13px}.warn{color:#ffc79d}.order-note{min-height:25px;padding-top:8px}.error{background:#4b2424;padding:12px;border-radius:7px}
@media(min-width:1600px){.image img{height:500px}}@media(max-width:1000px){main{padding:22px 16px}.judges{grid-template-columns:1fr}.image img{height:310px}}@media(max-width:600px){h1{font-size:26px}.stats{grid-template-columns:1fr}.images{grid-template-columns:1fr}.image img{height:auto}.toolbar label{margin-left:0}.summary-table{font-size:11px}.summary-table td,.summary-table th{padding:7px}}
</style></head><body><main>
<div class="eyebrow">RETAINED IMAGE JUDGE BENCH · 2026.09.05</div>
<h1>같은 이미지, 모델별 판단</h1>
<p>샷별 정순·역순 선택과 A/B 점수를 한 표로 비교합니다. 사진을 눌러 원본 크기로 볼 수 있습니다.</p>
<div class="note">Astra xhigh / Gemini 3.1 Pro high는 9월 5일 새 호출 · Muse / Gemini Flash는 9월 3일 응답 재사용 · 보관된 이미지와 같은 지시문·스키마 사용 · 이미지 새 생성 없음<br><span class="muted">정순/역순 일치는 판정 안정성이지 정확도가 아닙니다. 기존 선정도 정답 라벨이 아니며, ‘하드위반’은 모델의 주장입니다. 출력 상한은 이번 Astra·Pro 16,000(추론 포함), 과거 모델 8,000으로 다릅니다. API와 thinking 설정도 모델별로 다릅니다.</span></div>
<section id="stats" class="stats" aria-label="모델 집계"></section>
<details open><summary>6샷 전체 표 · 정순/역순 선택·점수</summary><div id="overview"></div><p class="muted">결합 규칙: 정·역 합의면 그 후보, 불일치면 점수 합, 동점이면 A. 모든 요약 A/B는 원본 이미지 기준입니다. 과거 실험에 이미지 SHA가 없어 당시 전송 바이트의 완전 동일성은 소급 증명하지 못합니다. 이번 입력은 모두 SHA를 남겼고 갤러리 이미지와 대조했습니다.</p><p><a href="summary.json">집계 JSON</a> · <a href="input_manifest.json">입력·이미지 지문</a> · <a href="results.json">전체 응답·토큰·시간</a> · <a href="judge_sys.txt">공통 심판 지시문</a></p></details>
<nav id="shots" aria-label="샷 선택"></nav>
<div class="toolbar"><h2 id="shot-title"></h2><label>제시 순서 <select id="order"><option value="forward">정순 · 원본 A → B</option><option value="reverse">역순 · 원본 B → A</option></select></label></div>
<div id="order-note" class="order-note muted"></div><section id="shot" class="shot"></section>
</main><script type="application/json" id="data">__DATA__</script><script>
const D=JSON.parse(document.getElementById('data').textContent);let selected=0;
const $=id=>document.getElementById(id), esc=x=>String(x??'').replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
function aggregate(m,tag){return D.summary.shots.find(s=>s.model===m&&s.tag===tag)}
$('stats').innerHTML=D.models.map(([m,name,note])=>{let s=D.summary.models[m]||{};return `<article class="stat"><h3>${esc(name)}</h3><div class="muted">${esc(note)}</div><div class="metric">${s.stable_pairs||0} / ${s.complete_pairs||0} <small>샷 · 정순/역순 같은 선택</small></div><div class="muted">성공 ${s.ok_calls||0}회 · 오류 ${s.errors||0}회 · 하드위반 주장 ${s.hard_violation_mentions||0}건${s.median_seconds?` · 중앙 ${s.median_seconds.toFixed(1)}초/호출`:''}</div></article>`}).join('');
function partialPair(orders){return ['forward','reverse'].map(o=>{let s=orders?.[o];return s?.ok?esc(s.normalized.winner):s?'실패':'미실행'}).join(' / ')+'<br><span class="warn">결합 불가</span>'}
$('overview').innerHTML='<table class="summary-table"><thead><tr><th>샷</th>'+D.models.map(m=>`<th>${esc(m[1])}<br>정순 / 역순 → 결합</th>`).join('')+'<th>기존 선정<br>(정답 아님)</th></tr></thead><tbody>'+D.shots.map((s,i)=>`<tr data-shot="${i}"><td>${esc(s.tag)}</td>`+D.models.map(([m])=>{let x=aggregate(m,s.tag);let scores=o=>{let v=s.judgments[m]?.[o]?.normalized?.verdicts;return v?['A','B'].map(l=>`${l} ${esc(v.find(v=>v.label===l)?.score)}`).join(' / '):'없음'};return `<td>${x?`${esc(x.forward)} / ${esc(x.reverse)} → <b>${esc(x.combined)}</b><br><span class="muted">정 ${scores('forward')}<br>역 ${scores('reverse')}</span>`:partialPair(s.judgments[m])}</td>`}).join('')+`<td>${esc(s.production_selected)}</td></tr>`).join('')+'</tbody></table>';
$('shots').innerHTML=D.shots.map((s,i)=>`<button data-shot="${i}" aria-pressed="false">${esc(s.tag)}</button>`).join('');
function render(){const s=D.shots[selected],order=$('order').value,labels=order==='forward'?['A','B']:['B','A'];
$('shot-title').textContent=s.tag;document.querySelectorAll('button[data-shot]').forEach(b=>b.setAttribute('aria-pressed',Number(b.dataset.shot)===selected));
$('order-note').textContent=order==='forward'?'정순: 표시 A = 원본 A · 표시 B = 원본 B. 아래 판정문은 모델 응답 원문입니다.':'역순: 표시 A = 원본 B · 표시 B = 원본 A. 이미지와 판정문 모두 이 표시 라벨을 따릅니다. 위 집계만 원본 기준입니다.';
let images=labels.map((orig,i)=>{let display=['A','B'][i],im=s.images[orig];return `<figure class="image"><a href="${esc(im.url)}" target="_blank" rel="noopener"><img src="${esc(im.url)}" alt="${esc(s.tag)} 표시 ${display}, 원본 ${orig}"></a><figcaption>표시 ${display} <span class="muted">원본 ${orig}</span><span class="hash">${im.sha256.slice(0,12)}</span></figcaption></figure>`}).join('');
let cards=D.models.map(([m,name,note])=>{let slot=s.judgments[m]?.[order],a=aggregate(m,s.tag);if(!slot?.ok)return `<article class="judge"><h3>${esc(name)}</h3><p class="error">${slot?esc(slot.error):'아직 응답 없음'}</p></article>`;let raw=slot.raw;
let verdicts=['A','B'].map(l=>{let v=raw.verdicts.find(v=>v.label===l),r=raw.readings.find(r=>r.label===l)||{},orig=labels[l==='A'?0:1];return `<section class="candidate"><strong>표시 ${l} · 원본 ${orig} <span class="pill">${esc(v?.score)}점</span></strong><p>${esc(v?.verdict_ko)}</p>${r.hard_violations?.length?`<div class="violation">하드위반 주장<ul>${r.hard_violations.map(t=>`<li>${esc(t)}</li>`).join('')}</ul></div>`:'<div class="nohv">하드위반 주장 없음</div>'}<details><summary>시각 판독 상세 · 표시 ${l}</summary><dl>${[['direction','방향·행동'],['built_space','장소·공간'],['entities','인물·의상·소품'],['physics','물리적 타당성']].map(([k,t])=>`<dt>${t}</dt><dd>${esc(r[k])}</dd>`).join('')}</dl></details></section>`}).join('');
let usage=slot.usage,meta=slot.elapsed_seconds?`${slot.elapsed_seconds.toFixed(1)}초 · 추론 ${usage?.completion_tokens_details?.reasoning_tokens?.toLocaleString()||0}토큰`:note;
return `<article class="judge"><h3>${esc(name)}</h3><div class="muted">${esc(meta)}</div><div class="winner"><b>선택: 표시 ${esc(raw.winner)} <span class="muted">(원본 ${labels[raw.winner==='A'?0:1]})</span></b><span class="pill">${a?`정·역 ${esc(a.forward)} / ${esc(a.reverse)}`:'1회'}</span></div>${raw.all_candidates_fail?'<p class="warn">모델이 두 후보 모두 실패라고 응답</p>':''}${verdicts}</article>`}).join('');
let refs=s.refs.map(r=>`<figure><a href="${esc(r.url)}" target="_blank" rel="noopener"><img src="${esc(r.url)}" alt="${esc(r.label)}"></a><figcaption>${esc(r.label)}<br>${r.sha256.slice(0,12)}</figcaption></figure>`).join('');
$('shot').innerHTML=`<p class="description">${esc(s.description)}</p><div class="images">${images}</div><div class="ribbon muted">기존 프로덕션 선정: 원본 ${esc(s.production_selected)} · 세 모델 정·역 결과는 위 요약에서 비교</div><div class="judges">${cards}</div><details><summary>모델에 함께 보낸 참조 이미지 (${s.refs.length}장)</summary><div class="refs">${refs}</div></details><details><summary>이 샷의 공통 생성 프롬프트 전문</summary><pre>${esc(s.prompt)}</pre></details><details><summary>기존 프로덕션 두 심판 기록 (별도 실험 · 순서와 모델이 묶여 있음)</summary><pre>${esc(JSON.stringify(s.production.map(p=>({model:p.model,order:p.order,normalized:p.normalized})),null,2))}</pre></details>`;
}
document.addEventListener('click',e=>{let b=e.target.closest('[data-shot]');if(b){selected=Number(b.dataset.shot);render()}});$('order').addEventListener('change',render);render();
</script></body></html>'''


SINGLE_TEMPLATE = r'''<!doctype html><html lang="ko"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>한 장씩 독립 평가 — 5모델 전체 비교</title><style>__STYLE__
.image.single{max-width:1050px;margin:auto}.image.single img{height:540px}.table-wrap{overflow-x:auto}.repeat{border-top:1px solid var(--line);padding-top:12px;margin-top:12px}.table-wrap table{min-width:920px}
</style></head><body><main><div class="eyebrow">SINGLE-IMAGE JUDGE BENCH · 2026.09.05</div><h1>한 장씩 보면 판단은 얼마나 같은가?</h1>
<div class="note">12장 × 5모델 × 독립 반복 2회 = 120회. 판정 대상은 한 장뿐이며 원래의 참조 사진·샷 설명은 유지했습니다. 모델에는 A/B 이름·다른 후보·앞선 판정을 주지 않았습니다. 같은 이미지의 두 요청은 입력 bytes가 같습니다.<br><span class="muted">Astra xhigh · 나머지 high · 출력 상한 16,000. 쌍 비교의 평가 기준을 유지하면서 비교/순위 지시만 단독 평가로 바꿨습니다. 점수는 모델 사이에 보정되지 않았으며, 반복 일치는 정확도가 아닙니다. 점수 동점은 그대로 동점으로 남깁니다. 시간에는 병렬 경합과 캐시가 포함됩니다.</span></div>
<p class="warn">후검증: JSON 형식이 맞아도 NUL 등 제어문자가 섞인 본문은 손상으로 따로 셌습니다. 모든 모델에 같은 기계 규칙을 적용했고, 원문·반환 점수는 보존하되 유효 점수 집계에서는 제외했습니다. 추가 유료 재호출 없음.</p>
<section id="stats" class="stats"></section>
<details open><summary>12장 전체 점수 · 1회 / 2회 · 평균</summary><div id="scores" class="table-wrap"></div></details>
<details open><summary>단독 점수로 복원한 A/B 비교 ↔ 이전 두 장 동시 평가</summary><p class="muted">각 반복에서 독립 점수의 대소만 비교한 값입니다. 모델이 두 장을 보고 고른 것이 아니며, 실패 판정은 별도 표시합니다. 동점을 A로 바꾸지 않습니다.</p><div id="rankings" class="table-wrap"></div></details>
<p><a href="summary.json">전체 집계</a> · <a href="results.json">전체 응답·사용량</a> · <a href="input_manifest.json">입력 지문</a> · <a href="judge_sys.txt">단독 평가 지시문</a> · <a href="prompt.diff">쌍 비교 대비 지시문 변경</a> · <a href="../20260905_vlm_high_parallel/index.html">기존 쌍 비교 갤러리</a></p>
<nav id="shots"></nav><div class="toolbar"><h2 id="shot-title"></h2><label>원본 이미지 <select id="candidate"><option value="A">A</option><option value="B">B</option></select></label></div><section id="shot" class="shot"></section>
</main><script type="application/json" id="data">__DATA__</script><script>
const D=JSON.parse(document.getElementById('data').textContent),$=id=>document.getElementById(id),esc=x=>String(x??'').replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));let selected=0;
const rank=x=>x==='tie'?'동점':x??'실패/미완';
$('stats').innerHTML=D.models.map(([m,name,effort])=>{let s=D.summary.models[m];return `<article class="stat"><h3>${esc(name)} · ${esc(effort)}</h3><div class="metric">${s.mean_absolute_score_delta?.toFixed(2)??'—'} <small>점 · 반복 간 평균 절대 차이</small></div><p>동점수 ${s.same_score_images}/${s.repeat_complete_images}장 · 실패 여부 동일 ${s.same_failure_flag_images}/${s.repeat_complete_images}장</p><div class="muted">유효 판정 ${s.ok}/${s.calls} · JSON 통과 ${s.schema_ok}/${s.calls} · 본문 손상 ${s.corrupt_outputs}<br>중앙 ${s.median_seconds.toFixed(1)}초 · 단독 점수 A/B 반복일치 ${s.score_rank_stable_shots}/${s.score_rank_complete_shots}샷 · 동점 ${s.derived_tie_runs}회</div></article>`}).join('');
$('scores').innerHTML='<table class="summary-table"><thead><tr><th>이미지</th>'+D.models.map(m=>`<th>${esc(m[1])}</th>`).join('')+'</tr></thead><tbody>'+D.shots.flatMap((s,i)=>['A','B'].map(c=>`<tr data-shot="${i}" data-candidate="${c}"><td>${esc(s.tag)} ${c}</td>`+D.models.map(([m])=>{let x=D.summary.images.find(x=>x.model===m&&x.tag===s.tag&&x.candidate===c);return `<td>${x.scores.map((v,j)=>v??(x.returned_scores[j]!==undefined?`${x.returned_scores[j]}* 본문 손상`:'실패')).join(' / ')} → <b>${x.mean_score?.toFixed(1)??'집계 제외'}</b><br><span class="muted">실패 판정 ${x.fails.map(v=>v===null?'—':v?'있음':'없음').join(' / ')}</span></td>`}).join('')+'</tr>')).join('')+'</tbody></table>';
$('rankings').innerHTML='<table class="summary-table"><thead><tr><th>샷</th>'+D.models.map(m=>`<th>${esc(m[1])}</th>`).join('')+'</tr></thead><tbody>'+D.shots.map((s,i)=>`<tr data-shot="${i}"><td>${esc(s.tag)}</td>`+D.models.map(([m])=>{let x=D.summary.shots.find(x=>x.model===m&&x.tag===s.tag);return `<td>단독 ${x.score_rankings.map(rank).join(' / ')}<br>평균점수 → <b>${rank(x.mean_score_winner)}</b><br><span class="muted">동시 ${x.paired_orders?.join(' / ')??'미완'} → ${rank(x.paired_combined)}</span></td>`}).join('')+'</tr>').join('')+'</tbody></table>';
$('shots').innerHTML=D.shots.map((s,i)=>`<button data-shot="${i}" aria-pressed="false">${esc(s.tag)}</button>`).join('');
function render(){let s=D.shots[selected],c=$('candidate').value,im=s.images[c];$('shot-title').textContent=s.tag+' · 원본 '+c;document.querySelectorAll('button[data-shot]').forEach(b=>b.setAttribute('aria-pressed',+b.dataset.shot===selected));
let cards=D.models.map(([m,name,effort])=>`<article class="judge"><h3>${esc(name)} · ${effort}</h3>`+[1,2].map(rep=>{let v=s.judgments[m][`${c}_r${rep}`];if(!v?.ok)return `<div class="repeat"><b>${rep}회</b><p class="error">${esc(v?.error??'미실행')}</p></div>`;let r=v.raw;return `<section class="repeat"><strong>${rep}회 · ${r.score}점</strong><span class="pill">${r.candidate_fails?'실패 판정':'실패 아님'}</span><p>${esc(r.verdict_ko)}</p><p class="muted">${v.elapsed_seconds.toFixed(1)}초 · 추론 ${v.usage?.completion_tokens_details?.reasoning_tokens?.toLocaleString()??'미기록'}토큰</p>${r.readings.hard_violations.length?'<div class="violation">하드위반 주장<ul>'+r.readings.hard_violations.map(x=>'<li>'+esc(x)+'</li>').join('')+'</ul></div>':'<p class="nohv">하드위반 주장 없음</p>'}<details><summary>4축 판독 전문</summary><dl>${[['direction','방향·행동'],['built_space','장소·공간'],['entities','인물·의상·소품'],['physics','물리']].map(([k,n])=>`<dt>${n}</dt><dd>${esc(r.readings[k])}</dd>`).join('')}</dl></details></section>`}).join('')+'</article>').join('');
$('shot').innerHTML=`<p class="description">${esc(s.description)}</p><figure class="image single"><a href="${esc(im.url)}" target="_blank" rel="noopener"><img src="${esc(im.url)}" alt="${s.tag} 원본 ${c} 단독 평가"></a><figcaption>원본 ${c} — 모델에는 A/B 이름이 전달되지 않음 <span class="hash">${im.sha256.slice(0,12)}</span></figcaption></figure><div class="judges">${cards}</div><details><summary>함께 보낸 참조 사진 ${s.refs.length}장</summary><div class="refs">${s.refs.map(r=>`<figure><a href="${esc(r.url)}" target="_blank"><img src="${esc(r.url)}" alt="${esc(r.label)}"></a><figcaption>${esc(r.label)}</figcaption></figure>`).join('')}</div></details><details><summary>같은 샷 설명·생성 프롬프트</summary><pre>${esc(s.prompt)}</pre></details>`}
document.addEventListener('click',e=>{let b=e.target.closest('[data-shot]');if(b){selected=+b.dataset.shot;if(b.dataset.candidate)$('candidate').value=b.dataset.candidate;render()}});$('candidate').addEventListener('change',render);render();
</script></body></html>'''


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("out", type=Path)
    parser.add_argument("--baseline", type=Path)
    parser.add_argument("--merge", type=Path, nargs="+")
    parser.add_argument("--single", action="store_true")
    args = parser.parse_args()
    if bool(args.baseline) != bool(args.merge):
        parser.error("--baseline and --merge must be provided together")
    if args.single and not args.merge:
        parser.error("--single requires --baseline and --merge")
    print(build_single(args.out, args.baseline, args.merge) if args.single else
          merge_runs(args.out, args.baseline, args.merge) if args.merge else build(args.out))