"""'사랑했지만' 최종 215샷 — OpenRouter Qwen 3.8 Max + Kimi K3 재선택 평가. 프로덕션 선정(G+Q: gemini-pro + DashScope qwen)이 육안과 어긋난 사례 (S1sh11 수정본 비율 붕괴·S4sh9 롤C 탈락·S5sh6 방향·S9 책상)가 나와, 같은 선정 계약(opus_pilot SELECT_SYS + select_schema — 금월도 재평가와 동일)으로 두 외부 심판에게 후보를 다시 고르게 해 프로덕션 선택과 비교한다. · Qwen 3.8 Max — openrouter `qwen/qwen3.8-max` ($2/$6 per M) · Kimi K3 — openrouter `moonshotai/kimi-k3` ($3/$15 per M) 돈 주의: 판정 호출만(이미지 생성 없음). 증분 저장 — 재실행 시 이미 판정된 (샷, 모델) 은 건너뛴다. 필요 env: OPENROUTER_API_KEY. 사용: .venv/bin/python build_qk_openrouter_eval.py --limit 3 # 소량 시험 .venv/bin/python build_qk_openrouter_eval.py # 전체 판정 .venv/bin/python build_qk_openrouter_eval.py --html # 갤러리만 """ from __future__ import annotations import argparse import json import os import sys import threading import time from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any, Dict, List, Tuple sys.path.insert(0, str(Path(__file__).resolve().parent)) from jsonschema import ValidationError, validate as _js_validate # noqa: E402 from app.modules.llm.image_tracer import record_provider_call # noqa: E402 from app.modules.pipeline.multiroll_gemini import png_part, ref_parts # noqa: E402 from app.modules.pipeline.multiroll_select import ( # noqa: E402 _compose_critique_prompt, ) from opus_pilot import SELECT_SYS, resolve_refs, select_schema # noqa: E402 from qwen_vlm_pilot import JSON_CLAUSE, extract_json # noqa: E402 PROJ = "c7e3b2e7-c545-4516-93b2-62a51a74d794" EPI = "7c902020-4451-4967-9eb6-1e53c2b9b717" ROOT = Path(__file__).resolve().parent.parent RECIPE = ROOT / f"projects/{PROJ}/images/{EPI}/scene/recipe" WEB = f"/projects/{PROJ}/images/{EPI}/scene/recipe" OUT_DIR = ROOT / "artifact" / "20260812_QK_openrouter_재평가" RESULTS = OUT_DIR / "results.json" INDEX = OUT_DIR / "index.html" WORKERS = 10 MODELS = { "qwen_or": "qwen/qwen3.8-max", "kimi_or": "moonshotai/kimi-k3", } # 08-12 사용자 육안 지적 — 선택/산출 문제가 걸린 샷 FLAGGED = {"S1sh11", "S4sh9", "S5sh6"} | {f"S9sh{i}" for i in range(13, 22)} def tag_key(tag: str): s, sh = tag[1:].split("sh") return (int(s), int(sh)) def build_size_index() -> Dict[int, List[Path]]: idx: Dict[int, List[Path]] = {} for p in (ROOT / f"projects/{PROJ}").rglob("*.png"): idx.setdefault(p.stat().st_size, []).append(p) return idx def build_parts(stem: str, rec: Dict[str, Any], size_idx) -> Tuple[List[Dict], List[str]]: """금월도 재평가와 동일한 제시 — 브리프 → 참조 → 후보 이미지.""" labels = sorted( p.stem.split("_")[-1].upper() for p in RECIPE.glob(f"{stem}_[abc].png")) if not labels: return [], [] roll_prompts = rec.get("roll_prompts") or {} parts: List[Dict[str, Any]] = [{ "type": "text", "text": ("THE BRIEF (every candidate was made from this):\n" + _compose_critique_prompt( rec.get("prompt", ""), roll_prompts, labels[0], shared_prompt=None)), }] ref0, _missing = resolve_refs( (rec.get("roll_refs") or {}).get(labels[0]) or rec.get("refs"), size_idx) parts += ref_parts(ref0) for lab in labels: parts.append({"type": "text", "text": f"Candidate {lab}:"}) parts.append(png_part(RECIPE / f"{stem}_{lab.lower()}.png")) return parts, labels def openrouter_client(): from openai import OpenAI key = os.environ.get("OPENROUTER_API_KEY", "").strip() if not key: raise RuntimeError("OPENROUTER_API_KEY 미설정") return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=key, timeout=180) def ask_openrouter( model: str, system: str, parts: List[Dict[str, Any]], schema: Dict[str, Any], *, shot_tag: str, max_retry: int = 1, ) -> Tuple[Dict[str, Any], Dict[str, Any]]: """qwen_vlm_pilot.ask_qwen 과 같은 계약 — client 만 OpenRouter.""" client = openrouter_client() sys_prompt = system + JSON_CLAUSE.format( schema=json.dumps(schema, ensure_ascii=False, indent=2)) messages: List[Dict[str, Any]] = [ {"role": "system", "content": sys_prompt}, {"role": "user", "content": parts}, ] meta: Dict[str, Any] = {"model": model, "attempts": []} for attempt in range(max_retry + 1): t0 = time.time() # json_object 미지원 배포본이면 400 — 그때는 빼고 한 번 더. resp = None for rf in ({"type": "json_object"}, None): try: resp = client.chat.completions.create( model=model, messages=messages, **({"response_format": rf} if rf else {}), ) break except Exception as exc: # noqa: BLE001 if rf is None or "400" not in repr(exc): raise meta["response_format"] = "unsupported" if resp is None: raise RuntimeError("openrouter: no response") content = resp.choices[0].message.content or "" usage = getattr(resp, "usage", None) rec = { "elapsed_s": round(time.time() - t0, 1), "prompt_tokens": getattr(usage, "prompt_tokens", None), "completion_tokens": getattr(usage, "completion_tokens", None), } ok = False err = "" payload: Dict[str, Any] = {} try: payload = extract_json(content) _js_validate(payload, schema) ok = True except (ValueError, ValidationError, json.JSONDecodeError) as exc: err = f"{type(exc).__name__}: {exc}"[:400] rec["ok"] = ok if err: rec["error"] = err meta["attempts"].append(rec) record_provider_call( step="qk_openrouter_reselect", model=model, prompt=f"reselect {shot_tag} ({len(parts)} parts)", status="success" if ok else "error", duration_ms=int((time.time() - t0) * 1000), meta={"project_id": PROJ, "episode_id": EPI, "shot_tag": shot_tag, "attempt": attempt}, provider="openrouter", output_text=(content[:500] if ok else None), error=(err or None), ) if ok: return payload, meta if attempt >= max_retry: raise ValueError(err) messages += [ {"role": "assistant", "content": content[:4000]}, {"role": "user", "content": ( "That reply did not satisfy the json schema: " f"{err}\nReturn the corrected json object only.")}, ] raise RuntimeError("unreachable") def load_records() -> dict: return json.loads((RECIPE / "records.json").read_text("utf-8")) def run_judges(limit: int = 0) -> None: records = load_records() size_idx = build_size_index() done: dict = {} if RESULTS.exists(): done = json.loads(RESULTS.read_text("utf-8")) tags = sorted( (t for t, r in records.items() if isinstance(r, dict) and "::" not in t and r.get("selected")), key=tag_key) jobs = [t for t in tags if any(done.get(t, {}).get(mk) is None for mk in MODELS)] if limit: jobs = jobs[:limit] print(f"판정 대상 {len(jobs)} / 전체 {len(tags)} (완료 " f"{len(tags) - len(jobs)})", flush=True) lock = threading.Lock() count = [0] def one(stem: str): rec = records[stem] parts, labels = build_parts(stem, rec, size_idx) entry = done.get(stem) or {} entry["labels"] = labels entry["baseline"] = rec.get("selected") if not labels: entry["skipped"] = "후보 롤 파일 없음" return stem, entry for mk, model in MODELS.items(): if entry.get(mk) is not None: continue try: payload, meta = ask_openrouter( model, SELECT_SYS, parts, select_schema(labels), shot_tag=stem) entry[mk] = payload entry[f"{mk}_meta"] = meta except Exception as exc: # noqa: BLE001 entry[f"{mk}_error"] = repr(exc)[:300] return stem, entry with ThreadPoolExecutor(max_workers=WORKERS) as ex: futs = {ex.submit(one, t): t for t in jobs} for fut in as_completed(futs): stem, entry = fut.result() with lock: done[stem] = entry count[0] += 1 OUT_DIR.mkdir(parents=True, exist_ok=True) RESULTS.write_text( json.dumps(done, ensure_ascii=False, indent=1), "utf-8") qw = (entry.get("qwen_or") or {}).get("winner") km = (entry.get("kimi_or") or {}).get("winner") print(f"[{count[0]}/{len(jobs)}] {stem} base=" f"{entry.get('baseline')} qwen={qw} kimi={km}", flush=True) def esc(x) -> str: import html as H return H.escape(str(x if x is not None else "")) def build_html() -> None: records = load_records() done = json.loads(RESULTS.read_text("utf-8")) if RESULTS.exists() else {} rows = [] n = {"both_agree": 0, "any_disagree": 0, "qwen_dis": 0, "kimi_dis": 0} for tag in sorted(done, key=tag_key): e = done[tag] base = e.get("baseline") qw = (e.get("qwen_or") or {}).get("winner") km = (e.get("kimi_or") or {}).get("winner") q_dis = bool(qw and base and qw != base) k_dis = bool(km and base and km != base) if q_dis: n["qwen_dis"] += 1 if k_dis: n["kimi_dis"] += 1 if q_dis or k_dis: n["any_disagree"] += 1 elif qw and km: n["both_agree"] += 1 cls = ["row"] if q_dis or k_dis: cls.append("disagree") if tag in FLAGGED: cls.append("flagged") figs = [] rec = records.get(tag) or {} for lab in e.get("labels") or []: marks = [] if base == lab: marks.append("프로덕션") if qw == lab: marks.append("Qwen") if km == lab: marks.append("Kimi") figs.append( f"
" f"
{lab} {' '.join(marks)}
") if (RECIPE / f"{tag}_fix.png").exists(): marks = "프로덕션(수정본)" if rec.get( "repair_mode") else "" figs.append( f"
" f"
수정본 {marks}
") det = [] for mk, label in (("qwen_or", "Qwen3.8-Max"), ("kimi_or", "Kimi-K3")): p = e.get(mk) if p: det.append( f"
{label}: winner=" f"{esc(p.get('winner'))} · {esc(p.get('reason_ko') or p.get('reason') or '')}" "
") elif e.get(f"{mk}_error"): det.append( f"
{label} 오류: " f"{esc(e.get(f'{mk}_error'))}
") flag_txt = (" 육안 지적" if tag in FLAGGED else "") rows.append( f"
" f"

{tag} — 프로덕션 {esc(base)} / Qwen {esc(qw)} / Kimi " f"{esc(km)}{flag_txt}

" f"
{''.join(figs)}
{''.join(det)}
") judged = [t for t in done if (done[t].get("qwen_or") or done[t].get("kimi_or"))] head = ( "\n" "사랑했지만 — OpenRouter Q+K 재선택 vs 프로덕션 G+Q\n" "\n") summary = ( f"

OpenRouter 재선택 (Qwen3.8-Max · Kimi-K3) vs 프로덕션 G+Q

" f"

판정 {len(judged)}샷 · 프로덕션과 불일치: Qwen {n['qwen_dis']}" f" · Kimi {n['kimi_dis']} · 어느 한쪽이라도 {n['any_disagree']}" f" · 셋 일치 {n['both_agree']}

" "

필터: 전체" " · 불일치만" " · 육안 지적만

" "\n") OUT_DIR.mkdir(parents=True, exist_ok=True) INDEX.write_text(head + summary + "\n".join(rows), "utf-8") print(f"갤러리: {INDEX} ({len(rows)}샷)") def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--limit", type=int, default=0) ap.add_argument("--html", action="store_true") args = ap.parse_args() if not args.html: run_judges(args.limit) build_html() if __name__ == "__main__": main()