"""experiment_floor_plan_readback_synthesis_slice — W17D.

Consume the W17C VLM readback JSON together with the W15e candidate
(joined via W17B3 → W17A → W15e run chain) and produce a deterministic
coarse layout synthesis + mismatch report. No new LLM call, no VLM
call, no image API call. Pure JSON-to-JSON synthesis.

Key principle (Codex spec): the W15e candidate mapping remains the
truth surface. The VLM readback is an observation layer. This script
joins observation vs truth and surfaces:
  - markers[] one row per legend marker: visible / confidence /
    observed_unit_label_or_area / expected_unit / expected_placement /
    approximate_region_or_cell / heuristic_unit_match (diagnostic) /
    bucket
  - mismatch_report: unit_mismatch_markers (heuristic substring
    mismatch) + vlm_reported_conflicts (verbatim VLM output) +
    missing_marker_count + summary counts

CLI:
  --derive-synthesis-from <W17C_run_dir>        (required)
  --target-fp-ids fp_l05_01                     (default, restricted)
  --output-root <path>
  --diag-print-imports

No --generate flag: the run is deterministic and runs every time.
"""
from __future__ import annotations

import argparse
import json
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple

_REPO_ROOT = Path(__file__).resolve().parents[2]
_SCRIPTS_DIR = _REPO_ROOT / "backend" / "scripts"
if str(_SCRIPTS_DIR) not in sys.path:
    sys.path.insert(0, str(_SCRIPTS_DIR))

from experiment_background_pipeline_slice import (  # type: ignore
    KST,
    PLAN_VERSION,
    _check_image_imports_present,
    _check_production_diff_empty,
    _maybe_print_imports,
)
from experiment_floor_plan_grid_layout_slice import (  # type: ignore
    W16_ALLOWED_TARGET_FP_IDS,
    _resolve_targets,
)
from experiment_floor_plan_image_smoke_slice import (  # type: ignore
    _load_w15e_candidate_index,
)
from experiment_floor_plan_vlm_readback_slice import (  # type: ignore
    _resolve_w17a_run_dir_from_w17b3,
)

W17D_STAGE = "w17d_floor_plan_readback_synthesis_slice"

_DEFAULT_OUTPUT_ROOT = (
    _REPO_ROOT / "scripts_output" / "floor_plan_readback_synthesis_slice_experiment"
)


def _run_id() -> str:
    import secrets

    return datetime.now(KST).strftime("%Y%m%d_%H%M") + "_" + secrets.token_hex(3)


def _parse_args(argv):
    p = argparse.ArgumentParser(
        description=(
            "W17D readback synthesis — consume W17C vlm_readback.json + "
            "W15e candidate join, emit coarse synthesis + heuristic "
            "mismatch report. No LLM/VLM/image API call."
        )
    )
    p.add_argument(
        "--derive-synthesis-from", required=True,
        help="Path to a prior W17C success run dir (containing "
             "vlm_readback.json + run_meta.json).",
    )
    p.add_argument(
        "--target-fp-ids", default="fp_l05_01",
        help="Comma-separated fp_id subset. Wave 1 only allows fp_l05_01.",
    )
    p.add_argument("--output-root", default=str(_DEFAULT_OUTPUT_ROOT))
    p.add_argument("--diag-print-imports", action="store_true")
    return p.parse_args(argv)


def _resolve_w17b3_run_dir_from_w17c(w17c_meta: Dict[str, Any]) -> Optional[Path]:
    args_dict = (w17c_meta or {}).get("args") or {}
    if not isinstance(args_dict, dict):
        return None
    raw = args_dict.get("derive_readback_from")
    if isinstance(raw, str) and raw:
        p = Path(raw)
        if not p.is_absolute():
            p = _REPO_ROOT / raw
        if (p / "run_meta.json").exists():
            return p
    derived_from = w17c_meta.get("derived_from") or ""
    if isinstance(derived_from, str) and derived_from:
        parent = (
            _REPO_ROOT / "scripts_output" / "floor_plan_image_smoke_slice_experiment"
        )
        if (parent / derived_from / "run_meta.json").exists():
            return parent / derived_from
    return None


def _heuristic_unit_substring_match(observed: str, expected_unit: str) -> bool:
    """Heuristic diagnostic ONLY — not a semantic proof. Substring
    containment between the W15e `unit_id_pointer` (with underscores
    converted to spaces) and the VLM-observed unit/area label, case-
    insensitive. The W17 spec is explicit: this is a `heuristic
    diagnostic + VLM-reported conflict surface`, never promoted to
    truth."""
    if not observed or not expected_unit:
        return False
    ob = observed.lower()
    eu = expected_unit.lower().replace("_", " ")
    if eu and eu in ob:
        return True
    if ob and ob in eu:
        return True
    return False


def _build_synthesis(
    *, fp_id: str,
    expected_legend: List[Dict[str, Any]],
    candidate_index: Dict[int, Dict[str, str]],
    vlm_readback: Dict[str, Any],
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
    """Return (synthesis_entry, mismatch_entry) for one fp_id."""
    read_by_n: Dict[int, Dict[str, Any]] = {}
    for entry in vlm_readback.get("read_markers") or []:
        if isinstance(entry, dict) and isinstance(entry.get("marker_number"), int):
            read_by_n[int(entry["marker_number"])] = entry
    missing_by_n: Dict[int, Dict[str, Any]] = {}
    for entry in vlm_readback.get("missing_or_ambiguous_markers") or []:
        if isinstance(entry, dict) and isinstance(entry.get("marker_number"), int):
            missing_by_n[int(entry["marker_number"])] = entry

    markers: List[Dict[str, Any]] = []
    unit_mismatch: List[Dict[str, Any]] = []
    matched_count = 0
    missing_count = 0
    mismatch_count = 0

    expected_numbers: List[int] = []
    legend_by_n: Dict[int, Dict[str, Any]] = {}
    for it in expected_legend or []:
        if isinstance(it, dict) and isinstance(it.get("marker_number"), int):
            n = int(it["marker_number"])
            expected_numbers.append(n)
            legend_by_n[n] = it

    for n in sorted(expected_numbers):
        legend_entry = legend_by_n[n]
        cand = candidate_index.get(n) or {}
        expected_unit = cand.get("unit") or ""
        expected_placement = cand.get("placement") or ""
        if n in read_by_n:
            r = read_by_n[n]
            observed = r.get("observed_unit_label_or_area") or ""
            visible = bool(r.get("visible"))
            confidence = r.get("confidence") or ""
            region = r.get("approximate_region_or_cell") or ""
            notes = r.get("notes") or ""
            heuristic_match = _heuristic_unit_substring_match(
                observed, expected_unit
            )
            if heuristic_match:
                bucket = "matched"
                matched_count += 1
            else:
                bucket = "unit_mismatch"
                mismatch_count += 1
                unit_mismatch.append({
                    "marker_number": n,
                    "expected_unit": expected_unit,
                    "expected_placement": expected_placement,
                    "observed_unit_label_or_area": observed,
                    "approximate_region_or_cell": region,
                    "confidence": confidence,
                    "label": legend_entry.get("label") or "",
                    "kind": legend_entry.get("element_kind") or "",
                })
        elif n in missing_by_n:
            r = missing_by_n[n]
            observed = ""
            visible = False
            confidence = "unknown"
            region = ""
            notes = r.get("reason") or ""
            heuristic_match = False
            bucket = "missing"
            missing_count += 1
        else:
            observed = ""
            visible = False
            confidence = "unknown"
            region = ""
            notes = ""
            heuristic_match = False
            bucket = "unaccounted_in_readback"
            missing_count += 1

        markers.append({
            "marker_number": n,
            "label": legend_entry.get("label") or "",
            "kind": legend_entry.get("element_kind") or "",
            "visual_encoding": legend_entry.get("visual_encoding") or "",
            "expected_unit": expected_unit,
            "expected_placement": expected_placement,
            "visible": visible,
            "confidence": confidence,
            "observed_unit_label_or_area": observed,
            "approximate_region_or_cell": region,
            "notes": notes,
            "heuristic_unit_match": heuristic_match,
            "bucket": bucket,
        })

    synthesis_entry = {
        "fp_id": fp_id,
        "markers": markers,
        "unit_boundary_summary": vlm_readback.get("unit_boundary_summary") or "",
        "summary_counts": {
            "expected_total": len(expected_numbers),
            "matched": matched_count,
            "unit_mismatch": mismatch_count,
            "missing": missing_count,
        },
    }
    mismatch_entry = {
        "fp_id": fp_id,
        "unit_mismatch_markers": unit_mismatch,
        "vlm_reported_conflicts": list(
            vlm_readback.get("readback_conflicts") or []
        ),
        "missing_marker_count": missing_count,
        "matched_marker_count": matched_count,
        "unit_mismatch_marker_count": mismatch_count,
        "diagnostic_note": (
            "heuristic substring match — diagnostic only, not a semantic "
            "proof. VLM-reported conflicts surfaced verbatim. Truth surface "
            "remains the W15e candidate mapping; this synthesis is an "
            "observation layer."
        ),
    }
    return synthesis_entry, mismatch_entry


def _build_w17d_compatibility_report(
    *, expected_legend: List[Dict[str, Any]],
    synthesis_entry: Dict[str, Any],
    mismatch_entry: Dict[str, Any],
    production_diff_empty: bool,
    db_write_count: int,
    image_import_seen: bool,
    image_api_call_count: int,
    vlm_api_call_count: int,
    missing_inputs: List[str],
    prev_run_id: str,
    target_fp_ids: Set[str],
) -> dict:
    inv: Dict[str, Dict[str, Any]] = {}

    inv["inputs_present"] = {
        "pass": (
            not missing_inputs
            and bool(expected_legend)
            and bool(synthesis_entry)
        ),
        "detail": {
            "missing_inputs": list(missing_inputs),
            "legend_entry_count": len(expected_legend),
            "prev_run_id": prev_run_id,
        },
    }

    inv["target_fp_only_fp_l05_01"] = {
        "pass": bool(target_fp_ids) and target_fp_ids.issubset(
            W16_ALLOWED_TARGET_FP_IDS
        ),
        "detail": {
            "received_target_fp_ids": sorted(target_fp_ids or set()),
            "allowed": sorted(W16_ALLOWED_TARGET_FP_IDS),
        },
    }

    expected_numbers: Set[int] = {
        int(it["marker_number"]) for it in expected_legend
        if isinstance(it, dict) and isinstance(it.get("marker_number"), int)
    }
    synth_numbers: Set[int] = {
        int(m["marker_number"]) for m in (synthesis_entry.get("markers") or [])
        if isinstance(m, dict) and isinstance(m.get("marker_number"), int)
    }
    inv["synthesis_covers_full_legend"] = {
        "pass": expected_numbers == synth_numbers,
        "detail": {
            "expected_count": len(expected_numbers),
            "synth_count": len(synth_numbers),
            "missing_marker_numbers": sorted(expected_numbers - synth_numbers),
            "extra_marker_numbers": sorted(synth_numbers - expected_numbers),
        },
    }

    inv["heuristic_match_is_diagnostic_only"] = {
        "pass": True,
        "detail": {
            "note": (
                "heuristic diagnostic + VLM-reported conflict surface; "
                "not a semantic proof"
            ),
            "matched_count": (synthesis_entry.get("summary_counts") or {}).get(
                "matched", 0
            ),
            "unit_mismatch_count": (synthesis_entry.get("summary_counts") or {}).get(
                "unit_mismatch", 0
            ),
            "missing_count": (synthesis_entry.get("summary_counts") or {}).get(
                "missing", 0
            ),
            "vlm_reported_conflict_count": len(
                mismatch_entry.get("vlm_reported_conflicts") or []
            ),
        },
    }

    inv["production_diff_zero_db_write_zero_image_api_call_zero"] = {
        "pass": (
            production_diff_empty
            and db_write_count == 0
            and not image_import_seen
            and image_api_call_count == 0
            and vlm_api_call_count == 0
        ),
        "detail": {
            "production_diff_empty": production_diff_empty,
            "db_write_count": db_write_count,
            "image_import_seen": image_import_seen,
            "image_api_call_count": image_api_call_count,
            "vlm_api_call_count": vlm_api_call_count,
        },
    }

    all_pass = all(v["pass"] for v in inv.values())
    return {"invariants": inv, "all_pass": all_pass}


def main(argv=None) -> int:
    args = _parse_args(argv)
    run_id = _run_id()
    out_root = Path(args.output_root)
    run_dir = out_root / run_id
    run_dir.mkdir(parents=True, exist_ok=True)

    prev_run_dir = Path(args.derive_synthesis_from)
    if not prev_run_dir.is_absolute():
        prev_run_dir = Path.cwd() / prev_run_dir

    target_fp_ids, invalid_targets = _resolve_targets(args.target_fp_ids)
    failed_invariants: List[str] = []
    run_status = "succeeded"
    exit_code = 0

    run_meta: Dict[str, Any] = {
        "run_id": run_id,
        "stage": W17D_STAGE,
        "plan_version": PLAN_VERSION,
        "generated_at": datetime.now(KST).isoformat(),
        "args": vars(args),
        "derived_from": prev_run_dir.name,
        "target_fp_ids": sorted(target_fp_ids),
        "invalid_targets": invalid_targets,
        "vlm_api_call_count": 0,
        "image_api_call_count": 0,
        "outputs": [],
        "run_status": run_status,
        "exit_code": exit_code,
        "failed_invariants": failed_invariants,
    }

    def _persist_and_exit(code: int) -> int:
        run_meta["exit_code"] = code
        if code != 0:
            run_meta["run_status"] = "validation_failed"
        run_meta["failed_invariants"] = failed_invariants
        (run_dir / "run_meta.json").write_text(
            json.dumps(run_meta, ensure_ascii=False, indent=2)
        )
        return code

    missing: List[str] = []
    w17c_readback_path = prev_run_dir / "vlm_readback.json"
    w17c_meta_path = prev_run_dir / "run_meta.json"
    if not w17c_readback_path.exists():
        missing.append("vlm_readback.json")
    if not w17c_meta_path.exists():
        missing.append("run_meta.json")
    if missing:
        failed_invariants.append("w17c_inputs_missing")
        run_meta["missing_inputs"] = missing
        return _persist_and_exit(1)
    if invalid_targets:
        failed_invariants.append("invalid_target_fp_ids")
        return _persist_and_exit(1)
    if not target_fp_ids.issubset(W16_ALLOWED_TARGET_FP_IDS):
        failed_invariants.append("target_fp_outside_allowed")
        return _persist_and_exit(1)

    vlm_readback = json.loads(w17c_readback_path.read_text())
    w17c_meta = json.loads(w17c_meta_path.read_text())

    # Walk W17C → W17B3 → W17A → W15e to gather the legend + candidate.
    w17b3_dir = _resolve_w17b3_run_dir_from_w17c(w17c_meta)
    expected_legend: List[Dict[str, Any]] = []
    candidate_index: Dict[int, Dict[str, str]] = {}
    if w17b3_dir is not None:
        try:
            w17b3_meta = json.loads((w17b3_dir / "run_meta.json").read_text())
            w17a_dir = _resolve_w17a_run_dir_from_w17b3(w17b3_meta)
        except Exception:  # noqa: BLE001
            w17a_dir = None
        if w17a_dir is not None:
            try:
                prompt_artifact = json.loads(
                    (w17a_dir / "floor_plan_image_prompt.json").read_text()
                )
                by_fp = (prompt_artifact or {}).get("floor_plan_image_prompt_by_fp") or {}
                fp_id_first = sorted(target_fp_ids)[0]
                fp_entry = by_fp.get(fp_id_first) or {}
                expected_legend = list(fp_entry.get("numbered_marker_legend") or [])
            except Exception:  # noqa: BLE001
                expected_legend = []
            try:
                w17a_meta = json.loads((w17a_dir / "run_meta.json").read_text())
                cand_idx_by_fp = _load_w15e_candidate_index(
                    w17a_run_dir=w17a_dir, w17a_run_meta=w17a_meta,
                )
                fp_id_first = sorted(target_fp_ids)[0]
                candidate_index = cand_idx_by_fp.get(fp_id_first) or {}
            except Exception:  # noqa: BLE001
                candidate_index = {}

    fp_id_first = sorted(target_fp_ids)[0]
    synthesis_entry, mismatch_entry = _build_synthesis(
        fp_id=fp_id_first,
        expected_legend=expected_legend,
        candidate_index=candidate_index,
        vlm_readback=vlm_readback,
    )

    synthesis = {"synthesis_by_fp": {fp_id_first: synthesis_entry}}
    mismatch = {"mismatch_by_fp": {fp_id_first: mismatch_entry}}

    (run_dir / "coarse_layout_synthesis.json").write_text(
        json.dumps(synthesis, ensure_ascii=False, indent=2)
    )
    run_meta["outputs"].append("coarse_layout_synthesis.json")
    (run_dir / "mismatch_report.json").write_text(
        json.dumps(mismatch, ensure_ascii=False, indent=2)
    )
    run_meta["outputs"].append("mismatch_report.json")

    report = _build_w17d_compatibility_report(
        expected_legend=expected_legend,
        synthesis_entry=synthesis_entry,
        mismatch_entry=mismatch_entry,
        production_diff_empty=_check_production_diff_empty(),
        db_write_count=0,
        image_import_seen=_check_image_imports_present(),
        image_api_call_count=0,
        vlm_api_call_count=0,
        missing_inputs=missing,
        prev_run_id=prev_run_dir.name,
        target_fp_ids=target_fp_ids,
    )
    (run_dir / "synthesis_compatibility_report.json").write_text(
        json.dumps(report, ensure_ascii=False, indent=2)
    )
    run_meta["outputs"].append("synthesis_compatibility_report.json")

    for name, v in report["invariants"].items():
        if not v["pass"] and name not in failed_invariants:
            failed_invariants.append(name)
    if failed_invariants and run_status == "succeeded":
        run_status = "validation_failed"
        exit_code = 1

    run_meta["run_status"] = run_status
    run_meta["exit_code"] = exit_code
    run_meta["failed_invariants"] = failed_invariants

    # Light HTML index.
    def esc(x):
        return (
            str(x).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
        )

    marker_rows = "".join(
        f"<tr><td>#{esc(m.get('marker_number'))}</td>"
        f"<td>{esc(m.get('label'))}</td>"
        f"<td>{esc(m.get('kind'))}</td>"
        f"<td>{esc(m.get('expected_unit'))}</td>"
        f"<td>{esc(m.get('observed_unit_label_or_area'))}</td>"
        f"<td>{esc(m.get('approximate_region_or_cell'))}</td>"
        f"<td>{esc(m.get('confidence'))}</td>"
        f"<td>{esc(m.get('heuristic_unit_match'))}</td>"
        f"<td><b>{esc(m.get('bucket'))}</b></td></tr>"
        for m in synthesis_entry.get("markers") or []
    )
    inv_rows = "".join(
        f"<tr><td>{esc(k)}</td>"
        f"<td class=\"{'pass' if v['pass'] else 'fail'}\">"
        f"{'PASS' if v['pass'] else 'FAIL'}</td>"
        f"<td><pre>{esc(json.dumps(v.get('detail'), ensure_ascii=False))[:600]}</pre></td>"
        f"</tr>"
        for k, v in (report.get("invariants") or {}).items()
    )
    summary = synthesis_entry.get("summary_counts") or {}
    (run_dir / "index.html").write_text(
        f"""<!doctype html><html><head><meta charset="utf-8">
<title>W17D readback_synthesis {esc(run_id)}</title>
<style>body{{font-family:sans-serif;margin:1.5em}}
table{{border-collapse:collapse;margin:0.5em 0}}
td,th{{border:1px solid #ccc;padding:4px 8px;vertical-align:top}}
.pass{{color:#080}} .fail{{color:#b00}}
pre{{white-space:pre-wrap;font-size:0.85em;max-width:84ch}}</style></head><body>
<h1>W17D — readback_synthesis {esc(run_id)}</h1>
<p>fp_id: <b>{esc(fp_id_first)}</b>
| derived_from(W17C): {esc(prev_run_dir.name)}
| vlm_api_call_count: <b>0</b> | image_api_call_count: <b>0</b>
| run_status: <b>{esc(run_status)}</b>
| exit_code: {esc(exit_code)}</p>

<section><h2>summary counts</h2>
<p>expected_total = <b>{esc(summary.get('expected_total'))}</b>
| matched = <b>{esc(summary.get('matched'))}</b>
| unit_mismatch = <b>{esc(summary.get('unit_mismatch'))}</b>
| missing = <b>{esc(summary.get('missing'))}</b></p>
<p>unit_boundary_summary:</p>
<pre>{esc(synthesis_entry.get('unit_boundary_summary') or '')}</pre></section>

<section><h2>markers</h2>
<table><tr><th>#</th><th>label</th><th>kind</th>
<th>expected_unit</th><th>observed_unit</th>
<th>region</th><th>confidence</th>
<th>heuristic_match</th><th>bucket</th></tr>
{marker_rows}</table></section>

<section><h2>mismatch_report.unit_mismatch_markers</h2>
<pre>{esc(json.dumps(mismatch_entry.get('unit_mismatch_markers') or [], ensure_ascii=False, indent=2))}</pre></section>

<section><h2>vlm_reported_conflicts</h2>
<pre>{esc(json.dumps(mismatch_entry.get('vlm_reported_conflicts') or [], ensure_ascii=False, indent=2))}</pre></section>

<section><h2>invariants</h2>
<table><tr><th>invariant</th><th>status</th><th>detail</th></tr>
{inv_rows}</table></section>

<details><summary>raw run_meta.json</summary>
<pre>{esc(json.dumps(run_meta, ensure_ascii=False, indent=2))}</pre></details>
</body></html>"""
    )
    run_meta["outputs"].append("index.html")

    (run_dir / "run_meta.json").write_text(
        json.dumps(run_meta, ensure_ascii=False, indent=2)
    )
    _maybe_print_imports(args)
    return exit_code


if __name__ == "__main__":
    sys.exit(main())