"""experiment_background_image_overlap_reference_slice — W18J.

Overlap-driven reference selection for the remaining L05 BGs after the
W18I L05B02 anchor was accepted. The planner uses *exact ID* overlap
between each target BG and the running reference catalog (no
regex/substring semantic extraction) to decide, per BG:

- if at least one catalog entry has *strong* overlap with the target
  (shared spatial units + shared base marker numbers from W18D), the
  target is generated in `reference_derived` mode: pass up to two
  catalog PNGs as `image=[ref1, ref2]` to ``images.edit`` and **do not
  include the base FP PNG**;
- otherwise the target is generated in `fp_seeded_anchor` mode: pass
  only the W18B base FP PNG so the BG becomes a fresh anchor under the
  modest style contract.

Guidance text is *mode-keyed*, not BG-id-keyed. BG-specific differences
emerge only from the deterministic overlap computation, not from any
hand-written per-BG sentence.

CLI:
  --derive-overlap-from <W18F_run_dir>   (required)
  --derive-region-ref-from <W18D_run_dir> (required — supplies the
                                           target_unit_refs +
                                           base_marker_refs that W18F
                                           does not carry verbatim)
  --derive-anchor-from <W18I_run_dir>    (required, accepted anchor source)
  --target-bg-ids L05B05,L05B04,L05B03,L05B01   (default, L05B02 excluded)
  --generate                              (default off → dry-run)
  --model gpt-image-2                    (fail-closed)
  --output-root <path>
  --diag-print-imports
"""
from __future__ import annotations

import argparse
import base64
import json
import os
import sys
import time
from datetime import datetime
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Set, Tuple

_REPO_ROOT = Path(__file__).resolve().parents[2]
_SCRIPTS_DIR = _REPO_ROOT / "backend" / "scripts"
if str(_SCRIPTS_DIR) not in sys.path:
    sys.path.insert(0, str(_SCRIPTS_DIR))

from experiment_actual_floor_plan_generation_slice import (  # type: ignore
    _create_openai_client,
    _to_jsonable,
)
from experiment_background_pipeline_slice import (  # type: ignore
    KST,
    PLAN_VERSION,
    _check_image_imports_present,
    _check_production_diff_empty,
    _load_backend_env,
    _maybe_print_imports,
)
from experiment_background_image_smoke_from_region_ref_slice import (  # type: ignore
    W18G_IMAGE_BACKEND,
    W18G_IMAGE_SIZE,
)


W18J_STAGE = "w18j_background_image_overlap_reference_slice"
W18J_ALLOWED_BG_IDS = {"L05B01", "L05B02", "L05B03", "L05B04", "L05B05"}
W18J_ANCHOR_BG_ID = "L05B02"
W18J_DEFAULT_TARGETS = "L05B05,L05B04,L05B03,L05B01"
W18J_MAX_REFS_PER_BG = 2

W18J_MODE_REFERENCE_DERIVED = "reference_derived"
W18J_MODE_FP_SEEDED_ANCHOR = "fp_seeded_anchor"
W18J_ALLOWED_MODES = {W18J_MODE_REFERENCE_DERIVED, W18J_MODE_FP_SEEDED_ANCHOR}

_DEFAULT_OUTPUT_ROOT = (
    _REPO_ROOT
    / "scripts_output"
    / "background_image_overlap_reference_slice_experiment"
)

W18J_MODEST_STYLE_PREAMBLE = (
    "Modest compact lived-in home, ordinary domestic finishes, "
    "practical low-budget materials, simple consistent indoor lighting, "
    "no luxury, no showroom, no hotel, no real-estate brochure styling, "
    "no marble, no chandelier, no designer furniture. "
    "Preserve the region and markers from the prompt body. "
    "Do not render any person, body, or corpse."
)

W18J_GUIDANCE_BY_MODE: Dict[str, str] = {
    W18J_MODE_REFERENCE_DERIVED: (
        "Inherit the home identity from the provided visual reference "
        "image(s). Match floor, wall, window-covering, lighting, and "
        "fixed-furniture palette so the room reads as the same "
        "dwelling. Do not redraw any transient cue that the reference "
        "already contains; only add the transient cues explicitly "
        "listed in the prompt body below."
    ),
    W18J_MODE_FP_SEEDED_ANCHOR: (
        "The provided reference is the base floor plan layout only. "
        "Use it as spatial guidance, not as a visible overlay. "
        "Establish a fresh ordinary lived-in interior under the modest "
        "style contract above. Render only the region and markers "
        "described in the prompt body below."
    ),
}


def _run_id() -> str:
    import secrets

    return datetime.now(KST).strftime("%Y%m%d_%H%M") + "_" + secrets.token_hex(3)


def _parse_args(argv):
    p = argparse.ArgumentParser(
        description=(
            "W18J overlap-driven reference selection — generates each "
            "remaining L05 BG with either visual references (no FP) "
            "or the base FP only, based on exact-ID overlap with the "
            "running reference catalog."
        )
    )
    p.add_argument(
        "--derive-overlap-from", required=True,
        help="Path to a prior W18F success run dir.",
    )
    p.add_argument(
        "--derive-region-ref-from", required=True,
        help=(
            "Path to a prior W18D success run dir (supplies "
            "target_unit_refs and base_marker_refs by bg_id)."
        ),
    )
    p.add_argument(
        "--derive-anchor-from", required=True,
        help=(
            "Path to a prior W18I success run dir that produced the "
            "accepted L05B02 anchor PNG."
        ),
    )
    p.add_argument(
        "--target-bg-ids", default=W18J_DEFAULT_TARGETS,
        help=(
            "CSV of BG ids to generate. L05B02 is excluded (anchor)."
        ),
    )
    p.add_argument("--generate", action="store_true")
    p.add_argument("--model", default=W18G_IMAGE_BACKEND)
    p.add_argument("--output-root", default=str(_DEFAULT_OUTPUT_ROOT))
    p.add_argument("--diag-print-imports", action="store_true")
    return p.parse_args(argv)


def _resolve_target_bg_ids(raw: str) -> Tuple[List[str], List[str]]:
    parts = [p.strip() for p in (raw or "").split(",") if p.strip()]
    seen: List[str] = []
    invalid: List[str] = []
    for p in parts:
        if p == W18J_ANCHOR_BG_ID:
            invalid.append(p)
            continue
        if p in W18J_ALLOWED_BG_IDS:
            if p not in seen:
                seen.append(p)
        else:
            invalid.append(p)
    return seen, invalid


def _bg_unit_set(w18d_bg: Dict[str, Any]) -> Set[str]:
    return {
        r.get("unit_id") or ""
        for r in (w18d_bg.get("target_unit_refs") or [])
        if isinstance(r, dict) and r.get("unit_id")
    }


def _bg_base_marker_set(w18d_bg: Dict[str, Any]) -> Set[int]:
    return {
        int(r.get("marker_number"))
        for r in (w18d_bg.get("base_marker_refs") or [])
        if isinstance(r, dict)
        and isinstance(r.get("marker_number"), int)
    }


def _load_w18d_region_ref(prev_run_dir: Path) -> Tuple[Dict[str, Any], List[str]]:
    missing: List[str] = []
    payload_path = prev_run_dir / "bg_to_fp_region_ref.json"
    if not payload_path.exists():
        missing.append("bg_to_fp_region_ref.json")
        return {}, missing
    doc = json.loads(payload_path.read_text())
    return (doc.get("bg_to_fp_region_ref_by_bg") or {}), missing


def _merge_w18d_into_target(
    *, w18f_bg: Dict[str, Any], w18d_bg: Dict[str, Any],
) -> Dict[str, Any]:
    """Combine the W18F assembled prompt entry with the W18D region/ref
    arrays so the overlap planner can see target_unit_refs +
    base_marker_refs."""
    merged: Dict[str, Any] = dict(w18f_bg or {})
    if w18d_bg:
        merged["target_unit_refs"] = list(
            w18d_bg.get("target_unit_refs") or []
        )
        merged["base_marker_refs"] = list(
            w18d_bg.get("base_marker_refs") or []
        )
        if not merged.get("transient_overlay_marker_numbers"):
            merged["transient_overlay_marker_numbers"] = [
                int(t.get("marker_number"))
                for t in (w18d_bg.get("transient_overlay_to_describe") or [])
                if isinstance(t, dict)
                and isinstance(t.get("marker_number"), int)
            ]
    return merged


def _score_pair(
    *, target_units: Set[str], target_markers: Set[int],
    cand_units: Set[str], cand_markers: Set[int],
) -> Tuple[float, Set[str], Set[int]]:
    shared_units = target_units & cand_units
    shared_markers = target_markers & cand_markers
    score = float(len(shared_units)) + 0.5 * float(len(shared_markers))
    return score, shared_units, shared_markers


def _is_strong_overlap(
    *, shared_units: Set[str], shared_markers: Set[int],
) -> bool:
    """A candidate ref counts as 'strong' iff at least one spatial unit
    is shared AND (≥2 shared units OR ≥1 shared base marker number).
    Pure unit-only overlap with no shared base marker is too weak — the
    target should be seeded from the FP layout instead."""
    if not shared_units:
        return False
    if len(shared_units) >= 2:
        return True
    return len(shared_markers) >= 1


def _compute_reference_decision(
    *, bg_id: str, target_w18d_bg: Dict[str, Any],
    catalog: List[Dict[str, Any]], base_fp_png_path: Path,
) -> Dict[str, Any]:
    """Deterministic planner output for a single target BG.

    The catalog entries each carry:
      bg_id, png_path, unit_set, base_marker_set, transient_marker_set.
    """
    target_units = _bg_unit_set(target_w18d_bg)
    target_markers = _bg_base_marker_set(target_w18d_bg)

    scored: List[Dict[str, Any]] = []
    for cand in catalog:
        score, shared_units, shared_markers = _score_pair(
            target_units=target_units, target_markers=target_markers,
            cand_units=set(cand["unit_set"]),
            cand_markers=set(cand["base_marker_set"]),
        )
        scored.append({
            "ref_bg_id": cand["bg_id"],
            "ref_png_path": cand["png_path"],
            "score": score,
            "shared_units": sorted(shared_units),
            "shared_base_markers": sorted(shared_markers),
            "is_strong_overlap": _is_strong_overlap(
                shared_units=shared_units, shared_markers=shared_markers,
            ),
        })

    strong = [s for s in scored if s["is_strong_overlap"]]
    if strong:
        # sort: score desc, then ref_bg_id for stability.
        strong.sort(key=lambda x: (-x["score"], x["ref_bg_id"]))
        selected = strong[: W18J_MAX_REFS_PER_BG]
        mode = W18J_MODE_REFERENCE_DERIVED
        reference_paths = [s["ref_png_path"] for s in selected]
        fp_included = False
        reason = (
            f"reference_derived: {len(strong)} catalog entr"
            f"{'y' if len(strong)==1 else 'ies'} reached strong overlap "
            f"(>=1 shared unit AND (>=2 shared units OR >=1 shared "
            f"base marker)); selected top-"
            f"{len(selected)} by score; FP excluded so reference "
            f"identity dominates."
        )
    else:
        mode = W18J_MODE_FP_SEEDED_ANCHOR
        selected = []
        reference_paths = [str(base_fp_png_path)]
        fp_included = True
        reason = (
            "fp_seeded_anchor: no catalog entry reached strong overlap "
            "(needs >=1 shared unit AND (>=2 shared units OR >=1 "
            "shared base marker)); base FP used as layout-only "
            "reference, BG must establish its own home identity under "
            "the modest style contract."
        )

    return {
        "bg_id": bg_id,
        "target_units": sorted(target_units),
        "target_base_markers": sorted(target_markers),
        "candidate_scores": scored,
        "selected_refs": selected,
        "mode": mode,
        "reference_paths": reference_paths,
        "fp_included": fp_included,
        "reference_decision_reason": reason,
    }


def _build_prompt(*, mode: str, assembled_preview: str) -> str:
    if mode not in W18J_ALLOWED_MODES:
        raise ValueError(f"unsupported mode: {mode}")
    return (
        W18J_MODEST_STYLE_PREAMBLE
        + "\n\nReference strategy: "
        + W18J_GUIDANCE_BY_MODE[mode]
        + "\n\n"
        + assembled_preview
    )


def _edit_png_via_openai_with_references(
    *, bg_id: str, prompt: str, model: str, size: str,
    reference_paths: List[Path], target_path: Path, client: Any,
) -> Dict[str, Any]:
    """images.edit call with either a single FP reference or a list of
    visual references. Returns JSON-safe fragment. Never raises."""
    t0 = time.monotonic()
    open_files: List[Any] = []
    try:
        for p in reference_paths:
            open_files.append(open(p, "rb"))
        image_arg: Any
        if len(open_files) == 1:
            image_arg = open_files[0]
        else:
            image_arg = open_files
        try:
            resp = client.images.edit(
                model=model, image=image_arg, prompt=prompt,
                size=size, n=1,
            )
        except Exception as exc:  # noqa: BLE001
            latency_ms = int((time.monotonic() - t0) * 1000)
            return {
                "status": "api_call_failed",
                "png_size_bytes": 0,
                "actual_api_response_meta": {"latency_ms": latency_ms},
                "error_meta": {
                    "status_code": getattr(exc, "status_code", None),
                    "message": str(exc)[:240],
                },
                "cost_meta": {},
            }
    finally:
        for f in open_files:
            try:
                f.close()
            except Exception:  # noqa: BLE001
                pass

    latency_ms = int((time.monotonic() - t0) * 1000)
    data = getattr(resp, "data", None) or []
    if not data:
        return {
            "status": "api_call_failed",
            "png_size_bytes": 0,
            "actual_api_response_meta": {"latency_ms": latency_ms},
            "error_meta": {"status_code": None,
                           "message": "empty response.data"},
            "cost_meta": {},
        }
    first = data[0]
    b64 = getattr(first, "b64_json", None)
    if not b64:
        return {
            "status": "api_call_failed",
            "png_size_bytes": 0,
            "actual_api_response_meta": {"latency_ms": latency_ms},
            "error_meta": {"status_code": None,
                           "message": "response missing b64_json payload"},
            "cost_meta": {},
        }
    target_path.parent.mkdir(parents=True, exist_ok=True)
    target_path.write_bytes(base64.b64decode(b64))

    api_meta: Dict[str, Any] = {"latency_ms": latency_ms}
    for fname in ("created", "request_id", "id", "model"):
        val = getattr(resp, fname, None)
        if val is not None:
            api_meta[fname] = _to_jsonable(val)
    usage = getattr(resp, "usage", None)
    cost_meta: Dict[str, Any] = {}
    if usage is not None:
        cost_meta["provider_usage"] = _to_jsonable(usage)
    return {
        "status": "success",
        "png_size_bytes": target_path.stat().st_size,
        "actual_api_response_meta": api_meta,
        "error_meta": {},
        "cost_meta": cost_meta,
    }


_openai_edit_caller: Callable = _edit_png_via_openai_with_references


def _build_w18j_compatibility_report(
    *, target_bg_ids: List[str], invalid_targets: List[str],
    decisions: Dict[str, Dict[str, Any]],
    per_bg_results: Dict[str, Dict[str, Any]],
    mode_summary: Dict[str, int], image_api_call_count: int,
    image_generation_count: int, mode: str,
    production_diff_empty: bool, db_write_count: int,
    image_asset_write_count: int, image_import_seen: bool,
    base_fp_png_path: Optional[Path], anchor_png_path: Optional[Path],
    missing_inputs: List[str],
) -> dict:
    inv: Dict[str, Dict[str, Any]] = {}

    inv["inputs_present"] = {
        "pass": (
            not missing_inputs
            and base_fp_png_path is not None and base_fp_png_path.exists()
            and anchor_png_path is not None and anchor_png_path.exists()
            and bool(decisions)
        ),
        "detail": {
            "missing_inputs": list(missing_inputs),
            "decisions_count": len(decisions),
            "base_fp_png_exists": (
                bool(base_fp_png_path and base_fp_png_path.exists())
            ),
            "anchor_png_exists": (
                bool(anchor_png_path and anchor_png_path.exists())
            ),
        },
    }

    inv["targets_within_allowed_and_exclude_anchor"] = {
        "pass": (
            bool(target_bg_ids)
            and not invalid_targets
            and set(target_bg_ids).issubset(W18J_ALLOWED_BG_IDS)
            and W18J_ANCHOR_BG_ID not in target_bg_ids
        ),
        "detail": {
            "target_bg_ids": list(target_bg_ids),
            "invalid_targets": list(invalid_targets),
            "anchor_excluded": W18J_ANCHOR_BG_ID not in target_bg_ids,
            "allowed": sorted(W18J_ALLOWED_BG_IDS - {W18J_ANCHOR_BG_ID}),
        },
    }

    decision_failures: List[dict] = []
    for bg_id, dec in decisions.items():
        if dec.get("mode") not in W18J_ALLOWED_MODES:
            decision_failures.append({"bg_id": bg_id,
                                      "reason": "invalid_mode",
                                      "mode": dec.get("mode")})
            continue
        if dec["mode"] == W18J_MODE_REFERENCE_DERIVED:
            if dec.get("fp_included"):
                decision_failures.append({
                    "bg_id": bg_id,
                    "reason": "reference_derived_must_not_include_fp",
                })
            n_refs = len(dec.get("selected_refs") or [])
            if n_refs < 1 or n_refs > W18J_MAX_REFS_PER_BG:
                decision_failures.append({
                    "bg_id": bg_id,
                    "reason": "reference_derived_ref_count_out_of_range",
                    "n_refs": n_refs,
                })
        elif dec["mode"] == W18J_MODE_FP_SEEDED_ANCHOR:
            if not dec.get("fp_included"):
                decision_failures.append({
                    "bg_id": bg_id,
                    "reason": "fp_seeded_anchor_must_include_fp",
                })
            if dec.get("selected_refs"):
                decision_failures.append({
                    "bg_id": bg_id,
                    "reason": "fp_seeded_anchor_must_not_select_visual_refs",
                })
    inv["reference_decision_shape_valid"] = {
        "pass": not decision_failures and bool(decisions),
        "detail": {
            "failures": decision_failures[:30],
            "failure_count": len(decision_failures),
            "mode_summary": mode_summary,
        },
    }

    if mode == "dry_run":
        png_all_absent = all(
            not (row or {}).get("png_exists_on_disk")
            for row in per_bg_results.values()
        )
        gen_ok = (
            image_api_call_count == 0
            and image_generation_count == 0
            and png_all_absent
        )
    else:
        n_targets = len(per_bg_results)
        png_all_present = all(
            (row or {}).get("png_exists_on_disk")
            for row in per_bg_results.values()
        )
        gen_ok = (
            image_api_call_count == n_targets
            and image_generation_count == n_targets
            and png_all_present
            and n_targets > 0
        )
    inv["image_generation_count_matches_targets_and_mode"] = {
        "pass": gen_ok,
        "detail": {
            "mode": mode,
            "image_api_call_count": image_api_call_count,
            "image_generation_count": image_generation_count,
            "per_bg_png_existence": {
                bg_id: bool((row or {}).get("png_exists_on_disk"))
                for bg_id, row in per_bg_results.items()
            },
        },
    }

    inv["production_diff_zero_db_write_zero_no_imageasset_write"] = {
        "pass": (
            production_diff_empty and db_write_count == 0
            and image_asset_write_count == 0 and not image_import_seen
        ),
        "detail": {
            "production_diff_empty": production_diff_empty,
            "db_write_count": db_write_count,
            "image_asset_write_count": image_asset_write_count,
            "image_import_seen": image_import_seen,
        },
    }

    return {"invariants": inv, "all_pass": all(v["pass"] for v in inv.values())}


def _load_w18f_assembly(prev_run_dir: Path) -> Tuple[Dict[str, Any], List[str]]:
    missing: List[str] = []
    payload_path = (
        prev_run_dir / "background_prompt_region_ref_assembly_preview.json"
    )
    meta_path = prev_run_dir / "run_meta.json"
    if not payload_path.exists():
        missing.append("background_prompt_region_ref_assembly_preview.json")
        return {}, missing
    if not meta_path.exists():
        missing.append("run_meta.json")
    payload = json.loads(payload_path.read_text())
    return payload, missing


def _resolve_w18d_bg_payload(
    *, w18f_payload: Dict[str, Any], bg_id: str,
) -> Dict[str, Any]:
    """W18F carries the W18D-derived per-bg ref data within
    ``assembly_preview_by_bg`` — target_unit_refs and base_marker_refs
    are propagated verbatim from W18E/W18D."""
    by_bg = (w18f_payload or {}).get("assembly_preview_by_bg") or {}
    return by_bg.get(bg_id) or {}


def _resolve_base_fp_png_path(
    *, w18f_payload: Dict[str, Any], target_bg_ids: List[str],
) -> Optional[Path]:
    for bg_id in target_bg_ids:
        pl = _resolve_w18d_bg_payload(
            w18f_payload=w18f_payload, bg_id=bg_id,
        )
        ref = pl.get("base_fp_ref_path") or ""
        if ref:
            p = Path(ref)
            if not p.is_absolute():
                p = _REPO_ROOT / ref
            if p.exists():
                return p
    return None


def _resolve_anchor_artifact(
    w18i_run_dir: Path,
) -> Tuple[Optional[Path], Dict[str, Any]]:
    """Locate the W18I anchor PNG + return its consistency_reference_plan
    entry for L05B02."""
    plan_path = w18i_run_dir / "consistency_reference_plan.json"
    if not plan_path.exists():
        return None, {}
    plan = json.loads(plan_path.read_text())
    per_bg = (plan or {}).get("per_bg_results") or {}
    anchor_row = per_bg.get(W18J_ANCHOR_BG_ID) or {}
    rel = anchor_row.get("png_relative_path") or ""
    if not rel:
        return None, anchor_row
    png_path = w18i_run_dir / rel
    if png_path.exists():
        return png_path, anchor_row
    return None, anchor_row


def main(argv=None) -> int:
    args = _parse_args(argv)
    if args.generate:
        _load_backend_env()

    run_id = _run_id()
    out_root = Path(args.output_root)
    run_dir = out_root / run_id
    run_dir.mkdir(parents=True, exist_ok=True)

    w18f_run_dir = Path(args.derive_overlap_from)
    if not w18f_run_dir.is_absolute():
        w18f_run_dir = Path.cwd() / w18f_run_dir
    w18d_run_dir = Path(args.derive_region_ref_from)
    if not w18d_run_dir.is_absolute():
        w18d_run_dir = Path.cwd() / w18d_run_dir
    w18i_run_dir = Path(args.derive_anchor_from)
    if not w18i_run_dir.is_absolute():
        w18i_run_dir = Path.cwd() / w18i_run_dir

    target_bg_ids, invalid_targets = _resolve_target_bg_ids(args.target_bg_ids)
    failed_invariants: List[str] = []
    run_status = "succeeded"
    exit_code = 0
    mode = "dry_run"
    image_api_call_count = 0
    image_generation_count = 0

    run_meta: Dict[str, Any] = {
        "run_id": run_id,
        "stage": W18J_STAGE,
        "plan_version": PLAN_VERSION,
        "generated_at": datetime.now(KST).isoformat(),
        "args": vars(args),
        "derived_from_w18f": w18f_run_dir.name,
        "derived_from_w18d": w18d_run_dir.name,
        "derived_from_w18i": w18i_run_dir.name,
        "target_bg_ids": list(target_bg_ids),
        "invalid_targets": list(invalid_targets),
        "model": args.model,
        "mode": mode,
        "image_api_call_count": image_api_call_count,
        "image_generation_count": image_generation_count,
        "outputs": [],
        "run_status": run_status,
        "exit_code": exit_code,
        "failed_invariants": failed_invariants,
    }

    def _persist_and_exit(code: int) -> int:
        run_meta["exit_code"] = code
        if code != 0:
            run_meta["run_status"] = "validation_failed"
        run_meta["failed_invariants"] = failed_invariants
        (run_dir / "run_meta.json").write_text(
            json.dumps(run_meta, ensure_ascii=False, indent=2)
        )
        return code

    if invalid_targets:
        failed_invariants.append("invalid_target_bg_ids_or_anchor_included")
        return _persist_and_exit(1)
    if not target_bg_ids:
        failed_invariants.append("no_target_bg_ids")
        return _persist_and_exit(1)

    w18f_payload, missing = _load_w18f_assembly(w18f_run_dir)
    if missing:
        failed_invariants.append("w18f_inputs_missing")
        run_meta["missing_inputs"] = missing
        return _persist_and_exit(1)
    w18d_by_bg, missing_d = _load_w18d_region_ref(w18d_run_dir)
    if missing_d:
        failed_invariants.append("w18d_inputs_missing")
        run_meta["missing_inputs"] = missing_d
        return _persist_and_exit(1)

    base_fp_png_path = _resolve_base_fp_png_path(
        w18f_payload=w18f_payload, target_bg_ids=target_bg_ids,
    )
    anchor_png_path, anchor_row = _resolve_anchor_artifact(w18i_run_dir)

    # Catalog seed: L05B02 anchor (use W18D-derived target/marker sets).
    anchor_w18f_payload = _resolve_w18d_bg_payload(
        w18f_payload=w18f_payload, bg_id=W18J_ANCHOR_BG_ID,
    )
    anchor_w18d_payload = w18d_by_bg.get(W18J_ANCHOR_BG_ID) or {}
    anchor_merged = _merge_w18d_into_target(
        w18f_bg=anchor_w18f_payload, w18d_bg=anchor_w18d_payload,
    )
    catalog: List[Dict[str, Any]] = []
    if anchor_png_path is not None:
        catalog.append({
            "bg_id": W18J_ANCHOR_BG_ID,
            "png_path": str(anchor_png_path),
            "unit_set": sorted(_bg_unit_set(anchor_merged)),
            "base_marker_set": sorted(_bg_base_marker_set(anchor_merged)),
            "transient_marker_set": sorted(
                int(t.get("marker_number"))
                for t in (
                    anchor_w18d_payload.get("transient_overlay_to_describe")
                    or []
                ) if isinstance(t, dict)
                and isinstance(t.get("marker_number"), int)
            ),
        })

    if base_fp_png_path is None or not base_fp_png_path.exists():
        failed_invariants.append("base_fp_png_unresolved")
        run_meta["base_fp_png_path"] = (
            str(base_fp_png_path) if base_fp_png_path else None
        )
        return _persist_and_exit(1)
    if anchor_png_path is None or not anchor_png_path.exists():
        failed_invariants.append("anchor_png_unresolved")
        run_meta["anchor_png_path"] = (
            str(anchor_png_path) if anchor_png_path else None
        )
        return _persist_and_exit(1)

    decisions: Dict[str, Dict[str, Any]] = {}
    per_bg_results: Dict[str, Dict[str, Any]] = {}
    mode_summary = {
        W18J_MODE_REFERENCE_DERIVED: 0, W18J_MODE_FP_SEEDED_ANCHOR: 0,
    }

    client = None
    if args.generate:
        if (args.model or "").strip() != W18G_IMAGE_BACKEND:
            failed_invariants.append("model_must_be_gpt_image_2")
            return _persist_and_exit(1)
        if not os.environ.get("OPENAI_API_KEY"):
            failed_invariants.append("openai_api_key_missing")
            return _persist_and_exit(1)
        try:
            client = _create_openai_client()
        except Exception as exc:  # noqa: BLE001
            failed_invariants.append("openai_client_unavailable")
            run_meta["openai_client_error"] = str(exc)[:240]
            return _persist_and_exit(1)
        mode = "generated"

    for bg_id in target_bg_ids:
        w18f_bg = _resolve_w18d_bg_payload(
            w18f_payload=w18f_payload, bg_id=bg_id,
        )
        w18d_bg = w18d_by_bg.get(bg_id) or {}
        target_w18d_bg = _merge_w18d_into_target(
            w18f_bg=w18f_bg, w18d_bg=w18d_bg,
        )
        decision = _compute_reference_decision(
            bg_id=bg_id, target_w18d_bg=target_w18d_bg,
            catalog=catalog, base_fp_png_path=base_fp_png_path,
        )
        decisions[bg_id] = decision
        mode_summary[decision["mode"]] = mode_summary.get(
            decision["mode"], 0,
        ) + 1

        assembled_preview = (
            target_w18d_bg.get("assembled_background_prompt_preview") or ""
        )
        prompt_text = (
            _build_prompt(
                mode=decision["mode"], assembled_preview=assembled_preview,
            ) if assembled_preview else ""
        )
        png_path = run_dir / "png" / f"{bg_id}.png"

        row: Dict[str, Any] = {
            "bg_id": bg_id,
            "fp_id": target_w18d_bg.get("fp_id") or "",
            "model": args.model,
            "size": W18G_IMAGE_SIZE,
            "mode_decided": decision["mode"],
            "fp_included": decision["fp_included"],
            "reference_paths": list(decision["reference_paths"]),
            "selected_refs": list(decision["selected_refs"]),
            "candidate_scores": list(decision["candidate_scores"]),
            "shared_units_per_ref": [
                {"ref_bg_id": s["ref_bg_id"],
                 "shared_units": list(s["shared_units"])}
                for s in decision["candidate_scores"]
            ],
            "shared_markers_per_ref": [
                {"ref_bg_id": s["ref_bg_id"],
                 "shared_base_markers": list(s["shared_base_markers"])}
                for s in decision["candidate_scores"]
            ],
            "reference_decision_reason": decision["reference_decision_reason"],
            "prompt_text_len_chars": len(prompt_text),
            "style_preamble": W18J_MODEST_STYLE_PREAMBLE,
            "mode_guidance": W18J_GUIDANCE_BY_MODE.get(decision["mode"], ""),
            "transient_overlay_marker_numbers": list(
                target_w18d_bg.get("transient_overlay_marker_numbers") or []
            ),
            "clean_background_expected": bool(
                target_w18d_bg.get("clean_background_expected")
            ),
            "png_relative_path": str(png_path.relative_to(run_dir)),
            "png_exists_on_disk": False,
            "status": "dry_run",
        }

        if args.generate:
            if not prompt_text:
                row["status"] = "skipped_missing_prompt"
                failed_invariants.append(f"prompt_missing_for_{bg_id}")
                per_bg_results[bg_id] = row
                continue
            ref_paths = [Path(p) for p in decision["reference_paths"]]
            if not all(p.exists() for p in ref_paths):
                row["status"] = "skipped_missing_reference"
                row["missing_reference_paths"] = [
                    str(p) for p in ref_paths if not p.exists()
                ]
                failed_invariants.append(f"reference_missing_for_{bg_id}")
                per_bg_results[bg_id] = row
                continue
            image_api_call_count += 1
            outcome = _openai_edit_caller(
                bg_id=bg_id, prompt=prompt_text, model=args.model,
                size=W18G_IMAGE_SIZE, reference_paths=ref_paths,
                target_path=png_path, client=client,
            )
            row.update(outcome)
            if outcome.get("status") == "success":
                image_generation_count += 1
                row["png_exists_on_disk"] = png_path.exists()
                # Append to catalog so subsequent BGs can reference this one.
                catalog.append({
                    "bg_id": bg_id,
                    "png_path": str(png_path),
                    "unit_set": sorted(_bg_unit_set(target_w18d_bg)),
                    "base_marker_set": sorted(
                        _bg_base_marker_set(target_w18d_bg)
                    ),
                    "transient_marker_set": sorted(
                        row["transient_overlay_marker_numbers"]
                    ),
                })
            else:
                failed_invariants.append(f"openai_call_failed_for_{bg_id}")
                run_status = "validation_failed"
                exit_code = 1
        else:
            # dry-run: still append to catalog so subsequent BGs see this
            # one as if it were generated, for planning visibility.
            catalog.append({
                "bg_id": bg_id, "png_path": str(png_path),
                "unit_set": sorted(_bg_unit_set(target_w18d_bg)),
                "base_marker_set": sorted(
                    _bg_base_marker_set(target_w18d_bg)
                ),
                "transient_marker_set": sorted(
                    row["transient_overlay_marker_numbers"]
                ),
            })

        per_bg_results[bg_id] = row

    overlap_plan = {
        "stage": W18J_STAGE,
        "anchor_bg_id": W18J_ANCHOR_BG_ID,
        "anchor_png_path": str(anchor_png_path) if anchor_png_path else "",
        "anchor_units": (
            sorted(_bg_unit_set(anchor_merged))
            if anchor_merged else []
        ),
        "anchor_base_markers": (
            sorted(_bg_base_marker_set(anchor_merged))
            if anchor_merged else []
        ),
        "base_fp_png_path": str(base_fp_png_path),
        "style_contract": W18J_MODEST_STYLE_PREAMBLE,
        "mode_guidance": W18J_GUIDANCE_BY_MODE,
        "decisions_by_bg": decisions,
        "per_bg_results": per_bg_results,
        "mode_summary": mode_summary,
        "final_catalog": [
            {k: v for k, v in c.items() if k != "png_path"} | {
                "png_path": c["png_path"],
            }
            for c in catalog
        ],
    }
    (run_dir / "overlap_reference_plan.json").write_text(
        json.dumps(overlap_plan, ensure_ascii=False, indent=2)
    )
    run_meta["outputs"].append("overlap_reference_plan.json")

    production_diff_empty = _check_production_diff_empty()
    image_import_seen = _check_image_imports_present()
    report = _build_w18j_compatibility_report(
        target_bg_ids=target_bg_ids, invalid_targets=invalid_targets,
        decisions=decisions, per_bg_results=per_bg_results,
        mode_summary=mode_summary,
        image_api_call_count=image_api_call_count,
        image_generation_count=image_generation_count, mode=mode,
        production_diff_empty=production_diff_empty,
        db_write_count=0, image_asset_write_count=0,
        image_import_seen=image_import_seen,
        base_fp_png_path=base_fp_png_path,
        anchor_png_path=anchor_png_path, missing_inputs=missing,
    )
    (run_dir / "w18j_compatibility_report.json").write_text(
        json.dumps(report, ensure_ascii=False, indent=2)
    )
    run_meta["outputs"].append("w18j_compatibility_report.json")

    for name, v in report["invariants"].items():
        if not v["pass"] and name not in failed_invariants:
            failed_invariants.append(name)
    if failed_invariants and run_status == "succeeded":
        run_status = "validation_failed"
        exit_code = 1

    run_meta.update({
        "mode": mode,
        "image_api_call_count": image_api_call_count,
        "image_generation_count": image_generation_count,
        "run_status": run_status,
        "exit_code": exit_code,
        "failed_invariants": failed_invariants,
        "mode_summary": mode_summary,
        "per_bg_png_size_bytes": {
            bg_id: (
                (run_dir / "png" / f"{bg_id}.png").stat().st_size
                if (run_dir / "png" / f"{bg_id}.png").exists() else 0
            )
            for bg_id in target_bg_ids
        },
    })

    def esc(x):
        return (
            str(x).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
        )

    inv_rows = "".join(
        f"<tr><td>{esc(k)}</td>"
        f"<td class=\"{'pass' if v['pass'] else 'fail'}\">"
        f"{'PASS' if v['pass'] else 'FAIL'}</td>"
        f"<td><pre>{esc(json.dumps(v.get('detail'), ensure_ascii=False))[:600]}</pre></td>"
        f"</tr>"
        for k, v in (report.get("invariants") or {}).items()
    )

    bg_sections = []
    for bg_id in target_bg_ids:
        row = per_bg_results.get(bg_id) or {}
        dec = decisions.get(bg_id) or {}
        png_rel = row.get("png_relative_path") or ""
        png_full = run_dir / png_rel
        png_html = (
            f'<img src="{esc(png_rel)}" '
            f'style="max-width:480px;border:1px solid #ccc"/>'
            if png_full.exists()
            else "<p><i>(no PNG — dry-run or failed call)</i></p>"
        )
        cand_rows = "".join(
            f"<tr><td>{esc(c.get('ref_bg_id'))}</td>"
            f"<td>{esc(c.get('score'))}</td>"
            f"<td>{esc(c.get('shared_units'))}</td>"
            f"<td>{esc(c.get('shared_base_markers'))}</td>"
            f"<td>{'YES' if c.get('is_strong_overlap') else 'no'}</td></tr>"
            for c in (dec.get("candidate_scores") or [])
        ) or "<tr><td colspan=5>(no catalog entries yet)</td></tr>"
        bg_sections.append(
            f"<section><h3>{esc(bg_id)}</h3>"
            f"<p><b>mode</b>: {esc(row.get('mode_decided'))}"
            f" · <b>fp_included</b>: {esc(row.get('fp_included'))}"
            f" · <b>transient</b>: <code>{esc(row.get('transient_overlay_marker_numbers'))}</code>"
            f" · <b>clean</b>: {esc(row.get('clean_background_expected'))}"
            f" · <b>prompt_len</b>: {esc(row.get('prompt_text_len_chars'))}"
            f"</p>"
            f"<p><b>reference_decision_reason</b>: {esc(row.get('reference_decision_reason'))}</p>"
            f"<p><b>selected_refs</b>: <code>{esc([s.get('ref_bg_id') for s in (dec.get('selected_refs') or [])])}</code></p>"
            f"<table><tr><th>candidate</th><th>score</th><th>shared_units</th><th>shared_base_markers</th><th>strong</th></tr>"
            f"{cand_rows}</table>"
            f"{png_html}"
            f"</section>"
        )

    (run_dir / "index.html").write_text(
        f"""<!doctype html><html><head><meta charset=\"utf-8\">
<title>W18J overlap_reference {esc(run_id)}</title>
<style>body{{font-family:sans-serif;margin:1.5em}}
table{{border-collapse:collapse;margin:0.5em 0}}
td,th{{border:1px solid #ccc;padding:4px 8px;vertical-align:top;font-size:0.85em}}
.pass{{color:#080}} .fail{{color:#b00}}
pre{{white-space:pre-wrap;font-size:0.78em;max-width:96ch}}
section{{margin:1.5em 0}}</style></head>
<body>
<h1>W18J — overlap_reference {esc(run_id)}</h1>
<p>mode: <b>{esc(mode)}</b>
| run_status: <b>{esc(run_status)}</b>
| exit_code: {esc(exit_code)}
| model: <b>{esc(args.model)}</b>
| image_api_call_count: <b>{esc(image_api_call_count)}</b>
| image_generation_count: <b>{esc(image_generation_count)}</b>
| derived_from_w18f: <b>{esc(w18f_run_dir.name)}</b>
| derived_from_w18i: <b>{esc(w18i_run_dir.name)}</b>
| anchor: <b>{esc(W18J_ANCHOR_BG_ID)}</b>
| anchor_png: <code>{esc(anchor_png_path)}</code>
| base_fp_png: <code>{esc(base_fp_png_path)}</code></p>

<section><h2>mode summary</h2><pre>{esc(json.dumps(mode_summary, ensure_ascii=False))}</pre></section>

<section><h2>per-BG decisions</h2>
{''.join(bg_sections)}
</section>

<section><h2>invariants</h2>
<table><tr><th>invariant</th><th>status</th><th>detail</th></tr>
{inv_rows}</table></section>

<details><summary>raw run_meta.json</summary>
<pre>{esc(json.dumps(run_meta, ensure_ascii=False, indent=2))}</pre></details>
</body></html>"""
    )
    run_meta["outputs"].append("index.html")
    for bg_id in target_bg_ids:
        p = run_dir / "png" / f"{bg_id}.png"
        if p.exists():
            run_meta["outputs"].append(str(p.relative_to(run_dir)))

    (run_dir / "run_meta.json").write_text(
        json.dumps(run_meta, ensure_ascii=False, indent=2)
    )
    _maybe_print_imports(args)
    return exit_code


if __name__ == "__main__":
    sys.exit(main())