"""W18E background prompt region/ref preview slice — minimal tests. 3 tests: 1. happy path: synthetic W18D-shaped input → preview covers bg set, carries transient none/markers correctly, all 5 invariants PASS. 2. broken: drop a bg from preview (cover invariant fails); inject a marker number into appendix that does not exist in the payload (appendix_references_resolve invariant fails); invalid decision enum also fails resolve invariant. 3. methodology grep + production/api zero guard variations. """ from __future__ import annotations import json import re import sys from pathlib import Path _REPO_ROOT = Path(__file__).resolve().parents[3] _SCRIPTS_DIR = _REPO_ROOT / "backend" / "scripts" if str(_SCRIPTS_DIR) not in sys.path: sys.path.insert(0, str(_SCRIPTS_DIR)) def _synthetic_w18d_region_ref_by_bg() -> dict: base_path = "scripts_output/fake/png/fp_l05_01.png" return { "BGa": { "bg_id": "BGa", "fp_id": "fp_l05_01", "base_fp_png_path_relative_to_repo": base_path, "target_unit_refs": [ {"unit_id": "U1", "marker_number": 1, "label": "u1", "approximate_10x10_rect": [0, 0, 4, 4], "observed_area_label": "u1", "adjacency_notes": "left"}, ], "base_marker_refs": [ {"marker_number": 1, "label": "u1", "base_layer_decision": "base_structural_unit", "visual_encoding": "filled", "visible": True, "confidence": "high", "approximate_10x10_rect": None, "approximate_10x10_cell": [2, 2], "observed_label_or_area": "u1"}, ], "transient_overlay_to_describe": [ {"marker_number": 14, "label": "Tx", "unit_id": "U1", "base_layer_decision": "state_overlay_transient_object", "overlay_instruction_hint": "describe in bg body"}, ], "region_prompt_hint": "Reference base FP units #1; ...", }, "BGb": { "bg_id": "BGb", "fp_id": "fp_l05_01", "base_fp_png_path_relative_to_repo": base_path, "target_unit_refs": [ {"unit_id": "U2", "marker_number": 2, "label": "u2", "approximate_10x10_rect": [5, 0, 9, 4], "observed_area_label": "u2", "adjacency_notes": "right"}, ], "base_marker_refs": [ {"marker_number": 2, "label": "u2", "base_layer_decision": "base_structural_unit", "visual_encoding": "filled", "visible": True, "confidence": "high", "approximate_10x10_rect": None, "approximate_10x10_cell": [7, 2], "observed_label_or_area": "u2"}, ], "transient_overlay_to_describe": [], "region_prompt_hint": "Reference base FP units #2; ...", }, } def _build_preview_via_helper(w18d_by_bg): from experiment_background_prompt_region_ref_preview_slice import ( _build_preview_for_fp, ) return _build_preview_for_fp( fp_id="fp_l05_01", w18d_region_ref_by_bg=w18d_by_bg, ) def _build_report(*, preview_by_bg, w18d_region_ref_by_bg, target_fp_ids={"fp_l05_01"}, production_diff_empty=True, image_import_seen=False, image_api_call_count=0, vlm_api_call_count=0, llm_api_call_count=0): from experiment_background_prompt_region_ref_preview_slice import ( _build_w18e_compatibility_report, ) return _build_w18e_compatibility_report( preview_by_bg=preview_by_bg, w18d_region_ref_by_bg=w18d_region_ref_by_bg, target_fp_ids=target_fp_ids, production_diff_empty=production_diff_empty, db_write_count=0, image_import_seen=image_import_seen, image_api_call_count=image_api_call_count, vlm_api_call_count=vlm_api_call_count, llm_api_call_count=llm_api_call_count, missing_inputs=[], prev_run_id="fakeW18D", ) def test_w18e_happy_path_preview_shape_and_invariants(): """Synthetic W18D input → W18E preview covers bg set; transient none/markers are carried; appendix references resolve; consumer_decision_preview is the READY enum; all 5 invariants pass.""" w18d = _synthetic_w18d_region_ref_by_bg() preview, diags = _build_preview_via_helper(w18d) assert diags == [] assert set(preview.keys()) == {"BGa", "BGb"} bga = preview["BGa"] assert bga["base_fp_ref_path"].endswith("/png/fp_l05_01.png") assert bga["consumer_decision_preview"] == "READY_FOR_BG_PROMPT_PREVIEW" # BGa carries 1 transient marker (#14) — appendix must mention it. appendix_a = bga["background_prompt_region_appendix"] assert "#1" in appendix_a assert "#14" in appendix_a assert "No transient overlay markers" not in appendix_a # Base FP image is declared as layout anchor only. assert "layout" in appendix_a.lower() assert "do not redraw" in appendix_a.lower() bgb = preview["BGb"] appendix_b = bgb["background_prompt_region_appendix"] # BGb has no transient — sentinel must appear. assert "No transient overlay markers" in appendix_b assert "#2" in appendix_b rep = _build_report( preview_by_bg=preview, w18d_region_ref_by_bg=w18d, ) assert rep["all_pass"] is True for k in ( "inputs_present", "target_fp_only_fp_l05_01", "preview_covers_all_w18d_bgs", "appendix_references_resolve", "production_diff_zero_db_write_zero_llm_vlm_image_zero", ): assert rep["invariants"][k]["pass"] is True def test_w18e_broken_coverage_or_marker_ref_fails_invariants(): """Drop a bg → cover invariant fails. Inject an unknown marker into the appendix → appendix_references_resolve fails. Bad decision enum → appendix_references_resolve fails.""" w18d = _synthetic_w18d_region_ref_by_bg() preview, _ = _build_preview_via_helper(w18d) # (a) drop BGb from W18E output → cover invariant fails. missing_bg = json.loads(json.dumps(preview)) del missing_bg["BGb"] rep_miss = _build_report( preview_by_bg=missing_bg, w18d_region_ref_by_bg=w18d, ) assert rep_miss["invariants"][ "preview_covers_all_w18d_bgs" ]["pass"] is False # (b) inject a marker number not present in the bg payload's allowed set. tampered = json.loads(json.dumps(preview)) tampered["BGa"]["background_prompt_region_appendix"] += ( "\n - injected ghost marker #99 not in payload" ) rep_ghost = _build_report( preview_by_bg=tampered, w18d_region_ref_by_bg=w18d, ) assert rep_ghost["invariants"][ "appendix_references_resolve" ]["pass"] is False failures = ( rep_ghost["invariants"]["appendix_references_resolve"]["detail"][ "failures" ] ) assert any( 99 in (f.get("unresolved_marker_numbers_in_appendix") or []) for f in failures ) # (c) invalid consumer_decision_preview enum. bad_enum = json.loads(json.dumps(preview)) bad_enum["BGa"]["consumer_decision_preview"] = "FREEFORM_NOT_ENUM" rep_enum = _build_report( preview_by_bg=bad_enum, w18d_region_ref_by_bg=w18d, ) assert rep_enum["invariants"][ "appendix_references_resolve" ]["pass"] is False def test_w18e_production_api_zero_guard_and_methodology_grep(): """All API call counts must be zero; production diff empty; image imports not seen. Plus script source must not embed scenario-specific tokens.""" w18d = _synthetic_w18d_region_ref_by_bg() preview, _ = _build_preview_via_helper(w18d) rep_ok = _build_report( preview_by_bg=preview, w18d_region_ref_by_bg=w18d, ) assert rep_ok["invariants"][ "production_diff_zero_db_write_zero_llm_vlm_image_zero" ]["pass"] is True for kw in ( {"llm_api_call_count": 1}, {"vlm_api_call_count": 1}, {"image_api_call_count": 1}, {"image_import_seen": True}, {"production_diff_empty": False}, ): rep_bad = _build_report( preview_by_bg=preview, w18d_region_ref_by_bg=w18d, **kw, ) assert rep_bad["invariants"][ "production_diff_zero_db_write_zero_llm_vlm_image_zero" ]["pass"] is False, kw script_path = ( _SCRIPTS_DIR / "experiment_background_prompt_region_ref_preview_slice.py" ) assert script_path.exists(), f"script missing: {script_path}" forbidden_tokens = [ "b" + "edroom", "ki" + "tchen", "blood" + "stain", "cur" + "tain", "coo" + "ktop", "tele" + "vision", "cri" + "me", "vi" + "lla", "roo" + "ftop", "foot" + "print", "pol" + "ice", "de" + "ck", "wheel" + "house", "ba" + "throom", "su" + "ri-young", ] pat = re.compile(r"(?i)\b(" + "|".join(forbidden_tokens) + r")\b") m = pat.search(script_path.read_text()) assert m is None, ( f"{script_path.name}: scenario-specific token leaked → " f"{m.group(0) if m else ''}" )