"""W18A base layout prompt + transient overlay slice tests — minimal. 3 tests per Codex spec: 1. partition coverage (included + excluded covers all candidate numbers). 2. base prompt does not reference excluded marker numbers AND bg state overlay payload carries every excluded marker. 3. production / API guard + methodology grep (no scenario-specific static tokens leaked). Token-assembly trick used in the methodology grep so this test source does not match the guard it builds. """ from __future__ import annotations import json import re import sys from pathlib import Path _REPO_ROOT = Path(__file__).resolve().parents[3] _SCRIPTS_DIR = _REPO_ROOT / "backend" / "scripts" if str(_SCRIPTS_DIR) not in sys.path: sys.path.insert(0, str(_SCRIPTS_DIR)) def _synthetic_candidate() -> dict: return { "candidate_floor_plans": { "FPx": { "fp_id": "FPx", "group_id_pointer": "Gx", "candidate_diagram_t2i_prompt": "schematic alpha.", "candidate_key_elements": [], "candidate_numbered_elements": [ {"number": 1, "label": "alpha", "category": "area", "position_hint": "p1", "unit_id_pointer": "U1"}, {"number": 2, "label": "beta", "category": "area", "position_hint": "p2", "unit_id_pointer": "U2"}, {"number": 3, "label": "gamma", "category": "area", "position_hint": "p3", "unit_id_pointer": "U3"}, {"number": 7, "label": "side opening", "category": "opening", "position_hint": "wall", "unit_id_pointer": "U3"}, {"number": 11, "label": "interior fixture", "category": "furniture", "position_hint": "perimeter", "unit_id_pointer": "U3"}, {"number": 14, "label": "displaced object", "category": "furniture", "position_hint": "floor", "unit_id_pointer": "U1"}, {"number": 18, "label": "surface state cue", "category": "plot_device", "position_hint": "floor area", "unit_id_pointer": "U3"}, ], "candidate_camera_recommendations": [], "reconciliation_notes_vs_production": "", }, }, } def _synthetic_bg_bindings() -> dict: return { "FPx": { "BGa": { "primary_unit_ids": ["U1"], "secondary_visible_unit_ids": [], "state_delta_unit_ids": [], "evidence_refs": [], }, "BGb": { "primary_unit_ids": ["U3"], "secondary_visible_unit_ids": [], "state_delta_unit_ids": ["U3"], "evidence_refs": [], }, } } def _synthetic_per_bg() -> dict: """W18A2: synthetic W15e per_bg with explicit use/ignore lists. BGa.use carries excluded marker 14 but ignores excluded marker 18; BGb.use carries excluded marker 18 but ignores excluded marker 14. A "clean" BGc carries no excluded marker in its use list — its overlay transient list must stay empty. """ return { "FPx": { "BGa": { "bg_id": "BGa", "fp_id": "FPx", "use_numbered_elements": [1, 11, 14], "ignore_numbered_elements": [3, 7, 18], "render_prompt_appendix": "anchor on living zone", "final_prompt_assembly_preview": "living scene", }, "BGb": { "bg_id": "BGb", "fp_id": "FPx", "use_numbered_elements": [3, 7, 11, 18], "ignore_numbered_elements": [1, 2, 14], "render_prompt_appendix": "anchor on private room", "final_prompt_assembly_preview": "private scene with cue", }, } } def _synthetic_per_bg_clean_only() -> dict: """W18A2: synthetic per_bg where the bg explicitly ignores every excluded marker (e.g. clean restoration shot). The overlay for this bg must remain empty; any transient_markers_to_describe value fails the invariant.""" return { "FPx": { "BGc": { "bg_id": "BGc", "fp_id": "FPx", "use_numbered_elements": [3, 7, 11], "ignore_numbered_elements": [1, 2, 14, 18], "render_prompt_appendix": "clean restored scene", "final_prompt_assembly_preview": "no evidence remains", } } } def _synthetic_w18a_llm_output_good() -> dict: return { "base_layout_prompt_by_fp": { "FPx": { "base_fp_t2i_prompt_text": ( "Schematic orthographic floor plan diagram on white " "background, thick black walls, simple lines, no " "perspective, no photorealism. Use large high-contrast " "circled marker numbers for every numbered element. " "Markers must be visible and non-overlapping where " "possible. Spatial units drawn as pale filled blocks; " "openings as wall-break with swing arc symbols; " "fixtures as outlined generic symbols. Draw exactly " "the listed marker numbers below. #1 #2 #3 #7 #11. " "Do not renumber, do not omit, do not invent extra " "markers." ), "included_marker_legend": [ {"marker_number": 1, "source_candidate_number": 1, "base_layer_decision": "base_structural_unit", "label": "alpha", "unit_id": "U1", "visual_encoding": "filled_area", "must_be_legible": True}, {"marker_number": 2, "source_candidate_number": 2, "base_layer_decision": "base_structural_unit", "label": "beta", "unit_id": "U2", "visual_encoding": "filled_area", "must_be_legible": True}, {"marker_number": 3, "source_candidate_number": 3, "base_layer_decision": "base_structural_unit", "label": "gamma", "unit_id": "U3", "visual_encoding": "filled_area", "must_be_legible": True}, {"marker_number": 7, "source_candidate_number": 7, "base_layer_decision": "base_opening", "label": "side opening", "unit_id": "U3", "visual_encoding": "thick_wall_opening", "must_be_legible": True}, {"marker_number": 11, "source_candidate_number": 11, "base_layer_decision": "base_persistent_furniture", "label": "interior fixture", "unit_id": "U3", "visual_encoding": "outlined_furniture_symbol", "must_be_legible": True}, ], "excluded_transient_elements": [ {"marker_number": 14, "source_candidate_number": 14, "base_layer_decision": "state_overlay_transient_object", "label": "displaced object", "unit_id": "U1", "excluded_reason": "scene-driven displaced object", "overlay_instruction_hint": ( "describe the displaced object in BG body" )}, {"marker_number": 18, "source_candidate_number": 18, "base_layer_decision": "state_overlay_plot_cue", "label": "surface state cue", "unit_id": "U3", "excluded_reason": "scene-driven floor surface cue", "overlay_instruction_hint": ( "describe the surface state cue in BG body" )}, ], "bg_state_overlay_payload_by_bg": { "BGa": { "bg_id": "BGa", "fp_id": "fp_l05_01", "target_unit_ids": ["U1"], "base_markers_to_reference": [1, 11], "transient_markers_to_describe": [14], "prompt_appendix_hint": "anchor on living zone", }, "BGb": { "bg_id": "BGb", "fp_id": "fp_l05_01", "target_unit_ids": ["U3"], "base_markers_to_reference": [3, 7, 11], "transient_markers_to_describe": [18], "prompt_appendix_hint": "anchor on private room", }, }, "base_fp_contract_notes": ( "open zones connect to neighbouring spatial units " "without internal partition walls; persistent " "fixtures anchor camera framing." ), "production_prompt_delta_recommendations": ( "review-only: future production prompt could add a " "persistence_class field to numbered elements." ), } } } def _build_report(*, llm_output, candidate, target_fp_ids={"fp_l05_01"}, bg_bindings_for_fp_by_target=None, per_bg_for_fp_by_target=None, llm_api_call_count=1, expected_llm_api_call_count=1, model_used="gpt-5.5", stage_status="generated", image_api_call_count=0, production_diff_empty=True, image_import_seen=False): from experiment_floor_plan_base_layout_prompt_slice import ( _build_w18a_compatibility_report, ) if bg_bindings_for_fp_by_target is None: bg_bindings_for_fp_by_target = {} if per_bg_for_fp_by_target is None: per_bg_for_fp_by_target = {} return _build_w18a_compatibility_report( llm_output=llm_output, candidate=candidate, target_fp_ids=target_fp_ids, bg_bindings_for_fp_by_target=bg_bindings_for_fp_by_target, per_bg_for_fp_by_target=per_bg_for_fp_by_target, production_diff_empty=production_diff_empty, db_write_count=0, image_import_seen=image_import_seen, image_api_call_count=image_api_call_count, vlm_api_call_count=0, llm_api_call_count=llm_api_call_count, expected_llm_api_call_count=expected_llm_api_call_count, model_used=model_used, stage_status=stage_status, missing_inputs=[], prev_run_id="fakeW15e", ) # ───────────────────────────────────────────────────────────────────────────── def test_w18a_partition_coverage_included_plus_excluded_equals_candidate(): """Every candidate numbered element appears EXACTLY ONCE in either `included_marker_legend` or `excluded_transient_elements`. Duplicate, missing, or extra → fail. Decisions must be from the proper enum.""" cand = _synthetic_candidate() out = _synthetic_w18a_llm_output_good() # Remap top-level fp key so synthetic candidate and llm_output align # under the wave-1 target id. out["base_layout_prompt_by_fp"] = { "fp_l05_01": out["base_layout_prompt_by_fp"]["FPx"] } cand["candidate_floor_plans"] = { "fp_l05_01": cand["candidate_floor_plans"]["FPx"] } bg = {"fp_l05_01": _synthetic_bg_bindings()["FPx"]} pbg = {"fp_l05_01": _synthetic_per_bg()["FPx"]} rep = _build_report( llm_output=out, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep["invariants"][ "included_plus_excluded_partitions_all_candidate_numbers" ]["pass"] is True # (a) drop excluded #18 → missing partition coverage fails. drop_excl = json.loads(json.dumps(out)) drop_excl["base_layout_prompt_by_fp"]["fp_l05_01"][ "excluded_transient_elements" ] = [ e for e in drop_excl["base_layout_prompt_by_fp"]["fp_l05_01"][ "excluded_transient_elements" ] if e["marker_number"] != 18 ] rep_drop = _build_report( llm_output=drop_excl, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep_drop["invariants"][ "included_plus_excluded_partitions_all_candidate_numbers" ]["pass"] is False # (b) put #14 in both lists → overlap fails. overlap = json.loads(json.dumps(out)) overlap["base_layout_prompt_by_fp"]["fp_l05_01"][ "included_marker_legend" ].append({ "marker_number": 14, "source_candidate_number": 14, "base_layer_decision": "base_persistent_furniture", "label": "displaced object", "unit_id": "U1", "visual_encoding": "outlined_furniture_symbol", "must_be_legible": True, }) rep_overlap = _build_report( llm_output=overlap, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep_overlap["invariants"][ "included_plus_excluded_partitions_all_candidate_numbers" ]["pass"] is False # (c) wrong enum value (included with state_overlay_*) → fails. bad_enum = json.loads(json.dumps(out)) bad_enum["base_layout_prompt_by_fp"]["fp_l05_01"][ "included_marker_legend" ][0]["base_layer_decision"] = "state_overlay_plot_cue" rep_enum = _build_report( llm_output=bad_enum, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep_enum["invariants"][ "included_plus_excluded_partitions_all_candidate_numbers" ]["pass"] is False def test_w18a_base_prompt_no_overlay_markers_and_overlay_covers_excluded(): """W18A2 strengthened overlay rules: - Excluded marker numbers must not appear as `#N` substring in the base prompt. - Every target bg must be present in `bg_state_overlay_payload_by_bg`. - For every overlay entry, every value in `transient_markers_to_describe` must be (a) in the global excluded set, (b) in that bg's W15e `use_numbered_elements`, and (c) NOT in that bg's W15e `ignore_numbered_elements`. - Coverage rule (relaxed): only excluded markers that appear in at least one bg's `use_numbered_elements` must be covered. - Clean bgs whose `use_numbered_elements` carry no excluded marker legitimately yield an empty `transient_markers_to_describe`.""" cand = _synthetic_candidate() out = _synthetic_w18a_llm_output_good() out["base_layout_prompt_by_fp"] = { "fp_l05_01": out["base_layout_prompt_by_fp"]["FPx"] } cand["candidate_floor_plans"] = { "fp_l05_01": cand["candidate_floor_plans"]["FPx"] } bg = {"fp_l05_01": _synthetic_bg_bindings()["FPx"]} pbg = {"fp_l05_01": _synthetic_per_bg()["FPx"]} rep = _build_report( llm_output=out, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep["invariants"][ "base_prompt_excludes_overlay_marker_numbers" ]["pass"] is True assert rep["invariants"][ "bg_state_overlay_covers_target_bgs_and_excluded_markers" ]["pass"] is True # (a) leak #18 into base prompt → invariant 3 fail. leak = json.loads(json.dumps(out)) leak["base_layout_prompt_by_fp"]["fp_l05_01"][ "base_fp_t2i_prompt_text" ] = ( leak["base_layout_prompt_by_fp"]["fp_l05_01"][ "base_fp_t2i_prompt_text" ] + " #18" ) rep_leak = _build_report( llm_output=leak, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep_leak["invariants"][ "base_prompt_excludes_overlay_marker_numbers" ]["pass"] is False # (b) drop transient #14 from BGa overlay → uncovered (14 is in # BGa.use_numbered_elements) → invariant 4 fail. uncovered = json.loads(json.dumps(out)) uncovered["base_layout_prompt_by_fp"]["fp_l05_01"][ "bg_state_overlay_payload_by_bg" ]["BGa"]["transient_markers_to_describe"] = [] rep_uncov = _build_report( llm_output=uncovered, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep_uncov["invariants"][ "bg_state_overlay_covers_target_bgs_and_excluded_markers" ]["pass"] is False # (c) drop BGb entry entirely → missing bg → invariant 4 fail. miss_bg = json.loads(json.dumps(out)) del miss_bg["base_layout_prompt_by_fp"]["fp_l05_01"][ "bg_state_overlay_payload_by_bg" ]["BGb"] rep_miss = _build_report( llm_output=miss_bg, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep_miss["invariants"][ "bg_state_overlay_covers_target_bgs_and_excluded_markers" ]["pass"] is False # (d) W18A2 ignore-list violation: BGa overlay adds #18, which is in # BGa.ignore_numbered_elements → invariant 4 fail. ignore_violation = json.loads(json.dumps(out)) ignore_violation["base_layout_prompt_by_fp"]["fp_l05_01"][ "bg_state_overlay_payload_by_bg" ]["BGa"]["transient_markers_to_describe"] = [14, 18] rep_ignore = _build_report( llm_output=ignore_violation, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep_ignore["invariants"][ "bg_state_overlay_covers_target_bgs_and_excluded_markers" ]["pass"] is False # (e) W18A2 clean-bg case: a single bg that explicitly ignores every # excluded marker. Overlay correctly leaves transient empty. clean_bg = {"BGc": { "primary_unit_ids": ["U3"], "secondary_visible_unit_ids": [], "state_delta_unit_ids": [], "evidence_refs": [], }} clean_bg_bindings = {"fp_l05_01": clean_bg} clean_per_bg = {"fp_l05_01": _synthetic_per_bg_clean_only()["FPx"]} clean_out = json.loads(json.dumps(out)) clean_out["base_layout_prompt_by_fp"]["fp_l05_01"][ "bg_state_overlay_payload_by_bg" ] = { "BGc": { "bg_id": "BGc", "fp_id": "fp_l05_01", "target_unit_ids": ["U3"], "base_markers_to_reference": [3, 7, 11], "transient_markers_to_describe": [], "prompt_appendix_hint": "clean restored scene", } } rep_clean_ok = _build_report( llm_output=clean_out, candidate=cand, bg_bindings_for_fp_by_target=clean_bg_bindings, per_bg_for_fp_by_target=clean_per_bg, ) assert rep_clean_ok["invariants"][ "bg_state_overlay_covers_target_bgs_and_excluded_markers" ]["pass"] is True # (f) Clean bg overlay incorrectly carries an excluded marker that is # in the bg's ignore list → invariant 4 fail. clean_out_bad = json.loads(json.dumps(clean_out)) clean_out_bad["base_layout_prompt_by_fp"]["fp_l05_01"][ "bg_state_overlay_payload_by_bg" ]["BGc"]["transient_markers_to_describe"] = [14] rep_clean_bad = _build_report( llm_output=clean_out_bad, candidate=cand, bg_bindings_for_fp_by_target=clean_bg_bindings, per_bg_for_fp_by_target=clean_per_bg, ) assert rep_clean_bad["invariants"][ "bg_state_overlay_covers_target_bgs_and_excluded_markers" ]["pass"] is False def test_w18a_production_image_vlm_guard_and_methodology_grep(): """Production diff + image/vlm api call + llm count guard surfaces every condition; source script must not embed scenario-specific tokens. Token list assembled per-character so this test does not self-match the methodology grep it builds.""" cand = _synthetic_candidate() out = _synthetic_w18a_llm_output_good() out["base_layout_prompt_by_fp"] = { "fp_l05_01": out["base_layout_prompt_by_fp"]["FPx"] } cand["candidate_floor_plans"] = { "fp_l05_01": cand["candidate_floor_plans"]["FPx"] } bg = {"fp_l05_01": _synthetic_bg_bindings()["FPx"]} pbg = {"fp_l05_01": _synthetic_per_bg()["FPx"]} rep_ok = _build_report( llm_output=out, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, ) assert rep_ok["invariants"][ "production_diff_zero_db_write_zero_llm_count_as_expected_image_zero_vlm_zero" ]["pass"] is True # any image api call → fail. rep_img = _build_report( llm_output=out, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, image_api_call_count=1, ) assert rep_img["invariants"][ "production_diff_zero_db_write_zero_llm_count_as_expected_image_zero_vlm_zero" ]["pass"] is False # llm count mismatch → fail. rep_llm = _build_report( llm_output=out, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, llm_api_call_count=2, expected_llm_api_call_count=1, ) assert rep_llm["invariants"][ "production_diff_zero_db_write_zero_llm_count_as_expected_image_zero_vlm_zero" ]["pass"] is False # production diff dirty → fail. rep_prod = _build_report( llm_output=out, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, production_diff_empty=False, ) assert rep_prod["invariants"][ "production_diff_zero_db_write_zero_llm_count_as_expected_image_zero_vlm_zero" ]["pass"] is False # image imports seen → fail. rep_imp = _build_report( llm_output=out, candidate=cand, bg_bindings_for_fp_by_target=bg, per_bg_for_fp_by_target=pbg, image_import_seen=True, ) assert rep_imp["invariants"][ "production_diff_zero_db_write_zero_llm_count_as_expected_image_zero_vlm_zero" ]["pass"] is False script_path = ( _SCRIPTS_DIR / "experiment_floor_plan_base_layout_prompt_slice.py" ) assert script_path.exists(), f"script missing: {script_path}" forbidden_tokens = [ "b" + "edroom", "ki" + "tchen", "blood" + "stain", "cur" + "tain", "coo" + "ktop", "tele" + "vision", "cri" + "me", "vi" + "lla", "roo" + "ftop", "foot" + "print", "pol" + "ice", "de" + "ck", "wheel" + "house", "ba" + "throom", "su" + "ri-young", ] pat = re.compile(r"(?i)\b(" + "|".join(forbidden_tokens) + r")\b") m = pat.search(script_path.read_text()) assert m is None, ( f"{script_path.name}: scenario-specific token leaked → " f"{m.group(0) if m else ''}" )