"""W17A floor-plan image prompt + legend slice tests — light coverage. No scenario-specific static tokens (rooms/props/colors/narrative literals) in test methodology. Forbidden-token strings are assembled per-character in the methodology grep so the test source itself does not match the guard it builds. """ from __future__ import annotations import json import re import sys from pathlib import Path import pytest _REPO_ROOT = Path(__file__).resolve().parents[3] _SCRIPTS_DIR = _REPO_ROOT / "backend" / "scripts" if str(_SCRIPTS_DIR) not in sys.path: sys.path.insert(0, str(_SCRIPTS_DIR)) # Synthetic generic fixture — opaque ids, no scenario tokens. Mirrors the # minimal shape the W17A builder consumes (topology + candidate numbered # elements). Per-char token assembly avoids self-matching the methodology # grep when this file is loaded by the test runner. def _synthetic_topology_brief() -> dict: return { "target_fp_ids": ["FPx"], "source_topology_by_fp": { "FPx": { "fp_id": "FPx", "spatial_units": [ {"unit_id": "U1", "unit_label": "zone alpha", "unit_kind": "living_zone", "is_enclosed_room": False, "evidence_refs": []}, {"unit_id": "U2", "unit_label": "zone beta", "unit_kind": "kitchen_zone", "is_enclosed_room": False, "evidence_refs": []}, {"unit_id": "U3", "unit_label": "room gamma", "unit_kind": "private_room", "is_enclosed_room": True, "evidence_refs": []}, ], "relationships": [ {"from_unit": "U1", "to_unit": "U2", "relation_kind": "open_connection"}, {"from_unit": "U1", "to_unit": "U3", "relation_kind": "door_between"}, ], "room_count_assessment": { "standalone_enclosed_room_count": 1, "rationale": "", }, "do_not_collapse_units": [], "conflicts_and_assumptions": [], }, }, } def _synthetic_candidate() -> dict: return { "candidate_floor_plans": { "FPx": { "fp_id": "FPx", "group_id_pointer": "Gx", "candidate_diagram_t2i_prompt": "schematic alpha.", "candidate_key_elements": [], "candidate_numbered_elements": [ {"number": 1, "label": "alpha", "category": "area", "position_hint": "p1", "unit_id_pointer": "U1"}, {"number": 2, "label": "beta", "category": "area", "position_hint": "p2", "unit_id_pointer": "U2"}, {"number": 3, "label": "gamma", "category": "area", "position_hint": "p3", "unit_id_pointer": "U3"}, {"number": 7, "label": "side opening", "category": "opening", "position_hint": "wall", "unit_id_pointer": "U3"}, {"number": 11, "label": "interior fixture", "category": "furniture", "position_hint": "perimeter", "unit_id_pointer": "U3"}, ], "candidate_camera_recommendations": [], "reconciliation_notes_vs_production": "", }, }, } def _synthetic_w17a_llm_output() -> dict: """Mirrors the shape the LLM is asked to emit. Marker numbers exactly match the candidate numbered elements (5 markers).""" return { "floor_plan_image_prompt_by_fp": { "FPx": { "t2i_prompt_text": ( "Schematic orthographic floor plan diagram on white " "background, thick black walls, simple lines, no " "perspective, no photorealism. Use large high-contrast " "circled marker numbers for every numbered element. " "Markers must be visible and non-overlapping where " "possible. Spatial units are drawn as filled blocks; " "other element kinds use distinguishable generic shape " "or color styles per legend. Numbered marker contract: " "draw circular markers with exactly the listed marker " "numbers below. #1 #2 #3 #7 #11. Do not renumber, " "do not omit, do not invent additional markers." ), "numbered_marker_legend": [ {"marker_number": 1, "source_candidate_number": 1, "element_kind": "spatial_unit", "visual_encoding": "filled_area", "label": "alpha", "priority": "must_show", "source_refs": [], "must_be_legible": True}, {"marker_number": 2, "source_candidate_number": 2, "element_kind": "spatial_unit", "visual_encoding": "filled_area", "label": "beta", "priority": "must_show", "source_refs": [], "must_be_legible": True}, {"marker_number": 3, "source_candidate_number": 3, "element_kind": "spatial_unit", "visual_encoding": "filled_area", "label": "gamma", "priority": "must_show", "source_refs": [], "must_be_legible": True}, {"marker_number": 7, "source_candidate_number": 7, "element_kind": "opening", "visual_encoding": "thick_wall_opening", "label": "side opening", "priority": "must_show", "source_refs": [], "must_be_legible": True}, {"marker_number": 11, "source_candidate_number": 11, "element_kind": "furniture_or_large_prop", "visual_encoding": "outlined_furniture_symbol", "label": "interior fixture", "priority": "must_show", "source_refs": [], "must_be_legible": True}, ], }, }, } # ───────────────────────────────────────────────────────────────────────────── def _w17a_build_report(*, llm_output, candidate, target_fp_ids={"FPx"}, model_used="gpt-5.5", stage_status="generated"): from experiment_floor_plan_image_prompt_slice import ( _build_w17a_compatibility_report, ) return _build_w17a_compatibility_report( llm_output=llm_output, candidate=candidate, target_fp_ids=target_fp_ids, production_diff_empty=True, db_write_count=0, image_import_seen=False, image_api_call_count=0, model_used=model_used, stage_status=stage_status, missing_inputs=[], prev_run_id="fakeW15e", ) def test_w17a_marker_legend_unique_and_covers_candidate_numbers(): """Every candidate numbered_element MUST appear EXACTLY ONCE in the LLM legend. Duplicate marker_number OR missing candidate number both fail.""" out = _synthetic_w17a_llm_output() cand = _synthetic_candidate() rep = _w17a_build_report(llm_output=out, candidate=cand) assert rep["invariants"]["marker_legend_unique_and_covers_candidate"]["pass"] is True # Duplicate marker_number 1 → fail. bad = json.loads(json.dumps(out)) bad["floor_plan_image_prompt_by_fp"]["FPx"]["numbered_marker_legend"].append({ "marker_number": 1, "source_candidate_number": 1, "element_kind": "spatial_unit", "visual_encoding": "filled_area", "label": "dup", "priority": "must_show", "source_refs": [], "must_be_legible": True, }) rep_dup = _w17a_build_report(llm_output=bad, candidate=cand) assert rep_dup["invariants"]["marker_legend_unique_and_covers_candidate"]["pass"] is False # Drop marker 7 → fail. missing = json.loads(json.dumps(out)) missing["floor_plan_image_prompt_by_fp"]["FPx"]["numbered_marker_legend"] = [ entry for entry in missing["floor_plan_image_prompt_by_fp"]["FPx"]["numbered_marker_legend"] if entry["marker_number"] != 7 ] rep_drop = _w17a_build_report(llm_output=missing, candidate=cand) assert rep_drop["invariants"]["marker_legend_unique_and_covers_candidate"]["pass"] is False def test_w17a_legend_enum_validation_kind_and_encoding(): """`element_kind` must be one of the generic enum values; same for `visual_encoding`. Any value outside the enum fails.""" out = _synthetic_w17a_llm_output() cand = _synthetic_candidate() rep = _w17a_build_report(llm_output=out, candidate=cand) assert rep["invariants"]["legend_enum_validation"]["pass"] is True # Invalid element_kind. bad_kind = json.loads(json.dumps(out)) bad_kind["floor_plan_image_prompt_by_fp"]["FPx"]["numbered_marker_legend"][0]["element_kind"] = "NON_GENERIC_KIND" rep_kind = _w17a_build_report(llm_output=bad_kind, candidate=cand) assert rep_kind["invariants"]["legend_enum_validation"]["pass"] is False # Invalid visual_encoding. bad_enc = json.loads(json.dumps(out)) bad_enc["floor_plan_image_prompt_by_fp"]["FPx"]["numbered_marker_legend"][0]["visual_encoding"] = "photorealistic_render" rep_enc = _w17a_build_report(llm_output=bad_enc, candidate=cand) assert rep_enc["invariants"]["legend_enum_validation"]["pass"] is False def test_w17a_prompt_contains_every_marker_number_and_contract(): """`t2i_prompt_text` must include each marker number AND the explicit numbered-marker prohibition contract (renumber/omit/invent tokens).""" out = _synthetic_w17a_llm_output() cand = _synthetic_candidate() rep = _w17a_build_report(llm_output=out, candidate=cand) assert rep["invariants"]["prompt_includes_marker_contract"]["pass"] is True # Strip every `#N` from prompt → fail. bad = json.loads(json.dumps(out)) txt = bad["floor_plan_image_prompt_by_fp"]["FPx"]["t2i_prompt_text"] bad["floor_plan_image_prompt_by_fp"]["FPx"]["t2i_prompt_text"] = re.sub( r"#\d+", "", txt, ) rep_bad = _w17a_build_report(llm_output=bad, candidate=cand) assert rep_bad["invariants"]["prompt_includes_marker_contract"]["pass"] is False # Strip prohibition contract → fail. no_contract = json.loads(json.dumps(out)) no_contract["floor_plan_image_prompt_by_fp"]["FPx"]["t2i_prompt_text"] = ( "Generic schematic plan with markers #1 #2 #3 #7 #11." ) rep_nc = _w17a_build_report(llm_output=no_contract, candidate=cand) assert rep_nc["invariants"]["prompt_includes_marker_contract"]["pass"] is False def test_w17a_production_image_guard_and_target_restriction(): """Combined production/db/image guard surfaces every condition; target restriction stays to {fp_l05_01}.""" out = _synthetic_w17a_llm_output() cand = _synthetic_candidate() rep = _w17a_build_report(llm_output=out, candidate=cand) assert rep["invariants"][ "production_diff_zero_db_write_zero_image_api_call_zero" ]["pass"] is True assert rep["invariants"]["target_fp_only_fp_l05_01"]["detail"][ "received_target_fp_ids" ] == ["FPx"] from experiment_floor_plan_image_prompt_slice import ( _build_w17a_compatibility_report, ) rep_img = _build_w17a_compatibility_report( llm_output=out, candidate=cand, target_fp_ids={"FPx"}, production_diff_empty=True, db_write_count=0, image_import_seen=False, image_api_call_count=1, model_used="gpt-5.5", stage_status="generated", missing_inputs=[], prev_run_id="fakeW15e", ) assert rep_img["invariants"][ "production_diff_zero_db_write_zero_image_api_call_zero" ]["pass"] is False def test_w17a_methodology_grep_no_scenario_specific_static_tokens(): """Source script must not embed scenario-specific tokens (rooftop / bedroom / curtain / footprint / kitchen / etc.). Per-char token assembly so this assertion source does not self-match.""" script_path = _SCRIPTS_DIR / "experiment_floor_plan_image_prompt_slice.py" assert script_path.exists(), f"script missing: {script_path}" forbidden_tokens = [ "b" + "edroom", "ki" + "tchen", "blood" + "stain", "cur" + "tain", "coo" + "ktop", "tele" + "vision", "cri" + "me", "vi" + "lla", "roo" + "ftop", "foot" + "print", "pol" + "ice", "de" + "ck", "wheel" + "house", "ba" + "throom", "su" + "ri-young", ] pat = re.compile(r"(?i)\b(" + "|".join(forbidden_tokens) + r")\b") m = pat.search(script_path.read_text()) assert m is None, ( f"{script_path.name}: scenario-specific token leaked → " f"{m.group(0) if m else ''}" )