"""W18C base-FP VLM readback — minimal safety tests only. 3 tests: 1. dry-run safety (no VLM call, no image generation). 2. partition invariant on synthetic VLM output (PASS + a missing case). 3. methodology grep — script source must not embed scenario-specific tokens. Per-char assembly so this assertion does not self-match. """ from __future__ import annotations import json import re import sys from pathlib import Path _REPO_ROOT = Path(__file__).resolve().parents[3] _SCRIPTS_DIR = _REPO_ROOT / "backend" / "scripts" if str(_SCRIPTS_DIR) not in sys.path: sys.path.insert(0, str(_SCRIPTS_DIR)) def _seed_w18b_chain(tmp_path: Path) -> Path: """Build a minimal W18A2 + W18B chain that W18C can resolve.""" w18a = tmp_path / "w18a_fake" w18a.mkdir() (w18a / "base_layout_prompt.json").write_text(json.dumps({ "base_layout_prompt_by_fp": { "fp_l05_01": { "base_fp_t2i_prompt_text": "schematic plan #1 #2 #3.", "included_marker_legend": [ {"marker_number": 1, "source_candidate_number": 1, "base_layer_decision": "base_structural_unit", "label": "u1", "unit_id": "U1", "visual_encoding": "filled_area", "must_be_legible": True}, {"marker_number": 2, "source_candidate_number": 2, "base_layer_decision": "base_structural_unit", "label": "u2", "unit_id": "U2", "visual_encoding": "filled_area", "must_be_legible": True}, {"marker_number": 3, "source_candidate_number": 3, "base_layer_decision": "base_structural_unit", "label": "u3", "unit_id": "U3", "visual_encoding": "filled_area", "must_be_legible": True}, ], "excluded_transient_elements": [ {"marker_number": 99, "source_candidate_number": 99, "base_layer_decision": "state_overlay_plot_cue", "label": "Tx", "unit_id": "U3", "excluded_reason": "scene-state", "overlay_instruction_hint": "bg overlay"}, ], "bg_state_overlay_payload_by_bg": {}, "base_fp_contract_notes": "compact zones", "production_prompt_delta_recommendations": "review", }, }, })) (w18a / "run_meta.json").write_text(json.dumps({ "run_id": "w18a_fake", "stage": "w18a_floor_plan_base_layout_prompt_slice", "run_status": "succeeded", "exit_code": 0, "stage_status": "generated", "model_used": "gpt-5.5", })) w18b = tmp_path / "w18b_fake" (w18b / "png").mkdir(parents=True, exist_ok=True) (w18b / "png" / "fp_l05_01.png").write_bytes( b"\x89PNG\r\n\x1a\nFAKE-W18B-PNG" ) (w18b / "run_meta.json").write_text(json.dumps({ "run_id": "w18b_fake", "stage": "w18b_floor_plan_base_layout_image_smoke_slice", "run_status": "succeeded", "exit_code": 0, "stage_status": "generated", "model": "gpt-image-2", "image_api_call_count": 1, "image_generation_count": 1, "args": {"derive_base_layout_image_smoke_from": str(w18a)}, "derived_from": w18a.name, "target_fp_ids": ["fp_l05_01"], })) return w18b def test_w18c_dry_run_makes_no_vlm_call_and_writes_placeholder(tmp_path): """Default (no --generate) must be a pure dry-run: no VLM call, no image API call, placeholder vlm_readback.json emitted.""" import experiment_floor_plan_base_layout_vlm_readback_slice as mod src = _seed_w18b_chain(tmp_path) out_root = tmp_path / "out" exit_code = mod.main([ "--derive-base-vlm-readback-from", str(src), "--target-fp-ids", "fp_l05_01", "--output-root", str(out_root), ]) assert exit_code == 0 runs = sorted(out_root.iterdir()) assert len(runs) == 1 run_dir = runs[0] meta = json.loads((run_dir / "run_meta.json").read_text()) assert meta["run_status"] == "succeeded" assert meta["stage_status"] == "dry_run" assert meta["vlm_api_call_count"] == 0 assert meta["image_api_call_count"] == 0 def test_w18c_partition_invariant_passes_on_good_output_and_fails_on_missing(): """W18C invariant 4 partitions #1..#N exactly into read or missing. PASS path: all base markers in read_markers. FAIL path: drop one marker so it appears in neither list.""" from experiment_floor_plan_base_layout_vlm_readback_slice import ( _build_w18c_compatibility_report, ) base_legend = [ {"marker_number": n, "label": f"u{n}", "unit_id": f"U{n}", "base_layer_decision": "base_structural_unit", "visual_encoding": "filled_area"} for n in (1, 2, 3) ] good = { "read_markers": [ {"marker_number": 1, "visible": True, "confidence": "high", "observed_label_or_area": "zone1", "approximate_10x10_rect": [0, 0, 4, 4], "notes": ""}, {"marker_number": 2, "visible": True, "confidence": "high", "observed_label_or_area": "zone2", "approximate_10x10_rect": [5, 0, 9, 4], "notes": ""}, {"marker_number": 3, "visible": True, "confidence": "medium", "observed_label_or_area": "zone3", "approximate_10x10_rect": [0, 5, 9, 9], "notes": ""}, ], "missing_or_ambiguous_markers": [], "base_unit_layout": [], "structural_relationship_summary": {}, "unexpected_transient_markers": [], "readback_conflicts": [], } rep_good = _build_w18c_compatibility_report( vlm_output=good, base_legend=base_legend, excluded_marker_numbers=[99], production_diff_empty=True, db_write_count=0, image_import_seen=False, image_api_call_count=0, vlm_api_call_count=1, expected_vlm_api_call_count=1, model_used="gpt-5.5", stage_status="generated", missing_inputs=[], prev_run_id="fakeW18B", target_fp_ids={"fp_l05_01"}, ) assert rep_good["invariants"][ "read_markers_partition_base_legend" ]["pass"] is True bad = json.loads(json.dumps(good)) bad["read_markers"] = [ e for e in bad["read_markers"] if e["marker_number"] != 2 ] rep_bad = _build_w18c_compatibility_report( vlm_output=bad, base_legend=base_legend, excluded_marker_numbers=[99], production_diff_empty=True, db_write_count=0, image_import_seen=False, image_api_call_count=0, vlm_api_call_count=1, expected_vlm_api_call_count=1, model_used="gpt-5.5", stage_status="generated", missing_inputs=[], prev_run_id="fakeW18B", target_fp_ids={"fp_l05_01"}, ) assert rep_bad["invariants"][ "read_markers_partition_base_legend" ]["pass"] is False # Confidence enum violation also fails the partition invariant. bad_conf = json.loads(json.dumps(good)) bad_conf["read_markers"][0]["confidence"] = "VERY_SURE" rep_conf = _build_w18c_compatibility_report( vlm_output=bad_conf, base_legend=base_legend, excluded_marker_numbers=[99], production_diff_empty=True, db_write_count=0, image_import_seen=False, image_api_call_count=0, vlm_api_call_count=1, expected_vlm_api_call_count=1, model_used="gpt-5.5", stage_status="generated", missing_inputs=[], prev_run_id="fakeW18B", target_fp_ids={"fp_l05_01"}, ) assert rep_conf["invariants"][ "read_markers_partition_base_legend" ]["pass"] is False # Invalid 10x10 rect (out of range) also fails. bad_rect = json.loads(json.dumps(good)) bad_rect["read_markers"][0]["approximate_10x10_rect"] = [0, 0, 15, 4] rep_rect = _build_w18c_compatibility_report( vlm_output=bad_rect, base_legend=base_legend, excluded_marker_numbers=[99], production_diff_empty=True, db_write_count=0, image_import_seen=False, image_api_call_count=0, vlm_api_call_count=1, expected_vlm_api_call_count=1, model_used="gpt-5.5", stage_status="generated", missing_inputs=[], prev_run_id="fakeW18B", target_fp_ids={"fp_l05_01"}, ) assert rep_rect["invariants"][ "read_markers_partition_base_legend" ]["pass"] is False def test_w18c_methodology_grep_no_scenario_specific_static_tokens(): """Script source must not embed scenario-specific tokens.""" script_path = ( _SCRIPTS_DIR / "experiment_floor_plan_base_layout_vlm_readback_slice.py" ) assert script_path.exists(), f"script missing: {script_path}" forbidden_tokens = [ "b" + "edroom", "ki" + "tchen", "blood" + "stain", "cur" + "tain", "coo" + "ktop", "tele" + "vision", "cri" + "me", "vi" + "lla", "roo" + "ftop", "foot" + "print", "pol" + "ice", "de" + "ck", "wheel" + "house", "ba" + "throom", "su" + "ri-young", ] pat = re.compile(r"(?i)\b(" + "|".join(forbidden_tokens) + r")\b") m = pat.search(script_path.read_text()) assert m is None, ( f"{script_path.name}: scenario-specific token leaked → " f"{m.group(0) if m else ''}" )