"""W18F background prompt region/ref assembly slice — minimal tests. 3 tests: 1. happy path: synthetic W18E preview + W15e per_bg → assembly carries source + appendix concat, READY decision, clean BG sentinel intact, invariant 4 passes when no transient markers leak. 2. clean BG with transient marker residue in W15e source preview → IMPORTANT diagnostic surfaced, invariant 4 still passes (residue is informational, not fail). Conversely, region_ref_appendix injected with #18 in clean BG → invariant 4 fails. 3. methodology grep + production/api zero guard variations. """ from __future__ import annotations import json import re import sys from pathlib import Path _REPO_ROOT = Path(__file__).resolve().parents[3] _SCRIPTS_DIR = _REPO_ROOT / "backend" / "scripts" if str(_SCRIPTS_DIR) not in sys.path: sys.path.insert(0, str(_SCRIPTS_DIR)) def _w18e_preview_by_bg() -> dict: base_path = "scripts_output/fake/png/fp_l05_01.png" bedroom_appendix = ( "[background_prompt_region_appendix] Preview only.\n" "Base FP image reference (layout / rooms / fixed anchors only): " f"{base_path}\n" "Do not redraw transient markers onto the base FP image.\n" "Target spatial units for this background:\n" " - #3 (U3) bedroom @ 10x10 rect [6,3,9,8]\n" "Base markers visible on the base FP image:\n" " - #3 bedroom [base_structural_unit] @ 10x10 cell (7,5)\n" "Transient overlay markers to describe in the prose:\n" " - #18 evidence [state_overlay_plot_cue] in unit 'U3' — " "overlay_hint: hint." ) clean_appendix = ( "[background_prompt_region_appendix] Preview only.\n" "Base FP image reference (layout / rooms / fixed anchors only): " f"{base_path}\n" "Do not redraw transient markers onto the base FP image.\n" "Target spatial units for this background:\n" " - #2 (U2) kitchen @ 10x10 rect [0,1,3,2]\n" "Base markers visible on the base FP image:\n" " - #2 kitchen [base_structural_unit] @ 10x10 cell (1,2)\n" "Transient overlay markers to describe in the prose:\n" " - No transient overlay markers for this background." ) return { "BGdirty": { "bg_id": "BGdirty", "fp_id": "fp_l05_01", "base_fp_ref_path": base_path, "target_unit_refs": [ {"unit_id": "U3", "marker_number": 3}, ], "base_marker_refs": [ {"marker_number": 3}, ], "transient_overlay_to_describe": [ {"marker_number": 18, "label": "evidence"}, ], "background_prompt_region_appendix": bedroom_appendix, "consumer_decision_preview": "READY_FOR_BG_PROMPT_PREVIEW", }, "BGclean": { "bg_id": "BGclean", "fp_id": "fp_l05_01", "base_fp_ref_path": base_path, "target_unit_refs": [ {"unit_id": "U2", "marker_number": 2}, ], "base_marker_refs": [ {"marker_number": 2}, ], "transient_overlay_to_describe": [], "background_prompt_region_appendix": clean_appendix, "consumer_decision_preview": "READY_FOR_BG_PROMPT_PREVIEW", }, } def _w15e_per_bg_clean() -> dict: return { "BGdirty": { "bg_id": "BGdirty", "final_prompt_assembly_preview": ( "Wide overhead bedroom prose, no marker numbers." ), }, "BGclean": { "bg_id": "BGclean", "final_prompt_assembly_preview": ( "Wide overhead kitchen prose, no marker numbers." ), }, } def _w15e_per_bg_with_residue() -> dict: return { "BGdirty": { "bg_id": "BGdirty", "final_prompt_assembly_preview": ( "Wide overhead bedroom prose, no marker numbers." ), }, "BGclean": { "bg_id": "BGclean", "final_prompt_assembly_preview": ( "Wide overhead kitchen prose mentioning marker #18 from " "an older spec — should be flagged." ), }, } def _transient_set() -> set: return {18, 19, 20, 21, 22, 23} def _build_assembly_via_helper(w18e_by_bg, w15e_by_bg): from experiment_background_prompt_region_ref_assembly_slice import ( _build_assembly_for_fp, ) return _build_assembly_for_fp( fp_id="fp_l05_01", w18e_preview_by_bg=w18e_by_bg, w15e_per_bg=w15e_by_bg, ) def _build_report(*, assembly_by_bg, w18e_preview_by_bg, transient_marker_set=None, target_fp_ids={"fp_l05_01"}, production_diff_empty=True, image_import_seen=False, image_api_call_count=0, vlm_api_call_count=0, llm_api_call_count=0): from experiment_background_prompt_region_ref_assembly_slice import ( _build_w18f_compatibility_report, ) return _build_w18f_compatibility_report( assembly_by_bg=assembly_by_bg, w18e_preview_by_bg=w18e_preview_by_bg, transient_marker_set=( transient_marker_set if transient_marker_set is not None else _transient_set() ), target_fp_ids=target_fp_ids, production_diff_empty=production_diff_empty, db_write_count=0, image_import_seen=image_import_seen, image_api_call_count=image_api_call_count, vlm_api_call_count=vlm_api_call_count, llm_api_call_count=llm_api_call_count, missing_inputs=[], prev_run_id="fakeW18E", ) def test_w18f_happy_path_assembly_concat_and_invariants(): """Synthetic W18E + W15e → assembly concat, clean BG sentinel intact, READY decision, 5 invariants pass.""" w18e = _w18e_preview_by_bg() w15e = _w15e_per_bg_clean() assembly, build_diags = _build_assembly_via_helper(w18e, w15e) assert build_diags == [] assert set(assembly.keys()) == {"BGdirty", "BGclean"} dirty = assembly["BGdirty"] assert dirty["consumer_decision_preview"] == ( "READY_FOR_BACKGROUND_PROMPT_DRY_RUN" ) assert dirty["clean_background_expected"] is False assert dirty["transient_overlay_marker_numbers"] == [18] # assembled prompt must contain both source preview and appendix. asm = dirty["assembled_background_prompt_preview"] assert "Wide overhead bedroom prose" in asm assert "Base FP image reference" in asm assert ( "----- background_prompt_region_appendix (W18E preview) -----" in asm ) clean = assembly["BGclean"] assert clean["clean_background_expected"] is True assert clean["transient_overlay_marker_numbers"] == [] assert "No transient overlay markers" in clean["region_ref_appendix"] assert ( "No transient overlay markers" in clean["assembled_background_prompt_preview"] ) rep, imp_diags = _build_report( assembly_by_bg=assembly, w18e_preview_by_bg=w18e, ) assert rep["all_pass"] is True, rep["invariants"] assert imp_diags == [] def test_w18f_clean_bg_source_residue_diagnostic_and_appendix_leak_fails(): """W15e source preview for a clean BG already references a transient marker (#18) → IMPORTANT diagnostic; invariant 4 still PASS because the appendix itself stays clean. Then inject #18 into the clean BG region_ref_appendix → invariant 4 FAILS.""" w18e = _w18e_preview_by_bg() # (a) clean BG with W15e residue → IMPORTANT diagnostic, invariant 4 PASS. w15e_res = _w15e_per_bg_with_residue() assembly_res, _ = _build_assembly_via_helper(w18e, w15e_res) rep_res, imp_res = _build_report( assembly_by_bg=assembly_res, w18e_preview_by_bg=w18e, ) assert ( rep_res["invariants"][ "clean_bg_has_no_transient_overlay_in_assembled_prompt" ]["pass"] is True ) assert any( d.get("bg_id") == "BGclean" and d.get("kind") == "clean_bg_source_preview_contains_transient_marker" and 18 in ( d.get("transient_marker_numbers_in_source_preview") or [] ) for d in imp_res ), imp_res # (b) inject transient #18 marker reference into the clean BG appendix # → invariant 4 FAILS (this is a real leak, not an old residue). w15e_clean = _w15e_per_bg_clean() assembly_clean, _ = _build_assembly_via_helper(w18e, w15e_clean) assembly_leak = json.loads(json.dumps(assembly_clean)) assembly_leak["BGclean"]["region_ref_appendix"] += ( "\n - injected leak #18 in clean BG" ) rep_leak, _ = _build_report( assembly_by_bg=assembly_leak, w18e_preview_by_bg=w18e, ) assert rep_leak["invariants"][ "clean_bg_has_no_transient_overlay_in_assembled_prompt" ]["pass"] is False # (c) clean BG appendix missing sentinel → invariant 4 FAILS. assembly_no_sentinel = json.loads(json.dumps(assembly_clean)) assembly_no_sentinel["BGclean"]["region_ref_appendix"] = ( assembly_no_sentinel["BGclean"]["region_ref_appendix"].replace( "No transient overlay markers for this background.", "" ) ) rep_no_sentinel, _ = _build_report( assembly_by_bg=assembly_no_sentinel, w18e_preview_by_bg=w18e, ) assert rep_no_sentinel["invariants"][ "clean_bg_has_no_transient_overlay_in_assembled_prompt" ]["pass"] is False def test_w18f_production_api_zero_guard_and_methodology_grep(): """All API call counts must be zero; production diff empty; image imports not seen. Plus script source must not embed scenario-specific tokens.""" w18e = _w18e_preview_by_bg() w15e = _w15e_per_bg_clean() assembly, _ = _build_assembly_via_helper(w18e, w15e) rep_ok, _ = _build_report( assembly_by_bg=assembly, w18e_preview_by_bg=w18e, ) assert rep_ok["invariants"][ "production_diff_zero_db_write_zero_llm_vlm_image_zero" ]["pass"] is True for kw in ( {"llm_api_call_count": 1}, {"vlm_api_call_count": 1}, {"image_api_call_count": 1}, {"image_import_seen": True}, {"production_diff_empty": False}, ): rep_bad, _ = _build_report( assembly_by_bg=assembly, w18e_preview_by_bg=w18e, **kw, ) assert rep_bad["invariants"][ "production_diff_zero_db_write_zero_llm_vlm_image_zero" ]["pass"] is False, kw script_path = ( _SCRIPTS_DIR / "experiment_background_prompt_region_ref_assembly_slice.py" ) assert script_path.exists(), f"script missing: {script_path}" forbidden_tokens = [ "b" + "edroom", "ki" + "tchen", "blood" + "stain", "cur" + "tain", "coo" + "ktop", "tele" + "vision", "cri" + "me", "vi" + "lla", "roo" + "ftop", "foot" + "print", "pol" + "ice", "de" + "ck", "wheel" + "house", "ba" + "throom", "su" + "ri-young", ] pat = re.compile(r"(?i)\b(" + "|".join(forbidden_tokens) + r")\b") m = pat.search(script_path.read_text()) assert m is None, ( f"{script_path.name}: scenario-specific token leaked → " f"{m.group(0) if m else ''}" )