"""W20E1: shot-aware BG preflight (readiness gate) tests. Pure, no-network, no-write validator. Verifies the preflight fails closed on every gate spelled out in the W20E1 task spec, and that no real-provider imports are required for the tests themselves. NO real API calls in any path of this file: - LLM call: 0 (no litellm / openai import path exercised). - VLM call: 0. - image gen call: 0. - DB write: 0. - ImageAsset / projects/ checkpoint write: 0. The preflight is pure — it consumes a duck-typed ``settings`` object and an explicit operator request and returns a structured diagnostics record. Exception-free for normal validation failures (the result aggregates failures). """ from __future__ import annotations import ast import inspect from types import SimpleNamespace import pytest from app.modules.pipeline.shot_aware_bg_preflight import ( SCOPE_FULL_FRESH_PROJECT_E2E, SCOPE_SHOT_AWARE_BG_E2E, SCOPE_SHOT_AWARE_BG_PLAN_ONLY, ShotAwareBgPreflightFailure, ShotAwareBgPreflightRequest, ShotAwareBgPreflightResult, evaluate_shot_aware_bg_preflight, ) # ───────────────────────── helpers ───────────────────────── def _settings_at_shot_aware_happy_path(**overrides) -> SimpleNamespace: """A settings object with every gate flipped to the shot-aware on happy path. Tests override fields one at a time to isolate a single failure code per case. """ base = dict( background_mode="on", floor_plan_prompt_version="6", background_prompt_version="7", background_render_reference_mode="shot_aware_plan", base_location_dossier_enabled=True, floor_plan_geometry_readback_enabled=True, shot_aware_bg_render_plan_enabled=True, floor_plan_vlm_readback_real_provider_enabled=False, shot_aware_bg_render_plan_real_provider_enabled=False, ) base.update(overrides) return SimpleNamespace(**base) def _request_plan_only(**overrides) -> ShotAwareBgPreflightRequest: base = dict( requested_scope=SCOPE_SHOT_AWARE_BG_PLAN_ONLY, approve_real_vlm=False, approve_real_shot_aware_planner_llm=False, approve_image_generation=False, image_call_cap=0, approve_full_fresh_project_e2e=False, ) base.update(overrides) return ShotAwareBgPreflightRequest(**base) def _request_e2e_with_image(**overrides) -> ShotAwareBgPreflightRequest: base = dict( requested_scope=SCOPE_SHOT_AWARE_BG_E2E, approve_real_vlm=False, approve_real_shot_aware_planner_llm=False, approve_image_generation=True, image_call_cap=3, approve_full_fresh_project_e2e=False, ) base.update(overrides) return ShotAwareBgPreflightRequest(**base) def _codes(res: ShotAwareBgPreflightResult) -> list[str]: return [f.code for f in res.failures] # ───────────────────────── happy paths ───────────────────────── def test_plan_only_happy_path_returns_ok(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=_request_plan_only(), ) assert isinstance(res, ShotAwareBgPreflightResult) assert res.ok is True, _codes(res) assert res.failures == () b = res.budget_summary assert b["scope_kind"] == SCOPE_SHOT_AWARE_BG_PLAN_ONLY assert b["estimated_image_api_calls_max"] == 0 assert b["real_vlm_enabled"] is False assert b["real_shot_aware_planner_llm_enabled"] is False # Preflight layer is always zero side effects. assert b["preflight_db_writes_expected"] == 0 assert b["preflight_image_api_calls_expected"] == 0 assert b["preflight_checkpoint_writes_expected"] == 0 # plan-only dispatches a real run (planner step) → writes step_run + # checkpoint rows but no ImageAsset rows. assert b["target_run_db_writes_expected"] is True assert b["target_run_checkpoint_writes_expected"] is True assert b["target_run_image_asset_writes_expected"] == 0 # The ambiguous v0 keys must be gone. assert "db_writes_expected" not in b assert "image_asset_writes_expected" not in b def test_e2e_happy_path_with_image_approval_returns_ok(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=_request_e2e_with_image(), ) assert res.ok is True, _codes(res) b = res.budget_summary assert b["scope_kind"] == SCOPE_SHOT_AWARE_BG_E2E assert b["estimated_image_api_calls_max"] == 3 assert b["image_generation_approved"] is True assert b["image_call_cap"] == 3 assert b["preflight_db_writes_expected"] == 0 assert b["preflight_image_api_calls_expected"] == 0 assert b["target_run_db_writes_expected"] is True assert b["target_run_checkpoint_writes_expected"] is True assert b["target_run_image_asset_writes_expected"] == 3 # ───────────────────────── background_mode ───────────────────────── @pytest.mark.parametrize("mode", ["off", "chain_only"]) def test_background_mode_incompatible_fails(mode): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(background_mode=mode), request=_request_plan_only(), ) assert res.ok is False assert "background_mode_incompatible" in _codes(res) def test_background_mode_floor_plan_anchored_alias_accepted(): """``floor_plan_anchored`` is an explicit alias for ``on`` in config.py and must also unlock the shot-aware path. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( background_mode="floor_plan_anchored" ), request=_request_plan_only(), ) assert res.ok is True, _codes(res) # ───────────────────────── version selectors ───────────────────────── def test_floor_plan_prompt_version_not_six_fails(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( floor_plan_prompt_version="5" ), request=_request_plan_only(), ) assert res.ok is False assert "floor_plan_prompt_version_not_six" in _codes(res) def test_background_prompt_version_not_seven_under_shot_aware_plan_fails(): """When render mode is ``shot_aware_plan``, background_prompt_version must be exactly ``"7"`` (the W19B-2 v7 pack). """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( background_prompt_version="6" ), request=_request_plan_only(), ) assert res.ok is False assert ( "background_prompt_version_not_seven_under_shot_aware_plan" in _codes(res) ) def test_background_prompt_version_check_only_fires_under_shot_aware_plan(): """If render mode is NOT shot_aware_plan, the v7-pair gate must not fire — that gate is contractually scoped to the shot-aware render path only. Other gates may still fail; only the version-pair code is asserted absent here. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( background_prompt_version="6", background_render_reference_mode="legacy", ), request=_request_plan_only(), ) assert ( "background_prompt_version_not_seven_under_shot_aware_plan" not in _codes(res) ) # ───────────────────────── shot-aware dependency selectors ───────────────────────── def test_base_location_dossier_disabled_fails(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( base_location_dossier_enabled=False ), request=_request_plan_only(), ) assert res.ok is False assert "base_location_dossier_disabled" in _codes(res) def test_floor_plan_geometry_readback_disabled_fails(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( floor_plan_geometry_readback_enabled=False ), request=_request_plan_only(), ) assert res.ok is False assert "floor_plan_geometry_readback_disabled" in _codes(res) def test_shot_aware_bg_render_plan_disabled_fails(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( shot_aware_bg_render_plan_enabled=False ), request=_request_plan_only(), ) assert res.ok is False assert "shot_aware_bg_render_plan_disabled" in _codes(res) # ───────────────────────── render_mode (W19B-3 / W20B / W20C) ───────────────────────── def test_render_mode_w18j_overlap_fails_with_w19b3_w20b_conflict_code(): """w18j_overlap (W19B-3) and shot_aware_bg_render_plan_enabled (W20B) are mutually exclusive — same reason as the existing in-step guard. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( background_render_reference_mode="w18j_overlap" ), request=_request_plan_only(), ) assert res.ok is False assert "w19b3_w20b_conflict" in _codes(res) def test_render_mode_legacy_under_e2e_scope_fails(): """For shot-aware BG E2E scope, render mode MUST be exactly ``shot_aware_plan`` so the W20C render path is the active one. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( background_render_reference_mode="legacy" ), request=_request_e2e_with_image(), ) assert res.ok is False assert ( "background_render_reference_mode_not_shot_aware_plan" in _codes(res) ) def test_render_mode_legacy_under_plan_only_scope_does_not_force_shot_aware_plan(): """For plan-only scope the planner step itself runs fine regardless of the downstream render mode; the e2e-specific render-mode gate must not fire here. Other gates may still pass, but the shot-aware-plan requirement is e2e-only. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( background_render_reference_mode="legacy", background_prompt_version="6", # ok because not shot_aware_plan ), request=_request_plan_only(), ) assert res.ok is True, _codes(res) # ───────────────────────── real-provider approvals ───────────────────────── def test_real_vlm_enabled_without_approval_fails(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( floor_plan_vlm_readback_real_provider_enabled=True, ), request=_request_plan_only(), # approve_real_vlm=False ) assert res.ok is False assert "real_vlm_not_approved" in _codes(res) def test_real_vlm_enabled_with_approval_ok(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( floor_plan_vlm_readback_real_provider_enabled=True, ), request=_request_plan_only(approve_real_vlm=True), ) assert res.ok is True, _codes(res) assert res.budget_summary["real_vlm_enabled"] is True def test_real_planner_llm_enabled_without_approval_fails(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( shot_aware_bg_render_plan_real_provider_enabled=True, ), request=_request_plan_only(), ) assert res.ok is False assert "real_shot_aware_planner_llm_not_approved" in _codes(res) def test_real_planner_llm_enabled_with_approval_ok(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( shot_aware_bg_render_plan_real_provider_enabled=True, ), request=_request_plan_only( approve_real_shot_aware_planner_llm=True, ), ) assert res.ok is True, _codes(res) assert res.budget_summary["real_shot_aware_planner_llm_enabled"] is True # ───────────────────────── image-gen approval & cap ───────────────────────── def test_image_generation_approved_without_cap_fails(): """approve_image_generation=True must come with image_call_cap>=1.""" res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=_request_e2e_with_image(image_call_cap=0), ) assert res.ok is False assert "image_call_cap_missing" in _codes(res) def test_e2e_scope_without_image_approval_fails(): """The shot-aware BG E2E scope drives W20C image generation; it must require explicit image approval + cap, never opt in implicitly. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=ShotAwareBgPreflightRequest( requested_scope=SCOPE_SHOT_AWARE_BG_E2E, approve_real_vlm=False, approve_real_shot_aware_planner_llm=False, approve_image_generation=False, image_call_cap=0, approve_full_fresh_project_e2e=False, ), ) assert res.ok is False assert "image_generation_not_approved_for_e2e" in _codes(res) def test_plan_only_scope_with_image_approval_fails(): """plan-only scope renders nothing; receiving image approval is a contract mismatch — fail closed so callers cannot bypass the e2e gate by smuggling image approval into the plan-only path. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=_request_plan_only( approve_image_generation=True, image_call_cap=2, ), ) assert res.ok is False assert "image_generation_unexpected_for_plan_only_scope" in _codes(res) # ───────────────────────── out-of-scope / W21 override ───────────────────────── def test_full_fresh_project_e2e_without_override_fails_as_out_of_scope(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=ShotAwareBgPreflightRequest( requested_scope=SCOPE_FULL_FRESH_PROJECT_E2E, approve_real_vlm=False, approve_real_shot_aware_planner_llm=False, approve_image_generation=True, image_call_cap=10, approve_full_fresh_project_e2e=False, ), ) assert res.ok is False assert "full_fresh_project_e2e_out_of_scope" in _codes(res) def test_full_fresh_project_e2e_with_override_ok_when_other_gates_pass(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=ShotAwareBgPreflightRequest( requested_scope=SCOPE_FULL_FRESH_PROJECT_E2E, approve_real_vlm=False, approve_real_shot_aware_planner_llm=False, approve_image_generation=True, image_call_cap=10, approve_full_fresh_project_e2e=True, ), ) assert res.ok is True, _codes(res) b = res.budget_summary assert b["target_run_db_writes_expected"] is True assert b["target_run_checkpoint_writes_expected"] is True assert b["target_run_image_asset_writes_expected"] == 10 def test_full_fresh_project_e2e_override_does_not_bypass_shot_aware_plan_render_mode(): """Codex-flagged: full E2E is BROADER than shot-aware BG E2E, not weaker. Even with the W21 override, render mode MUST still be ``shot_aware_plan`` exactly — the override only unlocks the scope, it does not waive the downstream render-path contract. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( background_render_reference_mode="legacy", ), request=ShotAwareBgPreflightRequest( requested_scope=SCOPE_FULL_FRESH_PROJECT_E2E, approve_real_vlm=False, approve_real_shot_aware_planner_llm=False, approve_image_generation=True, image_call_cap=10, approve_full_fresh_project_e2e=True, ), ) assert res.ok is False assert ( "background_render_reference_mode_not_shot_aware_plan" in _codes(res) ) def test_full_fresh_project_e2e_override_does_not_bypass_image_approval(): """Codex-flagged: even with the W21 override granted, image generation must still be explicitly approved with a positive cap — the override does not silently grant image budget. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=ShotAwareBgPreflightRequest( requested_scope=SCOPE_FULL_FRESH_PROJECT_E2E, approve_real_vlm=False, approve_real_shot_aware_planner_llm=False, approve_image_generation=False, image_call_cap=0, approve_full_fresh_project_e2e=True, ), ) assert res.ok is False assert "image_generation_not_approved_for_e2e" in _codes(res) def test_full_fresh_project_e2e_override_still_requires_positive_image_cap(): """Same finding, second axis: approve_image_generation=True with a zero/negative cap is fail-closed even under the W21 override. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=ShotAwareBgPreflightRequest( requested_scope=SCOPE_FULL_FRESH_PROJECT_E2E, approve_real_vlm=False, approve_real_shot_aware_planner_llm=False, approve_image_generation=True, image_call_cap=0, approve_full_fresh_project_e2e=True, ), ) assert res.ok is False assert "image_call_cap_missing" in _codes(res) def test_unknown_scope_value_fails(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=ShotAwareBgPreflightRequest( requested_scope="some_unknown_scope_label", approve_real_vlm=False, approve_real_shot_aware_planner_llm=False, approve_image_generation=False, image_call_cap=0, approve_full_fresh_project_e2e=False, ), ) assert res.ok is False assert "unknown_requested_scope" in _codes(res) # ───────────────────────── aggregation behaviour ───────────────────────── def test_multiple_failures_collected_without_raising(): """Failures must accumulate; the validator must not raise on normal validation misses and must not short-circuit after the first gate. """ res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( background_mode="off", floor_plan_prompt_version="5", base_location_dossier_enabled=False, ), request=_request_plan_only(), ) assert res.ok is False codes = _codes(res) assert "background_mode_incompatible" in codes assert "floor_plan_prompt_version_not_six" in codes assert "base_location_dossier_disabled" in codes assert len({c for c in codes}) >= 3 def test_failure_entries_carry_selector_path_and_message(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path( shot_aware_bg_render_plan_enabled=False ), request=_request_plan_only(), ) assert res.ok is False matches = [ f for f in res.failures if f.code == "shot_aware_bg_render_plan_disabled" ] assert len(matches) == 1 assert isinstance(matches[0], ShotAwareBgPreflightFailure) assert matches[0].selector_path == "shot_aware_bg_render_plan_enabled" assert matches[0].message # non-empty human-readable string # ───────────────────────── purity / no-real-provider safety ───────────────────────── def _collect_imported_module_names(module) -> list[str]: """AST-walk the module source and return every imported root module name (e.g. ``import litellm.foo`` → ``"litellm"``; ``from openai import bar`` → ``"openai"``). """ src = inspect.getsource(module) tree = ast.parse(src) names: list[str] = [] for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: names.append(alias.name.split(".")[0]) elif isinstance(node, ast.ImportFrom): if node.module: names.append(node.module.split(".")[0]) return names def test_module_imports_no_forbidden_provider_libs(): """Pure preflight must not import litellm / openai / fal / image-gen SDKs (directly or via re-export). AST-level guard — checks actual import statements, not docstring mentions. """ import app.modules.pipeline.shot_aware_bg_preflight as mod imported = set(_collect_imported_module_names(mod)) forbidden = { "litellm", "openai", "fal_client", "fal", "google", "anthropic", "PIL", "requests", "httpx", } leaked = imported & forbidden assert not leaked, ( f"shot_aware_bg_preflight imports forbidden module(s) {sorted(leaked)} " f"— preflight must stay pure" ) def test_module_does_not_import_from_app_namespace(): """No ``from app.*`` or ``import app.*`` at the module level — the pure preflight must stay self-contained so the test surface cannot transitively pull provider modules. """ import app.modules.pipeline.shot_aware_bg_preflight as mod imported = _collect_imported_module_names(mod) app_imports = [name for name in imported if name == "app"] assert not app_imports, ( f"shot_aware_bg_preflight pulls in app.* — pure helper must stay " f"self-contained ({app_imports!r})" ) # ───────────────────────── result type contract ───────────────────────── def test_result_and_failure_are_frozen_dataclasses(): res = evaluate_shot_aware_bg_preflight( settings=_settings_at_shot_aware_happy_path(), request=_request_plan_only(), ) with pytest.raises((AttributeError, Exception)): res.ok = False # type: ignore[misc] fail = ShotAwareBgPreflightFailure( code="x", message="y", selector_path="z" ) with pytest.raises((AttributeError, Exception)): fail.code = "other" # type: ignore[misc]