QuantEngineByItz/src/quant_engine/run_integration_test_v1.py

"""run_integration_test_v1.py — INTEGRATION_TEST_V1

§23 통합 테스트: 핵심 빌더를 순서대로 실행하고 각 산출물의 존재·schema·gate를 검증.

실행 순서 (실제 render-report-json 파이프라인 보완):
  1. build_scores_harness_v1
  2. build_strategy_routing_audit_v1
  3. build_sell_engine_audit_v1
  4. build_yaml_code_coverage_v1
  5. build_engine_audit_v1  (§3.10 집계)
  6. validate_engine_audit_v1  (검증)
  7. run_engine_audit_golden_cases_v1  (golden cases)

각 단계: 실행 성공 + 산출 JSON 존재 + 필수 필드 존재 + gate ≠ FAIL 검사.
WARN은 허용(미충족 항목이 있지만 감사 목적 허용), ERROR/FAIL은 실패.

종료코드: 하나라도 실패 시 1.
"""
from __future__ import annotations

import json
import subprocess
import sys
from pathlib import Path
from typing import Any

ROOT = Path(__file__).resolve().parents[2]
TEMP = ROOT / "Temp"
TOOLS = ROOT / "tools"

PYTHON = sys.executable

STEPS = [
    {
        "id": "STEP1_SCORES",
        "cmd": [PYTHON, str(TOOLS / "build_scores_harness_v1.py")],
        "output": TEMP / "scores_harness_v1.json",
        "required_fields": ["formula_id", "scores", "final_score"],
        "gate_field": None,  # 게이트 없음
        "allow_warn": True,
    },
    {
        "id": "STEP2_ROUTING",
        "cmd": [PYTHON, str(TOOLS / "build_strategy_routing_audit_v1.py")],
        "output": TEMP / "strategy_routing_audit_v1.json",
        "required_fields": ["formula_id", "selected_horizon", "horizon_conflict_count", "routing_confidence", "gate"],
        "gate_field": "gate",
        "allow_warn": True,  # FAIL(위반)도 audit 목적으로 허용
    },
    {
        "id": "STEP3_SELL",
        "cmd": [PYTHON, str(TOOLS / "build_sell_engine_audit_v1.py")],
        "output": TEMP / "sell_engine_audit_v1.json",
        "required_fields": ["formula_id", "sell_type_counts", "scr_plan", "gate"],
        "gate_field": "gate",
        "allow_warn": True,
    },
    {
        "id": "STEP4_YAML_COVERAGE",
        "cmd": [PYTHON, str(TOOLS / "build_yaml_code_coverage_v1.py")],
        "output": TEMP / "yaml_code_coverage_v1.json",
        "required_fields": ["formula_id", "yaml_formula_count", "coverage_ratio", "gate"],
        "gate_field": "gate",
        "allow_warn": True,
    },
    {
        "id": "STEP5_ENGINE_AUDIT",
        "cmd": [PYTHON, str(TOOLS / "build_engine_audit_v1.py")],
        "output": TEMP / "engine_audit_v1.json",
        "required_fields": ["meta", "data_quality", "routing", "scores", "decision",
                             "sell_plan", "evidence", "risk", "llm_control", "audit",
                             "imputed_data_exposure", "final_verdict"],
        "gate_field": None,
        "allow_warn": True,
    },
    {
        "id": "STEP6_VALIDATE",
        "cmd": [PYTHON, str(TOOLS / "validate_engine_audit_v1.py")],
        "output": None,  # validator는 JSON 미산출 (stdout 검사)
        "required_fields": [],
        "gate_field": None,
        "allow_warn": False,  # 검증기 실패는 실제 실패
    },
    {
        "id": "STEP7_GOLDEN",
        "cmd": [PYTHON, str(TOOLS / "run_engine_audit_golden_cases_v1.py")],
        "output": None,
        "required_fields": [],
        "gate_field": None,
        "allow_warn": False,
    },
    {
        "id": "STEP8_COMPLETION_GAP",
        "cmd": [PYTHON, str(TOOLS / "build_completion_gap_v1.py")],
        "output": TEMP / "completion_gap_v1.json",
        "required_fields": ["formula_id", "total_criteria", "pass_rate_pct", "criteria"],
        "gate_field": None,
        "allow_warn": True,
    },
    {
        "id": "STEP9_HORIZON_PLAN",
        "cmd": [PYTHON, str(TOOLS / "build_horizon_rebalance_plan_v1.py")],
        "output": TEMP / "horizon_rebalance_plan_v1.json",
        "required_fields": ["formula_id", "current_short_pct", "plan_rows", "gate_after_plan"],
        "gate_field": None,
        "allow_warn": True,
    },
    {
        "id": "STEP10_REALIZED_PERF",
        "cmd": [PYTHON, str(TOOLS / "build_realized_performance_v1.py")],
        "output": TEMP / "realized_performance_v1.json",
        "required_fields": ["formula_id", "performance_metrics", "summary"],
        "gate_field": None,
        "allow_warn": True,
    },
    {
        "id": "STEP11_COMPLETION_CRITERIA",
        "cmd": [PYTHON, str(TOOLS / "validate_completion_criteria_v1.py"),
                "--require-pass", "9"],  # 현재 달성 가능한 최소 PASS 기준
        "output": None,
        "required_fields": [],
        "gate_field": None,
        "allow_warn": False,
    },
]


def _load(path: Path) -> Any:
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except Exception:
        return None


def run_step(step: dict) -> tuple[bool, str]:
    cmd = step["cmd"]
    result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(ROOT))
    if result.returncode != 0:
        return False, f"exit={result.returncode} stderr={result.stderr[:200]}"

    out_path = step.get("output")
    if out_path and not out_path.exists():
        return False, f"output file missing: {out_path}"

    if out_path and step.get("required_fields"):
        data = _load(out_path)
        if data is None:
            return False, "output JSON parse failed"
        missing = [f for f in step["required_fields"] if f not in data]
        if missing:
            return False, f"missing required fields: {missing}"

    gate_field = step.get("gate_field")
    allow_warn = step.get("allow_warn", True)
    if out_path and gate_field:
        data = _load(out_path)
        if data:
            gate = str(data.get(gate_field, ""))
            if gate == "ERROR":
                return False, f"gate={gate}"
            if gate == "FAIL" and not allow_warn:
                return False, f"gate=FAIL (not allow_warn)"

    return True, "OK"


def main() -> int:
    failures: list[str] = []
    print(f"INTEGRATION_TEST_V1: running {len(STEPS)} steps\n{'='*50}")
    for step in STEPS:
        ok, msg = run_step(step)
        status = "PASS" if ok else "FAIL"
        print(f"[{status}] {step['id']}: {msg}")
        if not ok:
            failures.append(f"{step['id']}: {msg}")

    print(f"{'='*50}")
    if failures:
        print(f"INTEGRATION_TEST_V1: FAIL ({len(failures)} step(s) failed)")
        for f in failures:
            print(f"  - {f}")
        return 1
    print(f"INTEGRATION_TEST_V1: ALL {len(STEPS)} STEPS PASSED")
    return 0


if __name__ == "__main__":
    sys.exit(main())