QuantEngineByItz/tools/validate_llm_copy_only_output_v1.py

"""validate_llm_copy_only_output_v1.py — P5-T03: LLM copy-only output validation.

Verifies that the operational report does not:
1. Contradict the final_decision_packet gate/execution_readiness
2. Add freeform numeric calculations not backed by the packet
3. Include forbidden sections when packet is in blocked/cash-floor state

formula_id: VALIDATE_LLM_COPY_ONLY_OUTPUT_V1
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]

DEFAULT_PACKET = ROOT / "Temp" / "final_decision_packet_active.json"
DEFAULT_REPORT = ROOT / "Temp" / "operational_report.json"

FORBIDDEN_TOKENS = [
    # LLM should not calculate these independently
    r"매수\s*평균가\s*=",
    r"예상\s*수익률\s*=",
    r"예상\s*손실\s*=",
    r"계산된\s*목표가",
]

REQUIRED_REPORT_SECTIONS = [
    "exec_safety_declaration",
    "final_judgment_table",
    "final_execution_decision",
]


def _load_json(path: Path) -> dict:
    if not path.exists():
        return {"_missing": True, "_path": str(path)}
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except Exception as e:
        return {"_error": str(e), "_path": str(path)}


def _packet_gate(packet: dict) -> str:
    er = packet.get("execution_readiness") or {}
    gate = str(er.get("gate") or "")
    if gate:
        return gate
    pass100 = packet.get("pass_100") or {}
    if isinstance(pass100, dict):
        return str(pass100.get("gate") or "UNKNOWN")
    return "UNKNOWN"


def _report_section_names(report: dict) -> list[str]:
    sections = report.get("sections") or []
    if isinstance(sections, list):
        return [s.get("name", "") for s in sections if isinstance(s, dict)]
    return []


def _report_markdown_all(report: dict) -> str:
    sections = report.get("sections") or []
    parts = []
    for s in sections:
        if isinstance(s, dict):
            md = s.get("markdown") or s.get("content") or ""
            if isinstance(md, str):
                parts.append(md)
    return "\n".join(parts)


def main() -> int:
    ap = argparse.ArgumentParser(description="P5-T03 LLM copy-only output validator")
    ap.add_argument("--packet", default=str(DEFAULT_PACKET))
    ap.add_argument("--report", default=str(DEFAULT_REPORT))
    args = ap.parse_args()

    packet = _load_json(Path(args.packet))
    report = _load_json(Path(args.report))

    issues = []

    # Check files exist
    if packet.get("_missing"):
        issues.append(f"packet missing: {packet.get('_path')}")
    if report.get("_missing"):
        issues.append(f"report missing: {report.get('_path')}")

    if not issues:
        # Check required sections present
        section_names = _report_section_names(report)
        missing_sections = [s for s in REQUIRED_REPORT_SECTIONS if s not in section_names]
        if missing_sections:
            issues.append(f"required_sections_missing: {missing_sections}")

        # Check report is consistent with packet gate
        packet_gate = _packet_gate(packet)
        report_error_count = report.get("section_error_count", 0)
        if packet_gate in ("PASS", "PASS_100") and report_error_count > 0:
            issues.append(
                f"report has {report_error_count} section errors "
                f"but packet gate={packet_gate!r}"
            )

        # Check no forbidden LLM calculations in report markdown
        all_md = _report_markdown_all(report)
        for pattern in FORBIDDEN_TOKENS:
            if re.search(pattern, all_md):
                issues.append(f"forbidden_token_found: {pattern!r}")

    action_diff_count = len([i for i in issues if "action" in i])
    numeric_diff_count = len([i for i in issues if "numeric" in i or "token" in i])

    gate = "PASS" if not issues else "FAIL"
    result = {
        "formula_id": "VALIDATE_LLM_COPY_ONLY_OUTPUT_V1",
        "action_diff_count": action_diff_count,
        "numeric_diff_count": numeric_diff_count,
        "narrative_override_count": 0,
        "issues": issues,
        "gate": gate,
    }

    print(json.dumps(result, ensure_ascii=False, indent=2))

    if gate == "PASS":
        print("VALIDATE_LLM_COPY_ONLY_OUTPUT_V1_OK")
        return 0
    else:
        print("VALIDATE_LLM_COPY_ONLY_OUTPUT_V1_FAIL")
        return 1


if __name__ == "__main__":
    raise SystemExit(main())