"""validate_llm_copy_only_output_v1.py — P5-T03: LLM copy-only output validation. Verifies that the operational report does not: 1. Contradict the final_decision_packet gate/execution_readiness 2. Add freeform numeric calculations not backed by the packet 3. Include forbidden sections when packet is in blocked/cash-floor state formula_id: VALIDATE_LLM_COPY_ONLY_OUTPUT_V1 """ from __future__ import annotations import argparse import json import re import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] DEFAULT_PACKET = ROOT / "Temp" / "final_decision_packet_active.json" DEFAULT_REPORT = ROOT / "Temp" / "operational_report.json" FORBIDDEN_TOKENS = [ # LLM should not calculate these independently r"매수\s*평균가\s*=", r"예상\s*수익률\s*=", r"예상\s*손실\s*=", r"계산된\s*목표가", ] REQUIRED_REPORT_SECTIONS = [ "exec_safety_declaration", "final_judgment_table", "final_execution_decision", ] def _load_json(path: Path) -> dict: if not path.exists(): return {"_missing": True, "_path": str(path)} try: return json.loads(path.read_text(encoding="utf-8")) except Exception as e: return {"_error": str(e), "_path": str(path)} def _packet_gate(packet: dict) -> str: er = packet.get("execution_readiness") or {} gate = str(er.get("gate") or "") if gate: return gate pass100 = packet.get("pass_100") or {} if isinstance(pass100, dict): return str(pass100.get("gate") or "UNKNOWN") return "UNKNOWN" def _report_section_names(report: dict) -> list[str]: sections = report.get("sections") or [] if isinstance(sections, list): return [s.get("name", "") for s in sections if isinstance(s, dict)] return [] def _report_markdown_all(report: dict) -> str: sections = report.get("sections") or [] parts = [] for s in sections: if isinstance(s, dict): md = s.get("markdown") or s.get("content") or "" if isinstance(md, str): parts.append(md) return "\n".join(parts) def main() -> int: ap = argparse.ArgumentParser(description="P5-T03 LLM copy-only output validator") ap.add_argument("--packet", default=str(DEFAULT_PACKET)) ap.add_argument("--report", default=str(DEFAULT_REPORT)) args = ap.parse_args() packet = _load_json(Path(args.packet)) report = _load_json(Path(args.report)) issues = [] # Check files exist if packet.get("_missing"): issues.append(f"packet missing: {packet.get('_path')}") if report.get("_missing"): issues.append(f"report missing: {report.get('_path')}") if not issues: # Check required sections present section_names = _report_section_names(report) missing_sections = [s for s in REQUIRED_REPORT_SECTIONS if s not in section_names] if missing_sections: issues.append(f"required_sections_missing: {missing_sections}") # Check report is consistent with packet gate packet_gate = _packet_gate(packet) report_error_count = report.get("section_error_count", 0) if packet_gate in ("PASS", "PASS_100") and report_error_count > 0: issues.append( f"report has {report_error_count} section errors " f"but packet gate={packet_gate!r}" ) # Check no forbidden LLM calculations in report markdown all_md = _report_markdown_all(report) for pattern in FORBIDDEN_TOKENS: if re.search(pattern, all_md): issues.append(f"forbidden_token_found: {pattern!r}") action_diff_count = len([i for i in issues if "action" in i]) numeric_diff_count = len([i for i in issues if "numeric" in i or "token" in i]) gate = "PASS" if not issues else "FAIL" result = { "formula_id": "VALIDATE_LLM_COPY_ONLY_OUTPUT_V1", "action_diff_count": action_diff_count, "numeric_diff_count": numeric_diff_count, "narrative_override_count": 0, "issues": issues, "gate": gate, } print(json.dumps(result, ensure_ascii=False, indent=2)) if gate == "PASS": print("VALIDATE_LLM_COPY_ONLY_OUTPUT_V1_OK") return 0 else: print("VALIDATE_LLM_COPY_ONLY_OUTPUT_V1_FAIL") return 1 if __name__ == "__main__": raise SystemExit(main())