diff --git a/spec/llm_radar_trust_tiers_v1.yaml b/spec/llm_radar_trust_tiers_v1.yaml new file mode 100644 index 0000000..dd01a96 --- /dev/null +++ b/spec/llm_radar_trust_tiers_v1.yaml @@ -0,0 +1,159 @@ +meta: + title: LLM Radar Trust Tier System + version: 2026-06-23 + role: canonical + has_code_implementation: true + code_path: tools/build_final_context_for_llm_v5.py + purpose: "LLM의 개념 혼동 감소를 위한 신뢰도 기반 문서 로딩 시스템" + +trust_tier_system: + canonical: + name: "Canonical Reference" + trust_level: 100 + usage: "LLM의 단일 source of truth" + examples: + - spec/12_field_dictionary.yaml + - spec/14_raw_workbook_mapping.yaml + - spec/11_market_regime.yaml + characteristics: + - "정의가 명확하고 변경 불가" + - "다른 모든 문서가 참조" + - "테스트로 검증됨" + loading_priority: 1 + + adapter: + name: "Adapter (Bridging)" + trust_level: 80 + usage: "Canonical과 Reference 간 변환/매핑" + examples: + - spec/09_decision_flow.yaml + - spec/14_raw_workbook_mapping.yaml + - tools/build_formula_registry_sync_v1.py + characteristics: + - "1:1 매핑 관계 명확" + - "변환 규칙 문서화됨" + - "양쪽 참조 검증됨" + loading_priority: 2 + + reference: + name: "Reference (Context)" + trust_level: 60 + usage: "배경 지식 및 해석 가이드" + examples: + - docs/WBS_9_1_F14_MIGRATION_COMPLETE_2026_06_22.md + - docs/WBS_9_4_INCIDENT_RESPONSE_PLAYBOOK_2026_06_22.md + - docs/ROADMAP_WBS.md + characteristics: + - "해석의 여지가 있을 수 있음" + - "최신 상태를 반영할 수 있음" + - "보조 정보 역할" + loading_priority: 3 + + deprecated: + name: "Deprecated (Legacy)" + trust_level: 0 + usage: "사용 금지" + examples: + - "old_formula_v1.yaml" + - "retired_decision_tree_v2.yaml" + characteristics: + - "더 이상 유효하지 않음" + - "참조 시 경고 발생" + - "문서에는 보존하되 로딩 제외" + loading_priority: 0 + +document_classification: + core_logic: + tier: canonical + documents: + - spec/12_field_dictionary.yaml + - spec/14_raw_workbook_mapping.yaml + - spec/11_market_regime.yaml + - spec/09_decision_flow.yaml + + formula_mapping: + tier: adapter + documents: + - spec/13_formula_registry.yaml + - tools/build_formula_registry_sync_v1.py + + implementation_guide: + tier: reference + documents: + - docs/WBS_9_1_F14_MIGRATION_COMPLETE_2026_06_22.md + - docs/WBS_9_4_INCIDENT_RESPONSE_PLAYBOOK_2026_06_22.md + - prompts/engine_audit_master_prompt_v3.md + +loading_strategy: + phase_1_canonical: + name: "Phase 1: Load Canonical References" + trust_threshold: 100 + action: "Always load" + conflict_resolution: "Canonical wins" + example_documents: ["field_dictionary.yaml", "market_regime.yaml"] + + phase_2_adapter: + name: "Phase 2: Load Adapter Bridges" + trust_threshold: 80 + action: "Load if not contradicting canonical" + conflict_resolution: "Check 1:1 mapping validity" + example_documents: ["decision_flow.yaml", "formula_registry_sync"] + + phase_3_reference: + name: "Phase 3: Load Reference Context" + trust_threshold: 60 + action: "Load as secondary context" + conflict_resolution: "Reference yields to canonical/adapter" + example_documents: ["migration_reports", "playbooks"] + + phase_4_search: + name: "Phase 4: Search-Based Context" + trust_threshold: 50 + action: "Retrieve relevant docs by keyword" + conflict_resolution: "Higher tier overrides lower" + example_documents: ["any matching documents"] + + phase_5_fallback: + name: "Phase 5: LLM Knowledge Fallback" + trust_threshold: 0 + action: "Use LLM's training data only" + conflict_resolution: "Only if no explicit docs found" + example_documents: ["none - LLM internal only"] + +conflict_resolution_hierarchy: + 1: "Canonical (100) - always wins" + 2: "Adapter (80) - if not contradicted by canonical" + 3: "Reference (60) - if consistent with canonical + adapter" + 4: "Search results (50) - ranked by relevance + trust tier" + 5: "LLM knowledge (0) - fallback only" + +probability_weighting: + canonical_selection: 0.95 + adapter_selection: 0.80 + reference_selection: 0.60 + search_selection: 0.50 + fallback_selection: 0.05 + note: "선택 확률은 LLM confidence 보정에 사용" + +error_detection: + tier_mismatch: + description: "Canonical과 Reference가 모순될 때" + action: "Canonical 선택, Reference 무시, 경고 로그" + + missing_tier: + description: "Adapter tier 문서 부재" + action: "Canonical 직접 사용, 변환 생략" + + deprecated_reference: + description: "Deprecated 문서가 로드되려 할 때" + action: "로드 거부, canonical 대체 제시" + +success_metrics: + baseline_error_rate: "100%" + target_error_rate: "50%" + target_date: "2026-08-15" + measurement_method: "LLM 출력 검증 (개념 혼동 횟수)" + + phase_1_target: "20% 감소 (Phase 1 trust tier 적용)" + phase_2_target: "35% 감소 (Phase 2 load order 적용)" + phase_3_target: "50% 감소 (Phase 3-5 통합)" diff --git a/tools/llm_radar_phase1_phase2_v1.py b/tools/llm_radar_phase1_phase2_v1.py new file mode 100644 index 0000000..22356fc --- /dev/null +++ b/tools/llm_radar_phase1_phase2_v1.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +""" +WBS-9.6 Phase 1 & 2: LLM Radar Trust Tier + Loading Order + +Phase 1: Trust 라벨 시스템 정의 +Phase 2: 5-tier 로딩 순서 구현 +""" + +import yaml +import json +from pathlib import Path +from typing import Dict, List, Tuple +from datetime import datetime +import sys + +class LLMRadarPhase12: + """LLM Radar Trust Tier System (Phase 1 & 2)""" + + def __init__(self): + self.trust_tier_spec = Path("spec/llm_radar_trust_tiers_v1.yaml") + self.context_cache = {} + self.load_order_sequence = [] + self.results = { + "timestamp": datetime.now().isoformat(), + "phase_1": {}, + "phase_2": {}, + "summary": {} + } + + def load_trust_tier_spec(self) -> Dict: + """Trust tier 스펙 로드""" + if not self.trust_tier_spec.exists(): + print(f"[ERROR] Trust tier spec not found: {self.trust_tier_spec}") + return {} + + with open(self.trust_tier_spec, encoding='utf-8') as f: + spec = yaml.safe_load(f) + + return spec + + def phase_1_build_trust_labels(self) -> Dict: + """Phase 1: 모든 문서에 trust 라벨 지정""" + spec = self.load_trust_tier_spec() + + if not spec: + return {"status": "FAILED", "error": "Spec not loaded"} + + # Trust tier 정보 추출 + trust_tiers = spec.get("trust_tier_system", {}) + doc_classification = spec.get("document_classification", {}) + + trust_labels = {} + + # 각 분류에서 문서 추출 + for category, config in doc_classification.items(): + tier = config.get("tier") + documents = config.get("documents", []) + + if tier not in trust_tiers: + print(f"[WARNING] Unknown tier: {tier} for category {category}") + continue + + tier_info = trust_tiers[tier] + trust_level = tier_info.get("trust_level", 0) + + for doc in documents: + trust_labels[doc] = { + "tier": tier, + "trust_level": trust_level, + "category": category, + "priority": tier_info.get("loading_priority", 0) + } + + self.results["phase_1"]["total_documents"] = len(trust_labels) + self.results["phase_1"]["tier_distribution"] = self._count_by_tier(trust_labels) + self.results["phase_1"]["trust_labels"] = trust_labels + + print(f"[Phase 1] Trust labels created for {len(trust_labels)} documents") + return { + "status": "SUCCESS", + "documents_labeled": len(trust_labels), + "tiers": list(set(label["tier"] for label in trust_labels.values())) + } + + def _count_by_tier(self, labels: Dict) -> Dict: + """Tier별 문서 개수 계산""" + counts = {} + for label in labels.values(): + tier = label["tier"] + counts[tier] = counts.get(tier, 0) + 1 + return counts + + def phase_2_build_loading_order(self) -> Dict: + """Phase 2: 5-tier 로딩 순서 정의""" + spec = self.load_trust_tier_spec() + + if not spec: + return {"status": "FAILED", "error": "Spec not loaded"} + + loading_strategy = spec.get("loading_strategy", {}) + trust_tiers = spec.get("trust_tier_system", {}) + + # 로딩 순서 정의 + load_sequence = [] + + # Phase 1: Canonical (trust_level=100) + canonical_docs = [ + "spec/12_field_dictionary.yaml", + "spec/14_raw_workbook_mapping.yaml", + "spec/11_market_regime.yaml" + ] + load_sequence.append({ + "phase": 1, + "name": "Canonical References", + "tier": "canonical", + "trust_threshold": 100, + "documents": canonical_docs, + "action": "Always load" + }) + + # Phase 2: Adapter (trust_level=80) + adapter_docs = [ + "spec/09_decision_flow.yaml", + "spec/13_formula_registry.yaml" + ] + load_sequence.append({ + "phase": 2, + "name": "Adapter Bridges", + "tier": "adapter", + "trust_threshold": 80, + "documents": adapter_docs, + "action": "Load if no conflict with canonical" + }) + + # Phase 3: Reference (trust_level=60) + reference_docs = [ + "docs/WBS_9_1_F14_MIGRATION_COMPLETE_2026_06_22.md", + "docs/WBS_9_4_INCIDENT_RESPONSE_PLAYBOOK_2026_06_22.md" + ] + load_sequence.append({ + "phase": 3, + "name": "Reference Context", + "tier": "reference", + "trust_threshold": 60, + "documents": reference_docs, + "action": "Load as secondary context" + }) + + # Phase 4: Search-Based + load_sequence.append({ + "phase": 4, + "name": "Search-Based Context", + "tier": "search", + "trust_threshold": 50, + "documents": [], # Dynamic - determined by query + "action": "Retrieve by relevance + tier" + }) + + # Phase 5: Fallback + load_sequence.append({ + "phase": 5, + "name": "LLM Knowledge Fallback", + "tier": "fallback", + "trust_threshold": 0, + "documents": [], # LLM internal + "action": "Use LLM training data only" + }) + + self.load_order_sequence = load_sequence + self.results["phase_2"]["loading_phases"] = load_sequence + self.results["phase_2"]["total_phases"] = len(load_sequence) + + print(f"[Phase 2] Loading order defined for {len(load_sequence)} phases") + return { + "status": "SUCCESS", + "phases": len(load_sequence), + "total_documents_to_load": sum(len(p.get("documents", [])) for p in load_sequence) + } + + def generate_llm_context_builder_pseudo_code(self) -> str: + """LLM context builder를 위한 의사 코드 생성""" + pseudo_code = """ +// LLM Radar Phase 1 & 2 Context Builder +function buildContextWithTrustTiers(query, userContext) { + context = [] + loadedDocs = set() + + // Phase 1: Load Canonical (100% trust) + for (doc in canonicalDocuments) { + if (doc.exists()) { + content = loadDocument(doc) + context.append({ + tier: "canonical", + trust_level: 100, + content: content, + loaded_at: phase_1 + }) + loadedDocs.add(doc) + } + } + + // Phase 2: Load Adapter (80% trust) + for (doc in adapterDocuments) { + if (doc.exists() && doc not in loadedDocs) { + content = loadDocument(doc) + if (!conflictsWithCanonical(content, context)) { + context.append({ + tier: "adapter", + trust_level: 80, + content: content, + loaded_at: phase_2 + }) + loadedDocs.add(doc) + } + } + } + + // Phase 3: Load Reference (60% trust) + for (doc in referenceDocuments) { + if (doc.exists() && doc not in loadedDocs) { + content = loadDocument(doc) + context.append({ + tier: "reference", + trust_level: 60, + content: content, + loaded_at: phase_3 + }) + loadedDocs.add(doc) + } + } + + // Phase 4: Search-Based (50% trust) + relevantDocs = searchDocuments(query, threshold=50) + for (doc in relevantDocs) { + if (doc not in loadedDocs) { + content = loadDocument(doc) + context.append({ + tier: "search", + trust_level: 50, + content: content, + relevance_score: doc.score, + loaded_at: phase_4 + }) + loadedDocs.add(doc) + } + } + + // Phase 5: Fallback (0% trust - use LLM knowledge) + if (context.isEmpty()) { + context.append({ + tier: "fallback", + trust_level: 0, + source: "llm_training_data", + loaded_at: phase_5 + }) + } + + return context +} + +// Conflict detection +function conflictsWithCanonical(adapterDoc, canonicalContext) { + for (canonical in canonicalContext) { + if (contradicts(adapterDoc, canonical)) { + logWarning("Adapter contradicts canonical") + return true + } + } + return false +} +""" + return pseudo_code + + def generate_report(self) -> Dict: + """전체 리포트 생성""" + print("\n" + "="*80) + print("WBS-9.6 Phase 1 & 2: LLM Radar Trust Tier System") + print("="*80) + + # Phase 1 + phase1_result = self.phase_1_build_trust_labels() + print(f"\n[Phase 1 Result] {phase1_result}") + + # Phase 2 + phase2_result = self.phase_2_build_loading_order() + print(f"[Phase 2 Result] {phase2_result}") + + # Summary + self.results["summary"] = { + "phase_1_status": phase1_result.get("status"), + "phase_2_status": phase2_result.get("status"), + "total_trust_labels": self.results["phase_1"].get("total_documents", 0), + "loading_phases_defined": self.results["phase_2"].get("total_phases", 0), + "next_phase": "Phase 3: Dependency Graph", + "target_completion": "2026-08-15", + "error_rate_target": "50% reduction from baseline" + } + + print("\n[Summary]") + print(f" Phase 1 (Trust Labels): {self.results['phase_1'].get('total_documents', 0)} docs labeled") + print(f" Phase 2 (Load Order): {self.results['phase_2'].get('total_phases', 0)} phases defined") + print(f" Tiers: {', '.join(self.results['phase_1'].get('tier_distribution', {}).keys())}") + + return self.results + + def save_report(self, output_file: str = None) -> None: + """리포트 저장""" + if not output_file: + output_file = f"Temp/llm_radar_phase12_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + + Path(output_file).parent.mkdir(parents=True, exist_ok=True) + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(self.results, f, indent=2, ensure_ascii=False) + + print(f"\n[Save] Report saved: {output_file}") + +if __name__ == "__main__": + radar = LLMRadarPhase12() + radar.generate_report() + radar.save_report() + + # 의사 코드 출력 + print("\n" + "="*80) + print("Pseudo Code: LLM Context Builder with Trust Tiers") + print("="*80) + print(radar.generate_llm_context_builder_pseudo_code())