WBS-9.6 Phase 1 & 2: LLM Radar Trust Tier System 구현

Phase 1: Trust 라벨 시스템 - Canonical (100%), Adapter (80%), Reference (60%), Deprecated (0%) 4계층 - 9개 핵심 문서에 신뢰도 메타데이터 지정 - 문서 분류 체계 정의 Phase 2: 5-Tier 로딩 순서 - Phase 1: Canonical References (무조건 로드) - Phase 2: Adapter Bridges (Canonical과 모순 확인) - Phase 3: Reference Context (보조 정보) - Phase 4: Search-Based (관련성 기반 검색) - Phase 5: LLM Fallback (마지막 수단) 의사 코드: LLM Context Builder with Trust Tier conflict detection 목표: 오류율 50% 감소 (2026-08-15) Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
2026-06-23 00:24:47 +09:00
parent 6c7bdd35c7
commit 3a94b45e9e
2 changed files with 485 additions and 0 deletions
@@ -0,0 +1,159 @@
+meta:
+  title: LLM Radar Trust Tier System
+  version: 2026-06-23
+  role: canonical
+  has_code_implementation: true
+  code_path: tools/build_final_context_for_llm_v5.py
+  purpose: "LLM의 개념 혼동 감소를 위한 신뢰도 기반 문서 로딩 시스템"
+
+trust_tier_system:
+  canonical:
+    name: "Canonical Reference"
+    trust_level: 100
+    usage: "LLM의 단일 source of truth"
+    examples:
+      - spec/12_field_dictionary.yaml
+      - spec/14_raw_workbook_mapping.yaml
+      - spec/11_market_regime.yaml
+    characteristics:
+      - "정의가 명확하고 변경 불가"
+      - "다른 모든 문서가 참조"
+      - "테스트로 검증됨"
+    loading_priority: 1
+
+  adapter:
+    name: "Adapter (Bridging)"
+    trust_level: 80
+    usage: "Canonical과 Reference 간 변환/매핑"
+    examples:
+      - spec/09_decision_flow.yaml
+      - spec/14_raw_workbook_mapping.yaml
+      - tools/build_formula_registry_sync_v1.py
+    characteristics:
+      - "1:1 매핑 관계 명확"
+      - "변환 규칙 문서화됨"
+      - "양쪽 참조 검증됨"
+    loading_priority: 2
+
+  reference:
+    name: "Reference (Context)"
+    trust_level: 60
+    usage: "배경 지식 및 해석 가이드"
+    examples:
+      - docs/WBS_9_1_F14_MIGRATION_COMPLETE_2026_06_22.md
+      - docs/WBS_9_4_INCIDENT_RESPONSE_PLAYBOOK_2026_06_22.md
+      - docs/ROADMAP_WBS.md
+    characteristics:
+      - "해석의 여지가 있을 수 있음"
+      - "최신 상태를 반영할 수 있음"
+      - "보조 정보 역할"
+    loading_priority: 3
+
+  deprecated:
+    name: "Deprecated (Legacy)"
+    trust_level: 0
+    usage: "사용 금지"
+    examples:
+      - "old_formula_v1.yaml"
+      - "retired_decision_tree_v2.yaml"
+    characteristics:
+      - "더 이상 유효하지 않음"
+      - "참조 시 경고 발생"
+      - "문서에는 보존하되 로딩 제외"
+    loading_priority: 0
+
+document_classification:
+  core_logic:
+    tier: canonical
+    documents:
+      - spec/12_field_dictionary.yaml
+      - spec/14_raw_workbook_mapping.yaml
+      - spec/11_market_regime.yaml
+      - spec/09_decision_flow.yaml
+
+  formula_mapping:
+    tier: adapter
+    documents:
+      - spec/13_formula_registry.yaml
+      - tools/build_formula_registry_sync_v1.py
+
+  implementation_guide:
+    tier: reference
+    documents:
+      - docs/WBS_9_1_F14_MIGRATION_COMPLETE_2026_06_22.md
+      - docs/WBS_9_4_INCIDENT_RESPONSE_PLAYBOOK_2026_06_22.md
+      - prompts/engine_audit_master_prompt_v3.md
+
+loading_strategy:
+  phase_1_canonical:
+    name: "Phase 1: Load Canonical References"
+    trust_threshold: 100
+    action: "Always load"
+    conflict_resolution: "Canonical wins"
+    example_documents: ["field_dictionary.yaml", "market_regime.yaml"]
+
+  phase_2_adapter:
+    name: "Phase 2: Load Adapter Bridges"
+    trust_threshold: 80
+    action: "Load if not contradicting canonical"
+    conflict_resolution: "Check 1:1 mapping validity"
+    example_documents: ["decision_flow.yaml", "formula_registry_sync"]
+
+  phase_3_reference:
+    name: "Phase 3: Load Reference Context"
+    trust_threshold: 60
+    action: "Load as secondary context"
+    conflict_resolution: "Reference yields to canonical/adapter"
+    example_documents: ["migration_reports", "playbooks"]
+
+  phase_4_search:
+    name: "Phase 4: Search-Based Context"
+    trust_threshold: 50
+    action: "Retrieve relevant docs by keyword"
+    conflict_resolution: "Higher tier overrides lower"
+    example_documents: ["any matching documents"]
+
+  phase_5_fallback:
+    name: "Phase 5: LLM Knowledge Fallback"
+    trust_threshold: 0
+    action: "Use LLM's training data only"
+    conflict_resolution: "Only if no explicit docs found"
+    example_documents: ["none - LLM internal only"]
+
+conflict_resolution_hierarchy:
+  1: "Canonical (100) - always wins"
+  2: "Adapter (80) - if not contradicted by canonical"
+  3: "Reference (60) - if consistent with canonical + adapter"
+  4: "Search results (50) - ranked by relevance + trust tier"
+  5: "LLM knowledge (0) - fallback only"
+
+probability_weighting:
+  canonical_selection: 0.95
+  adapter_selection: 0.80
+  reference_selection: 0.60
+  search_selection: 0.50
+  fallback_selection: 0.05
+  note: "선택 확률은 LLM confidence 보정에 사용"
+
+error_detection:
+  tier_mismatch:
+    description: "Canonical과 Reference가 모순될 때"
+    action: "Canonical 선택, Reference 무시, 경고 로그"
+
+  missing_tier:
+    description: "Adapter tier 문서 부재"
+    action: "Canonical 직접 사용, 변환 생략"
+
+  deprecated_reference:
+    description: "Deprecated 문서가 로드되려 할 때"
+    action: "로드 거부, canonical 대체 제시"
+
+success_metrics:
+  baseline_error_rate: "100%"
+  target_error_rate: "50%"
+  target_date: "2026-08-15"
+  measurement_method: "LLM 출력 검증 (개념 혼동 횟수)"
+
+  phase_1_target: "20% 감소 (Phase 1 trust tier 적용)"
+  phase_2_target: "35% 감소 (Phase 2 load order 적용)"
+  phase_3_target: "50% 감소 (Phase 3-5 통합)"