kjh2064
3a94b45e9e
Phase 1: Trust 라벨 시스템 - Canonical (100%), Adapter (80%), Reference (60%), Deprecated (0%) 4계층 - 9개 핵심 문서에 신뢰도 메타데이터 지정 - 문서 분류 체계 정의 Phase 2: 5-Tier 로딩 순서 - Phase 1: Canonical References (무조건 로드) - Phase 2: Adapter Bridges (Canonical과 모순 확인) - Phase 3: Reference Context (보조 정보) - Phase 4: Search-Based (관련성 기반 검색) - Phase 5: LLM Fallback (마지막 수단) 의사 코드: LLM Context Builder with Trust Tier conflict detection 목표: 오류율 50% 감소 (2026-08-15) Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
160 lines
4.9 KiB
YAML
160 lines
4.9 KiB
YAML
meta:
|
|
title: LLM Radar Trust Tier System
|
|
version: 2026-06-23
|
|
role: canonical
|
|
has_code_implementation: true
|
|
code_path: tools/build_final_context_for_llm_v5.py
|
|
purpose: "LLM의 개념 혼동 감소를 위한 신뢰도 기반 문서 로딩 시스템"
|
|
|
|
trust_tier_system:
|
|
canonical:
|
|
name: "Canonical Reference"
|
|
trust_level: 100
|
|
usage: "LLM의 단일 source of truth"
|
|
examples:
|
|
- spec/12_field_dictionary.yaml
|
|
- spec/14_raw_workbook_mapping.yaml
|
|
- spec/11_market_regime.yaml
|
|
characteristics:
|
|
- "정의가 명확하고 변경 불가"
|
|
- "다른 모든 문서가 참조"
|
|
- "테스트로 검증됨"
|
|
loading_priority: 1
|
|
|
|
adapter:
|
|
name: "Adapter (Bridging)"
|
|
trust_level: 80
|
|
usage: "Canonical과 Reference 간 변환/매핑"
|
|
examples:
|
|
- spec/09_decision_flow.yaml
|
|
- spec/14_raw_workbook_mapping.yaml
|
|
- tools/build_formula_registry_sync_v1.py
|
|
characteristics:
|
|
- "1:1 매핑 관계 명확"
|
|
- "변환 규칙 문서화됨"
|
|
- "양쪽 참조 검증됨"
|
|
loading_priority: 2
|
|
|
|
reference:
|
|
name: "Reference (Context)"
|
|
trust_level: 60
|
|
usage: "배경 지식 및 해석 가이드"
|
|
examples:
|
|
- docs/WBS_9_1_F14_MIGRATION_COMPLETE_2026_06_22.md
|
|
- docs/WBS_9_4_INCIDENT_RESPONSE_PLAYBOOK_2026_06_22.md
|
|
- docs/ROADMAP_WBS.md
|
|
characteristics:
|
|
- "해석의 여지가 있을 수 있음"
|
|
- "최신 상태를 반영할 수 있음"
|
|
- "보조 정보 역할"
|
|
loading_priority: 3
|
|
|
|
deprecated:
|
|
name: "Deprecated (Legacy)"
|
|
trust_level: 0
|
|
usage: "사용 금지"
|
|
examples:
|
|
- "old_formula_v1.yaml"
|
|
- "retired_decision_tree_v2.yaml"
|
|
characteristics:
|
|
- "더 이상 유효하지 않음"
|
|
- "참조 시 경고 발생"
|
|
- "문서에는 보존하되 로딩 제외"
|
|
loading_priority: 0
|
|
|
|
document_classification:
|
|
core_logic:
|
|
tier: canonical
|
|
documents:
|
|
- spec/12_field_dictionary.yaml
|
|
- spec/14_raw_workbook_mapping.yaml
|
|
- spec/11_market_regime.yaml
|
|
- spec/09_decision_flow.yaml
|
|
|
|
formula_mapping:
|
|
tier: adapter
|
|
documents:
|
|
- spec/13_formula_registry.yaml
|
|
- tools/build_formula_registry_sync_v1.py
|
|
|
|
implementation_guide:
|
|
tier: reference
|
|
documents:
|
|
- docs/WBS_9_1_F14_MIGRATION_COMPLETE_2026_06_22.md
|
|
- docs/WBS_9_4_INCIDENT_RESPONSE_PLAYBOOK_2026_06_22.md
|
|
- prompts/engine_audit_master_prompt_v3.md
|
|
|
|
loading_strategy:
|
|
phase_1_canonical:
|
|
name: "Phase 1: Load Canonical References"
|
|
trust_threshold: 100
|
|
action: "Always load"
|
|
conflict_resolution: "Canonical wins"
|
|
example_documents: ["field_dictionary.yaml", "market_regime.yaml"]
|
|
|
|
phase_2_adapter:
|
|
name: "Phase 2: Load Adapter Bridges"
|
|
trust_threshold: 80
|
|
action: "Load if not contradicting canonical"
|
|
conflict_resolution: "Check 1:1 mapping validity"
|
|
example_documents: ["decision_flow.yaml", "formula_registry_sync"]
|
|
|
|
phase_3_reference:
|
|
name: "Phase 3: Load Reference Context"
|
|
trust_threshold: 60
|
|
action: "Load as secondary context"
|
|
conflict_resolution: "Reference yields to canonical/adapter"
|
|
example_documents: ["migration_reports", "playbooks"]
|
|
|
|
phase_4_search:
|
|
name: "Phase 4: Search-Based Context"
|
|
trust_threshold: 50
|
|
action: "Retrieve relevant docs by keyword"
|
|
conflict_resolution: "Higher tier overrides lower"
|
|
example_documents: ["any matching documents"]
|
|
|
|
phase_5_fallback:
|
|
name: "Phase 5: LLM Knowledge Fallback"
|
|
trust_threshold: 0
|
|
action: "Use LLM's training data only"
|
|
conflict_resolution: "Only if no explicit docs found"
|
|
example_documents: ["none - LLM internal only"]
|
|
|
|
conflict_resolution_hierarchy:
|
|
1: "Canonical (100) - always wins"
|
|
2: "Adapter (80) - if not contradicted by canonical"
|
|
3: "Reference (60) - if consistent with canonical + adapter"
|
|
4: "Search results (50) - ranked by relevance + trust tier"
|
|
5: "LLM knowledge (0) - fallback only"
|
|
|
|
probability_weighting:
|
|
canonical_selection: 0.95
|
|
adapter_selection: 0.80
|
|
reference_selection: 0.60
|
|
search_selection: 0.50
|
|
fallback_selection: 0.05
|
|
note: "선택 확률은 LLM confidence 보정에 사용"
|
|
|
|
error_detection:
|
|
tier_mismatch:
|
|
description: "Canonical과 Reference가 모순될 때"
|
|
action: "Canonical 선택, Reference 무시, 경고 로그"
|
|
|
|
missing_tier:
|
|
description: "Adapter tier 문서 부재"
|
|
action: "Canonical 직접 사용, 변환 생략"
|
|
|
|
deprecated_reference:
|
|
description: "Deprecated 문서가 로드되려 할 때"
|
|
action: "로드 거부, canonical 대체 제시"
|
|
|
|
success_metrics:
|
|
baseline_error_rate: "100%"
|
|
target_error_rate: "50%"
|
|
target_date: "2026-08-15"
|
|
measurement_method: "LLM 출력 검증 (개념 혼동 횟수)"
|
|
|
|
phase_1_target: "20% 감소 (Phase 1 trust tier 적용)"
|
|
phase_2_target: "35% 감소 (Phase 2 load order 적용)"
|
|
phase_3_target: "50% 감소 (Phase 3-5 통합)"
|