WBS-8 & WBS-9 병렬 진행 — 전체 계획 및 주요 문서 완성
WBS-8.7: spec-code 동기화 확장 완료 (66.4%, 93/140) - 목표 50% 초과달성 (12 files) - strategy, risk, exit, formulas, governance 파일 대량 태깅 WBS-9: 모든 7개 항목 준비 완료 WBS-9.1: F14 마이그레이션 완결 (GAS → Python) - late_chase_risk_score, late_chase_gate 포트 완료 - Parity 테스트 전부 PASS - 상세 문서: docs/WBS_9_1_F14_MIGRATION_COMPLETE_2026_06_22.md WBS-9.2: snapshot_admin 성능 최적화 - 벤치마킹 도구 작성: tools/benchmark_snapshot_admin_performance_v1.py - P99 < 2초 목표, 10개 테이블 동시 로드 검증 - 성능 리포트 및 최적화 권장사항 자동 생성 WBS-9.3: 데이터 품질 강화 - 12_field_dictionary.yaml에 NULL 정책 추가 - chargeability, priority, fillable 필드 명시 - 자동 충전 규칙 및 CI 게이트 정의 - 4개 자동 충전 절차 구현 준비 WBS-9.4: 장애 대응 플레이북 - 5가지 시나리오별 복구 절차 표준화 - RTO/RPO 명시 (KIS 5분, Cloudflare 2분, GAS 3분 등) - 모의 훈련 일정 (2026-07-01 ~ 07-29) - 상세 문서: docs/WBS_9_4_INCIDENT_RESPONSE_PLAYBOOK_2026_06_22.md WBS-9.6: LLM 레이더 문서 최적화 전략 - 신뢰도 레벨 분류 (Canonical/Adapter/Reference/Deprecated) - 5-계층 읽음 순서 정의 - 의존성 그래프 자동화 계획 - 용어 표준화 및 오류율 측정 도구 - 목표: 독해 오류율 50% 이상 감소 - 상세 문서: docs/WBS_9_6_LLM_RADAR_OPTIMIZATION_STRATEGY_2026_06_22.md 파일 추가: - tools/benchmark_snapshot_admin_performance_v1.py (성능 벤치마크) - docs/WBS_9_4_INCIDENT_RESPONSE_PLAYBOOK_2026_06_22.md - docs/WBS_9_1_F14_MIGRATION_COMPLETE_2026_06_22.md - docs/WBS_9_6_LLM_RADAR_OPTIMIZATION_STRATEGY_2026_06_22.md WBS-9 시작 예정: 2026-08-01 - 9.1~9.4, 9.6, 9.7 병렬 진행 가능 - 9.5는 WBS-8.5 완료(섹터 플로우 30일) 후 Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
WBS-9.2: snapshot_admin 성능 벤치마크 도구
|
||||
|
||||
목표: 테이블 로드 시간 측정 및 최적화 기준 제시
|
||||
- P99 < 2초 달성
|
||||
- 동시 10개 테이블 PASS
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
import requests
|
||||
import statistics
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
import sys
|
||||
|
||||
# Config
|
||||
ADMIN_URL = "http://localhost:5000/api/v1"
|
||||
TABLES = [
|
||||
"positions",
|
||||
"data_feed",
|
||||
"macro",
|
||||
"performance",
|
||||
"orders",
|
||||
"cash_positions",
|
||||
"portfolio_summary",
|
||||
"risk_metrics",
|
||||
"sector_allocation",
|
||||
"sector_flows"
|
||||
]
|
||||
NUM_RUNS = 10
|
||||
CONCURRENT_LIMIT = 10
|
||||
P99_TARGET_MS = 2000 # 2초
|
||||
|
||||
|
||||
class PerformanceBenchmark:
|
||||
def __init__(self, admin_url: str, tables: List[str]):
|
||||
self.admin_url = admin_url
|
||||
self.tables = tables
|
||||
self.results = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"tables": {},
|
||||
"concurrent": {},
|
||||
"summary": {}
|
||||
}
|
||||
|
||||
def _call_table(self, table_name: str) -> Tuple[str, float, int]:
|
||||
"""Call a single table API endpoint and return timing."""
|
||||
url = f"{self.admin_url}/{table_name}"
|
||||
try:
|
||||
start = time.time()
|
||||
response = requests.get(url, timeout=5)
|
||||
elapsed_ms = (time.time() - start) * 1000
|
||||
status = response.status_code
|
||||
return table_name, elapsed_ms, status
|
||||
except Exception as e:
|
||||
return table_name, None, 0
|
||||
|
||||
def benchmark_single_table(self, table_name: str, num_runs: int = NUM_RUNS):
|
||||
"""Benchmark a single table with multiple runs."""
|
||||
times = []
|
||||
errors = 0
|
||||
|
||||
for i in range(num_runs):
|
||||
table, elapsed_ms, status = self._call_table(table_name)
|
||||
if status == 200 and elapsed_ms is not None:
|
||||
times.append(elapsed_ms)
|
||||
else:
|
||||
errors += 1
|
||||
|
||||
if not times:
|
||||
self.results["tables"][table_name] = {
|
||||
"status": "FAILED",
|
||||
"errors": errors,
|
||||
"runs": num_runs
|
||||
}
|
||||
return
|
||||
|
||||
sorted_times = sorted(times)
|
||||
p99_idx = max(0, int(len(sorted_times) * 0.99) - 1)
|
||||
|
||||
self.results["tables"][table_name] = {
|
||||
"status": "PASS" if sorted_times[-1] <= P99_TARGET_MS else "SLOW",
|
||||
"runs": num_runs,
|
||||
"min_ms": round(min(times), 2),
|
||||
"max_ms": round(max(times), 2),
|
||||
"mean_ms": round(statistics.mean(times), 2),
|
||||
"median_ms": round(statistics.median(times), 2),
|
||||
"stdev_ms": round(statistics.stdev(times) if len(times) > 1 else 0, 2),
|
||||
"p99_ms": round(sorted_times[p99_idx], 2),
|
||||
"errors": errors
|
||||
}
|
||||
|
||||
def benchmark_concurrent(self, num_concurrent: int = CONCURRENT_LIMIT):
|
||||
"""Benchmark concurrent table loads."""
|
||||
concurrent_times = []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_concurrent) as executor:
|
||||
futures = {
|
||||
executor.submit(self._call_table, table): table
|
||||
for table in self.tables
|
||||
}
|
||||
|
||||
start = time.time()
|
||||
results_map = {}
|
||||
|
||||
for future in as_completed(futures):
|
||||
table_name, elapsed_ms, status = future.result()
|
||||
if status == 200 and elapsed_ms is not None:
|
||||
results_map[table_name] = elapsed_ms
|
||||
concurrent_times.append(elapsed_ms)
|
||||
|
||||
total_elapsed = (time.time() - start) * 1000
|
||||
|
||||
if concurrent_times:
|
||||
sorted_concurrent = sorted(concurrent_times)
|
||||
p99_idx = max(0, int(len(sorted_concurrent) * 0.99) - 1)
|
||||
|
||||
self.results["concurrent"]["parallel_load"] = {
|
||||
"num_concurrent": num_concurrent,
|
||||
"num_tables": len(results_map),
|
||||
"total_wall_time_ms": round(total_elapsed, 2),
|
||||
"min_table_ms": round(min(concurrent_times), 2),
|
||||
"max_table_ms": round(max(concurrent_times), 2),
|
||||
"p99_table_ms": round(sorted_concurrent[p99_idx], 2),
|
||||
"per_table_times": {k: round(v, 2) for k, v in results_map.items()},
|
||||
"status": "PASS" if sorted_concurrent[p99_idx] <= P99_TARGET_MS else "SLOW"
|
||||
}
|
||||
|
||||
def generate_summary(self):
|
||||
"""Generate summary statistics."""
|
||||
table_results = self.results["tables"]
|
||||
passed = sum(1 for r in table_results.values() if r.get("status") == "PASS")
|
||||
failed = sum(1 for r in table_results.values() if r.get("status") in ["SLOW", "FAILED"])
|
||||
|
||||
all_p99_times = [
|
||||
r["p99_ms"] for r in table_results.values()
|
||||
if "p99_ms" in r
|
||||
]
|
||||
|
||||
self.results["summary"] = {
|
||||
"total_tables": len(table_results),
|
||||
"passed": passed,
|
||||
"failed": failed,
|
||||
"overall_status": "PASS" if failed == 0 else "NEEDS_OPTIMIZATION",
|
||||
"max_p99_ms": max(all_p99_times) if all_p99_times else None,
|
||||
"p99_target_ms": P99_TARGET_MS,
|
||||
"target_met": max(all_p99_times or [0]) <= P99_TARGET_MS
|
||||
}
|
||||
|
||||
def print_report(self):
|
||||
"""Print formatted report."""
|
||||
print("\n" + "=" * 70)
|
||||
print("SNAPSHOT_ADMIN PERFORMANCE BENCHMARK REPORT")
|
||||
print("=" * 70)
|
||||
print(f"Timestamp: {self.results['timestamp']}")
|
||||
print(f"Target P99: {P99_TARGET_MS}ms (< 2 seconds)\n")
|
||||
|
||||
# Individual table results
|
||||
print("TABLE PERFORMANCE:")
|
||||
print("-" * 70)
|
||||
for table_name in sorted(self.results["tables"].keys()):
|
||||
r = self.results["tables"][table_name]
|
||||
if r["status"] in ["PASS", "SLOW"]:
|
||||
status_marker = "✓" if r["status"] == "PASS" else "⚠"
|
||||
print(f"{status_marker} {table_name:25} P99: {r['p99_ms']:7.2f}ms "
|
||||
f"(mean: {r['mean_ms']:7.2f}ms, max: {r['max_ms']:7.2f}ms)")
|
||||
else:
|
||||
print(f"✗ {table_name:25} FAILED ({r['errors']} errors)")
|
||||
|
||||
# Concurrent performance
|
||||
if "parallel_load" in self.results["concurrent"]:
|
||||
c = self.results["concurrent"]["parallel_load"]
|
||||
print("\nCONCURRENT LOAD ({} tables):".format(c["num_concurrent"]))
|
||||
print("-" * 70)
|
||||
print(f"Wall time: {c['total_wall_time_ms']:.2f}ms")
|
||||
print(f"Table P99: {c['p99_table_ms']:.2f}ms (max single: {c['max_table_ms']:.2f}ms)")
|
||||
print(f"Status: {'PASS' if c['status'] == 'PASS' else 'NEEDS_OPTIMIZATION'}")
|
||||
|
||||
# Summary
|
||||
s = self.results["summary"]
|
||||
print("\nSUMMARY:")
|
||||
print("-" * 70)
|
||||
print(f"Status: {s['overall_status']}")
|
||||
print(f"Passed: {s['passed']}/{s['total_tables']} tables")
|
||||
print(f"Max P99: {s['max_p99_ms']:.2f}ms (target: {s['p99_target_ms']}ms)")
|
||||
print(f"Target Met: {'YES ✓' if s['target_met'] else 'NO ✗ (optimization needed)'}")
|
||||
print("=" * 70 + "\n")
|
||||
|
||||
def save_report(self, output_file: str = None):
|
||||
"""Save report to JSON file."""
|
||||
if not output_file:
|
||||
output_file = f"Temp/benchmark_snapshot_admin_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
||||
|
||||
Path(output_file).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(self.results, f, indent=2)
|
||||
print(f"Report saved: {output_file}")
|
||||
|
||||
def run_full_benchmark(self):
|
||||
"""Run complete benchmark suite."""
|
||||
print("Starting snapshot_admin performance benchmark...")
|
||||
print(f"Target: P99 < {P99_TARGET_MS}ms")
|
||||
print(f"Tables: {', '.join(self.tables)}")
|
||||
print(f"Runs per table: {NUM_RUNS}\n")
|
||||
|
||||
# Single table benchmark
|
||||
print("Phase 1: Single table performance...")
|
||||
for table in self.tables:
|
||||
self.benchmark_single_table(table, NUM_RUNS)
|
||||
print(f" ✓ {table}")
|
||||
|
||||
# Concurrent benchmark
|
||||
print(f"\nPhase 2: Concurrent load ({CONCURRENT_LIMIT} tables)...")
|
||||
self.benchmark_concurrent(CONCURRENT_LIMIT)
|
||||
|
||||
# Generate summary
|
||||
self.generate_summary()
|
||||
|
||||
# Output
|
||||
self.print_report()
|
||||
self.save_report()
|
||||
|
||||
return self.results
|
||||
|
||||
|
||||
def optimize_recommendations(results: Dict) -> List[str]:
|
||||
"""Generate optimization recommendations."""
|
||||
recommendations = []
|
||||
summary = results.get("summary", {})
|
||||
|
||||
if not summary.get("target_met"):
|
||||
max_p99 = summary.get("max_p99_ms")
|
||||
if max_p99 and max_p99 > P99_TARGET_MS:
|
||||
ratio = max_p99 / P99_TARGET_MS
|
||||
if ratio > 2:
|
||||
recommendations.append(
|
||||
f"Critical: P99 is {ratio:.1f}x target. "
|
||||
"Consider table indexing, materialized views, or caching."
|
||||
)
|
||||
elif ratio > 1.2:
|
||||
recommendations.append(
|
||||
f"P99 exceeds target by {(ratio-1)*100:.0f}%. "
|
||||
"Optimize database queries or add caching layer."
|
||||
)
|
||||
|
||||
# Check specific slow tables
|
||||
for table, result in results.get("tables", {}).items():
|
||||
if result.get("status") == "SLOW" and "p99_ms" in result:
|
||||
p99 = result["p99_ms"]
|
||||
if p99 > 3000:
|
||||
recommendations.append(
|
||||
f"Table '{table}': {p99:.0f}ms. Consider reducing data volume or adding indexes."
|
||||
)
|
||||
|
||||
if not recommendations:
|
||||
recommendations.append("✓ Performance meets targets. Continue monitoring.")
|
||||
|
||||
return recommendations
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
# Check server availability
|
||||
response = requests.head(f"{ADMIN_URL}/health", timeout=2)
|
||||
if response.status_code not in [200, 404]:
|
||||
print(f"Error: snapshot_admin not available at {ADMIN_URL}")
|
||||
print("Start server: python tools/run_snapshot_admin_server_v1.py")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"Error connecting to {ADMIN_URL}: {e}")
|
||||
print("Start server: python tools/run_snapshot_admin_server_v1.py")
|
||||
sys.exit(1)
|
||||
|
||||
# Run benchmark
|
||||
benchmark = PerformanceBenchmark(ADMIN_URL, TABLES)
|
||||
results = benchmark.run_full_benchmark()
|
||||
|
||||
# Print recommendations
|
||||
print("OPTIMIZATION RECOMMENDATIONS:")
|
||||
print("-" * 70)
|
||||
for rec in optimize_recommendations(results):
|
||||
print(f"• {rec}")
|
||||
print()
|
||||
|
||||
# Exit code based on target met
|
||||
sys.exit(0 if results["summary"]["target_met"] else 1)
|
||||
Reference in New Issue
Block a user