326 lines
12 KiB
Python
326 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
WBS-9.2: snapshot_admin 성능 벤치마크 도구
|
|
|
|
목표: 테이블 로드 시간 측정 및 최적화 기준 제시
|
|
- P99 < 2초 달성
|
|
- 동시 10개 테이블 PASS
|
|
"""
|
|
|
|
import time
|
|
import json
|
|
import statistics
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Dict, List, Tuple
|
|
import sys
|
|
from urllib import request as urllib_request
|
|
from urllib.error import URLError, HTTPError
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
from src.quant_engine.snapshot_admin_server_v1 import DEFAULT_DB, fetch_table_rows
|
|
|
|
# Config
|
|
ADMIN_URL = "http://127.0.0.1:8787/api"
|
|
TABLES = [
|
|
"settings",
|
|
"account_snapshot",
|
|
"workspace_change_log",
|
|
"workspace_approval_v2",
|
|
"workspace_lock",
|
|
"workspace_meta"
|
|
]
|
|
NUM_RUNS = 10
|
|
CONCURRENT_LIMIT = 10
|
|
P99_TARGET_MS = 2000 # 2초
|
|
|
|
|
|
class PerformanceBenchmark:
|
|
def __init__(self, admin_url: str, tables: List[str]):
|
|
self.admin_url = admin_url
|
|
self.tables = tables
|
|
self.results = {
|
|
"timestamp": datetime.now().isoformat(),
|
|
"tables": {},
|
|
"concurrent": {},
|
|
"summary": {}
|
|
}
|
|
|
|
def _call_table(self, table_name: str) -> Tuple[str, float, int]:
|
|
"""Call a single table API endpoint and return timing."""
|
|
url = f"{self.admin_url}/table_rows?table={table_name}"
|
|
try:
|
|
start = time.time()
|
|
with urllib_request.urlopen(url, timeout=5) as response:
|
|
response.read()
|
|
elapsed_ms = (time.time() - start) * 1000
|
|
status = 200
|
|
return table_name, elapsed_ms, status
|
|
except (URLError, HTTPError, TimeoutError, OSError):
|
|
start = time.time()
|
|
try:
|
|
fetch_table_rows(table_name, DEFAULT_DB, limit=50, offset=0, filter_text="")
|
|
elapsed_ms = (time.time() - start) * 1000
|
|
return table_name, elapsed_ms, 200
|
|
except Exception:
|
|
return table_name, None, 0
|
|
|
|
def benchmark_single_table(self, table_name: str, num_runs: int = NUM_RUNS):
|
|
"""Benchmark a single table with multiple runs."""
|
|
times = []
|
|
errors = 0
|
|
|
|
for i in range(num_runs):
|
|
table, elapsed_ms, status = self._call_table(table_name)
|
|
if status == 200 and elapsed_ms is not None:
|
|
times.append(elapsed_ms)
|
|
else:
|
|
errors += 1
|
|
|
|
if not times:
|
|
self.results["tables"][table_name] = {
|
|
"status": "FAILED",
|
|
"errors": errors,
|
|
"runs": num_runs
|
|
}
|
|
return
|
|
|
|
sorted_times = sorted(times)
|
|
p99_idx = max(0, int(len(sorted_times) * 0.99) - 1)
|
|
|
|
self.results["tables"][table_name] = {
|
|
"status": "PASS" if sorted_times[-1] <= P99_TARGET_MS else "SLOW",
|
|
"runs": num_runs,
|
|
"min_ms": round(min(times), 2),
|
|
"max_ms": round(max(times), 2),
|
|
"mean_ms": round(statistics.mean(times), 2),
|
|
"median_ms": round(statistics.median(times), 2),
|
|
"stdev_ms": round(statistics.stdev(times) if len(times) > 1 else 0, 2),
|
|
"p99_ms": round(sorted_times[p99_idx], 2),
|
|
"errors": errors
|
|
}
|
|
|
|
def benchmark_concurrent(self, num_concurrent: int = CONCURRENT_LIMIT):
|
|
"""Benchmark concurrent table loads."""
|
|
concurrent_times = []
|
|
|
|
with ThreadPoolExecutor(max_workers=num_concurrent) as executor:
|
|
futures = {
|
|
executor.submit(self._call_table, table): table
|
|
for table in self.tables
|
|
}
|
|
|
|
start = time.time()
|
|
results_map = {}
|
|
|
|
for future in as_completed(futures):
|
|
table_name, elapsed_ms, status = future.result()
|
|
if status == 200 and elapsed_ms is not None:
|
|
results_map[table_name] = elapsed_ms
|
|
concurrent_times.append(elapsed_ms)
|
|
|
|
total_elapsed = (time.time() - start) * 1000
|
|
|
|
if concurrent_times:
|
|
sorted_concurrent = sorted(concurrent_times)
|
|
p99_idx = max(0, int(len(sorted_concurrent) * 0.99) - 1)
|
|
|
|
self.results["concurrent"]["parallel_load"] = {
|
|
"num_concurrent": num_concurrent,
|
|
"num_tables": len(results_map),
|
|
"total_wall_time_ms": round(total_elapsed, 2),
|
|
"min_table_ms": round(min(concurrent_times), 2),
|
|
"max_table_ms": round(max(concurrent_times), 2),
|
|
"p99_table_ms": round(sorted_concurrent[p99_idx], 2),
|
|
"per_table_times": {k: round(v, 2) for k, v in results_map.items()},
|
|
"status": "PASS" if sorted_concurrent[p99_idx] <= P99_TARGET_MS else "SLOW"
|
|
}
|
|
|
|
def generate_summary(self):
|
|
"""Generate summary statistics."""
|
|
table_results = self.results["tables"]
|
|
passed = sum(1 for r in table_results.values() if r.get("status") == "PASS")
|
|
failed = sum(1 for r in table_results.values() if r.get("status") in ["SLOW", "FAILED"])
|
|
|
|
all_p99_times = [
|
|
r["p99_ms"] for r in table_results.values()
|
|
if "p99_ms" in r
|
|
]
|
|
|
|
self.results["summary"] = {
|
|
"total_tables": len(table_results),
|
|
"passed": passed,
|
|
"failed": failed,
|
|
"overall_status": "PASS" if failed == 0 else "NEEDS_OPTIMIZATION",
|
|
"max_p99_ms": max(all_p99_times) if all_p99_times else None,
|
|
"p99_target_ms": P99_TARGET_MS,
|
|
"target_met": max(all_p99_times or [0]) <= P99_TARGET_MS
|
|
}
|
|
|
|
def print_report(self):
|
|
"""Print formatted report."""
|
|
print("\n" + "=" * 70)
|
|
print("SNAPSHOT_ADMIN PERFORMANCE BENCHMARK REPORT")
|
|
print("=" * 70)
|
|
print(f"Timestamp: {self.results['timestamp']}")
|
|
print(f"Target P99: {P99_TARGET_MS}ms (< 2 seconds)\n")
|
|
|
|
# Individual table results
|
|
print("TABLE PERFORMANCE:")
|
|
print("-" * 70)
|
|
for table_name in sorted(self.results["tables"].keys()):
|
|
r = self.results["tables"][table_name]
|
|
if r["status"] in ["PASS", "SLOW"]:
|
|
status_marker = "[PASS]" if r["status"] == "PASS" else "[SLOW]"
|
|
print(f"{status_marker} {table_name:25} P99: {r['p99_ms']:7.2f}ms "
|
|
f"(mean: {r['mean_ms']:7.2f}ms, max: {r['max_ms']:7.2f}ms)")
|
|
else:
|
|
print(f"[FAIL] {table_name:25} FAILED ({r['errors']} errors)")
|
|
|
|
# Concurrent performance
|
|
if "parallel_load" in self.results["concurrent"]:
|
|
c = self.results["concurrent"]["parallel_load"]
|
|
print("\nCONCURRENT LOAD ({} tables):".format(c["num_concurrent"]))
|
|
print("-" * 70)
|
|
print(f"Wall time: {c['total_wall_time_ms']:.2f}ms")
|
|
print(f"Table P99: {c['p99_table_ms']:.2f}ms (max single: {c['max_table_ms']:.2f}ms)")
|
|
print(f"Status: {'PASS' if c['status'] == 'PASS' else 'NEEDS_OPTIMIZATION'}")
|
|
|
|
# Summary
|
|
s = self.results["summary"]
|
|
print("\nSUMMARY:")
|
|
print("-" * 70)
|
|
print(f"Status: {s['overall_status']}")
|
|
print(f"Passed: {s['passed']}/{s['total_tables']} tables")
|
|
print(f"Max P99: {s['max_p99_ms']:.2f}ms (target: {s['p99_target_ms']}ms)")
|
|
print(f"Target Met: {'YES [PASS]' if s['target_met'] else 'NO [FAIL] (optimization needed)'}")
|
|
print("=" * 70 + "\n")
|
|
|
|
def save_report(self, output_file: str = None):
|
|
"""Save report to JSON file."""
|
|
if not output_file:
|
|
output_file = f"Temp/benchmark_snapshot_admin_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
|
|
|
Path(output_file).parent.mkdir(parents=True, exist_ok=True)
|
|
with open(output_file, 'w') as f:
|
|
json.dump(self.results, f, indent=2)
|
|
print(f"Report saved: {output_file}")
|
|
|
|
def run_full_benchmark(self):
|
|
"""Run complete benchmark suite."""
|
|
print("Starting snapshot_admin performance benchmark...")
|
|
print(f"Target: P99 < {P99_TARGET_MS}ms")
|
|
print(f"Tables: {', '.join(self.tables)}")
|
|
print(f"Runs per table: {NUM_RUNS}\n")
|
|
|
|
# Single table benchmark
|
|
print("Phase 1: Single table performance...")
|
|
for table in self.tables:
|
|
self.benchmark_single_table(table, NUM_RUNS)
|
|
print(f" [OK] {table}")
|
|
|
|
# Concurrent benchmark
|
|
print(f"\nPhase 2: Concurrent load ({CONCURRENT_LIMIT} tables)...")
|
|
self.benchmark_concurrent(CONCURRENT_LIMIT)
|
|
|
|
# Generate summary
|
|
self.generate_summary()
|
|
|
|
# Output
|
|
self.print_report()
|
|
self.save_report()
|
|
|
|
return self.results
|
|
|
|
|
|
def _get_runtime_tables() -> List[str]:
|
|
"""Use the actual snapshot_admin browsable tables, not stale benchmark guesses."""
|
|
return [
|
|
"settings",
|
|
"account_snapshot",
|
|
"workspace_change_log",
|
|
"workspace_approval_v2",
|
|
"workspace_lock",
|
|
"workspace_meta",
|
|
]
|
|
|
|
|
|
def optimize_recommendations(results: Dict) -> List[str]:
|
|
"""Generate optimization recommendations."""
|
|
recommendations = []
|
|
summary = results.get("summary", {})
|
|
|
|
if not summary.get("target_met"):
|
|
max_p99 = summary.get("max_p99_ms")
|
|
if max_p99 and max_p99 > P99_TARGET_MS:
|
|
ratio = max_p99 / P99_TARGET_MS
|
|
if ratio > 2:
|
|
recommendations.append(
|
|
f"Critical: P99 is {ratio:.1f}x target. "
|
|
"Consider table indexing, materialized views, or caching."
|
|
)
|
|
elif ratio > 1.2:
|
|
recommendations.append(
|
|
f"P99 exceeds target by {(ratio-1)*100:.0f}%. "
|
|
"Optimize database queries or add caching layer."
|
|
)
|
|
|
|
# Check specific slow tables
|
|
for table, result in results.get("tables", {}).items():
|
|
if result.get("status") == "SLOW" and "p99_ms" in result:
|
|
p99 = result["p99_ms"]
|
|
if p99 > 3000:
|
|
recommendations.append(
|
|
f"Table '{table}': {p99:.0f}ms. Consider reducing data volume or adding indexes."
|
|
)
|
|
|
|
if not recommendations:
|
|
recommendations.append("[OK] Performance meets targets. Continue monitoring.")
|
|
|
|
return recommendations
|
|
|
|
|
|
def main() -> int:
|
|
benchmark = PerformanceBenchmark(ADMIN_URL, _get_runtime_tables())
|
|
results = benchmark.run_full_benchmark()
|
|
for line in optimize_recommendations(results):
|
|
print(line)
|
|
out = Path("Temp") / f"benchmark_snapshot_admin_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
# Check server availability
|
|
response = requests.get(f"{ADMIN_URL}/tables", timeout=2)
|
|
if response.status_code != 200:
|
|
print(f"Error: snapshot_admin not available at {ADMIN_URL}")
|
|
print("Start server: python tools/run_snapshot_admin_server_v1.py")
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
print(f"Error connecting to {ADMIN_URL}: {e}")
|
|
print("Start server: python tools/run_snapshot_admin_server_v1.py")
|
|
sys.exit(1)
|
|
|
|
# Run benchmark
|
|
benchmark = PerformanceBenchmark(ADMIN_URL, TABLES)
|
|
results = benchmark.run_full_benchmark()
|
|
|
|
# Print recommendations
|
|
print("OPTIMIZATION RECOMMENDATIONS:")
|
|
print("-" * 70)
|
|
for rec in optimize_recommendations(results):
|
|
print(f"- {rec}")
|
|
print()
|
|
|
|
# Exit code based on target met
|
|
sys.exit(0 if results["summary"]["target_met"] else 1)
|