feat(db): fully deprecate and delete legacy Python SQLite databases and tools, consolidating into PostgreSQL

spec/61_dotnet_postgresql_json_cutover.yaml

@@ -8,8 +8,8 @@ canonical_runtime:
   migration: src/dotnet/QuantEngine.Infrastructure/Migrations/V5__Add_Normalized_Learning_History.sql
   output: Temp/kis_dotnet_collection_v1.json
 legacy_policy:
-  python_collector: migration_only
-  sqlite_store: migration_only
+  python_collector: forbidden
+  sqlite_store: forbidden
   xlsx_runtime_input: forbidden
 gates:
   - dotnet_collector_registered

src/quant_engine/data_collection_store_v1.py

@@ -1,464 +0,0 @@
-"""SQLite store for platform-transition data collection outputs.
-
-This store is intentionally small and backend-agnostic enough to be upgraded to
-PostgreSQL later without changing the row contract. The canonical payload is the
-normalized factor row plus provenance metadata.
-"""
-from __future__ import annotations
-
-import json
-import sqlite3
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Iterable
-
-
-SCHEMA = """
-PRAGMA journal_mode=WAL;
-
-CREATE TABLE IF NOT EXISTS collection_runs (
-    run_id TEXT PRIMARY KEY,
-    collector_name TEXT NOT NULL,
-    started_at TEXT NOT NULL,
-    finished_at TEXT,
-    status TEXT NOT NULL,
-    input_source TEXT,
-    output_json_path TEXT,
-    output_db_path TEXT,
-    notes TEXT,
-    created_at TEXT DEFAULT (datetime('now'))
-);
-
-CREATE TABLE IF NOT EXISTS collection_snapshots (
-    run_id TEXT NOT NULL,
-    dataset_name TEXT NOT NULL,
-    ticker TEXT NOT NULL,
-    name TEXT,
-    sector TEXT,
-    as_of_date TEXT,
-    source_priority TEXT,
-    source_status TEXT,
-    payload_json TEXT NOT NULL,
-    provenance_json TEXT NOT NULL,
-    created_at TEXT DEFAULT (datetime('now')),
-    PRIMARY KEY (run_id, dataset_name, ticker)
-);
-
-CREATE TABLE IF NOT EXISTS collection_source_errors (
-    run_id TEXT NOT NULL,
-    ticker TEXT,
-    source_name TEXT NOT NULL,
-    error_kind TEXT NOT NULL,
-    error_message TEXT NOT NULL,
-    payload_json TEXT,
-    created_at TEXT DEFAULT (datetime('now'))
-);
-
-CREATE INDEX IF NOT EXISTS idx_collection_snapshots_ticker_time
-    ON collection_snapshots(ticker, created_at DESC);
-
-CREATE INDEX IF NOT EXISTS idx_collection_source_errors_run
-    ON collection_source_errors(run_id, source_name);
-"""
-
-
-@dataclass(frozen=True)
-class CollectionRun:
-    run_id: str
-    collector_name: str
-    started_at: str
-    status: str
-    input_source: str | None = None
-    output_json_path: str | None = None
-    output_db_path: str | None = None
-    notes: str | None = None
-
-
-# SQLite와 PostgreSQL 연결을 동적으로 감지하여 연결 인스턴스를 리턴하는 헬퍼
-def _get_connection(db_target: Path | str) -> Any:
-    db_str = str(db_target)
-    if db_str.startswith("postgresql://") or db_str.startswith("postgres://"):
-        try:
-            import psycopg2
-            from psycopg2.extras import RealDictCursor
-            conn = psycopg2.connect(db_str)
-            # SQLite의 row_factory = Row 처럼 dict 접근을 가능하게 설정
-            return conn
-        except ImportError:
-            raise ImportError("PostgreSQL DSN이 제공되었으나 psycopg2 패키지가 설치되어 있지 않습니다.")
-    else:
-        return sqlite3.connect(Path(db_target))
-
-
-def init_db(db_target: Path | str) -> None:
-    db_str = str(db_target)
-    if db_str.startswith("postgresql://") or db_str.startswith("postgres://"):
-        # PostgreSQL은 DB 서버 측에서 직접 Schema 생성을 관리하므로, CLI 도구가 생성한 DDL 마이그레이션 스텁을 사용합니다.
-        # 런타임 수집 중 자동 DDL 실행은 락 이슈 예방을 위해 스킵하고 트랜잭션 연결만 보장합니다.
-        conn = _get_connection(db_target)
-        conn.close()
-        return
-
-    db_path = Path(db_target)
-    db_path.parent.mkdir(parents=True, exist_ok=True)
-    conn = sqlite3.connect(db_path)
-    try:
-        conn.executescript(SCHEMA)
-        conn.commit()
-    finally:
-        conn.close()
-
-
-def upsert_collection_run(db_target: Path | str, run: CollectionRun, finished_at: str | None = None) -> None:
-    init_db(db_target)
-    conn = _get_connection(db_target)
-    db_str = str(db_target)
-    is_pg = db_str.startswith("postgresql://") or db_str.startswith("postgres://")
-    try:
-        # SQLite와 PostgreSQL 쿼리 바인딩 플레이스홀더 분기 (? vs %s)
-        param_char = "%s" if is_pg else "?"
-        query = f"""
-            INSERT INTO collection_runs (
-                run_id, collector_name, started_at, finished_at, status,
-                input_source, output_json_path, output_db_path, notes
-            ) VALUES ({', '.join([param_char]*9)})
-            ON CONFLICT(run_id) DO UPDATE SET
-                collector_name=EXCLUDED.collector_name,
-                started_at=EXCLUDED.started_at,
-                finished_at=EXCLUDED.finished_at,
-                status=EXCLUDED.status,
-                input_source=EXCLUDED.input_source,
-                output_json_path=EXCLUDED.output_json_path,
-                output_db_path=EXCLUDED.output_db_path,
-                notes=EXCLUDED.notes
-        """
-        # PostgreSQL은 ON CONFLICT 테이블명 제외, EXCLUDED는 대소문자 무관하지만 PostgreSQL의 표준은 대문자 EXCLUDED를 권장
-        cursor = conn.cursor()
-        cursor.execute(
-            query,
-            (
-                run.run_id,
-                run.collector_name,
-                run.started_at,
-                finished_at,
-                run.status,
-                run.input_source,
-                run.output_json_path,
-                run.output_db_path,
-                run.notes,
-            ),
-        )
-        conn.commit()
-    finally:
-        conn.close()
-
-
-def upsert_collection_snapshot(
-    db_target: Path | str,
-    *,
-    run_id: str,
-    dataset_name: str,
-    ticker: str,
-    name: str | None,
-    sector: str | None,
-    as_of_date: str | None,
-    source_priority: str,
-    source_status: str,
-    payload: dict[str, Any],
-    provenance: dict[str, Any],
-) -> None:
-    init_db(db_target)
-    conn = _get_connection(db_target)
-    db_str = str(db_target)
-    is_pg = db_str.startswith("postgresql://") or db_str.startswith("postgres://")
-    try:
-        param_char = "%s" if is_pg else "?"
-        query = f"""
-            INSERT INTO collection_snapshots (
-                run_id, dataset_name, ticker, name, sector, as_of_date,
-                source_priority, source_status, payload_json, provenance_json
-            ) VALUES ({', '.join([param_char]*10)})
-            ON CONFLICT(run_id, dataset_name, ticker) DO UPDATE SET
-                name=EXCLUDED.name,
-                sector=EXCLUDED.sector,
-                as_of_date=EXCLUDED.as_of_date,
-                source_priority=EXCLUDED.source_priority,
-                source_status=EXCLUDED.source_status,
-                payload_json=EXCLUDED.payload_json,
-                provenance_json=EXCLUDED.provenance_json
-        """
-        cursor = conn.cursor()
-        cursor.execute(
-            query,
-            (
-                run_id,
-                dataset_name,
-                ticker,
-                name,
-                sector,
-                as_of_date,
-                source_priority,
-                source_status,
-                json.dumps(payload, ensure_ascii=False, default=str),
-                json.dumps(provenance, ensure_ascii=False, default=str),
-            ),
-        )
-        conn.commit()
-    finally:
-        conn.close()
-
-
-def append_collection_error(
-    db_target: Path | str,
-    *,
-    run_id: str,
-    source_name: str,
-    error_kind: str,
-    error_message: str,
-    ticker: str | None = None,
-    payload: dict[str, Any] | None = None,
-) -> None:
-    init_db(db_target)
-    conn = _get_connection(db_target)
-    db_str = str(db_target)
-    is_pg = db_str.startswith("postgresql://") or db_str.startswith("postgres://")
-    try:
-        param_char = "%s" if is_pg else "?"
-        query = f"""
-            INSERT INTO collection_source_errors (
-                run_id, ticker, source_name, error_kind, error_message, payload_json
-            ) VALUES ({', '.join([param_char]*6)})
-        """
-        cursor = conn.cursor()
-        cursor.execute(
-            query,
-            (
-                run_id,
-                ticker,
-                source_name,
-                error_kind,
-                error_message,
-                json.dumps(payload or {}, ensure_ascii=False, default=str),
-            ),
-        )
-        conn.commit()
-    finally:
-        conn.close()
-
-
-def fetch_latest_snapshots(db_target: Path | str, ticker: str, dataset_name: str | None = None) -> list[dict[str, Any]]:
-    db_str = str(db_target)
-    is_pg = db_str.startswith("postgresql://") or db_str.startswith("postgres://")
-    if not is_pg and not Path(db_target).exists():
-        return []
-
-    conn = _get_connection(db_target)
-    if not is_pg:
-        conn.row_factory = sqlite3.Row
-    try:
-        param_char = "%s" if is_pg else "?"
-        cursor = conn.cursor()
-        if dataset_name:
-            cursor.execute(
-                f"""
-                SELECT * FROM collection_snapshots
-                WHERE ticker = {param_char} AND dataset_name = {param_char}
-                ORDER BY created_at DESC
-                """,
-                (ticker, dataset_name),
-            )
-        else:
-            cursor.execute(
-                f"""
-                SELECT * FROM collection_snapshots
-                WHERE ticker = {param_char}
-                ORDER BY created_at DESC
-                """,
-                (ticker,),
-            )
-        rows = cursor.fetchall()
-        return [dict(row) for row in rows]
-    finally:
-        conn.close()
-
-
-def iter_recent_snapshots(db_target: Path | str, limit: int = 50) -> Iterable[dict[str, Any]]:
-    db_str = str(db_target)
-    is_pg = db_str.startswith("postgresql://") or db_str.startswith("postgres://")
-    if not is_pg and not Path(db_target).exists():
-        return []
-
-    conn = _get_connection(db_target)
-    if not is_pg:
-        conn.row_factory = sqlite3.Row
-    try:
-        param_char = "%s" if is_pg else "?"
-        cursor = conn.cursor()
-        cursor.execute(
-            f"SELECT * FROM collection_snapshots ORDER BY created_at DESC LIMIT {param_char}",
-            (limit,),
-        )
-        rows = cursor.fetchall()
-        return [dict(row) for row in rows]
-    finally:
-        conn.close()
-
-
-def load_collection_runs(db_target: Path | str, limit: int = 20) -> list[dict[str, Any]]:
-    db_str = str(db_target)
-    is_pg = db_str.startswith("postgresql://") or db_str.startswith("postgres://")
-    if not is_pg and not Path(db_target).exists():
-        return []
-
-    conn = _get_connection(db_target)
-    if not is_pg:
-        conn.row_factory = sqlite3.Row
-    try:
-        param_char = "%s" if is_pg else "?"
-        cursor = conn.cursor()
-        cursor.execute(
-            f"""
-            SELECT run_id, collector_name, started_at, finished_at, status,
-                   input_source, output_json_path, output_db_path, notes, created_at
-            FROM collection_runs
-            ORDER BY started_at DESC, created_at DESC
-            LIMIT {param_char}
-            """,
-            (int(limit),),
-        )
-        rows = cursor.fetchall()
-        return [dict(row) for row in rows]
-    finally:
-        conn.close()
-
-
-def load_collection_errors(db_target: Path | str, limit: int = 20) -> list[dict[str, Any]]:
-    db_str = str(db_target)
-    is_pg = db_str.startswith("postgresql://") or db_str.startswith("postgres://")
-    if not is_pg and not Path(db_target).exists():
-        return []
-
-    conn = _get_connection(db_target)
-    if not is_pg:
-        conn.row_factory = sqlite3.Row
-    try:
-        param_char = "%s" if is_pg else "?"
-        cursor = conn.cursor()
-        cursor.execute(
-            f"""
-            SELECT run_id, ticker, source_name, error_kind, error_message, payload_json, created_at
-            FROM collection_source_errors
-            ORDER BY created_at DESC
-            LIMIT {param_char}
-            """,
-            (int(limit),),
-        )
-        rows = cursor.fetchall()
-        return [dict(row) for row in rows]
-    finally:
-        conn.close()
-
-
-def load_collection_dashboard_state(
-    db_target: Path | str | None = None,
-    output_json_path: Path | str | None = None,
-    *,
-    limit: int = 8,
-) -> dict[str, Any]:
-    db_str = str(db_target or "")
-    is_pg = db_str.startswith("postgresql://") or db_str.startswith("postgres://")
-    db = Path(db_target) if db_target and not is_pg else Path()
-    report = Path(output_json_path) if output_json_path else Path()
-    state: dict[str, Any] = {
-        "db_path": db_str,
-        "output_json_path": str(report) if output_json_path else "",
-        "runs": [],
-        "recent_snapshots": [],
-        "recent_errors": [],
-        "counts": {
-            "collection_runs": 0,
-            "collection_snapshots": 0,
-            "collection_source_errors": 0,
-        },
-        "latest_run": {},
-        "latest_report": {},
-    }
-    if report.exists():
-        try:
-            state["latest_report"] = json.loads(report.read_text(encoding="utf-8"))
-        except Exception:
-            state["latest_report"] = {}
-
-    if not is_pg and (not db_target or not db.exists()):
-        return state
-
-    conn = _get_connection(db_target)
-    if not is_pg:
-        conn.row_factory = sqlite3.Row
-    try:
-        cursor = conn.cursor()
-        state["counts"] = {
-            "collection_runs": cursor.execute("SELECT COUNT(*) FROM collection_runs").fetchone()[0] if not is_pg else cursor.execute("SELECT COUNT(*) FROM collection_runs") or 0,
-            "collection_snapshots": cursor.execute("SELECT COUNT(*) FROM collection_snapshots").fetchone()[0] if not is_pg else cursor.execute("SELECT COUNT(*) FROM collection_snapshots") or 0,
-            "collection_source_errors": cursor.execute("SELECT COUNT(*) FROM collection_source_errors").fetchone()[0] if not is_pg else cursor.execute("SELECT COUNT(*) FROM collection_source_errors") or 0,
-        }
-        # PostgreSQL인 경우 단순 fetchone() 보완
-        if is_pg:
-            # PostgreSQL count 처리
-            cursor.execute("SELECT COUNT(*) FROM collection_runs")
-            state["counts"]["collection_runs"] = cursor.fetchone()[0]
-            cursor.execute("SELECT COUNT(*) FROM collection_snapshots")
-            state["counts"]["collection_snapshots"] = cursor.fetchone()[0]
-            cursor.execute("SELECT COUNT(*) FROM collection_source_errors")
-            state["counts"]["collection_source_errors"] = cursor.fetchone()[0]
-
-        cursor.execute(
-            """
-            SELECT run_id, collector_name, started_at, finished_at, status,
-                   input_source, output_json_path, output_db_path, notes, created_at
-            FROM collection_runs
-            ORDER BY started_at DESC, created_at DESC
-            LIMIT 1
-            """
-        )
-        run_row = cursor.fetchone()
-        state["latest_run"] = dict(run_row) if run_row is not None else {}
-
-        param_char = "%s" if is_pg else "?"
-        cursor.execute(
-            f"""
-            SELECT run_id, collector_name, started_at, finished_at, status,
-                   input_source, output_json_path, output_db_path, notes, created_at
-            FROM collection_runs
-            ORDER BY started_at DESC, created_at DESC
-            LIMIT {param_char}
-            """,
-            (int(limit),),
-        )
-        state["runs"] = [dict(row) for row in cursor.fetchall()]
-
-        cursor.execute(
-            f"""
-            SELECT run_id, dataset_name, ticker, name, sector, as_of_date,
-                   source_priority, source_status, created_at
-            FROM collection_snapshots
-            ORDER BY created_at DESC
-            LIMIT {param_char}
-            """,
-            (int(limit),),
-        )
-        state["recent_snapshots"] = [dict(row) for row in cursor.fetchall()]
-
-        cursor.execute(
-            f"""
-            SELECT run_id, ticker, source_name, error_kind, error_message, created_at
-            FROM collection_source_errors
-            ORDER BY created_at DESC
-            LIMIT {param_char}
-            """,
-            (int(limit),),
-        )
-        state["recent_errors"] = [dict(row) for row in cursor.fetchall()]
-    finally:
-        conn.close()
-    return state

tools/initialize_databases.py

@@ -1,142 +0,0 @@
-#!/usr/bin/env python3
-"""
-데이터베이스 초기화 도구 (2개 DB)
-1. kis_data_collection.db - KIS API 데이터 수집
-2. snapshot_admin.db - 성능/포지션 관리
-"""
-
-import sqlite3
-from pathlib import Path
-from datetime import datetime
-
-DB1_PATH = "src/quant_engine/kis_data_collection.db"
-DB2_PATH = "src/quant_engine/snapshot_admin.db"
-
-def create_kis_db():
-    """kis_data_collection.db: KIS API 데이터 스키마"""
-    conn = sqlite3.connect(DB1_PATH)
-    cursor = conn.cursor()
-
-    cursor.execute("""
-        CREATE TABLE IF NOT EXISTS data_feed (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            ticker TEXT NOT NULL,
-            name TEXT,
-            close_price REAL,
-            entry_price REAL,
-            quantity INTEGER,
-            stop_price REAL,
-            target_price REAL,
-            entry_stage TEXT,
-            account TEXT,
-            entry_date TEXT,
-            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            velocity_1d REAL,
-            velocity_5d REAL,
-            ma20 REAL,
-            atr20 REAL,
-            rsi_14 REAL,
-            volume INTEGER,
-            avg_trade_value_5d REAL,
-            sector TEXT,
-            beta REAL,
-            UNIQUE(ticker, entry_date)
-        )
-    """)
-
-    cursor.execute("CREATE INDEX IF NOT EXISTS idx_data_feed_ticker ON data_feed(ticker)")
-    cursor.execute("CREATE INDEX IF NOT EXISTS idx_data_feed_entry_date ON data_feed(entry_date)")
-
-    conn.commit()
-    conn.close()
-    print(f"[OK] kis_data_collection.db: data_feed 테이블 생성")
-
-def create_snapshot_admin_db():
-    """snapshot_admin.db: 성능/포지션 스키마"""
-    conn = sqlite3.connect(DB2_PATH)
-    cursor = conn.cursor()
-
-    cursor.execute("""
-        CREATE TABLE IF NOT EXISTS performance (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            ticker TEXT NOT NULL,
-            name TEXT,
-            entry_date TEXT NOT NULL,
-            entry_price REAL NOT NULL,
-            quantity INTEGER,
-            stop_price REAL,
-            target_price REAL,
-            exit_date TEXT,
-            current_price REAL,
-            pnl_pct REAL,
-            status TEXT,
-            t20_milestone TEXT,
-            entry_stage TEXT,
-            account TEXT,
-            recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            UNIQUE(ticker, entry_date)
-        )
-    """)
-
-    cursor.execute("""
-        CREATE TABLE IF NOT EXISTS positions (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            ticker TEXT NOT NULL UNIQUE,
-            name TEXT,
-            quantity INTEGER,
-            entry_price REAL,
-            current_price REAL,
-            average_cost REAL,
-            sector TEXT,
-            weight_pct REAL,
-            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-        )
-    """)
-
-    cursor.execute("CREATE INDEX IF NOT EXISTS idx_performance_ticker ON performance(ticker)")
-    cursor.execute("CREATE INDEX IF NOT EXISTS idx_performance_entry_date ON performance(entry_date)")
-    cursor.execute("CREATE INDEX IF NOT EXISTS idx_positions_ticker ON positions(ticker)")
-
-    conn.commit()
-    conn.close()
-    print(f"[OK] snapshot_admin.db: performance, positions 테이블 생성")
-
-def verify():
-    """검증"""
-    print("\n" + "="*80)
-    print("데이터베이스 구조 확인")
-    print("="*80)
-
-    for db_path, db_name in [(DB1_PATH, "kis_data_collection.db"), (DB2_PATH, "snapshot_admin.db")]:
-        conn = sqlite3.connect(db_path)
-        cursor = conn.cursor()
-
-        cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
-        tables = [row[0] for row in cursor.fetchall()]
-
-        file_size = Path(db_path).stat().st_size / 1024
-        print(f"\n[{db_name}]")
-        print(f"  크기: {file_size:.2f} KB")
-        print(f"  테이블: {', '.join(tables)}")
-
-        for table in tables:
-            if table != 'sqlite_sequence':
-                cursor.execute(f"SELECT COUNT(*) FROM {table}")
-                row_count = cursor.fetchone()[0]
-                cursor.execute(f"PRAGMA table_info({table})")
-                col_count = len(cursor.fetchall())
-                print(f"    - {table}: {col_count}개 컬럼, {row_count}개 행")
-
-        conn.close()
-
-if __name__ == "__main__":
-    print("2개 데이터베이스 초기화 중...\n")
-
-    create_kis_db()
-    create_snapshot_admin_db()
-
-    verify()
-
-    print("\n[완료] 2개 DB 초기화 완료!")
-    print(f"  1. kis_data_collection.db (KIS API)")
-    print(f"  2. snapshot_admin.db (성능/포지션)")