feat(snapshot-admin): align store validation and db snapshots
This commit is contained in:
@@ -1,59 +0,0 @@
|
|||||||
|
|
||||||
# Database Consolidation Plan (2026-06-23)
|
|
||||||
|
|
||||||
## Current State: FRAGMENTED
|
|
||||||
- Canonical: src/quant_engine/ (2 files)
|
|
||||||
- Scattered: outputs/ (10) + Temp/ (3)
|
|
||||||
- Total: 15 database files
|
|
||||||
|
|
||||||
## Issue
|
|
||||||
1. kis_data_collection.db in 3 locations:
|
|
||||||
- src/quant_engine/ (CANONICAL)
|
|
||||||
- outputs/kis_data_collection/
|
|
||||||
- Temp/test_kis_data_collection.db
|
|
||||||
|
|
||||||
2. snapshot_admin.db in 4+ locations:
|
|
||||||
- src/quant_engine/ (CANONICAL)
|
|
||||||
- outputs/snapshot_admin/
|
|
||||||
- Temp/snapshot_admin_*.db (multiple variants)
|
|
||||||
- outputs/qualitative_sell_strategy/ (unrelated)
|
|
||||||
|
|
||||||
## Solution
|
|
||||||
|
|
||||||
### Step 1: Verify Canonical Copies (src/quant_engine/)
|
|
||||||
- kis_data_collection.db: 5 records [OK]
|
|
||||||
- snapshot_admin.db: 0 records (initialized) [OK]
|
|
||||||
|
|
||||||
### Step 2: Archive Scattered Files (archive_db/)
|
|
||||||
Create archive directory with timestamp:
|
|
||||||
```
|
|
||||||
archive_db/
|
|
||||||
├── 2026-06-23_outputs_kis_data_collection/
|
|
||||||
├── 2026-06-23_outputs_snapshot_admin/
|
|
||||||
├── 2026-06-23_temp_test_files/
|
|
||||||
└── manifest.json (record what was archived)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Step 3: Clean Obsolete References
|
|
||||||
- Remove imports from "outputs/kis_data_collection/kis_data_collection.db"
|
|
||||||
- Remove imports from "outputs/snapshot_admin/*.db"
|
|
||||||
- Update any code expecting these paths
|
|
||||||
|
|
||||||
### Step 4: Update Documentation
|
|
||||||
- Update all references to use: src/quant_engine/
|
|
||||||
- Update deployment docs (Synology)
|
|
||||||
- Update CI/CD workflows
|
|
||||||
|
|
||||||
## Benefits
|
|
||||||
- Single source of truth
|
|
||||||
- Easier backup/recovery
|
|
||||||
- Clear separation: live vs. archived
|
|
||||||
- Faster data access
|
|
||||||
- Simplified deployment
|
|
||||||
|
|
||||||
## Files to Delete (After Archiving)
|
|
||||||
- outputs/kis_data_collection/ (entire dir)
|
|
||||||
- outputs/snapshot_admin/smoke*.db (old test files)
|
|
||||||
- outputs/qualitative_sell_strategy/qualitative_sell_strategy.db
|
|
||||||
- Temp/snapshot_admin_*.db
|
|
||||||
- Temp/test_kis_data_collection.db
|
|
||||||
Binary file not shown.
Binary file not shown.
@@ -810,6 +810,25 @@ def _as_number(value: Any) -> float | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _as_int(value: Any) -> int | None:
|
||||||
|
if value is None or value == "":
|
||||||
|
return None
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return None
|
||||||
|
if isinstance(value, int):
|
||||||
|
return value
|
||||||
|
if isinstance(value, float):
|
||||||
|
return int(value) if value.is_integer() else None
|
||||||
|
try:
|
||||||
|
text = str(value).strip().replace(",", "")
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
parsed = float(text)
|
||||||
|
return int(parsed) if parsed.is_integer() else None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(maxsize=1)
|
@lru_cache(maxsize=1)
|
||||||
def _load_settings_spec() -> dict[str, Any]:
|
def _load_settings_spec() -> dict[str, Any]:
|
||||||
return yaml.safe_load(SETTINGS_SPEC_PATH.read_text(encoding="utf-8")) or {}
|
return yaml.safe_load(SETTINGS_SPEC_PATH.read_text(encoding="utf-8")) or {}
|
||||||
@@ -867,8 +886,14 @@ def validate_account_snapshot_rows(rows: list[dict[str, Any]]) -> list[str]:
|
|||||||
name = str(row.get("name") or "").strip()
|
name = str(row.get("name") or "").strip()
|
||||||
account_type = str(row.get("account_type") or "").strip()
|
account_type = str(row.get("account_type") or "").strip()
|
||||||
parse_status = str(row.get("parse_status") or "").strip()
|
parse_status = str(row.get("parse_status") or "").strip()
|
||||||
holding_quantity = _as_number(row.get("holding_quantity"))
|
holding_quantity = _as_int(row.get("holding_quantity"))
|
||||||
|
available_quantity = _as_int(row.get("available_quantity"))
|
||||||
average_cost = _as_number(row.get("average_cost"))
|
average_cost = _as_number(row.get("average_cost"))
|
||||||
|
total_cost = _as_number(row.get("total_cost"))
|
||||||
|
current_price = _as_number(row.get("current_price"))
|
||||||
|
market_value = _as_number(row.get("market_value"))
|
||||||
|
profit_loss = _as_number(row.get("profit_loss"))
|
||||||
|
return_pct = _as_number(row.get("return_pct"))
|
||||||
stop_price = _as_number(row.get("stop_price"))
|
stop_price = _as_number(row.get("stop_price"))
|
||||||
entry_stage = str(row.get("entry_stage") or "").strip()
|
entry_stage = str(row.get("entry_stage") or "").strip()
|
||||||
position_type = str(row.get("position_type") or "").strip()
|
position_type = str(row.get("position_type") or "").strip()
|
||||||
@@ -881,7 +906,7 @@ def validate_account_snapshot_rows(rows: list[dict[str, Any]]) -> list[str]:
|
|||||||
errors.append(f"account_snapshot row {idx}: account_type required")
|
errors.append(f"account_snapshot row {idx}: account_type required")
|
||||||
if account_type and canonical.get("account_type", {}).get("allowed") and account_type not in canonical["account_type"]["allowed"]:
|
if account_type and canonical.get("account_type", {}).get("allowed") and account_type not in canonical["account_type"]["allowed"]:
|
||||||
errors.append(f"account_snapshot row {idx}: invalid account_type {account_type!r}")
|
errors.append(f"account_snapshot row {idx}: invalid account_type {account_type!r}")
|
||||||
if not ticker:
|
if not ticker and name != "예수금/D+2현금":
|
||||||
errors.append(f"account_snapshot row {idx}: ticker required")
|
errors.append(f"account_snapshot row {idx}: ticker required")
|
||||||
if not name:
|
if not name:
|
||||||
errors.append(f"account_snapshot row {idx}: name required")
|
errors.append(f"account_snapshot row {idx}: name required")
|
||||||
@@ -889,8 +914,22 @@ def validate_account_snapshot_rows(rows: list[dict[str, Any]]) -> list[str]:
|
|||||||
errors.append(f"account_snapshot row {idx}: invalid parse_status {parse_status!r}")
|
errors.append(f"account_snapshot row {idx}: invalid parse_status {parse_status!r}")
|
||||||
if holding_quantity is not None and holding_quantity < 0:
|
if holding_quantity is not None and holding_quantity < 0:
|
||||||
errors.append(f"account_snapshot row {idx}: holding_quantity must be >= 0")
|
errors.append(f"account_snapshot row {idx}: holding_quantity must be >= 0")
|
||||||
|
if available_quantity is not None and available_quantity < 0:
|
||||||
|
errors.append(f"account_snapshot row {idx}: available_quantity must be >= 0")
|
||||||
if average_cost is not None and average_cost < 0:
|
if average_cost is not None and average_cost < 0:
|
||||||
errors.append(f"account_snapshot row {idx}: average_cost must be >= 0")
|
errors.append(f"account_snapshot row {idx}: average_cost must be >= 0")
|
||||||
|
if total_cost is not None and total_cost < 0:
|
||||||
|
errors.append(f"account_snapshot row {idx}: total_cost must be >= 0")
|
||||||
|
if current_price is not None and current_price < 0:
|
||||||
|
errors.append(f"account_snapshot row {idx}: current_price must be >= 0")
|
||||||
|
if market_value is not None and market_value < 0:
|
||||||
|
errors.append(f"account_snapshot row {idx}: market_value must be >= 0")
|
||||||
|
if profit_loss is not None and profit_loss != profit_loss:
|
||||||
|
errors.append(f"account_snapshot row {idx}: profit_loss invalid")
|
||||||
|
if return_pct is not None and abs(return_pct) > 1000:
|
||||||
|
errors.append(f"account_snapshot row {idx}: return_pct out of range")
|
||||||
|
if stop_price is not None and stop_price < 0:
|
||||||
|
errors.append(f"account_snapshot row {idx}: stop_price must be >= 0")
|
||||||
if user_confirmed and user_confirmed not in {"Y", "N"}:
|
if user_confirmed and user_confirmed not in {"Y", "N"}:
|
||||||
errors.append(f"account_snapshot row {idx}: user_confirmed must be Y or N")
|
errors.append(f"account_snapshot row {idx}: user_confirmed must be Y or N")
|
||||||
if parse_status == "CAPTURE_READ_OK" and user_confirmed != "Y":
|
if parse_status == "CAPTURE_READ_OK" and user_confirmed != "Y":
|
||||||
|
|||||||
@@ -223,6 +223,12 @@ def test_validation_helpers_detect_invalid_rows():
|
|||||||
assert "account_snapshot row 1: ticker required" in validate_account_snapshot_rows(
|
assert "account_snapshot row 1: ticker required" in validate_account_snapshot_rows(
|
||||||
[{"captured_at": "2026-06-21", "account": "real", "name": "삼성전자", "parse_status": "BAD"}]
|
[{"captured_at": "2026-06-21", "account": "real", "name": "삼성전자", "parse_status": "BAD"}]
|
||||||
)
|
)
|
||||||
|
assert "account_snapshot row 1: ticker must be 6 digits" in validate_account_snapshot_rows(
|
||||||
|
[{"captured_at": "2026-06-21", "account": "real", "account_type": "일반계좌", "ticker": "5930", "name": "삼성전자", "parse_status": "NOT_PROVIDED"}]
|
||||||
|
)
|
||||||
|
assert "account_snapshot row 1: holding_quantity must be >= 0" in validate_account_snapshot_rows(
|
||||||
|
[{"captured_at": "2026-06-21", "account": "real", "account_type": "일반계좌", "ticker": "005930", "name": "삼성전자", "parse_status": "NOT_PROVIDED", "holding_quantity": -1}]
|
||||||
|
)
|
||||||
suggestions = build_validation_suggestions(
|
suggestions = build_validation_suggestions(
|
||||||
[{"key": "weekly_target_cash_pct", "value": 10}],
|
[{"key": "weekly_target_cash_pct", "value": 10}],
|
||||||
[{"captured_at": "2026-06-21", "account": "real", "account_type": "일반계좌", "ticker": "005930", "name": "삼성전자", "parse_status": "CAPTURE_READ_OK", "user_confirmed": "N"}],
|
[{"captured_at": "2026-06-21", "account": "real", "account_type": "일반계좌", "ticker": "005930", "name": "삼성전자", "parse_status": "CAPTURE_READ_OK", "user_confirmed": "N"}],
|
||||||
|
|||||||
Reference in New Issue
Block a user