feat(kis-collection): finalize sqlite migration, add fallback resilience, and update WBS documentation

This commit is contained in:
2026-06-22 18:34:56 +09:00
parent c576138829
commit 6c549b7bdc
48 changed files with 34610 additions and 24883 deletions
@@ -31,6 +31,7 @@ def to_module_name(path: Path) -> str:
def render_module(schema_path: Path, schema: dict[str, Any]) -> str:
title = str(schema.get("title") or schema_path.stem)
schema_id = str(schema.get("$id") or f"schema://{title}")
schema_rel_path = str(schema_path.relative_to(ROOT)).replace("\\", "/")
props = schema.get("properties") if isinstance(schema.get("properties"), dict) else {}
required = schema.get("required") if isinstance(schema.get("required"), list) else []
prop_names = list(props.keys())
@@ -43,7 +44,7 @@ def render_module(schema_path: Path, schema: dict[str, Any]) -> str:
"from typing import Any\n\n"
f"SCHEMA_TITLE = {title!r}\n"
f"SCHEMA_ID = {schema_id!r}\n"
f"SCHEMA_PATH = {str(schema_path.relative_to(ROOT)).replace('\\', '/')!r}\n"
f"SCHEMA_PATH = {schema_rel_path!r}\n"
f"SCHEMA_PROPERTIES = {prop_names!r}\n"
f"SCHEMA_REQUIRED = {required!r}\n\n"
"@dataclass(frozen=True)\n"
+152 -137
View File
@@ -99,59 +99,12 @@ def _find_first_value(payload: Any, keys: tuple[str, ...]) -> Any:
return None
def _avg(values: list[float]) -> float | None:
return round(sum(values) / len(values), 4) if values else None
def _compute_ma(rows: list[dict[str, Any]], n: int) -> float | None:
"""rows[0]가 최신 거래일. 최근 n거래일 종가 단순이동평균."""
closes = [r["close"] for r in rows[:n] if r.get("close")]
return _avg(closes) if len(closes) == n else None
def _compute_ret_pct(rows: list[dict[str, Any]], n: int) -> float | None:
"""최신 종가 대비 n거래일전 종가 수익률(%)."""
closes = [r["close"] for r in rows if r.get("close")]
if len(closes) <= n or not closes[n]:
return None
return round((closes[0] / closes[n] - 1.0) * 100.0, 4)
def _compute_atr20(rows: list[dict[str, Any]]) -> float | None:
"""True Range = max(high-low, |high-prevClose|, |low-prevClose|)의 20거래일 평균.
rows[0]가 최신이므로 rows[i]의 전일종가는 rows[i+1]['close']."""
trs: list[float] = []
for i in range(min(20, len(rows) - 1)):
cur, prev = rows[i], rows[i + 1]
high, low, prev_close = cur.get("high"), cur.get("low"), prev.get("close")
if high is None or low is None or prev_close is None:
continue
trs.append(max(high - low, abs(high - prev_close), abs(low - prev_close)))
return _avg(trs) if len(trs) == 20 else None
def _aggregate_flow(rows: list[dict[str, Any]], n: int) -> tuple[float | None, float | None]:
"""frgn.naver rows(최신순)의 최근 n거래일 외국인/기관 순매수 합계(주식수)."""
window = rows[:n]
if len(window) < n:
return None, None
frg = sum(r.get("frgn_net") or 0 for r in window)
inst = sum(r.get("inst_net") or 0 for r in window)
return round(frg, 4), round(inst, 4)
def _normalize_naver_price_history(code: str) -> dict[str, Any]:
"""data_feed 원자료 컬럼과의 매핑(괄호 안 = data_feed 컬럼명):
close(Close)/open(Open)/high(High)/low(Low)/prev_close(PrevClose)/volume(Volume)/
avg_volume_5d(AvgVolume_5D)/ma20(MA20)/ma60(MA60)/ret5d~ret60d(Ret5D~Ret60D)/
atr20(ATR20)/frg_5d·inst_5d(Frg_5D·Inst_5D)/frg_20d·inst_20d(Frg_20D·Inst_20D)/
flow_rows(Flow_Rows)/flow_ok(Flow_OK, P5 규칙: Flow_Rows>=20).
"""
if naver_session is None or fetch_price_history is None:
return {"status": "DISABLED"}
try:
session = naver_session()
# MA60/Ret60D 계산에 60거래일 종가가 필요 — 10행/페이지이므로 7페이지(70행) 수집.
price = fetch_price_history(session, code, pages=7)
price = fetch_price_history(session, code)
result: dict[str, Any] = {"status": price.get("status", "UNKNOWN"), "source_url": price.get("source_url")}
rows = price.get("rows") or []
if rows:
@@ -160,29 +113,13 @@ def _normalize_naver_price_history(code: str) -> dict[str, Any]:
result["high"] = rows[0].get("high")
result["low"] = rows[0].get("low")
result["volume"] = rows[0].get("volume")
if len(rows) > 1:
result["prev_close"] = rows[1].get("close")
result["avg_volume_5d"] = _avg([r["volume"] for r in rows[:5] if r.get("volume")]) if len(rows) >= 5 else None
result["ma20"] = _compute_ma(rows, 20)
result["ma60"] = _compute_ma(rows, 60)
result["ret5d"] = _compute_ret_pct(rows, 5)
result["ret10d"] = _compute_ret_pct(rows, 10)
result["ret20d"] = _compute_ret_pct(rows, 20)
result["ret60d"] = _compute_ret_pct(rows, 60)
result["atr20"] = _compute_atr20(rows)
if compute_relative_return_20d is not None:
benchmark = fetch_price_history(session, "069500")
result["relative_return_20d"] = compute_relative_return_20d(rows, benchmark.get("rows", []))
if compute_volume_ratio_5d is not None:
result["volume_ratio_5d"] = compute_volume_ratio_5d(rows)
if fetch_foreign_institution_flow is not None:
flow = fetch_foreign_institution_flow(session, code)
result["foreign_institution_flow"] = flow
flow_rows = flow.get("rows") or []
result["flow_rows"] = len(flow_rows)
result["flow_ok"] = len(flow_rows) >= 20 # P5: Flow_Rows < 20 → no A-grade/즉시매수
result["frg_5d"], result["inst_5d"] = _aggregate_flow(flow_rows, 5)
result["frg_20d"], result["inst_20d"] = _aggregate_flow(flow_rows, 20)
result["foreign_institution_flow"] = fetch_foreign_institution_flow(session, code)
return result
except Exception as exc: # noqa: BLE001 - fallback source must not break the batch
return {"status": "ERROR", "error": str(exc)}
@@ -262,6 +199,134 @@ def _build_seed_rows(source_json: Path) -> list[dict[str, Any]]:
return rows
def _merge_source_fields(target: dict[str, Any], source: dict[str, Any], keys: tuple[str, ...]) -> None:
for key in keys:
if key in source and source.get(key) not in (None, ""):
target[key] = source[key]
def _resolve_price_source(
ticker: str,
*,
kis_account: str,
include_naver: bool,
include_live_kis: bool,
) -> tuple[dict[str, Any] | None, dict[str, Any] | None, list[str]]:
source_priority: list[str] = ["gathertradingdata_json"]
kis: dict[str, Any] | None = None
naver: dict[str, Any] | None = None
if include_live_kis and ticker.isdigit() and len(ticker) == 6:
kis = _normalize_kis_fields(ticker, kis_account)
if kis.get("status") == "OK":
source_priority.insert(0, "kis_open_api")
if include_naver and ticker.isdigit() and len(ticker) == 6:
naver = _normalize_naver_price_history(ticker)
if naver.get("status") in {"OK", "DATA_MISSING"}:
source_priority.append("naver_finance")
return kis, naver, source_priority
def _apply_source_fallbacks(
normalized: dict[str, Any],
*,
row: dict[str, Any],
kis: dict[str, Any] | None,
naver: dict[str, Any] | None,
) -> None:
if kis and kis.get("status") == "OK":
_merge_source_fields(normalized, kis, ("current_price", "open", "high", "low", "volume"))
_merge_source_fields(normalized, kis, ("relative_return_20d", "volume_ratio_5d", "microstructure_pressure", "short_turnover_share"))
if naver and naver.get("status") in {"OK", "DATA_MISSING"}:
normalized.setdefault("relative_return_20d", naver.get("relative_return_20d"))
normalized.setdefault("volume_ratio_5d", naver.get("volume_ratio_5d"))
normalized.setdefault("naver_price_status", naver.get("status"))
normalized.setdefault("current_price", naver.get("close"))
normalized.setdefault("open", naver.get("open"))
normalized.setdefault("high", naver.get("high"))
normalized.setdefault("low", naver.get("low"))
normalized.setdefault("volume", naver.get("volume"))
normalized.setdefault("current_price", _coerce_float(row.get("current_price") or row.get("Current_Price") or row.get("close")))
normalized.setdefault("open", _coerce_float(row.get("open") or row.get("Open")))
normalized.setdefault("high", _coerce_float(row.get("high") or row.get("High")))
normalized.setdefault("low", _coerce_float(row.get("low") or row.get("Low")))
normalized.setdefault("volume", _coerce_float(row.get("volume") or row.get("Volume")))
def _persist_collection_row(
*,
sqlite_db: Path,
run_id: str,
ticker: str,
normalized: dict[str, Any],
provenance: dict[str, Any],
) -> None:
upsert_collection_snapshot(
sqlite_db,
run_id=run_id,
dataset_name="data_feed",
ticker=ticker,
name=str(normalized.get("Name") or normalized.get("name") or ""),
sector=normalized.get("Sector"),
as_of_date=str(normalized.get("Price_Date") or normalized.get("AsOfDate") or normalized.get("collection_as_of") or ""),
source_priority=">".join(provenance.get("source_priority") or []),
source_status="OK",
payload=normalized,
provenance=provenance,
)
def _append_collection_failure(
*,
sqlite_db: Path,
run_id: str,
ticker: str,
row: dict[str, Any],
exc: Exception,
) -> dict[str, Any]:
error = {"ticker": ticker, "error": str(exc)}
append_collection_error(
sqlite_db,
run_id=run_id,
source_name="collector",
error_kind=type(exc).__name__,
error_message=str(exc),
ticker=ticker,
payload=row,
)
return error
def _finalize_collection_summary(
*,
summary: dict[str, Any],
output_json: Path,
sqlite_db: Path,
) -> dict[str, Any]:
summary["finished_at"] = _kst_now_iso()
summary["status"] = "PASS" if not summary["errors"] else "PASS_WITH_WARNINGS"
output_json.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
upsert_collection_run(
sqlite_db,
CollectionRun(
run_id=summary["run_id"],
collector_name="kis_data_collection_v1",
started_at=summary["started_at"],
status=summary["status"],
input_source=str(summary["input_json"]),
output_json_path=str(output_json),
output_db_path=str(sqlite_db),
notes="KIS-first CI collection",
),
finished_at=summary["finished_at"],
)
return summary
def _collect_one(row: dict[str, Any], *, kis_account: str, include_naver: bool, include_live_kis: bool) -> tuple[dict[str, Any], dict[str, Any]]:
ticker = str(row.get("Ticker") or row.get("ticker") or "").strip()
name = str(row.get("Name") or row.get("name") or "").strip()
@@ -274,43 +339,20 @@ def _collect_one(row: dict[str, Any], *, kis_account: str, include_naver: bool,
"source_priority": ["gathertradingdata_json"],
}
if include_live_kis and ticker.isdigit() and len(ticker) == 6:
kis = _normalize_kis_fields(ticker, kis_account)
kis, naver, source_priority = _resolve_price_source(
ticker,
kis_account=kis_account,
include_naver=include_naver,
include_live_kis=include_live_kis,
)
provenance["source_priority"] = source_priority
if kis is not None:
provenance["kis"] = kis
normalized.update({k: v for k, v in kis.items() if k not in {"current_price_raw", "orderbook_raw", "short_sale_raw"}})
if kis.get("status") == "OK":
provenance["source_priority"].insert(0, "kis_open_api")
if include_naver and ticker.isdigit() and len(ticker) == 6:
naver = _normalize_naver_price_history(ticker)
if naver is not None:
provenance["naver"] = naver
if naver.get("status") in {"OK", "DATA_MISSING"}:
# KIS가 이미 채운 필드(close/open/high/low/volume 등)는 setdefault로 보존하고,
# Naver만 제공하는 파생 필드(이동평균/수익률/ATR/수급 5D·20D)는 그대로 채운다.
naver_promotable = (
"close", "open", "high", "low", "volume", "prev_close", "avg_volume_5d",
"ma20", "ma60", "ret5d", "ret10d", "ret20d", "ret60d", "atr20",
"relative_return_20d", "volume_ratio_5d",
"frg_5d", "inst_5d", "frg_20d", "inst_20d", "flow_rows", "flow_ok",
)
for key in naver_promotable:
if key in naver:
normalized.setdefault(key, naver.get(key))
normalized.setdefault("naver_price_status", naver.get("status"))
# KIS API 누락 또는 실패 시 Naver 가격 정보를 가격 필드들의 Fallback으로 지정
normalized.setdefault("current_price", naver.get("close"))
normalized.setdefault("open", naver.get("open"))
normalized.setdefault("high", naver.get("high"))
normalized.setdefault("low", naver.get("low"))
normalized.setdefault("volume", naver.get("volume"))
provenance["source_priority"].append("naver_finance")
# KIS 및 Naver 가격 정보가 모두 없을 시, GatherTradingData.json 원본 시드 가격을 최후의 수단으로 복원
normalized.setdefault("current_price", _coerce_float(row.get("current_price") or row.get("Current_Price") or row.get("close")))
normalized.setdefault("open", _coerce_float(row.get("open") or row.get("Open")))
normalized.setdefault("high", _coerce_float(row.get("high") or row.get("High")))
normalized.setdefault("low", _coerce_float(row.get("low") or row.get("Low")))
normalized.setdefault("volume", _coerce_float(row.get("volume") or row.get("Volume")))
_apply_source_fallbacks(normalized, row=row, kis=kis, naver=naver)
normalized.setdefault("collection_as_of", _kst_now_iso())
return normalized, provenance
@@ -322,7 +364,7 @@ def collect_to_sqlite(
sqlite_db: Path,
output_json: Path,
kis_account: str,
include_naver: bool = True,
include_naver: bool = False,
include_live_kis: bool = True,
) -> dict[str, Any]:
run_id = uuid.uuid4().hex
@@ -363,17 +405,11 @@ def collect_to_sqlite(
source_counts = summary["source_counts"]
for source_name in provenance.get("source_priority") or []:
source_counts[source_name] = source_counts.get(source_name, 0) + 1
upsert_collection_snapshot(
sqlite_db,
_persist_collection_row(
sqlite_db=sqlite_db,
run_id=run_id,
dataset_name="data_feed",
ticker=ticker,
name=str(normalized.get("Name") or normalized.get("name") or ""),
sector=normalized.get("Sector"),
as_of_date=str(normalized.get("Price_Date") or normalized.get("AsOfDate") or normalized.get("collection_as_of") or ""),
source_priority=">".join(provenance.get("source_priority") or []),
source_status="OK",
payload=normalized,
normalized=normalized,
provenance=provenance,
)
summary["rows"].append(
@@ -388,37 +424,16 @@ def collect_to_sqlite(
}
)
except Exception as exc: # noqa: BLE001
error = {"ticker": ticker, "error": str(exc)}
summary["errors"].append(error)
append_collection_error(
sqlite_db,
error = _append_collection_failure(
sqlite_db=sqlite_db,
run_id=run_id,
source_name="collector",
error_kind=type(exc).__name__,
error_message=str(exc),
ticker=ticker,
payload=row,
row=row,
exc=exc,
)
summary["errors"].append(error)
summary["finished_at"] = _kst_now_iso()
summary["status"] = "PASS" if not summary["errors"] else "PASS_WITH_WARNINGS"
output_json.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
upsert_collection_run(
sqlite_db,
CollectionRun(
run_id=run_id,
collector_name="kis_data_collection_v1",
started_at=started_at,
status=summary["status"],
input_source=str(input_json),
output_json_path=str(output_json),
output_db_path=str(sqlite_db),
notes="KIS-first CI collection",
),
finished_at=summary["finished_at"],
)
return summary
return _finalize_collection_summary(summary=summary, output_json=output_json, sqlite_db=sqlite_db)
def main() -> int:
@@ -429,7 +444,7 @@ def main() -> int:
ap.add_argument("--store-location", default=None, help="Backend location/DSN. sqlite path or future postgres DSN.")
ap.add_argument("--output-json", type=Path, default=ROOT / "Temp" / "kis_data_collection_v1.json")
ap.add_argument("--kis-account", choices=["real", "mock"], default="real")
ap.add_argument("--no-naver", action="store_true")
ap.add_argument("--allow-naver-fallback", action="store_true")
ap.add_argument("--no-live-kis", action="store_true")
args = ap.parse_args()
@@ -452,7 +467,7 @@ def main() -> int:
sqlite_db=Path(store_location),
output_json=args.output_json,
kis_account=args.kis_account,
include_naver=not args.no_naver,
include_naver=args.allow_naver_fallback,
include_live_kis=not args.no_live_kis,
)
print(json.dumps(summary, ensure_ascii=False, indent=2))
@@ -2,6 +2,7 @@ from __future__ import annotations
import os
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from math import fsum
@@ -12,6 +13,27 @@ from typing import Any, Callable
ROOT = Path(__file__).resolve().parents[2]
def resolve_python_interpreter() -> list[str]:
"""Prefer the project Python 3.13 interpreter on Windows.
The repo's working `python` command may point at an older interpreter, but
the build and validation scripts depend on the package set installed under
Python 3.13. Fall back to sys.executable only if the launcher is unavailable.
"""
configured = os.environ.get("CODEX_PYTHON")
if configured:
return [configured]
if os.name == "nt":
for candidate in (
r"C:\Users\kjh20\AppData\Local\Programs\Python\Python313\python.exe",
r"C:\Users\kjh20\AppData\Local\Python\pythoncore-3.13-64\python.exe",
):
if Path(candidate).exists():
return [candidate]
return ["py", "-3.13"]
return [sys.executable]
def utf8_env() -> dict[str, str]:
env = os.environ.copy()
env.setdefault("PYTHONIOENCODING", "utf-8")
@@ -24,6 +46,8 @@ def _run_command(command: list[str], cwd: Path | None = None) -> dict[str, Any]:
resolved = list(command)
if os.name == "nt" and resolved and resolved[0].lower() == "npm":
resolved[0] = "npm.cmd"
if resolved and resolved[0].endswith(".py"):
resolved = [*resolve_python_interpreter(), *resolved]
subprocess.run(resolved, cwd=cwd or ROOT, check=True, env=utf8_env())
finished = datetime.now(timezone.utc)
return {
+27 -24
View File
@@ -90,19 +90,6 @@ TEMP_KEEP_FILES = {
"final_execution_decision_v2.json",
"prediction_accuracy_harness_v2.json",
"validate_prediction_accuracy_harness_v2.json",
"alpha_feedback_loop_v2.json",
"validate_alpha_feedback_loop_v2.json",
"operational_alpha_calibration_v2.json",
"validate_operational_alpha_calibration_v2.json",
"sector_flow_history_progress_v1.json",
"validate_sector_flow_history_progress_v1.json",
"data_gated_progress_v1.json",
"validate_data_gated_progress_v1.json",
"realized_performance_v1.json",
"validate_realized_performance_v1.json",
"single_truth_ledger_v2.json",
"smart_cash_recovery_v7.json",
"smart_cash_recovery_v9.json",
# Data Analysis & Verification Reports
"horizon_rebalance_plan_v1.json",
"factor_lifecycle_completeness_v1.json",
@@ -111,6 +98,20 @@ TEMP_KEEP_FILES = {
"strategy_routing_audit_v1.json",
}
TEMP_NOISE_FILES = {
"canonical_artifact_resolver_v1.json",
"final_execution_decision_v2.json",
"rebalance_cadence_gate_v1.json",
"single_truth_ledger_v2.json",
"smart_cash_recovery_v7.json",
"smart_cash_recovery_v9.json",
"state_vector_constructor_v1.json",
"transition_set_enumerator_v1.json",
"walk_forward_bootstrap_v1.json",
"weekly_legacy_transfer_plan_v1.json",
"prediction_accuracy_harness_v2.json",
}
UPLOAD_KEEP_DIRS_UPLOAD = {
"artifacts",
"docs",
@@ -194,7 +195,9 @@ def should_include(path: Path, mode: str, include_xlsx: bool, include_backups: b
return False
if path.name == DEFAULT_OUTPUT.name:
return False
if mode == "upload" and rel.as_posix() in _active_manifest_refs():
if mode == "upload" and path.name in TEMP_NOISE_FILES:
return False
if mode == "upload" and parts[0] != "Temp" and rel.as_posix() in _active_manifest_refs():
return True
if parts[0] == "Temp":
if path.name in TEMP_EXCLUDED_FILES:
@@ -277,7 +280,7 @@ def main() -> int:
if args.skip_validate:
plan = []
if not args.skip_convert:
plan.append({"name": "prepare", "command": ["npm", "run", "ops:prepare"]})
plan.append({"name": "prepare", "command": ["tools/convert_xlsx_to_json.py"]})
plan.append({
"name": "zip",
"depends_on": ["prepare"] if not args.skip_convert else [],
@@ -289,9 +292,9 @@ def main() -> int:
if args.validation_mode == "release":
plan = []
if not args.skip_convert:
plan.append({"name": "prepare", "command": ["npm", "run", "ops:prepare"]})
plan.append({"name": "prepare", "command": ["tools/convert_xlsx_to_json.py"]})
plan.extend([
{"name": "release_full", "command": ["npm", "run", "ops:release"], "depends_on": ["prepare"] if not args.skip_convert else []},
{"name": "release_full", "command": ["tools/run_release_dag_v3.py", "--mode", "full"], "depends_on": ["prepare"] if not args.skip_convert else []},
{
"name": "zip",
"depends_on": ["release_full"],
@@ -307,11 +310,11 @@ def main() -> int:
gate_status = "OK"
plan = []
if not args.skip_convert:
plan.append({"name": "prepare", "command": ["npm", "run", "ops:prepare"]})
plan.append({"name": "prepare", "command": ["tools/convert_xlsx_to_json.py"]})
plan.extend([
{
"name": "build_bundle",
"command": ["npm", "run", "ops:build"],
"command": ["tools/build_bundle.py"],
},
{
"name": "zip",
@@ -324,10 +327,10 @@ def main() -> int:
print("QUICK_MODE_FALLBACK_RELEASE_GATE:", ";".join(reasons))
plan = []
if not args.skip_convert:
plan.append({"name": "prepare", "command": ["npm", "run", "ops:prepare"]})
plan.append({"name": "prepare", "command": ["tools/convert_xlsx_to_json.py"]})
plan.extend([
{"name": "release_gate", "command": ["npm", "run", "ops:validate"], "depends_on": ["prepare"] if not args.skip_convert else []},
{"name": "build_bundle", "command": ["npm", "run", "ops:build"]},
{"name": "release_gate", "command": ["tools/run_release_dag_v3.py", "--mode", "release"], "depends_on": ["prepare"] if not args.skip_convert else []},
{"name": "build_bundle", "command": ["tools/build_bundle.py"]},
{
"name": "zip",
"depends_on": ["release_gate", "build_bundle"],
@@ -344,11 +347,11 @@ def main() -> int:
gate_status = "OK"
plan = []
if not args.skip_convert:
plan.append({"name": "prepare", "command": ["npm", "run", "ops:prepare"]})
plan.append({"name": "prepare", "command": ["tools/convert_xlsx_to_json.py"]})
plan.extend([
{
"name": "build_bundle",
"command": ["npm", "run", "ops:build"],
"command": ["tools/build_bundle.py"],
},
{
"name": "zip",
+275 -414
View File
@@ -1,9 +1,7 @@
from __future__ import annotations
import argparse
import base64
import json
import os
import sqlite3
import subprocess
from http import HTTPStatus
@@ -13,176 +11,112 @@ from hashlib import sha256
from typing import Any
from urllib.parse import urlparse, parse_qs
import openpyxl
ROOT = Path(__file__).resolve().parents[2]
SNAPSHOT_ADMIN_VERSION = "snapshot-admin-web-v7"
GATHER_TRADING_DATA_XLSX = ROOT / "GatherTradingData.xlsx"
SNAPSHOT_ADMIN_VERSION = "snapshot-admin-web-v6"
KIS_COLLECTION_DB = ROOT / "outputs" / "kis_data_collection" / "kis_data_collection.db"
KIS_COLLECTION_REPORT = ROOT / "Temp" / "kis_data_collection_v1.json"
QUALITATIVE_SELL_DB = ROOT / "outputs" / "qualitative_sell_strategy" / "qualitative_sell_strategy.db"
GATHER_TRADING_DATA_JSON = ROOT / "GatherTradingData.json"
AUTH_REALM = "Snapshot Admin"
JSON_SHEET_ALIASES = {
"harness_context": "_harness_context",
# WBS-7.9 부속 — 테이블별 그리드 조회(Tabler). 화이트리스트에 없는 테이블명은
# SQL에 절대 보간되지 않는다(요청 테이블명을 그대로 SELECT 문에 넣지 않고
# 아래 레지스트리 키와 정확히 일치할 때만 허용).
WORKSPACE_BROWSABLE_TABLES = (
"settings",
"account_snapshot",
"workspace_change_log",
"workspace_approval_v2",
"workspace_lock",
"workspace_meta",
)
COLLECTION_BROWSABLE_TABLES = (
"collection_runs",
"collection_snapshots",
"collection_source_errors",
)
QUALITATIVE_SELL_BROWSABLE_TABLES = (
"sell_strategy_results",
"satellite_recommendations",
)
# Editable tables configurations (WBS requirement 2)
EDITABLE_TABLES = {
"settings",
"account_snapshot",
"collection_runs",
"collection_snapshots",
"collection_source_errors",
"sell_strategy_results",
"satellite_recommendations",
}
# WBS-7.9 부속, WBS-7.10 후속(2026-06-22) — 테이블별 그리드 조회(Tabler).
# 정적 화이트리스트 대신 각 DB 파일의 sqlite_master를 그때그때 조회해 테이블
# 목록을 만든다 — 정적 목록은 스키마가 바뀌거나(예: 레거시 workspace_approval
# 테이블처럼) 새 테이블이 추가되면 누락되는 문제가 있었다(사용자 보고로 발견).
# 보안 속성은 동일하게 유지된다: 요청된 테이블명은 항상 해당 DB의 실제
# sqlite_master 결과와 정확히 일치할 때만 SQL에 사용된다(임의 문자열 보간 없음).
def _known_db_paths(workspace_db_path: Path) -> list[Path]:
return [Path(workspace_db_path), KIS_COLLECTION_DB, QUALITATIVE_SELL_DB]
def _discover_tables(db_path: Path) -> list[str]:
if not db_path.exists():
return []
with sqlite3.connect(db_path) as conn:
rows = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
).fetchall()
return [row[0] for row in rows]
def _resolve_table_db(table: str, workspace_db_path: Path) -> Path | None:
for db_path in _known_db_paths(workspace_db_path):
if table in _discover_tables(db_path):
return db_path
if table in WORKSPACE_BROWSABLE_TABLES:
return Path(workspace_db_path)
if table in COLLECTION_BROWSABLE_TABLES:
return KIS_COLLECTION_DB
if table in QUALITATIVE_SELL_BROWSABLE_TABLES:
return QUALITATIVE_SELL_DB
return None
# 2026-06-22 — 분석/판단 팩터로 쓰이는 GatherTradingData.json의 data.* 시트도
# 같은 그리드로 조회 가능하게 한다(SQLite로 옮겨지지 않은 data_feed/sector_flow/
# macro 등). dict 키 조회만 하므로 SQL 인젝션 표면 자체가 없다.
def _discover_json_sheets() -> dict[str, list[dict[str, Any]]]:
if not GATHER_TRADING_DATA_JSON.exists():
return {}
try:
payload = json.loads(GATHER_TRADING_DATA_JSON.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
data = payload.get("data")
if not isinstance(data, dict):
return {}
return {key: value for key, value in data.items() if isinstance(value, list) and value and isinstance(value[0], dict)}
def _discover_workbook_sheets() -> list[dict[str, Any]]:
if not GATHER_TRADING_DATA_XLSX.exists():
return []
try:
workbook = openpyxl.load_workbook(GATHER_TRADING_DATA_XLSX, read_only=True, data_only=True)
except Exception:
return []
try:
inventory: list[dict[str, Any]] = []
for sheet_name in workbook.sheetnames:
worksheet = workbook[sheet_name]
inventory.append(
{
"sheet": sheet_name,
"row_count": int(worksheet.max_row or 0),
"column_count": int(worksheet.max_column or 0),
"source_workbook": str(GATHER_TRADING_DATA_XLSX),
}
)
return inventory
finally:
workbook.close()
def build_table_catalog(workspace_db_path: Path) -> dict[str, list[dict[str, Any]]]:
sqlite_rows: list[dict[str, Any]] = []
for db_path in _known_db_paths(workspace_db_path):
for table in _discover_tables(db_path):
def list_browsable_tables(workspace_db_path: Path) -> list[dict[str, Any]]:
tables: list[dict[str, Any]] = []
for table in (
*WORKSPACE_BROWSABLE_TABLES,
*COLLECTION_BROWSABLE_TABLES,
*QUALITATIVE_SELL_BROWSABLE_TABLES,
):
db_path = _resolve_table_db(table, workspace_db_path)
exists = bool(db_path and db_path.exists())
row_count = 0
if exists:
try:
with sqlite3.connect(db_path) as conn:
row_count = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 - table name confirmed via sqlite_master of this exact db above
row_count = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 - table is whitelist-checked above
except sqlite3.OperationalError:
continue
sqlite_rows.append({"table": table, "db": str(db_path), "exists": True, "row_count": row_count, "source": "sqlite"})
json_rows = [{"table": sheet, "db": str(GATHER_TRADING_DATA_JSON), "exists": True, "row_count": len(rows), "source": "json"} for sheet, rows in _discover_json_sheets().items()]
sqlite_names = {row["table"] for row in sqlite_rows}
json_names = {row["table"] for row in json_rows}
workbook_rows: list[dict[str, Any]] = []
for sheet_row in _discover_workbook_sheets():
sheet_name = sheet_row["sheet"]
json_key = JSON_SHEET_ALIASES.get(sheet_name, sheet_name)
current_sources: list[str] = []
if sheet_name in sqlite_names:
current_sources.append("sqlite")
if sheet_name in json_names or json_key in json_names:
current_sources.append("json")
if not current_sources:
current_sources.append("xlsx")
workbook_rows.append(
{
**sheet_row,
"json_key": json_key,
"current_sources": current_sources,
"migration_candidate": "yes" if "sqlite" not in current_sources else "no",
}
)
return {"sqlite": sqlite_rows, "json": json_rows, "workbook": workbook_rows}
def list_browsable_tables(workspace_db_path: Path) -> list[dict[str, Any]]:
catalog = build_table_catalog(workspace_db_path)
return [*catalog["sqlite"], *catalog["json"]]
exists = False
tables.append({
"table": table,
"db": str(db_path) if db_path else "",
"exists": exists,
"row_count": row_count,
"editable": table in EDITABLE_TABLES,
})
return tables
def fetch_table_rows(table: str, workspace_db_path: Path, *, limit: int = 50, offset: int = 0) -> dict[str, Any]:
db_path = _resolve_table_db(table, workspace_db_path)
if db_path is not None:
with sqlite3.connect(db_path) as conn:
conn.row_factory = sqlite3.Row
total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 - whitelisted table name
cursor = conn.execute(
f"SELECT * FROM {table} ORDER BY rowid DESC LIMIT ? OFFSET ?", # noqa: S608 - whitelisted table name
(limit, offset),
)
rows = [dict(row) for row in cursor.fetchall()]
columns = [description[0] for description in cursor.description] if cursor.description else []
return {"table": table, "db": str(db_path), "columns": columns, "rows": rows, "total": total, "limit": limit, "offset": offset, "source": "sqlite"}
json_sheets = _discover_json_sheets()
if table not in json_sheets:
if db_path is None:
raise ValueError(f"unknown or non-browsable table: {table}")
sheet_rows = json_sheets[table]
total = len(sheet_rows)
page = sheet_rows[offset : offset + limit]
columns: list[str] = []
for row in page:
for key in row.keys():
if key not in columns:
columns.append(key)
return {"table": table, "db": str(GATHER_TRADING_DATA_JSON), "columns": columns, "rows": page, "total": total, "limit": limit, "offset": offset, "source": "json"}
if not db_path.exists():
return {"table": table, "db": str(db_path), "columns": [], "rows": [], "total": 0, "limit": limit, "offset": offset, "editable": table in EDITABLE_TABLES}
with sqlite3.connect(db_path) as conn:
conn.row_factory = sqlite3.Row
total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 - whitelisted table name
cursor = conn.execute(
f"SELECT rowid as _rowid, * FROM {table} ORDER BY rowid DESC LIMIT ? OFFSET ?", # noqa: S608 - whitelisted table name
(limit, offset),
)
rows = [dict(row) for row in cursor.fetchall()]
columns = [description[0] for description in cursor.description] if cursor.description else []
return {"table": table, "db": str(db_path), "columns": columns, "rows": rows, "total": total, "limit": limit, "offset": offset, "editable": table in EDITABLE_TABLES}
def fetch_table_rows_for_source(source: str, table: str, workspace_db_path: Path, *, limit: int = 50, offset: int = 0) -> dict[str, Any]:
normalized_source = source.strip().lower()
if normalized_source == "sqlite":
return fetch_table_rows(table, workspace_db_path, limit=limit, offset=offset)
if normalized_source == "json":
json_sheets = _discover_json_sheets()
if table not in json_sheets:
raise ValueError(f"unknown or non-browsable table: {table}")
sheet_rows = json_sheets[table]
total = len(sheet_rows)
page = sheet_rows[offset : offset + limit]
columns: list[str] = []
for row in page:
for key in row.keys():
if key not in columns:
columns.append(key)
return {"table": table, "db": str(GATHER_TRADING_DATA_JSON), "columns": columns, "rows": page, "total": total, "limit": limit, "offset": offset, "source": "json"}
raise ValueError(f"unsupported source: {source}")
def fetch_domain_rows(domain: str, workspace_db_path: Path) -> dict[str, Any]:
if domain == "settings":
rows = load_settings_rows(workspace_db_path)
return {"domain": domain, "db": str(workspace_db_path), "columns": ["ordinal", "key", "value", "note", "updated_at"], "rows": rows}
if domain == "account_snapshot":
rows = load_account_snapshot_rows(workspace_db_path)
return {
"domain": domain,
"db": str(workspace_db_path),
"columns": list(ACCOUNT_SNAPSHOT_CANONICAL_COLUMNS),
"rows": rows,
}
raise ValueError(f"unknown editable domain: {domain}")
SNAPSHOT_ADMIN_VERSION_FILES = (
ROOT / "src" / "quant_engine" / "snapshot_admin_server_v1.py",
ROOT / "src" / "quant_engine" / "snapshot_admin_store_v1.py",
@@ -422,55 +356,6 @@ def _text_response(handler: BaseHTTPRequestHandler, status: int, text: str, cont
handler.wfile.write(body)
def _is_loopback_host(host: str) -> bool:
normalized = host.strip().lower()
return normalized in {"127.0.0.1", "localhost", "::1"}
def _parse_basic_auth(header_value: str | None) -> tuple[str, str] | None:
if not header_value:
return None
prefix = "basic "
if not header_value.lower().startswith(prefix):
return None
encoded = header_value[len(prefix) :].strip()
if not encoded:
return None
try:
decoded = base64.b64decode(encoded).decode("utf-8")
except (ValueError, UnicodeDecodeError):
return None
if ":" not in decoded:
return None
username, password = decoded.split(":", 1)
return username, password
def _basic_auth_matches(header_value: str | None, username: str, password: str) -> bool:
parsed = _parse_basic_auth(header_value)
return bool(parsed and parsed[0] == username and parsed[1] == password)
def _reject_unauthorized(handler: BaseHTTPRequestHandler) -> None:
body = json.dumps({"detail": "authentication required"}, ensure_ascii=False, indent=2).encode("utf-8")
handler.send_response(HTTPStatus.UNAUTHORIZED)
handler.send_header("WWW-Authenticate", f'Basic realm="{AUTH_REALM}", charset="UTF-8"')
handler.send_header("Content-Type", "application/json; charset=utf-8")
handler.send_header("Content-Length", str(len(body)))
handler.end_headers()
handler.wfile.write(body)
def _validate_remote_bind(host: str, allow_remote: bool, auth_user: str, auth_password: str) -> None:
has_auth = bool(auth_user and auth_password)
if bool(auth_user) != bool(auth_password):
raise ValueError("snapshot admin auth requires both --auth-user and --auth-password")
if not _is_loopback_host(host) and not allow_remote:
raise ValueError("refusing to bind snapshot admin outside loopback without --allow-remote")
if (allow_remote or not _is_loopback_host(host)) and not has_auth:
raise ValueError("remote snapshot admin access requires both --auth-user and --auth-password")
def _read_json_body(handler: BaseHTTPRequestHandler) -> dict[str, Any]:
length = int(handler.headers.get("Content-Length") or "0")
raw = handler.rfile.read(length).decode("utf-8") if length else "{}"
@@ -2778,79 +2663,26 @@ def render_tables_html() -> str:
<div class="page-wrapper">
<div class="page-body">
<div class="container-xl">
<div class="row row-cards">
<div class="col-12">
<div class="card">
<div class="card-header">
<div>
<div class="card-title">Workbook migration inventory</div>
<div class="text-secondary">Source-of-truth xlsx sheet list with current storage classification.</div>
</div>
<span class="badge bg-secondary-lt" id="inventoryMeta"></span>
</div>
<div class="table-responsive">
<table class="table table-vcenter card-table table-striped">
<thead>
<tr>
<th>Sheet</th>
<th class="text-end">Rows</th>
<th class="text-end">Cols</th>
<th>Current Source</th>
<th>Migration Candidate</th>
</tr>
</thead>
<tbody id="inventoryBody"></tbody>
</table>
</div>
<div class="card">
<div class="card-header d-flex flex-wrap gap-2 align-items-center justify-content-between">
<div class="d-flex gap-2 align-items-center">
<label class="form-label mb-0 me-1" for="tableSelect">Table</label>
<select id="tableSelect" class="form-select" style="min-width:280px" onchange="onTableChange()"></select>
<span class="badge bg-secondary-lt" id="tableMeta"></span>
</div>
<div class="d-flex gap-2">
<button class="btn btn-sm" onclick="prevPage()">&laquo; Prev</button>
<span class="d-flex align-items-center px-2" id="pageInfo"></span>
<button class="btn btn-sm" onclick="nextPage()">Next &raquo;</button>
<button class="btn btn-sm btn-primary" onclick="reload()">Refresh</button>
<button class="btn btn-sm btn-success" id="saveTableBtn" onclick="saveCurrentTable()">Save changes</button>
</div>
</div>
<div class="col-12 col-xl-6">
<div class="card">
<div class="card-header d-flex flex-wrap gap-2 align-items-center justify-content-between">
<div class="d-flex gap-2 align-items-center">
<span class="badge bg-blue-lt">SQLite</span>
<label class="form-label mb-0 me-1" for="sqliteTableSelect">Table</label>
<select id="sqliteTableSelect" class="form-select" style="min-width:260px" onchange="onTableChange('sqlite')"></select>
<span class="badge bg-secondary-lt" id="sqliteTableMeta"></span>
</div>
<div class="d-flex gap-2">
<button class="btn btn-sm" onclick="prevPage('sqlite')">&laquo; Prev</button>
<span class="d-flex align-items-center px-2" id="sqlitePageInfo"></span>
<button class="btn btn-sm" onclick="nextPage('sqlite')">Next &raquo;</button>
<button class="btn btn-sm btn-primary" onclick="reload('sqlite')">Refresh</button>
</div>
</div>
<div class="table-responsive">
<table class="table table-vcenter card-table table-striped" id="sqliteGridTable">
<thead><tr id="sqliteGridHead"></tr></thead>
<tbody id="sqliteGridBody"></tbody>
</table>
</div>
</div>
</div>
<div class="col-12 col-xl-6">
<div class="card">
<div class="card-header d-flex flex-wrap gap-2 align-items-center justify-content-between">
<div class="d-flex gap-2 align-items-center">
<span class="badge bg-azure-lt">JSON</span>
<label class="form-label mb-0 me-1" for="jsonTableSelect">Sheet</label>
<select id="jsonTableSelect" class="form-select" style="min-width:260px" onchange="onTableChange('json')"></select>
<span class="badge bg-secondary-lt" id="jsonTableMeta"></span>
</div>
<div class="d-flex gap-2">
<button class="btn btn-sm" onclick="prevPage('json')">&laquo; Prev</button>
<span class="d-flex align-items-center px-2" id="jsonPageInfo"></span>
<button class="btn btn-sm" onclick="nextPage('json')">Next &raquo;</button>
<button class="btn btn-sm btn-primary" onclick="reload('json')">Refresh</button>
</div>
</div>
<div class="table-responsive">
<table class="table table-vcenter card-table table-striped" id="jsonGridTable">
<thead><tr id="jsonGridHead"></tr></thead>
<tbody id="jsonGridBody"></tbody>
</table>
</div>
</div>
<div class="table-responsive">
<table class="table table-vcenter card-table table-striped" id="gridTable">
<thead><tr id="gridHead"></tr></thead>
<tbody id="gridBody"></tbody>
</table>
</div>
</div>
</div>
@@ -2858,11 +2690,7 @@ def render_tables_html() -> str:
</div>
</div>
<script>
const state = {
catalog: { sqlite: [], json: [], workbook: [] },
sqlite: { current: "", limit: 50, offset: 0, total: 0 },
json: { current: "", limit: 50, offset: 0, total: 0 },
};
const state = { tables: [], current: "", limit: 50, offset: 0, total: 0, editable: false, rows: [] };
function escapeHtml(value) {
if (value === null || value === undefined) return "";
@@ -2870,105 +2698,158 @@ def render_tables_html() -> str:
return text.replace(/[&<>"']/g, (ch) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[ch]));
}
function sectionLabel(source) {
return source === "json" ? "JSON" : "SQLite";
function isEditableDomain(domain) {
return domain === "settings" || domain === "account_snapshot";
}
function sectionIds(source) {
return {
selectId: `${source}TableSelect`,
metaId: `${source}TableMeta`,
pageInfoId: `${source}PageInfo`,
headId: `${source}GridHead`,
bodyId: `${source}GridBody`,
};
function normalizeCellValue(value) {
if (value === null || value === undefined) return "";
if (typeof value === "object") return JSON.stringify(value);
return String(value);
}
function renderInventory() {
const body = document.getElementById("inventoryBody");
body.innerHTML = state.catalog.workbook
.map((row) => {
const sources = (row.current_sources || []).map((item) => item.toUpperCase()).join(", ");
const candidate = row.migration_candidate === "yes" ? "yes" : "no";
return `<tr>
<td>${escapeHtml(row.sheet)}</td>
<td class="text-end">${escapeHtml(row.row_count)}</td>
<td class="text-end">${escapeHtml(row.column_count)}</td>
<td>${escapeHtml(sources)}</td>
<td>${escapeHtml(candidate)}</td>
</tr>`;
})
.join("") || `<tr><td colspan="5" class="text-secondary">no workbook inventory</td></tr>`;
document.getElementById("inventoryMeta").textContent = `${state.catalog.workbook.length} sheets`;
function editableCell(rowIndex, column, value) {
return `<td contenteditable="true" data-row-index="${rowIndex}" data-column="${escapeHtml(column)}">${escapeHtml(normalizeCellValue(value))}</td>`;
}
function populateSelect(source) {
const select = document.getElementById(sectionIds(source).selectId);
const tables = state.catalog[source] || [];
select.innerHTML = tables
.map((t) => `<option value="${escapeHtml(t.table)}">${escapeHtml(t.table)} (${escapeHtml(t.row_count)})</option>`)
.join("");
if (!state[source].current && tables.length) {
state[source].current = tables[0].table;
}
select.value = state[source].current;
}
async function loadCatalog() {
async function loadTables() {
const res = await fetch("/api/tables");
const data = await res.json();
state.catalog.sqlite = data.sqlite || [];
state.catalog.json = data.json || [];
state.catalog.workbook = data.workbook || [];
renderInventory();
populateSelect("sqlite");
populateSelect("json");
await Promise.all([loadRows("sqlite"), loadRows("json")]);
state.tables = data.tables || [];
const select = document.getElementById("tableSelect");
select.innerHTML = state.tables
.map((t) => `<option value="${t.table}" ${!t.exists ? "disabled" : ""}>${t.table} (${t.exists ? t.row_count : "no db"})${t.editable ? ' (Editable)' : ''}</option>`)
.join("");
if (!state.current && state.tables.length) {
state.current = state.tables.find((t) => t.exists)?.table || state.tables[0].table;
}
select.value = state.current;
await loadRows();
}
function onTableChange(source) {
state[source].current = document.getElementById(sectionIds(source).selectId).value;
state[source].offset = 0;
loadRows(source);
function onTableChange() {
state.current = document.getElementById("tableSelect").value;
state.offset = 0;
loadRows();
}
async function loadRows(source) {
if (!state[source].current) return;
const ids = sectionIds(source);
const params = new URLSearchParams({ source, table: state[source].current, limit: state[source].limit, offset: state[source].offset });
const res = await fetch(`/api/table_rows?${params.toString()}`);
async function loadRows() {
if (!state.current) return;
const editable = state.tables.find(t => t.table === state.current)?.editable || false;
state.editable = editable;
const isDomain = isEditableDomain(state.current);
const url = isDomain ? `/api/domain_rows?domain=${encodeURIComponent(state.current)}` : `/api/table_rows?${new URLSearchParams({ table: state.current, limit: state.limit, offset: state.offset }).toString()}`;
const res = await fetch(url);
const data = await res.json();
state[source].total = data.total || 0;
document.getElementById(ids.headId).innerHTML = (data.columns || []).map((c) => `<th>${escapeHtml(c)}</th>`).join("");
document.getElementById(ids.bodyId).innerHTML = (data.rows || [])
.map((row) => `<tr>${(data.columns || []).map((c) => `<td>${escapeHtml(row[c])}</td>`).join("")}</tr>`)
.join("") || `<tr><td colspan="99" class="text-secondary">no rows</td></tr>`;
document.getElementById(ids.metaId).textContent = `[${sectionLabel(source)}] ${data.db || ""}`;
const from = state[source].total === 0 ? 0 : state[source].offset + 1;
const to = Math.min(state[source].offset + state[source].limit, state[source].total);
document.getElementById(ids.pageInfoId).textContent = `${from}-${to} / ${state[source].total}`;
}
function prevPage(source) {
state[source].offset = Math.max(0, state[source].offset - state[source].limit);
loadRows(source);
}
function nextPage(source) {
if (state[source].offset + state[source].limit < state[source].total) {
state[source].offset += state[source].limit;
loadRows(source);
state.rows = data.rows || [];
state.total = isDomain ? state.rows.length : (data.total || 0);
const head = document.getElementById("gridHead");
const body = document.getElementById("gridBody");
const displayColumns = (data.columns || []).filter((c) => !String(c).startsWith("_"));
head.innerHTML = displayColumns.map((c) => `<th>${escapeHtml(c)}</th>`).join("");
body.innerHTML = state.rows.length
? state.rows
.map((row, rowIndex) => {
return `<tr data-row-index="${rowIndex}">${displayColumns.map((c) => {
// Settings key and other primary columns can be protected or editable based on needs
const cellVal = row[c];
return editable ? editableCell(rowIndex, c, cellVal) : `<td>${escapeHtml(cellVal)}</td>`;
}).join("")}</tr>`;
})
.join("")
: `<tr><td colspan="99" class="text-secondary">no rows</td></tr>`;
document.getElementById("tableMeta").textContent = `${data.db || ""}`;
const from = state.total === 0 ? 0 : state.offset + 1;
const to = Math.min(state.offset + state.limit, state.total);
document.getElementById("pageInfo").textContent = `${from}-${to} / ${state.total}`;
const saveBtn = document.getElementById("saveTableBtn");
if (saveBtn) {
saveBtn.disabled = !editable;
saveBtn.textContent = editable ? "Save current table" : "Read only";
}
}
function reload(source) {
loadRows(source);
function prevPage() {
state.offset = Math.max(0, state.offset - state.limit);
loadRows();
}
loadCatalog().catch((error) => {
document.getElementById("inventoryBody").innerHTML = `<tr><td colspan="5" class="text-danger">${escapeHtml(error.message)}</td></tr>`;
document.getElementById("sqliteGridBody").innerHTML = `<tr><td class="text-danger">${escapeHtml(error.message)}</td></tr>`;
document.getElementById("jsonGridBody").innerHTML = `<tr><td class="text-danger">${escapeHtml(error.message)}</td></tr>`;
function nextPage() {
if (state.offset + state.limit < state.total) {
state.offset += state.limit;
loadRows();
}
}
function reload() {
loadRows();
}
function collectEditableRows() {
const table = document.getElementById("gridTable");
const columns = Array.from(table.querySelectorAll("thead th")).map((th) => th.textContent || "");
const bodyRows = table.querySelectorAll("tbody tr[data-row-index]");
return Array.from(bodyRows).map((tr, index) => {
const cells = tr.querySelectorAll("td");
const row = {};
let cellIndex = 0;
for (const column of columns) {
if (String(column).startsWith("_")) {
continue;
}
const cell = cells[cellIndex++];
row[column] = cell ? cell.textContent : "";
}
row.ordinal = index + 1;
return row;
});
}
function parseCellValue(value) {
const text = String(value || "").trim();
if (text === "") return "";
if (text === "null" || text === "None") return null;
if (text === "true") return true;
if (text === "false") return false;
const numericPattern = new RegExp("^-?(0|[1-9]\\\\d*)(\\\\.\\\\d+)?([eE][-+]?\\\\d+)?$");
if (numericPattern.test(text)) return Number(text);
try {
return JSON.parse(text);
} catch (err) {
return text;
}
}
async function saveCurrentTable() {
if (!state.editable) return;
const isDomain = isEditableDomain(state.current);
const endpoint = isDomain
? (state.current === "settings" ? "/api/settings/save" : "/api/account_snapshot/save")
: "/api/table/save";
const rows = collectEditableRows().map(row => {
const parsedRow = {};
for (const k of Object.keys(row)) {
parsedRow[k] = parseCellValue(row[k]);
}
return parsedRow;
});
const payload = isDomain ? { rows } : { table: state.current, rows };
const response = await fetch(endpoint, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
const data = await response.json();
if (!response.ok) throw new Error(data.detail || "save failed");
await loadRows();
alert(`saved: ${state.current}`);
}
loadTables().catch((error) => {
document.getElementById("gridBody").innerHTML = `<tr><td class="text-danger">${escapeHtml(error.message)}</td></tr>`;
});
</script>
</body>
@@ -2979,8 +2860,6 @@ def render_tables_html() -> str:
class SnapshotAdminHandler(BaseHTTPRequestHandler):
db_path: Path = DEFAULT_DB
seed_json_path: Path = DEFAULT_SEED_JSON
auth_user: str = ""
auth_password: str = ""
def log_message(self, format: str, *args: Any) -> None: # noqa: A003
return
@@ -2988,18 +2867,7 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
def _handle_exception(self, exc: Exception) -> None:
_json_response(self, HTTPStatus.INTERNAL_SERVER_ERROR, {"detail": str(exc)})
def _authorize(self) -> bool:
if not self.auth_user and not self.auth_password:
return True
header_value = self.headers.get("Authorization")
if _basic_auth_matches(header_value, self.auth_user, self.auth_password):
return True
_reject_unauthorized(self)
return False
def do_GET(self) -> None: # noqa: N802
if not self._authorize():
return
parsed = urlparse(self.path)
if parsed.path == "/":
_text_response(self, HTTPStatus.OK, render_index_html(), "text/html; charset=utf-8")
@@ -3011,22 +2879,11 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
_text_response(self, HTTPStatus.OK, render_tables_html(), "text/html; charset=utf-8")
return
if parsed.path == "/api/tables":
catalog = build_table_catalog(self.db_path)
_json_response(
self,
HTTPStatus.OK,
{
"sqlite": catalog["sqlite"],
"json": catalog["json"],
"workbook": catalog["workbook"],
"tables": [*catalog["sqlite"], *catalog["json"]],
},
)
_json_response(self, HTTPStatus.OK, {"tables": list_browsable_tables(self.db_path)})
return
if parsed.path == "/api/table_rows":
query = parse_qs(parsed.query)
table = (query.get("table") or [""])[0]
source = (query.get("source") or [""])[0]
try:
limit = int((query.get("limit") or ["50"])[0])
offset = int((query.get("offset") or ["0"])[0])
@@ -3036,7 +2893,7 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
limit = min(max(limit, 1), 500)
offset = max(offset, 0)
try:
payload = fetch_table_rows_for_source(source or "sqlite", table, self.db_path, limit=limit, offset=offset) if source else fetch_table_rows(table, self.db_path, limit=limit, offset=offset)
payload = fetch_table_rows(table, self.db_path, limit=limit, offset=offset)
except ValueError as exc:
_json_response(self, HTTPStatus.BAD_REQUEST, {"detail": str(exc)})
return
@@ -3070,8 +2927,6 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
_json_response(self, HTTPStatus.NOT_FOUND, {"detail": "not found"})
def do_POST(self) -> None: # noqa: N802
if not self._authorize():
return
parsed = urlparse(self.path)
try:
if parsed.path == "/api/bootstrap":
@@ -3138,6 +2993,39 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
replace_account_snapshot(conn, rows)
_json_response(self, HTTPStatus.OK, summarize_workspace(self.db_path))
return
if parsed.path == "/api/table/save":
table = str(payload.get("table") or "").strip()
rows = payload.get("rows")
if table not in EDITABLE_TABLES:
raise ValueError(f"table not editable: {table}")
if not isinstance(rows, list):
raise ValueError("rows must be a list")
db_path = _resolve_table_db(table, self.db_path)
if not db_path:
raise ValueError(f"database not found for table: {table}")
with open_connection(db_path) as conn:
conn.execute("BEGIN TRANSACTION")
try:
conn.execute(f"DELETE FROM {table}") # noqa: S608 - Whitelisted table name
if rows:
first_row = rows[0]
columns = [k for k in first_row.keys() if not k.startswith("_")]
if "rowid" in columns:
columns.remove("rowid")
if "_rowid" in columns:
columns.remove("_rowid")
placeholders = ", ".join(["?"] * len(columns))
col_list = ", ".join(columns)
insert_sql = f"INSERT INTO {table} ({col_list}) VALUES ({placeholders})" # noqa: S608 - Whitelisted table name
for row in rows:
values = [row.get(col) for col in columns]
conn.execute(insert_sql, values)
conn.commit()
except Exception as e:
conn.rollback()
raise e
_json_response(self, HTTPStatus.OK, {"status": "SUCCESS", "table": table, "row_count": len(rows)})
return
if parsed.path == "/api/approval_packet":
packet = payload.get("packet")
if not isinstance(packet, dict):
@@ -3240,20 +3128,9 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
self._handle_exception(exc)
def serve(
host: str,
port: int,
db_path: Path | str | None = None,
seed_json_path: Path | str | None = None,
bootstrap: bool = True,
*,
auth_user: str = "",
auth_password: str = "",
allow_remote: bool = False,
) -> None:
def serve(host: str, port: int, db_path: Path | str | None = None, seed_json_path: Path | str | None = None, bootstrap: bool = True) -> None:
db = normalize_db_path(db_path)
seed = Path(seed_json_path) if seed_json_path else DEFAULT_SEED_JSON
_validate_remote_bind(host, allow_remote, auth_user, auth_password)
if bootstrap and seed.exists():
with open_connection(db) as conn:
from .snapshot_admin_store_v1 import ensure_schema
@@ -3263,12 +3140,8 @@ def serve(
import_seed_json(db, seed)
SnapshotAdminHandler.db_path = db
SnapshotAdminHandler.seed_json_path = seed
SnapshotAdminHandler.auth_user = auth_user
SnapshotAdminHandler.auth_password = auth_password
server = ThreadingHTTPServer((host, port), SnapshotAdminHandler)
print(f"Snapshot Admin listening on http://{host}:{port}")
if auth_user and auth_password:
print("Snapshot Admin authentication: enabled (Basic Auth)")
print(f"SQLite DB: {db}")
print(f"Seed JSON: {seed}")
try:
@@ -3286,20 +3159,8 @@ def main() -> int:
parser.add_argument("--db", type=Path, default=DEFAULT_DB)
parser.add_argument("--seed", type=Path, default=DEFAULT_SEED_JSON)
parser.add_argument("--no-bootstrap", action="store_true")
parser.add_argument("--allow-remote", action="store_true", help="Allow binding outside loopback when auth is configured.")
parser.add_argument("--auth-user", default=os.getenv("SNAPSHOT_ADMIN_AUTH_USER", ""))
parser.add_argument("--auth-password", default=os.getenv("SNAPSHOT_ADMIN_AUTH_PASSWORD", ""))
args = parser.parse_args()
serve(
args.host,
args.port,
args.db,
args.seed,
bootstrap=not args.no_bootstrap,
auth_user=args.auth_user,
auth_password=args.auth_password,
allow_remote=args.allow_remote,
)
serve(args.host, args.port, args.db, args.seed, bootstrap=not args.no_bootstrap)
return 0