feat(kis-collection): finalize sqlite migration, add fallback resilience, and update WBS documentation

This commit is contained in:
2026-06-22 18:34:56 +09:00
parent c576138829
commit 6c549b7bdc
48 changed files with 34610 additions and 24883 deletions
+275 -414
View File
@@ -1,9 +1,7 @@
from __future__ import annotations
import argparse
import base64
import json
import os
import sqlite3
import subprocess
from http import HTTPStatus
@@ -13,176 +11,112 @@ from hashlib import sha256
from typing import Any
from urllib.parse import urlparse, parse_qs
import openpyxl
ROOT = Path(__file__).resolve().parents[2]
SNAPSHOT_ADMIN_VERSION = "snapshot-admin-web-v7"
GATHER_TRADING_DATA_XLSX = ROOT / "GatherTradingData.xlsx"
SNAPSHOT_ADMIN_VERSION = "snapshot-admin-web-v6"
KIS_COLLECTION_DB = ROOT / "outputs" / "kis_data_collection" / "kis_data_collection.db"
KIS_COLLECTION_REPORT = ROOT / "Temp" / "kis_data_collection_v1.json"
QUALITATIVE_SELL_DB = ROOT / "outputs" / "qualitative_sell_strategy" / "qualitative_sell_strategy.db"
GATHER_TRADING_DATA_JSON = ROOT / "GatherTradingData.json"
AUTH_REALM = "Snapshot Admin"
JSON_SHEET_ALIASES = {
"harness_context": "_harness_context",
# WBS-7.9 부속 — 테이블별 그리드 조회(Tabler). 화이트리스트에 없는 테이블명은
# SQL에 절대 보간되지 않는다(요청 테이블명을 그대로 SELECT 문에 넣지 않고
# 아래 레지스트리 키와 정확히 일치할 때만 허용).
WORKSPACE_BROWSABLE_TABLES = (
"settings",
"account_snapshot",
"workspace_change_log",
"workspace_approval_v2",
"workspace_lock",
"workspace_meta",
)
COLLECTION_BROWSABLE_TABLES = (
"collection_runs",
"collection_snapshots",
"collection_source_errors",
)
QUALITATIVE_SELL_BROWSABLE_TABLES = (
"sell_strategy_results",
"satellite_recommendations",
)
# Editable tables configurations (WBS requirement 2)
EDITABLE_TABLES = {
"settings",
"account_snapshot",
"collection_runs",
"collection_snapshots",
"collection_source_errors",
"sell_strategy_results",
"satellite_recommendations",
}
# WBS-7.9 부속, WBS-7.10 후속(2026-06-22) — 테이블별 그리드 조회(Tabler).
# 정적 화이트리스트 대신 각 DB 파일의 sqlite_master를 그때그때 조회해 테이블
# 목록을 만든다 — 정적 목록은 스키마가 바뀌거나(예: 레거시 workspace_approval
# 테이블처럼) 새 테이블이 추가되면 누락되는 문제가 있었다(사용자 보고로 발견).
# 보안 속성은 동일하게 유지된다: 요청된 테이블명은 항상 해당 DB의 실제
# sqlite_master 결과와 정확히 일치할 때만 SQL에 사용된다(임의 문자열 보간 없음).
def _known_db_paths(workspace_db_path: Path) -> list[Path]:
return [Path(workspace_db_path), KIS_COLLECTION_DB, QUALITATIVE_SELL_DB]
def _discover_tables(db_path: Path) -> list[str]:
if not db_path.exists():
return []
with sqlite3.connect(db_path) as conn:
rows = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
).fetchall()
return [row[0] for row in rows]
def _resolve_table_db(table: str, workspace_db_path: Path) -> Path | None:
for db_path in _known_db_paths(workspace_db_path):
if table in _discover_tables(db_path):
return db_path
if table in WORKSPACE_BROWSABLE_TABLES:
return Path(workspace_db_path)
if table in COLLECTION_BROWSABLE_TABLES:
return KIS_COLLECTION_DB
if table in QUALITATIVE_SELL_BROWSABLE_TABLES:
return QUALITATIVE_SELL_DB
return None
# 2026-06-22 — 분석/판단 팩터로 쓰이는 GatherTradingData.json의 data.* 시트도
# 같은 그리드로 조회 가능하게 한다(SQLite로 옮겨지지 않은 data_feed/sector_flow/
# macro 등). dict 키 조회만 하므로 SQL 인젝션 표면 자체가 없다.
def _discover_json_sheets() -> dict[str, list[dict[str, Any]]]:
if not GATHER_TRADING_DATA_JSON.exists():
return {}
try:
payload = json.loads(GATHER_TRADING_DATA_JSON.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
data = payload.get("data")
if not isinstance(data, dict):
return {}
return {key: value for key, value in data.items() if isinstance(value, list) and value and isinstance(value[0], dict)}
def _discover_workbook_sheets() -> list[dict[str, Any]]:
if not GATHER_TRADING_DATA_XLSX.exists():
return []
try:
workbook = openpyxl.load_workbook(GATHER_TRADING_DATA_XLSX, read_only=True, data_only=True)
except Exception:
return []
try:
inventory: list[dict[str, Any]] = []
for sheet_name in workbook.sheetnames:
worksheet = workbook[sheet_name]
inventory.append(
{
"sheet": sheet_name,
"row_count": int(worksheet.max_row or 0),
"column_count": int(worksheet.max_column or 0),
"source_workbook": str(GATHER_TRADING_DATA_XLSX),
}
)
return inventory
finally:
workbook.close()
def build_table_catalog(workspace_db_path: Path) -> dict[str, list[dict[str, Any]]]:
sqlite_rows: list[dict[str, Any]] = []
for db_path in _known_db_paths(workspace_db_path):
for table in _discover_tables(db_path):
def list_browsable_tables(workspace_db_path: Path) -> list[dict[str, Any]]:
tables: list[dict[str, Any]] = []
for table in (
*WORKSPACE_BROWSABLE_TABLES,
*COLLECTION_BROWSABLE_TABLES,
*QUALITATIVE_SELL_BROWSABLE_TABLES,
):
db_path = _resolve_table_db(table, workspace_db_path)
exists = bool(db_path and db_path.exists())
row_count = 0
if exists:
try:
with sqlite3.connect(db_path) as conn:
row_count = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 - table name confirmed via sqlite_master of this exact db above
row_count = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 - table is whitelist-checked above
except sqlite3.OperationalError:
continue
sqlite_rows.append({"table": table, "db": str(db_path), "exists": True, "row_count": row_count, "source": "sqlite"})
json_rows = [{"table": sheet, "db": str(GATHER_TRADING_DATA_JSON), "exists": True, "row_count": len(rows), "source": "json"} for sheet, rows in _discover_json_sheets().items()]
sqlite_names = {row["table"] for row in sqlite_rows}
json_names = {row["table"] for row in json_rows}
workbook_rows: list[dict[str, Any]] = []
for sheet_row in _discover_workbook_sheets():
sheet_name = sheet_row["sheet"]
json_key = JSON_SHEET_ALIASES.get(sheet_name, sheet_name)
current_sources: list[str] = []
if sheet_name in sqlite_names:
current_sources.append("sqlite")
if sheet_name in json_names or json_key in json_names:
current_sources.append("json")
if not current_sources:
current_sources.append("xlsx")
workbook_rows.append(
{
**sheet_row,
"json_key": json_key,
"current_sources": current_sources,
"migration_candidate": "yes" if "sqlite" not in current_sources else "no",
}
)
return {"sqlite": sqlite_rows, "json": json_rows, "workbook": workbook_rows}
def list_browsable_tables(workspace_db_path: Path) -> list[dict[str, Any]]:
catalog = build_table_catalog(workspace_db_path)
return [*catalog["sqlite"], *catalog["json"]]
exists = False
tables.append({
"table": table,
"db": str(db_path) if db_path else "",
"exists": exists,
"row_count": row_count,
"editable": table in EDITABLE_TABLES,
})
return tables
def fetch_table_rows(table: str, workspace_db_path: Path, *, limit: int = 50, offset: int = 0) -> dict[str, Any]:
db_path = _resolve_table_db(table, workspace_db_path)
if db_path is not None:
with sqlite3.connect(db_path) as conn:
conn.row_factory = sqlite3.Row
total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 - whitelisted table name
cursor = conn.execute(
f"SELECT * FROM {table} ORDER BY rowid DESC LIMIT ? OFFSET ?", # noqa: S608 - whitelisted table name
(limit, offset),
)
rows = [dict(row) for row in cursor.fetchall()]
columns = [description[0] for description in cursor.description] if cursor.description else []
return {"table": table, "db": str(db_path), "columns": columns, "rows": rows, "total": total, "limit": limit, "offset": offset, "source": "sqlite"}
json_sheets = _discover_json_sheets()
if table not in json_sheets:
if db_path is None:
raise ValueError(f"unknown or non-browsable table: {table}")
sheet_rows = json_sheets[table]
total = len(sheet_rows)
page = sheet_rows[offset : offset + limit]
columns: list[str] = []
for row in page:
for key in row.keys():
if key not in columns:
columns.append(key)
return {"table": table, "db": str(GATHER_TRADING_DATA_JSON), "columns": columns, "rows": page, "total": total, "limit": limit, "offset": offset, "source": "json"}
if not db_path.exists():
return {"table": table, "db": str(db_path), "columns": [], "rows": [], "total": 0, "limit": limit, "offset": offset, "editable": table in EDITABLE_TABLES}
with sqlite3.connect(db_path) as conn:
conn.row_factory = sqlite3.Row
total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: S608 - whitelisted table name
cursor = conn.execute(
f"SELECT rowid as _rowid, * FROM {table} ORDER BY rowid DESC LIMIT ? OFFSET ?", # noqa: S608 - whitelisted table name
(limit, offset),
)
rows = [dict(row) for row in cursor.fetchall()]
columns = [description[0] for description in cursor.description] if cursor.description else []
return {"table": table, "db": str(db_path), "columns": columns, "rows": rows, "total": total, "limit": limit, "offset": offset, "editable": table in EDITABLE_TABLES}
def fetch_table_rows_for_source(source: str, table: str, workspace_db_path: Path, *, limit: int = 50, offset: int = 0) -> dict[str, Any]:
normalized_source = source.strip().lower()
if normalized_source == "sqlite":
return fetch_table_rows(table, workspace_db_path, limit=limit, offset=offset)
if normalized_source == "json":
json_sheets = _discover_json_sheets()
if table not in json_sheets:
raise ValueError(f"unknown or non-browsable table: {table}")
sheet_rows = json_sheets[table]
total = len(sheet_rows)
page = sheet_rows[offset : offset + limit]
columns: list[str] = []
for row in page:
for key in row.keys():
if key not in columns:
columns.append(key)
return {"table": table, "db": str(GATHER_TRADING_DATA_JSON), "columns": columns, "rows": page, "total": total, "limit": limit, "offset": offset, "source": "json"}
raise ValueError(f"unsupported source: {source}")
def fetch_domain_rows(domain: str, workspace_db_path: Path) -> dict[str, Any]:
if domain == "settings":
rows = load_settings_rows(workspace_db_path)
return {"domain": domain, "db": str(workspace_db_path), "columns": ["ordinal", "key", "value", "note", "updated_at"], "rows": rows}
if domain == "account_snapshot":
rows = load_account_snapshot_rows(workspace_db_path)
return {
"domain": domain,
"db": str(workspace_db_path),
"columns": list(ACCOUNT_SNAPSHOT_CANONICAL_COLUMNS),
"rows": rows,
}
raise ValueError(f"unknown editable domain: {domain}")
SNAPSHOT_ADMIN_VERSION_FILES = (
ROOT / "src" / "quant_engine" / "snapshot_admin_server_v1.py",
ROOT / "src" / "quant_engine" / "snapshot_admin_store_v1.py",
@@ -422,55 +356,6 @@ def _text_response(handler: BaseHTTPRequestHandler, status: int, text: str, cont
handler.wfile.write(body)
def _is_loopback_host(host: str) -> bool:
normalized = host.strip().lower()
return normalized in {"127.0.0.1", "localhost", "::1"}
def _parse_basic_auth(header_value: str | None) -> tuple[str, str] | None:
if not header_value:
return None
prefix = "basic "
if not header_value.lower().startswith(prefix):
return None
encoded = header_value[len(prefix) :].strip()
if not encoded:
return None
try:
decoded = base64.b64decode(encoded).decode("utf-8")
except (ValueError, UnicodeDecodeError):
return None
if ":" not in decoded:
return None
username, password = decoded.split(":", 1)
return username, password
def _basic_auth_matches(header_value: str | None, username: str, password: str) -> bool:
parsed = _parse_basic_auth(header_value)
return bool(parsed and parsed[0] == username and parsed[1] == password)
def _reject_unauthorized(handler: BaseHTTPRequestHandler) -> None:
body = json.dumps({"detail": "authentication required"}, ensure_ascii=False, indent=2).encode("utf-8")
handler.send_response(HTTPStatus.UNAUTHORIZED)
handler.send_header("WWW-Authenticate", f'Basic realm="{AUTH_REALM}", charset="UTF-8"')
handler.send_header("Content-Type", "application/json; charset=utf-8")
handler.send_header("Content-Length", str(len(body)))
handler.end_headers()
handler.wfile.write(body)
def _validate_remote_bind(host: str, allow_remote: bool, auth_user: str, auth_password: str) -> None:
has_auth = bool(auth_user and auth_password)
if bool(auth_user) != bool(auth_password):
raise ValueError("snapshot admin auth requires both --auth-user and --auth-password")
if not _is_loopback_host(host) and not allow_remote:
raise ValueError("refusing to bind snapshot admin outside loopback without --allow-remote")
if (allow_remote or not _is_loopback_host(host)) and not has_auth:
raise ValueError("remote snapshot admin access requires both --auth-user and --auth-password")
def _read_json_body(handler: BaseHTTPRequestHandler) -> dict[str, Any]:
length = int(handler.headers.get("Content-Length") or "0")
raw = handler.rfile.read(length).decode("utf-8") if length else "{}"
@@ -2778,79 +2663,26 @@ def render_tables_html() -> str:
<div class="page-wrapper">
<div class="page-body">
<div class="container-xl">
<div class="row row-cards">
<div class="col-12">
<div class="card">
<div class="card-header">
<div>
<div class="card-title">Workbook migration inventory</div>
<div class="text-secondary">Source-of-truth xlsx sheet list with current storage classification.</div>
</div>
<span class="badge bg-secondary-lt" id="inventoryMeta"></span>
</div>
<div class="table-responsive">
<table class="table table-vcenter card-table table-striped">
<thead>
<tr>
<th>Sheet</th>
<th class="text-end">Rows</th>
<th class="text-end">Cols</th>
<th>Current Source</th>
<th>Migration Candidate</th>
</tr>
</thead>
<tbody id="inventoryBody"></tbody>
</table>
</div>
<div class="card">
<div class="card-header d-flex flex-wrap gap-2 align-items-center justify-content-between">
<div class="d-flex gap-2 align-items-center">
<label class="form-label mb-0 me-1" for="tableSelect">Table</label>
<select id="tableSelect" class="form-select" style="min-width:280px" onchange="onTableChange()"></select>
<span class="badge bg-secondary-lt" id="tableMeta"></span>
</div>
<div class="d-flex gap-2">
<button class="btn btn-sm" onclick="prevPage()">&laquo; Prev</button>
<span class="d-flex align-items-center px-2" id="pageInfo"></span>
<button class="btn btn-sm" onclick="nextPage()">Next &raquo;</button>
<button class="btn btn-sm btn-primary" onclick="reload()">Refresh</button>
<button class="btn btn-sm btn-success" id="saveTableBtn" onclick="saveCurrentTable()">Save changes</button>
</div>
</div>
<div class="col-12 col-xl-6">
<div class="card">
<div class="card-header d-flex flex-wrap gap-2 align-items-center justify-content-between">
<div class="d-flex gap-2 align-items-center">
<span class="badge bg-blue-lt">SQLite</span>
<label class="form-label mb-0 me-1" for="sqliteTableSelect">Table</label>
<select id="sqliteTableSelect" class="form-select" style="min-width:260px" onchange="onTableChange('sqlite')"></select>
<span class="badge bg-secondary-lt" id="sqliteTableMeta"></span>
</div>
<div class="d-flex gap-2">
<button class="btn btn-sm" onclick="prevPage('sqlite')">&laquo; Prev</button>
<span class="d-flex align-items-center px-2" id="sqlitePageInfo"></span>
<button class="btn btn-sm" onclick="nextPage('sqlite')">Next &raquo;</button>
<button class="btn btn-sm btn-primary" onclick="reload('sqlite')">Refresh</button>
</div>
</div>
<div class="table-responsive">
<table class="table table-vcenter card-table table-striped" id="sqliteGridTable">
<thead><tr id="sqliteGridHead"></tr></thead>
<tbody id="sqliteGridBody"></tbody>
</table>
</div>
</div>
</div>
<div class="col-12 col-xl-6">
<div class="card">
<div class="card-header d-flex flex-wrap gap-2 align-items-center justify-content-between">
<div class="d-flex gap-2 align-items-center">
<span class="badge bg-azure-lt">JSON</span>
<label class="form-label mb-0 me-1" for="jsonTableSelect">Sheet</label>
<select id="jsonTableSelect" class="form-select" style="min-width:260px" onchange="onTableChange('json')"></select>
<span class="badge bg-secondary-lt" id="jsonTableMeta"></span>
</div>
<div class="d-flex gap-2">
<button class="btn btn-sm" onclick="prevPage('json')">&laquo; Prev</button>
<span class="d-flex align-items-center px-2" id="jsonPageInfo"></span>
<button class="btn btn-sm" onclick="nextPage('json')">Next &raquo;</button>
<button class="btn btn-sm btn-primary" onclick="reload('json')">Refresh</button>
</div>
</div>
<div class="table-responsive">
<table class="table table-vcenter card-table table-striped" id="jsonGridTable">
<thead><tr id="jsonGridHead"></tr></thead>
<tbody id="jsonGridBody"></tbody>
</table>
</div>
</div>
<div class="table-responsive">
<table class="table table-vcenter card-table table-striped" id="gridTable">
<thead><tr id="gridHead"></tr></thead>
<tbody id="gridBody"></tbody>
</table>
</div>
</div>
</div>
@@ -2858,11 +2690,7 @@ def render_tables_html() -> str:
</div>
</div>
<script>
const state = {
catalog: { sqlite: [], json: [], workbook: [] },
sqlite: { current: "", limit: 50, offset: 0, total: 0 },
json: { current: "", limit: 50, offset: 0, total: 0 },
};
const state = { tables: [], current: "", limit: 50, offset: 0, total: 0, editable: false, rows: [] };
function escapeHtml(value) {
if (value === null || value === undefined) return "";
@@ -2870,105 +2698,158 @@ def render_tables_html() -> str:
return text.replace(/[&<>"']/g, (ch) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[ch]));
}
function sectionLabel(source) {
return source === "json" ? "JSON" : "SQLite";
function isEditableDomain(domain) {
return domain === "settings" || domain === "account_snapshot";
}
function sectionIds(source) {
return {
selectId: `${source}TableSelect`,
metaId: `${source}TableMeta`,
pageInfoId: `${source}PageInfo`,
headId: `${source}GridHead`,
bodyId: `${source}GridBody`,
};
function normalizeCellValue(value) {
if (value === null || value === undefined) return "";
if (typeof value === "object") return JSON.stringify(value);
return String(value);
}
function renderInventory() {
const body = document.getElementById("inventoryBody");
body.innerHTML = state.catalog.workbook
.map((row) => {
const sources = (row.current_sources || []).map((item) => item.toUpperCase()).join(", ");
const candidate = row.migration_candidate === "yes" ? "yes" : "no";
return `<tr>
<td>${escapeHtml(row.sheet)}</td>
<td class="text-end">${escapeHtml(row.row_count)}</td>
<td class="text-end">${escapeHtml(row.column_count)}</td>
<td>${escapeHtml(sources)}</td>
<td>${escapeHtml(candidate)}</td>
</tr>`;
})
.join("") || `<tr><td colspan="5" class="text-secondary">no workbook inventory</td></tr>`;
document.getElementById("inventoryMeta").textContent = `${state.catalog.workbook.length} sheets`;
function editableCell(rowIndex, column, value) {
return `<td contenteditable="true" data-row-index="${rowIndex}" data-column="${escapeHtml(column)}">${escapeHtml(normalizeCellValue(value))}</td>`;
}
function populateSelect(source) {
const select = document.getElementById(sectionIds(source).selectId);
const tables = state.catalog[source] || [];
select.innerHTML = tables
.map((t) => `<option value="${escapeHtml(t.table)}">${escapeHtml(t.table)} (${escapeHtml(t.row_count)})</option>`)
.join("");
if (!state[source].current && tables.length) {
state[source].current = tables[0].table;
}
select.value = state[source].current;
}
async function loadCatalog() {
async function loadTables() {
const res = await fetch("/api/tables");
const data = await res.json();
state.catalog.sqlite = data.sqlite || [];
state.catalog.json = data.json || [];
state.catalog.workbook = data.workbook || [];
renderInventory();
populateSelect("sqlite");
populateSelect("json");
await Promise.all([loadRows("sqlite"), loadRows("json")]);
state.tables = data.tables || [];
const select = document.getElementById("tableSelect");
select.innerHTML = state.tables
.map((t) => `<option value="${t.table}" ${!t.exists ? "disabled" : ""}>${t.table} (${t.exists ? t.row_count : "no db"})${t.editable ? ' (Editable)' : ''}</option>`)
.join("");
if (!state.current && state.tables.length) {
state.current = state.tables.find((t) => t.exists)?.table || state.tables[0].table;
}
select.value = state.current;
await loadRows();
}
function onTableChange(source) {
state[source].current = document.getElementById(sectionIds(source).selectId).value;
state[source].offset = 0;
loadRows(source);
function onTableChange() {
state.current = document.getElementById("tableSelect").value;
state.offset = 0;
loadRows();
}
async function loadRows(source) {
if (!state[source].current) return;
const ids = sectionIds(source);
const params = new URLSearchParams({ source, table: state[source].current, limit: state[source].limit, offset: state[source].offset });
const res = await fetch(`/api/table_rows?${params.toString()}`);
async function loadRows() {
if (!state.current) return;
const editable = state.tables.find(t => t.table === state.current)?.editable || false;
state.editable = editable;
const isDomain = isEditableDomain(state.current);
const url = isDomain ? `/api/domain_rows?domain=${encodeURIComponent(state.current)}` : `/api/table_rows?${new URLSearchParams({ table: state.current, limit: state.limit, offset: state.offset }).toString()}`;
const res = await fetch(url);
const data = await res.json();
state[source].total = data.total || 0;
document.getElementById(ids.headId).innerHTML = (data.columns || []).map((c) => `<th>${escapeHtml(c)}</th>`).join("");
document.getElementById(ids.bodyId).innerHTML = (data.rows || [])
.map((row) => `<tr>${(data.columns || []).map((c) => `<td>${escapeHtml(row[c])}</td>`).join("")}</tr>`)
.join("") || `<tr><td colspan="99" class="text-secondary">no rows</td></tr>`;
document.getElementById(ids.metaId).textContent = `[${sectionLabel(source)}] ${data.db || ""}`;
const from = state[source].total === 0 ? 0 : state[source].offset + 1;
const to = Math.min(state[source].offset + state[source].limit, state[source].total);
document.getElementById(ids.pageInfoId).textContent = `${from}-${to} / ${state[source].total}`;
}
function prevPage(source) {
state[source].offset = Math.max(0, state[source].offset - state[source].limit);
loadRows(source);
}
function nextPage(source) {
if (state[source].offset + state[source].limit < state[source].total) {
state[source].offset += state[source].limit;
loadRows(source);
state.rows = data.rows || [];
state.total = isDomain ? state.rows.length : (data.total || 0);
const head = document.getElementById("gridHead");
const body = document.getElementById("gridBody");
const displayColumns = (data.columns || []).filter((c) => !String(c).startsWith("_"));
head.innerHTML = displayColumns.map((c) => `<th>${escapeHtml(c)}</th>`).join("");
body.innerHTML = state.rows.length
? state.rows
.map((row, rowIndex) => {
return `<tr data-row-index="${rowIndex}">${displayColumns.map((c) => {
// Settings key and other primary columns can be protected or editable based on needs
const cellVal = row[c];
return editable ? editableCell(rowIndex, c, cellVal) : `<td>${escapeHtml(cellVal)}</td>`;
}).join("")}</tr>`;
})
.join("")
: `<tr><td colspan="99" class="text-secondary">no rows</td></tr>`;
document.getElementById("tableMeta").textContent = `${data.db || ""}`;
const from = state.total === 0 ? 0 : state.offset + 1;
const to = Math.min(state.offset + state.limit, state.total);
document.getElementById("pageInfo").textContent = `${from}-${to} / ${state.total}`;
const saveBtn = document.getElementById("saveTableBtn");
if (saveBtn) {
saveBtn.disabled = !editable;
saveBtn.textContent = editable ? "Save current table" : "Read only";
}
}
function reload(source) {
loadRows(source);
function prevPage() {
state.offset = Math.max(0, state.offset - state.limit);
loadRows();
}
loadCatalog().catch((error) => {
document.getElementById("inventoryBody").innerHTML = `<tr><td colspan="5" class="text-danger">${escapeHtml(error.message)}</td></tr>`;
document.getElementById("sqliteGridBody").innerHTML = `<tr><td class="text-danger">${escapeHtml(error.message)}</td></tr>`;
document.getElementById("jsonGridBody").innerHTML = `<tr><td class="text-danger">${escapeHtml(error.message)}</td></tr>`;
function nextPage() {
if (state.offset + state.limit < state.total) {
state.offset += state.limit;
loadRows();
}
}
function reload() {
loadRows();
}
function collectEditableRows() {
const table = document.getElementById("gridTable");
const columns = Array.from(table.querySelectorAll("thead th")).map((th) => th.textContent || "");
const bodyRows = table.querySelectorAll("tbody tr[data-row-index]");
return Array.from(bodyRows).map((tr, index) => {
const cells = tr.querySelectorAll("td");
const row = {};
let cellIndex = 0;
for (const column of columns) {
if (String(column).startsWith("_")) {
continue;
}
const cell = cells[cellIndex++];
row[column] = cell ? cell.textContent : "";
}
row.ordinal = index + 1;
return row;
});
}
function parseCellValue(value) {
const text = String(value || "").trim();
if (text === "") return "";
if (text === "null" || text === "None") return null;
if (text === "true") return true;
if (text === "false") return false;
const numericPattern = new RegExp("^-?(0|[1-9]\\\\d*)(\\\\.\\\\d+)?([eE][-+]?\\\\d+)?$");
if (numericPattern.test(text)) return Number(text);
try {
return JSON.parse(text);
} catch (err) {
return text;
}
}
async function saveCurrentTable() {
if (!state.editable) return;
const isDomain = isEditableDomain(state.current);
const endpoint = isDomain
? (state.current === "settings" ? "/api/settings/save" : "/api/account_snapshot/save")
: "/api/table/save";
const rows = collectEditableRows().map(row => {
const parsedRow = {};
for (const k of Object.keys(row)) {
parsedRow[k] = parseCellValue(row[k]);
}
return parsedRow;
});
const payload = isDomain ? { rows } : { table: state.current, rows };
const response = await fetch(endpoint, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
const data = await response.json();
if (!response.ok) throw new Error(data.detail || "save failed");
await loadRows();
alert(`saved: ${state.current}`);
}
loadTables().catch((error) => {
document.getElementById("gridBody").innerHTML = `<tr><td class="text-danger">${escapeHtml(error.message)}</td></tr>`;
});
</script>
</body>
@@ -2979,8 +2860,6 @@ def render_tables_html() -> str:
class SnapshotAdminHandler(BaseHTTPRequestHandler):
db_path: Path = DEFAULT_DB
seed_json_path: Path = DEFAULT_SEED_JSON
auth_user: str = ""
auth_password: str = ""
def log_message(self, format: str, *args: Any) -> None: # noqa: A003
return
@@ -2988,18 +2867,7 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
def _handle_exception(self, exc: Exception) -> None:
_json_response(self, HTTPStatus.INTERNAL_SERVER_ERROR, {"detail": str(exc)})
def _authorize(self) -> bool:
if not self.auth_user and not self.auth_password:
return True
header_value = self.headers.get("Authorization")
if _basic_auth_matches(header_value, self.auth_user, self.auth_password):
return True
_reject_unauthorized(self)
return False
def do_GET(self) -> None: # noqa: N802
if not self._authorize():
return
parsed = urlparse(self.path)
if parsed.path == "/":
_text_response(self, HTTPStatus.OK, render_index_html(), "text/html; charset=utf-8")
@@ -3011,22 +2879,11 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
_text_response(self, HTTPStatus.OK, render_tables_html(), "text/html; charset=utf-8")
return
if parsed.path == "/api/tables":
catalog = build_table_catalog(self.db_path)
_json_response(
self,
HTTPStatus.OK,
{
"sqlite": catalog["sqlite"],
"json": catalog["json"],
"workbook": catalog["workbook"],
"tables": [*catalog["sqlite"], *catalog["json"]],
},
)
_json_response(self, HTTPStatus.OK, {"tables": list_browsable_tables(self.db_path)})
return
if parsed.path == "/api/table_rows":
query = parse_qs(parsed.query)
table = (query.get("table") or [""])[0]
source = (query.get("source") or [""])[0]
try:
limit = int((query.get("limit") or ["50"])[0])
offset = int((query.get("offset") or ["0"])[0])
@@ -3036,7 +2893,7 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
limit = min(max(limit, 1), 500)
offset = max(offset, 0)
try:
payload = fetch_table_rows_for_source(source or "sqlite", table, self.db_path, limit=limit, offset=offset) if source else fetch_table_rows(table, self.db_path, limit=limit, offset=offset)
payload = fetch_table_rows(table, self.db_path, limit=limit, offset=offset)
except ValueError as exc:
_json_response(self, HTTPStatus.BAD_REQUEST, {"detail": str(exc)})
return
@@ -3070,8 +2927,6 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
_json_response(self, HTTPStatus.NOT_FOUND, {"detail": "not found"})
def do_POST(self) -> None: # noqa: N802
if not self._authorize():
return
parsed = urlparse(self.path)
try:
if parsed.path == "/api/bootstrap":
@@ -3138,6 +2993,39 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
replace_account_snapshot(conn, rows)
_json_response(self, HTTPStatus.OK, summarize_workspace(self.db_path))
return
if parsed.path == "/api/table/save":
table = str(payload.get("table") or "").strip()
rows = payload.get("rows")
if table not in EDITABLE_TABLES:
raise ValueError(f"table not editable: {table}")
if not isinstance(rows, list):
raise ValueError("rows must be a list")
db_path = _resolve_table_db(table, self.db_path)
if not db_path:
raise ValueError(f"database not found for table: {table}")
with open_connection(db_path) as conn:
conn.execute("BEGIN TRANSACTION")
try:
conn.execute(f"DELETE FROM {table}") # noqa: S608 - Whitelisted table name
if rows:
first_row = rows[0]
columns = [k for k in first_row.keys() if not k.startswith("_")]
if "rowid" in columns:
columns.remove("rowid")
if "_rowid" in columns:
columns.remove("_rowid")
placeholders = ", ".join(["?"] * len(columns))
col_list = ", ".join(columns)
insert_sql = f"INSERT INTO {table} ({col_list}) VALUES ({placeholders})" # noqa: S608 - Whitelisted table name
for row in rows:
values = [row.get(col) for col in columns]
conn.execute(insert_sql, values)
conn.commit()
except Exception as e:
conn.rollback()
raise e
_json_response(self, HTTPStatus.OK, {"status": "SUCCESS", "table": table, "row_count": len(rows)})
return
if parsed.path == "/api/approval_packet":
packet = payload.get("packet")
if not isinstance(packet, dict):
@@ -3240,20 +3128,9 @@ class SnapshotAdminHandler(BaseHTTPRequestHandler):
self._handle_exception(exc)
def serve(
host: str,
port: int,
db_path: Path | str | None = None,
seed_json_path: Path | str | None = None,
bootstrap: bool = True,
*,
auth_user: str = "",
auth_password: str = "",
allow_remote: bool = False,
) -> None:
def serve(host: str, port: int, db_path: Path | str | None = None, seed_json_path: Path | str | None = None, bootstrap: bool = True) -> None:
db = normalize_db_path(db_path)
seed = Path(seed_json_path) if seed_json_path else DEFAULT_SEED_JSON
_validate_remote_bind(host, allow_remote, auth_user, auth_password)
if bootstrap and seed.exists():
with open_connection(db) as conn:
from .snapshot_admin_store_v1 import ensure_schema
@@ -3263,12 +3140,8 @@ def serve(
import_seed_json(db, seed)
SnapshotAdminHandler.db_path = db
SnapshotAdminHandler.seed_json_path = seed
SnapshotAdminHandler.auth_user = auth_user
SnapshotAdminHandler.auth_password = auth_password
server = ThreadingHTTPServer((host, port), SnapshotAdminHandler)
print(f"Snapshot Admin listening on http://{host}:{port}")
if auth_user and auth_password:
print("Snapshot Admin authentication: enabled (Basic Auth)")
print(f"SQLite DB: {db}")
print(f"Seed JSON: {seed}")
try:
@@ -3286,20 +3159,8 @@ def main() -> int:
parser.add_argument("--db", type=Path, default=DEFAULT_DB)
parser.add_argument("--seed", type=Path, default=DEFAULT_SEED_JSON)
parser.add_argument("--no-bootstrap", action="store_true")
parser.add_argument("--allow-remote", action="store_true", help="Allow binding outside loopback when auth is configured.")
parser.add_argument("--auth-user", default=os.getenv("SNAPSHOT_ADMIN_AUTH_USER", ""))
parser.add_argument("--auth-password", default=os.getenv("SNAPSHOT_ADMIN_AUTH_PASSWORD", ""))
args = parser.parse_args()
serve(
args.host,
args.port,
args.db,
args.seed,
bootstrap=not args.no_bootstrap,
auth_user=args.auth_user,
auth_password=args.auth_password,
allow_remote=args.allow_remote,
)
serve(args.host, args.port, args.db, args.seed, bootstrap=not args.no_bootstrap)
return 0