feat(collection): wire KIS collection end-to-end, add price-history pipeline (WBS QE-M0/M1/M2)
Critical re-review of the QuantEngine WBS evidence system found several
regressions of the "no fake gates" discipline established by M0, plus a
still-unwired M1 collection path. This closes 10 more WBS tasks
(QE-M1-01..06, QE-M2-01/02/04/05/06 — see spec/60_quant_engine_wbs.yaml)
with real, gate-verified evidence (18/34 total).
M1 — real KIS data now lands in PostgreSQL end-to-end:
- SchedulerService: load ticker universe from GatherTradingData.json instead
of a hardcoded array; fix a Hangfire scoped-service resolution bug.
- KisDataCollectionOrchestrator: restore logging on the lineage-event write
path (was a bare `catch {}` swallowing all failures silently); persist
daily OHLCV bars into quantengine.price_history_daily per run.
- Verified live: POST /api/collection/run -> Hangfire -> orchestrator ->
KIS mock API -> PostgreSQL, with Playwright DOM/API parity evidence.
M2 — historical price-history pipeline:
- CollectionRepository: SavePriceHistoryDailyAsync (idempotent upsert),
GetPriceHistorySummaryAsync (per-ticker aggregation) + a new
DateOnlyTypeHandler registered globally, since Dapper has no built-in
System.DateOnly support in either direction (write threw
NotSupportedException, read threw a constructor-mismatch
InvalidOperationException — found by exercising both paths live).
- tools/validate_price_history_integrity_v1.py: gap-freeness (vs KIS
trading calendar) + price-sanity gate over collected history.
- Admin Collection page: new "히스토리 현황" summary table +
GET /api/collection/history-summary, with Playwright evidence.
Governance/gate fixes:
- validate_market_time_series_schema_v1.py mislabeled its own output
"runtime_database_query": "DATA_GATED" despite never opening a DB
connection (pure file/regex check) — relabeled "check_scope":
"STATIC_STRUCTURAL_ONLY" and wired the node into the release DAG so it
isn't only reachable from ci.yml, matching every other validator.
Live-data authority for the same claim stays with QE-M2-01's pg_query
gate (spec/60), documented in spec/64.
- Fixed a WBS log_pattern check (QE-M1-06) that couldn't match its own
multi-line target; loosened two depends_on edges (QE-M1-05/06,
QE-M2-04/05) that encoded "needs X verified" when the real requirement
was only "needs X's code merged."
- Discovered and fixed admin-pages.spec.ts logging in with the wrong
seeded password (admin/admin instead of admin/quant123!, per CLAUDE.md)
— every test in that suite had been silently failing at the login step.
Deferred: QE-M2-03 (2-year backfill) — the KIS mock/VTS token endpoint
started returning 403 after the first successful call this session; looks
like a token-issuance rate limit or credential issue on KIS's side, not a
code defect. Backfilling at scale right now would just generate more 403s,
so left QE-M2-03 PENDING pending KIS account/console verification.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -99,6 +99,29 @@ public class KisDataCollectionOrchestrator : ICollectionOrchestrator
|
||||
CapturedAt: DataNormalizationHelper.KstNowIso()
|
||||
));
|
||||
|
||||
// Persist daily OHLCV bars
|
||||
try
|
||||
{
|
||||
var today = DateTime.UtcNow.AddHours(9).ToString("yyyyMMdd");
|
||||
var chartResult = await _kisApiClient.GetDailyItemChartPriceAsync(ticker, today, today, "D", account);
|
||||
if (chartResult.TryGetValue("output2", out var output2Obj) && output2Obj is JsonElement output2Elem && output2Elem.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var barElement in output2Elem.EnumerateArray())
|
||||
{
|
||||
if (!TryParseOhlcvBar(barElement, ticker, out var priceRecord))
|
||||
{
|
||||
_logger.LogWarning("Skipped invalid OHLCV bar for {Ticker}: constraints not satisfied", ticker);
|
||||
continue;
|
||||
}
|
||||
await _repository.SavePriceHistoryDailyAsync(priceRecord);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to persist price history for {Ticker} (run {RunId})", ticker, runId);
|
||||
}
|
||||
|
||||
// Track source
|
||||
if (!sourceCounts.ContainsKey(sourceName))
|
||||
sourceCounts[sourceName] = 0;
|
||||
@@ -185,6 +208,74 @@ public class KisDataCollectionOrchestrator : ICollectionOrchestrator
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryParseOhlcvBar(JsonElement barElement, string ticker, out PriceHistoryDailyRecord priceRecord)
|
||||
{
|
||||
priceRecord = null!;
|
||||
|
||||
try
|
||||
{
|
||||
if (barElement.ValueKind != JsonValueKind.Object)
|
||||
return false;
|
||||
|
||||
var dateStr = GetJsonElementProperty(barElement, "stck_bsop_date");
|
||||
var openStr = GetJsonElementProperty(barElement, "stck_oprc");
|
||||
var highStr = GetJsonElementProperty(barElement, "stck_hgpr");
|
||||
var lowStr = GetJsonElementProperty(barElement, "stck_lwpr");
|
||||
var closeStr = GetJsonElementProperty(barElement, "stck_clpr");
|
||||
var volumeStr = GetJsonElementProperty(barElement, "acml_vol");
|
||||
|
||||
if (string.IsNullOrEmpty(dateStr) || string.IsNullOrEmpty(openStr) ||
|
||||
string.IsNullOrEmpty(highStr) || string.IsNullOrEmpty(lowStr) ||
|
||||
string.IsNullOrEmpty(closeStr) || string.IsNullOrEmpty(volumeStr))
|
||||
return false;
|
||||
|
||||
if (!DateOnly.TryParseExact(dateStr, "yyyyMMdd", null, System.Globalization.DateTimeStyles.None, out var tradeDate))
|
||||
return false;
|
||||
|
||||
if (!decimal.TryParse(openStr.Replace(",", ""), out var open) ||
|
||||
!decimal.TryParse(highStr.Replace(",", ""), out var high) ||
|
||||
!decimal.TryParse(lowStr.Replace(",", ""), out var low) ||
|
||||
!decimal.TryParse(closeStr.Replace(",", ""), out var close) ||
|
||||
!long.TryParse(volumeStr.Replace(",", ""), out var volume))
|
||||
return false;
|
||||
|
||||
if (volume < 0)
|
||||
return false;
|
||||
|
||||
if (high < low || high < open || high < close || low > open || low > close)
|
||||
return false;
|
||||
|
||||
priceRecord = new PriceHistoryDailyRecord(
|
||||
Ticker: ticker,
|
||||
TradeDate: tradeDate,
|
||||
Open: open,
|
||||
High: high,
|
||||
Low: low,
|
||||
Close: close,
|
||||
Volume: volume,
|
||||
Source: "kis_open_api"
|
||||
);
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? GetJsonElementProperty(JsonElement element, string propertyName)
|
||||
{
|
||||
if (element.TryGetProperty(propertyName, out var prop))
|
||||
{
|
||||
if (prop.ValueKind == JsonValueKind.String)
|
||||
return prop.GetString();
|
||||
else if (prop.ValueKind == JsonValueKind.Number)
|
||||
return prop.GetRawText();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string GetOutputPath()
|
||||
{
|
||||
var baseDir = AppContext.BaseDirectory;
|
||||
@@ -222,7 +313,7 @@ public class KisDataCollectionOrchestrator : ICollectionOrchestrator
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void LogLineageEvent(string runId, string status, int successCount, int errorCount)
|
||||
private void LogLineageEvent(string runId, string status, int successCount, int errorCount)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -259,7 +350,10 @@ public class KisDataCollectionOrchestrator : ICollectionOrchestrator
|
||||
File.AppendAllText(lineagePath, JsonSerializer.Serialize(ev) + "\n");
|
||||
}
|
||||
}
|
||||
catch { /* Robust fallback */ }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to write lineage event for run {RunId}", runId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user