feat(collection): wire KIS collection end-to-end, add price-history pipeline (WBS QE-M0/M1/M2)
Critical re-review of the QuantEngine WBS evidence system found several
regressions of the "no fake gates" discipline established by M0, plus a
still-unwired M1 collection path. This closes 10 more WBS tasks
(QE-M1-01..06, QE-M2-01/02/04/05/06 — see spec/60_quant_engine_wbs.yaml)
with real, gate-verified evidence (18/34 total).
M1 — real KIS data now lands in PostgreSQL end-to-end:
- SchedulerService: load ticker universe from GatherTradingData.json instead
of a hardcoded array; fix a Hangfire scoped-service resolution bug.
- KisDataCollectionOrchestrator: restore logging on the lineage-event write
path (was a bare `catch {}` swallowing all failures silently); persist
daily OHLCV bars into quantengine.price_history_daily per run.
- Verified live: POST /api/collection/run -> Hangfire -> orchestrator ->
KIS mock API -> PostgreSQL, with Playwright DOM/API parity evidence.
M2 — historical price-history pipeline:
- CollectionRepository: SavePriceHistoryDailyAsync (idempotent upsert),
GetPriceHistorySummaryAsync (per-ticker aggregation) + a new
DateOnlyTypeHandler registered globally, since Dapper has no built-in
System.DateOnly support in either direction (write threw
NotSupportedException, read threw a constructor-mismatch
InvalidOperationException — found by exercising both paths live).
- tools/validate_price_history_integrity_v1.py: gap-freeness (vs KIS
trading calendar) + price-sanity gate over collected history.
- Admin Collection page: new "히스토리 현황" summary table +
GET /api/collection/history-summary, with Playwright evidence.
Governance/gate fixes:
- validate_market_time_series_schema_v1.py mislabeled its own output
"runtime_database_query": "DATA_GATED" despite never opening a DB
connection (pure file/regex check) — relabeled "check_scope":
"STATIC_STRUCTURAL_ONLY" and wired the node into the release DAG so it
isn't only reachable from ci.yml, matching every other validator.
Live-data authority for the same claim stays with QE-M2-01's pg_query
gate (spec/60), documented in spec/64.
- Fixed a WBS log_pattern check (QE-M1-06) that couldn't match its own
multi-line target; loosened two depends_on edges (QE-M1-05/06,
QE-M2-04/05) that encoded "needs X verified" when the real requirement
was only "needs X's code merged."
- Discovered and fixed admin-pages.spec.ts logging in with the wrong
seeded password (admin/admin instead of admin/quant123!, per CLAUDE.md)
— every test in that suite had been silently failing at the login step.
Deferred: QE-M2-03 (2-year backfill) — the KIS mock/VTS token endpoint
started returning 403 after the first successful call this session; looks
like a token-issuance rate limit or credential issue on KIS's side, not a
code defect. Backfilling at scale right now would just generate more 403s,
so left QE-M2-03 PENDING pending KIS account/console verification.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -216,6 +216,46 @@ public class GetLatestSnapshotsEndpoint : Endpoint<GetLatestSnapshotsRequest, Ge
|
||||
}
|
||||
}
|
||||
|
||||
public class GetPriceHistorySummaryResponse
|
||||
{
|
||||
public List<PriceHistorySummaryRecord> Tickers { get; set; } = new();
|
||||
}
|
||||
|
||||
public class GetPriceHistorySummaryEndpoint : EndpointWithoutRequest<GetPriceHistorySummaryResponse>
|
||||
{
|
||||
private readonly ICollectionRepository _repo;
|
||||
private readonly ILogger<GetPriceHistorySummaryEndpoint> _logger;
|
||||
|
||||
public GetPriceHistorySummaryEndpoint(ICollectionRepository repo, ILogger<GetPriceHistorySummaryEndpoint> logger)
|
||||
{
|
||||
_repo = repo;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Configure()
|
||||
{
|
||||
Get("/api/collection/history-summary");
|
||||
AllowAnonymous();
|
||||
Description(d => d
|
||||
.Produces<GetPriceHistorySummaryResponse>(200)
|
||||
.Produces(500));
|
||||
}
|
||||
|
||||
public override async Task HandleAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var summary = await _repo.GetPriceHistorySummaryAsync();
|
||||
await SendOkAsync(new GetPriceHistorySummaryResponse { Tickers = summary }, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to fetch price history summary");
|
||||
await SendErrorsAsync(500, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class StartCollectionRunResponse
|
||||
{
|
||||
public string RunId { get; set; } = "";
|
||||
|
||||
@@ -86,4 +86,46 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row row-deck row-cards">
|
||||
<div class="col-12">
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<h3 class="card-title">히스토리 현황</h3>
|
||||
</div>
|
||||
<div class="table-responsive">
|
||||
<table class="table card-table table-vcenter">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>티커</th>
|
||||
<th>데이터 수</th>
|
||||
<th>시작일</th>
|
||||
<th>종료일</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@if (Model.HistorySummary?.Any() == true)
|
||||
{
|
||||
@foreach (var summary in Model.HistorySummary)
|
||||
{
|
||||
<tr>
|
||||
<td>@summary.Ticker</td>
|
||||
<td>@summary.RowCount</td>
|
||||
<td>@summary.FirstDate:yyyy-MM-dd</td>
|
||||
<td>@summary.LastDate:yyyy-MM-dd</td>
|
||||
</tr>
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
<tr>
|
||||
<td colspan="4" class="text-center text-muted">데이터가 없습니다</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -12,6 +12,7 @@ public class IndexModel : PageModel
|
||||
private readonly ILogger<IndexModel> _logger;
|
||||
|
||||
public List<CollectionRunRecord>? Runs { get; set; }
|
||||
public List<PriceHistorySummaryRecord>? HistorySummary { get; set; }
|
||||
public string? Message { get; set; }
|
||||
|
||||
public IndexModel(ICollectionRepository collectionRepository, ILogger<IndexModel> logger)
|
||||
@@ -25,10 +26,11 @@ public class IndexModel : PageModel
|
||||
try
|
||||
{
|
||||
Runs = await _collectionRepository.GetRecentRunsAsync(limit: 20);
|
||||
HistorySummary = await _collectionRepository.GetPriceHistorySummaryAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Collection runs loading failed");
|
||||
_logger.LogError(ex, "Collection data loading failed");
|
||||
Message = "데이터 수집 현황을 불러올 수 없습니다.";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,9 @@ Log.Logger = new LoggerConfiguration()
|
||||
.WriteTo.File("logs/quantengine-.log", rollingInterval: RollingInterval.Day)
|
||||
.CreateLogger();
|
||||
|
||||
// Dapper has no built-in handler for System.DateOnly (params or result mapping) — register once globally.
|
||||
Dapper.SqlMapper.AddTypeHandler(new DateOnlyTypeHandler());
|
||||
|
||||
try
|
||||
{
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
@@ -21,19 +21,78 @@ public class SchedulerService
|
||||
private readonly IRecurringJobManager _recurringJobManager;
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly GatherTradingDataParser _parser;
|
||||
|
||||
public SchedulerService(
|
||||
ILogger<SchedulerService> logger,
|
||||
IBackgroundJobClient jobClient,
|
||||
IRecurringJobManager recurringJobManager,
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IConfiguration configuration)
|
||||
IConfiguration configuration,
|
||||
GatherTradingDataParser parser)
|
||||
{
|
||||
_logger = logger;
|
||||
_jobClient = jobClient;
|
||||
_recurringJobManager = recurringJobManager;
|
||||
_scopeFactory = scopeFactory;
|
||||
_configuration = configuration;
|
||||
_parser = parser;
|
||||
}
|
||||
|
||||
private List<string> LoadTickersFromJson()
|
||||
{
|
||||
try
|
||||
{
|
||||
var jsonPath = FindGatherTradingDataJson();
|
||||
if (string.IsNullOrEmpty(jsonPath))
|
||||
{
|
||||
_logger.LogWarning("GatherTradingData.json not found, falling back to default universe");
|
||||
return new List<string> { "005930" };
|
||||
}
|
||||
|
||||
var data = _parser.ParseGatherTradingData(jsonPath);
|
||||
var tickers = new HashSet<string>();
|
||||
|
||||
foreach (var row in data)
|
||||
{
|
||||
if (row.TryGetValue("Ticker", out var tickerObj) && tickerObj is string tickerRaw && !string.IsNullOrEmpty(tickerRaw))
|
||||
{
|
||||
var ticker = tickerRaw.Trim('"');
|
||||
if (!string.IsNullOrEmpty(ticker))
|
||||
{
|
||||
tickers.Add(ticker);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var result = tickers.ToList();
|
||||
_logger.LogInformation("Loaded {Count} tickers from GatherTradingData.json", result.Count);
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error loading tickers from GatherTradingData.json, falling back to default universe");
|
||||
return new List<string> { "005930" };
|
||||
}
|
||||
}
|
||||
|
||||
private static string? FindGatherTradingDataJson()
|
||||
{
|
||||
var baseDir = AppContext.BaseDirectory;
|
||||
var current = new DirectoryInfo(baseDir);
|
||||
|
||||
while (current != null)
|
||||
{
|
||||
var gatherPath = Path.Combine(current.FullName, "GatherTradingData.json");
|
||||
if (Directory.Exists(Path.Combine(current.FullName, ".git"))
|
||||
|| File.Exists(gatherPath))
|
||||
{
|
||||
return File.Exists(gatherPath) ? gatherPath : null;
|
||||
}
|
||||
current = current.Parent;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -94,8 +153,7 @@ public class SchedulerService
|
||||
{
|
||||
_logger.LogInformation("Starting daily data collection job at {Time}", DateTime.Now);
|
||||
|
||||
// List of tickers to collect
|
||||
var tickers = new[] { "005930", "000660", "051910", "005380", "010140", "005490" };
|
||||
var tickers = LoadTickersFromJson();
|
||||
|
||||
// Create scope for scoped services
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
@@ -108,7 +166,7 @@ public class SchedulerService
|
||||
var accountMode = _configuration["Kis:AccountMode"] ?? "mock";
|
||||
|
||||
// Execute collection
|
||||
var result = await orchestrator.RunCollectionAsync(runId, accountMode, tickers.ToList());
|
||||
var result = await orchestrator.RunCollectionAsync(runId, accountMode, tickers);
|
||||
|
||||
// Log completion
|
||||
_logger.LogInformation("Collection run {RunId} completed: {Snapshots} snapshots, {Errors} errors",
|
||||
@@ -129,7 +187,7 @@ public class SchedulerService
|
||||
{
|
||||
_logger.LogInformation("Starting hourly price update at {Time}", DateTime.Now);
|
||||
|
||||
var tickers = new[] { "005930", "000660", "051910" };
|
||||
var tickers = LoadTickersFromJson();
|
||||
|
||||
foreach (var ticker in tickers)
|
||||
{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user