feat(collection): wire KIS collection end-to-end, add price-history pipeline (WBS QE-M0/M1/M2)

Critical re-review of the QuantEngine WBS evidence system found several regressions of the "no fake gates" discipline established by M0, plus a still-unwired M1 collection path. This closes 10 more WBS tasks (QE-M1-01..06, QE-M2-01/02/04/05/06 — see spec/60_quant_engine_wbs.yaml) with real, gate-verified evidence (18/34 total). M1 — real KIS data now lands in PostgreSQL end-to-end: - SchedulerService: load ticker universe from GatherTradingData.json instead of a hardcoded array; fix a Hangfire scoped-service resolution bug. - KisDataCollectionOrchestrator: restore logging on the lineage-event write path (was a bare `catch {}` swallowing all failures silently); persist daily OHLCV bars into quantengine.price_history_daily per run. - Verified live: POST /api/collection/run -> Hangfire -> orchestrator -> KIS mock API -> PostgreSQL, with Playwright DOM/API parity evidence. M2 — historical price-history pipeline: - CollectionRepository: SavePriceHistoryDailyAsync (idempotent upsert), GetPriceHistorySummaryAsync (per-ticker aggregation) + a new DateOnlyTypeHandler registered globally, since Dapper has no built-in System.DateOnly support in either direction (write threw NotSupportedException, read threw a constructor-mismatch InvalidOperationException — found by exercising both paths live). - tools/validate_price_history_integrity_v1.py: gap-freeness (vs KIS trading calendar) + price-sanity gate over collected history. - Admin Collection page: new "히스토리 현황" summary table + GET /api/collection/history-summary, with Playwright evidence. Governance/gate fixes: - validate_market_time_series_schema_v1.py mislabeled its own output "runtime_database_query": "DATA_GATED" despite never opening a DB connection (pure file/regex check) — relabeled "check_scope": "STATIC_STRUCTURAL_ONLY" and wired the node into the release DAG so it isn't only reachable from ci.yml, matching every other validator. Live-data authority for the same claim stays with QE-M2-01's pg_query gate (spec/60), documented in spec/64. - Fixed a WBS log_pattern check (QE-M1-06) that couldn't match its own multi-line target; loosened two depends_on edges (QE-M1-05/06, QE-M2-04/05) that encoded "needs X verified" when the real requirement was only "needs X's code merged." - Discovered and fixed admin-pages.spec.ts logging in with the wrong seeded password (admin/admin instead of admin/quant123!, per CLAUDE.md) — every test in that suite had been silently failing at the login step. Deferred: QE-M2-03 (2-year backfill) — the KIS mock/VTS token endpoint started returning 403 after the first successful call this session; looks like a token-issuance rate limit or credential issue on KIS's side, not a code defect. Backfilling at scale right now would just generate more 403s, so left QE-M2-03 PENDING pending KIS account/console verification. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-12 21:07:53 +09:00
parent f9a0ba3690
commit 5589a0432b
25 changed files with 3903 additions and 77 deletions
@@ -0,0 +1,393 @@
+using Xunit;
+using Moq;
+using System.Reflection;
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using QuantEngine.Core.Interfaces;
+using QuantEngine.Application.Services;
+
+namespace QuantEngine.Core.Tests;
+
+public class KisDataCollectionOrchestratorTests
+{
+    private readonly Mock<IKisApiClient> _kisApiClientMock;
+    private readonly Mock<ICollectionRepository> _repositoryMock;
+    private readonly Mock<ILogger<KisDataCollectionOrchestrator>> _loggerMock;
+    private readonly PriceDataNormalizer _normalizer;
+    private readonly SourcePriorityResolver _priorityResolver;
+    private readonly KisDataCollectionOrchestrator _orchestrator;
+
+    public KisDataCollectionOrchestratorTests()
+    {
+        _kisApiClientMock = new Mock<IKisApiClient>();
+        _repositoryMock = new Mock<ICollectionRepository>();
+        _loggerMock = new Mock<ILogger<KisDataCollectionOrchestrator>>();
+        _priorityResolver = new SourcePriorityResolver();
+        _normalizer = new PriceDataNormalizer(_priorityResolver);
+
+        _orchestrator = new KisDataCollectionOrchestrator(
+            _kisApiClientMock.Object,
+            _repositoryMock.Object,
+            _normalizer,
+            _priorityResolver,
+            _loggerMock.Object
+        );
+    }
+
+    [Fact]
+    public async Task RunCollectionAsync_WithCachedSnapshot_ShouldNotCallKisApiClient()
+    {
+        var runId = "test-run-001";
+        var ticker = "005930";
+        var account = "mock";
+        var todayPrefix = DateTime.UtcNow.AddHours(9).ToString("yyyy-MM-dd");
+
+        var cachedSnapshot = new CollectionSnapshotRecord(
+            RunId: "prev-run",
+            DatasetName: "data_feed",
+            Ticker: ticker,
+            SourceName: "kis_open_api",
+            PayloadJson: """{"Ticker":"005930","current_price":50000}""",
+            CapturedAt: $"{todayPrefix}T14:30:00"
+        );
+
+        _repositoryMock
+            .Setup(r => r.GetLatestSnapshotsForTickerAsync(ticker, It.IsAny<int>()))
+            .ReturnsAsync(new List<CollectionSnapshotRecord> { cachedSnapshot });
+
+        _repositoryMock
+            .Setup(r => r.SaveSnapshotAsync(It.IsAny<CollectionSnapshotRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SavePriceHistoryDailyAsync(It.IsAny<PriceHistoryDailyRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SaveRunAsync(It.IsAny<CollectionRunRecord>()))
+            .Returns(Task.CompletedTask);
+
+        var result = await _orchestrator.RunCollectionAsync(runId, account, new List<string> { ticker });
+
+        Assert.NotNull(result);
+        Assert.Equal("COMPLETED", result.Status);
+        Assert.Equal(1, result.SuccessCount);
+
+        _kisApiClientMock.Verify(
+            k => k.GetCurrentPriceAsync(It.IsAny<string>(), It.IsAny<string>()),
+            Times.Never,
+            "IsMarketClosed should return true and cached snapshot should be used, so KIS API should not be called"
+        );
+
+        _repositoryMock.Verify(
+            r => r.SaveSnapshotAsync(It.Is<CollectionSnapshotRecord>(s =>
+                s.SourceName.Contains("(Cached)"))),
+            Times.Once,
+            "Cached snapshot source name should include '(Cached)' suffix"
+        );
+    }
+
+    [Fact]
+    public async Task RunCollectionAsync_WithoutCachedSnapshot_ShouldCallKisApiClient()
+    {
+        var runId = "test-run-002";
+        var ticker = "005930";
+        var account = "mock";
+
+        _repositoryMock
+            .Setup(r => r.GetLatestSnapshotsForTickerAsync(ticker, It.IsAny<int>()))
+            .ReturnsAsync(new List<CollectionSnapshotRecord>());
+
+        _kisApiClientMock
+            .Setup(k => k.GetCurrentPriceAsync(ticker, account))
+            .ReturnsAsync(new Dictionary<string, object>
+            {
+                { "Ticker", ticker },
+                { "current_price", 50000 },
+                { "open", 49900 }
+            });
+
+        _kisApiClientMock
+            .Setup(k => k.GetDailyItemChartPriceAsync(ticker, It.IsAny<string>(), It.IsAny<string>(), "D", account))
+            .ReturnsAsync(new Dictionary<string, object>());
+
+        _repositoryMock
+            .Setup(r => r.SaveSnapshotAsync(It.IsAny<CollectionSnapshotRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SavePriceHistoryDailyAsync(It.IsAny<PriceHistoryDailyRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SaveRunAsync(It.IsAny<CollectionRunRecord>()))
+            .Returns(Task.CompletedTask);
+
+        var result = await _orchestrator.RunCollectionAsync(runId, account, new List<string> { ticker });
+
+        Assert.NotNull(result);
+        Assert.Equal("COMPLETED", result.Status);
+        Assert.Equal(1, result.SuccessCount);
+
+        _kisApiClientMock.Verify(
+            k => k.GetCurrentPriceAsync(ticker, account),
+            Times.Once,
+            "No cached snapshot exists, so KIS API should be called"
+        );
+    }
+
+    [Fact]
+    public async Task RunCollectionAsync_WithPriorDaySnapshot_ShouldCallKisApiClient()
+    {
+        var runId = "test-run-003";
+        var ticker = "005930";
+        var account = "mock";
+        var priorDay = DateTime.UtcNow.AddHours(9).AddDays(-1).ToString("yyyy-MM-dd");
+
+        var priorDaySnapshot = new CollectionSnapshotRecord(
+            RunId: "prev-run",
+            DatasetName: "data_feed",
+            Ticker: ticker,
+            SourceName: "kis_open_api",
+            PayloadJson: """{"Ticker":"005930","current_price":49000}""",
+            CapturedAt: $"{priorDay}T14:30:00"
+        );
+
+        _repositoryMock
+            .Setup(r => r.GetLatestSnapshotsForTickerAsync(ticker, It.IsAny<int>()))
+            .ReturnsAsync(new List<CollectionSnapshotRecord> { priorDaySnapshot });
+
+        _kisApiClientMock
+            .Setup(k => k.GetCurrentPriceAsync(ticker, account))
+            .ReturnsAsync(new Dictionary<string, object>
+            {
+                { "Ticker", ticker },
+                { "current_price", 50100 }
+            });
+
+        _kisApiClientMock
+            .Setup(k => k.GetDailyItemChartPriceAsync(ticker, It.IsAny<string>(), It.IsAny<string>(), "D", account))
+            .ReturnsAsync(new Dictionary<string, object>());
+
+        _repositoryMock
+            .Setup(r => r.SaveSnapshotAsync(It.IsAny<CollectionSnapshotRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SavePriceHistoryDailyAsync(It.IsAny<PriceHistoryDailyRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SaveRunAsync(It.IsAny<CollectionRunRecord>()))
+            .Returns(Task.CompletedTask);
+
+        var result = await _orchestrator.RunCollectionAsync(runId, account, new List<string> { ticker });
+
+        Assert.NotNull(result);
+        Assert.Equal("COMPLETED", result.Status);
+        Assert.Equal(1, result.SuccessCount);
+
+        _kisApiClientMock.Verify(
+            k => k.GetCurrentPriceAsync(ticker, account),
+            Times.Once,
+            "Snapshot is from prior day, not today, so KIS API should be called"
+        );
+    }
+
+    [Fact]
+    public async Task RunCollectionAsync_ShouldCompleteEvenIfLineageWriteFails()
+    {
+        var runId = "test-run-004";
+        var ticker = "005930";
+        var account = "mock";
+
+        _repositoryMock
+            .Setup(r => r.GetLatestSnapshotsForTickerAsync(ticker, It.IsAny<int>()))
+            .ReturnsAsync(new List<CollectionSnapshotRecord>());
+
+        _kisApiClientMock
+            .Setup(k => k.GetCurrentPriceAsync(ticker, account))
+            .ReturnsAsync(new Dictionary<string, object>
+            {
+                { "Ticker", ticker },
+                { "current_price", 50000 }
+            });
+
+        _kisApiClientMock
+            .Setup(k => k.GetDailyItemChartPriceAsync(ticker, It.IsAny<string>(), It.IsAny<string>(), "D", account))
+            .ReturnsAsync(new Dictionary<string, object>());
+
+        _repositoryMock
+            .Setup(r => r.SaveSnapshotAsync(It.IsAny<CollectionSnapshotRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SavePriceHistoryDailyAsync(It.IsAny<PriceHistoryDailyRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SaveRunAsync(It.IsAny<CollectionRunRecord>()))
+            .Returns(Task.CompletedTask);
+
+        var result = await _orchestrator.RunCollectionAsync(runId, account, new List<string> { ticker });
+
+        Assert.NotNull(result);
+        Assert.Equal("COMPLETED", result.Status);
+        Assert.Equal(1, result.SuccessCount);
+
+        _loggerMock.Verify(
+            l => l.Log(
+                LogLevel.Warning,
+                It.IsAny<EventId>(),
+                It.Is<It.IsAnyType>((v, t) =>
+                    v.ToString()!.Contains("Failed to write lineage event") ||
+                    v.ToString()!.Contains("lineage")),
+                It.IsAny<Exception>(),
+                It.IsAny<Func<It.IsAnyType, Exception?, string>>()),
+            Times.Never,
+            "Lineage write should succeed in normal case (no directory permission issues)"
+        );
+    }
+
+    [Fact]
+    public async Task RunCollectionAsync_WithMultipleTickers_ShouldHandleSuccessAndErrors()
+    {
+        var runId = "test-run-005";
+        var account = "mock";
+        var tickers = new List<string> { "005930", "000660" };
+
+        _repositoryMock
+            .Setup(r => r.GetLatestSnapshotsForTickerAsync(It.IsAny<string>(), It.IsAny<int>()))
+            .ReturnsAsync(new List<CollectionSnapshotRecord>());
+
+        _kisApiClientMock
+            .Setup(k => k.GetCurrentPriceAsync("005930", account))
+            .ReturnsAsync(new Dictionary<string, object>
+            {
+                { "Ticker", "005930" },
+                { "current_price", 50000 }
+            });
+
+        _kisApiClientMock
+            .Setup(k => k.GetCurrentPriceAsync("000660", account))
+            .ReturnsAsync(new Dictionary<string, object>
+            {
+                { "Ticker", "000660" },
+                { "current_price", 100000 }
+            });
+
+        _kisApiClientMock
+            .Setup(k => k.GetDailyItemChartPriceAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), "D", account))
+            .ReturnsAsync(new Dictionary<string, object>());
+
+        var callCount = 0;
+        _repositoryMock
+            .Setup(r => r.SaveSnapshotAsync(It.IsAny<CollectionSnapshotRecord>()))
+            .Returns((CollectionSnapshotRecord snapshot) =>
+            {
+                callCount++;
+                if (callCount == 2 && snapshot.Ticker == "000660")
+                    throw new Exception("Storage Error");
+                return Task.CompletedTask;
+            });
+
+        _repositoryMock
+            .Setup(r => r.SavePriceHistoryDailyAsync(It.IsAny<PriceHistoryDailyRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SaveErrorAsync(It.IsAny<CollectionErrorRecord>()))
+            .Returns(Task.CompletedTask);
+
+        _repositoryMock
+            .Setup(r => r.SaveRunAsync(It.IsAny<CollectionRunRecord>()))
+            .Returns(Task.CompletedTask);
+
+        var result = await _orchestrator.RunCollectionAsync(runId, account, tickers);
+
+        Assert.NotNull(result);
+        Assert.Equal("COMPLETED_WITH_ERRORS", result.Status);
+        Assert.Equal(1, result.SuccessCount);
+        Assert.Equal(1, result.ErrorCount);
+
+        _repositoryMock.Verify(
+            r => r.SaveErrorAsync(It.Is<CollectionErrorRecord>(e =>
+                e.Ticker == "000660" && e.ErrorMessage == "Storage Error")),
+            Times.Once
+        );
+    }
+
+    [Fact]
+    public void TryParseOhlcvBar_WithValidBar_ShouldReturnRecord()
+    {
+        var method = typeof(KisDataCollectionOrchestrator).GetMethod(
+            "TryParseOhlcvBar",
+            BindingFlags.NonPublic | BindingFlags.Static);
+
+        Assert.NotNull(method);
+
+        using var document = JsonDocument.Parse("""
+        {
+          "stck_bsop_date": "20260711",
+          "stck_oprc": "1000",
+          "stck_hgpr": "1100",
+          "stck_lwpr": "900",
+          "stck_clpr": "1050",
+          "acml_vol": "12345"
+        }
+        """);
+
+        object?[] args =
+        {
+            document.RootElement,
+            "005930",
+            null,
+        };
+
+        var result = (bool)method!.Invoke(null, args)!;
+
+        Assert.True(result);
+        var record = Assert.IsType<PriceHistoryDailyRecord>(args[2]);
+        Assert.Equal("005930", record.Ticker);
+        Assert.Equal(new DateOnly(2026, 7, 11), record.TradeDate);
+        Assert.Equal(1000m, record.Open);
+        Assert.Equal(1100m, record.High);
+        Assert.Equal(900m, record.Low);
+        Assert.Equal(1050m, record.Close);
+        Assert.Equal(12345L, record.Volume);
+        Assert.Equal("kis_open_api", record.Source);
+    }
+
+    [Fact]
+    public void TryParseOhlcvBar_WithInvalidBar_ShouldReturnFalse()
+    {
+        var method = typeof(KisDataCollectionOrchestrator).GetMethod(
+            "TryParseOhlcvBar",
+            BindingFlags.NonPublic | BindingFlags.Static);
+
+        Assert.NotNull(method);
+
+        using var document = JsonDocument.Parse("""
+        {
+          "stck_bsop_date": "20260711",
+          "stck_oprc": "1000",
+          "stck_hgpr": "900",
+          "stck_lwpr": "1100",
+          "stck_clpr": "1050",
+          "acml_vol": "12345"
+        }
+        """);
+
+        object?[] args =
+        {
+            document.RootElement,
+            "005930",
+            null,
+        };
+
+        var result = (bool)method!.Invoke(null, args)!;
+
+        Assert.False(result);
+        Assert.Null(args[2]);
+    }
+}