using KArtSell.BuildingBlocks.Time; using Microsoft.Extensions.Logging; namespace KArtSell.Modules.ModelOperations.ShadowRun; /// /// Backfills historical OHLCV and FeeSchedule data for shadow run period. /// Data fetched from KRX API and normalized to trading-session boundaries. /// public sealed class DataBackfiller( IMarketCalendarService marketCalendar, IKrxDataService krxData, ILogger logger) { public record OhlcvBar( DateOnly Date, string Ticker, decimal Open, decimal High, decimal Low, decimal Close, long Volume); public record FeeScheduleEntry( DateOnly EffectiveDate, decimal TransactionFeePercent, decimal SlippagePercent); /// /// Fetch OHLCV for all tickers in portfolio across shadow run window. /// public async Task> BackfillOhlcvAsync( DateOnly windowStart, DateOnly windowEnd, IReadOnlyList tickers, CancellationToken cancellationToken) { // Validate window against market calendar var tradingSessions = await marketCalendar.GetTradingSessionsAsync( windowStart, windowEnd, cancellationToken); logger.LogInformation( "Backfilling OHLCV: {TickerCount} tickers, {TradingDays} trading days ({Start:yyyy-MM-dd} to {End:yyyy-MM-dd})", tickers.Count, tradingSessions.Count, windowStart, windowEnd); const int BatchDays = 30; // Batch size: ~252 days / 30 = 9 calls (vs 252) var bars = new List(); var barLock = new object(); // Fetch all tickers in parallel (5 concurrent) to maximize throughput await Parallel.ForEachAsync(tickers, new ParallelOptions { MaxDegreeOfParallelism = 5, CancellationToken = cancellationToken }, async (ticker, ct) => { var tickerBars = new List(); // Fetch in 30-day batches for (var batchStart = windowStart; batchStart <= windowEnd; batchStart = batchStart.AddDays(BatchDays)) { var batchEnd = batchStart.AddDays(BatchDays - 1) > windowEnd ? windowEnd : batchStart.AddDays(BatchDays - 1); // 100ms throttle between batches await Task.Delay(100, ct); var batchBars = await krxData.GetDailyOhlcvAsync( ticker, batchStart, batchEnd, ct); tickerBars.AddRange(batchBars); } lock (barLock) { bars.AddRange(tickerBars); } }); logger.LogInformation("Backfilled {BarCount} OHLCV bars (parallel mode: 5 tickers, 30-day chunks)", bars.Count); return bars; } /// /// Fetch transaction fee schedule for window. /// public async Task> BackfillFeeScheduleAsync( DateOnly windowStart, DateOnly windowEnd, CancellationToken cancellationToken) { logger.LogInformation( "Backfilling fee schedule ({Start:yyyy-MM-dd} to {End:yyyy-MM-dd})", windowStart, windowEnd); var schedule = await krxData.GetFeeScheduleAsync(windowStart, windowEnd, cancellationToken); logger.LogInformation("Backfilled {ScheduleEntries} fee schedule entries", schedule.Count); return schedule; } /// /// Validate data completeness: no gaps, all tickers present, fee schedule continuous. /// public async Task ValidateAsync( IReadOnlyList bars, IReadOnlyList fees, IReadOnlyList expectedTickers, DateOnly windowStart, DateOnly windowEnd, CancellationToken cancellationToken) { var tradingSessions = await marketCalendar.GetTradingSessionsAsync( windowStart, windowEnd, cancellationToken); var result = new DataBackfillValidationResult( IsValid: true, TradingDaysProcessed: 0, MissingTickers: new List(), DataGaps: new List()); // Check OHLCV completeness var tickersBars = bars.GroupBy(b => b.Ticker).ToDictionary(g => g.Key, g => g.ToList()); var missingTickers = expectedTickers.Where(t => !tickersBars.ContainsKey(t)).ToList(); if (missingTickers.Any()) { result = result with { MissingTickers = missingTickers }; } // Check for gaps in each ticker foreach (var (ticker, tickerBars) in tickersBars) { var tickerDates = tickerBars.Select(b => b.Date).OrderBy(d => d).ToList(); var sessionDates = tradingSessions.ToList(); var gaps = sessionDates.Where(s => !tickerDates.Contains(s)).ToList(); if (gaps.Any()) { var updatedGaps = (result.DataGaps ?? new List()).Concat( gaps.Select(g => $"{ticker}:{g:yyyy-MM-dd}")).ToList(); result = result with { DataGaps = updatedGaps }; } } // Check fee schedule continuity var feesByDate = fees.GroupBy(f => f.EffectiveDate).ToDictionary(g => g.Key); var feeDates = feesByDate.Keys.OrderBy(d => d).ToList(); if (!feeDates.Any()) { result = result with { IsValid = false }; } result = result with { TradingDaysProcessed = tradingSessions.Count }; return result; } } public sealed record DataBackfillValidationResult( bool IsValid = true, int TradingDaysProcessed = 0, List? MissingTickers = null, List? DataGaps = null) { public bool HasIssues => !IsValid || (MissingTickers?.Any() ?? false) || (DataGaps?.Any() ?? false); } /// /// Market calendar service: trading sessions, holidays, special sessions. /// public interface IMarketCalendarService { Task> GetTradingSessionsAsync( DateOnly startDate, DateOnly endDate, CancellationToken cancellationToken); } /// /// KRX data service: OHLCV, fee schedule. /// public interface IKrxDataService { Task> GetDailyOhlcvAsync( string ticker, DateOnly startDate, DateOnly endDate, CancellationToken cancellationToken); Task> GetFeeScheduleAsync( DateOnly startDate, DateOnly endDate, CancellationToken cancellationToken); }