using KArtSell.BuildingBlocks.Time;
using Microsoft.Extensions.Logging;
namespace KArtSell.Modules.ModelOperations.ShadowRun;
///
/// Backfills historical OHLCV and FeeSchedule data for shadow run period.
/// Data fetched from KRX API and normalized to trading-session boundaries.
///
public sealed class DataBackfiller(
IMarketCalendarService marketCalendar,
IKrxDataService krxData,
ILogger logger)
{
public record OhlcvBar(
DateOnly Date,
string Ticker,
decimal Open,
decimal High,
decimal Low,
decimal Close,
long Volume);
public record FeeScheduleEntry(
DateOnly EffectiveDate,
decimal TransactionFeePercent,
decimal SlippagePercent);
///
/// Fetch OHLCV for all tickers in portfolio across shadow run window.
///
public async Task> BackfillOhlcvAsync(
DateOnly windowStart,
DateOnly windowEnd,
IReadOnlyList tickers,
CancellationToken cancellationToken)
{
// Validate window against market calendar
var tradingSessions = await marketCalendar.GetTradingSessionsAsync(
windowStart, windowEnd, cancellationToken);
logger.LogInformation(
"Backfilling OHLCV: {TickerCount} tickers, {TradingDays} trading days ({Start:yyyy-MM-dd} to {End:yyyy-MM-dd})",
tickers.Count, tradingSessions.Count, windowStart, windowEnd);
const int BatchDays = 30; // Batch size: ~252 days / 30 = 9 calls (vs 252)
var bars = new List();
var barLock = new object();
// Fetch all tickers in parallel (5 concurrent) to maximize throughput
await Parallel.ForEachAsync(tickers, new ParallelOptions { MaxDegreeOfParallelism = 5, CancellationToken = cancellationToken },
async (ticker, ct) =>
{
var tickerBars = new List();
// Fetch in 30-day batches
for (var batchStart = windowStart; batchStart <= windowEnd; batchStart = batchStart.AddDays(BatchDays))
{
var batchEnd = batchStart.AddDays(BatchDays - 1) > windowEnd
? windowEnd
: batchStart.AddDays(BatchDays - 1);
// 100ms throttle between batches
await Task.Delay(100, ct);
var batchBars = await krxData.GetDailyOhlcvAsync(
ticker, batchStart, batchEnd, ct);
tickerBars.AddRange(batchBars);
}
lock (barLock)
{
bars.AddRange(tickerBars);
}
});
logger.LogInformation("Backfilled {BarCount} OHLCV bars (parallel mode: 5 tickers, 30-day chunks)", bars.Count);
return bars;
}
///
/// Fetch transaction fee schedule for window.
///
public async Task> BackfillFeeScheduleAsync(
DateOnly windowStart,
DateOnly windowEnd,
CancellationToken cancellationToken)
{
logger.LogInformation(
"Backfilling fee schedule ({Start:yyyy-MM-dd} to {End:yyyy-MM-dd})",
windowStart, windowEnd);
var schedule = await krxData.GetFeeScheduleAsync(windowStart, windowEnd, cancellationToken);
logger.LogInformation("Backfilled {ScheduleEntries} fee schedule entries", schedule.Count);
return schedule;
}
///
/// Validate data completeness: no gaps, all tickers present, fee schedule continuous.
///
public async Task ValidateAsync(
IReadOnlyList bars,
IReadOnlyList fees,
IReadOnlyList expectedTickers,
DateOnly windowStart,
DateOnly windowEnd,
CancellationToken cancellationToken)
{
var tradingSessions = await marketCalendar.GetTradingSessionsAsync(
windowStart, windowEnd, cancellationToken);
var result = new DataBackfillValidationResult(
IsValid: true,
TradingDaysProcessed: 0,
MissingTickers: new List(),
DataGaps: new List());
// Check OHLCV completeness
var tickersBars = bars.GroupBy(b => b.Ticker).ToDictionary(g => g.Key, g => g.ToList());
var missingTickers = expectedTickers.Where(t => !tickersBars.ContainsKey(t)).ToList();
if (missingTickers.Any())
{
result = result with { MissingTickers = missingTickers };
}
// Check for gaps in each ticker
foreach (var (ticker, tickerBars) in tickersBars)
{
var tickerDates = tickerBars.Select(b => b.Date).OrderBy(d => d).ToList();
var sessionDates = tradingSessions.ToList();
var gaps = sessionDates.Where(s => !tickerDates.Contains(s)).ToList();
if (gaps.Any())
{
var updatedGaps = (result.DataGaps ?? new List()).Concat(
gaps.Select(g => $"{ticker}:{g:yyyy-MM-dd}")).ToList();
result = result with { DataGaps = updatedGaps };
}
}
// Check fee schedule continuity
var feesByDate = fees.GroupBy(f => f.EffectiveDate).ToDictionary(g => g.Key);
var feeDates = feesByDate.Keys.OrderBy(d => d).ToList();
if (!feeDates.Any())
{
result = result with { IsValid = false };
}
result = result with { TradingDaysProcessed = tradingSessions.Count };
return result;
}
}
public sealed record DataBackfillValidationResult(
bool IsValid = true,
int TradingDaysProcessed = 0,
List? MissingTickers = null,
List? DataGaps = null)
{
public bool HasIssues => !IsValid || (MissingTickers?.Any() ?? false) || (DataGaps?.Any() ?? false);
}
///
/// Market calendar service: trading sessions, holidays, special sessions.
///
public interface IMarketCalendarService
{
Task> GetTradingSessionsAsync(
DateOnly startDate,
DateOnly endDate,
CancellationToken cancellationToken);
}
///
/// KRX data service: OHLCV, fee schedule.
///
public interface IKrxDataService
{
Task> GetDailyOhlcvAsync(
string ticker,
DateOnly startDate,
DateOnly endDate,
CancellationToken cancellationToken);
Task> GetFeeScheduleAsync(
DateOnly startDate,
DateOnly endDate,
CancellationToken cancellationToken);
}