KArtSell.Aegis/src/KArtSell.Modules.ModelOperations/ShadowRun/DataBackfiller.cs

using KArtSell.BuildingBlocks.Time;
using Microsoft.Extensions.Logging;

namespace KArtSell.Modules.ModelOperations.ShadowRun;

/// <summary>
/// Backfills historical OHLCV and FeeSchedule data for shadow run period.
/// Data fetched from KRX API and normalized to trading-session boundaries.
/// </summary>
public sealed class DataBackfiller(
    IMarketCalendarService marketCalendar,
    IKrxDataService krxData,
    ILogger<DataBackfiller> logger)
{
    public record OhlcvBar(
        DateOnly Date,
        string Ticker,
        decimal Open,
        decimal High,
        decimal Low,
        decimal Close,
        long Volume);

    public record FeeScheduleEntry(
        DateOnly EffectiveDate,
        decimal TransactionFeePercent,
        decimal SlippagePercent);

    /// <summary>
    /// Fetch OHLCV for all tickers in portfolio across shadow run window.
    /// </summary>
    public async Task<IReadOnlyList<OhlcvBar>> BackfillOhlcvAsync(
        DateOnly windowStart,
        DateOnly windowEnd,
        IReadOnlyList<string> tickers,
        CancellationToken cancellationToken)
    {
        // Validate window against market calendar
        var tradingSessions = await marketCalendar.GetTradingSessionsAsync(
            windowStart, windowEnd, cancellationToken);

        logger.LogInformation(
            "Backfilling OHLCV: {TickerCount} tickers, {TradingDays} trading days ({Start:yyyy-MM-dd} to {End:yyyy-MM-dd})",
            tickers.Count, tradingSessions.Count, windowStart, windowEnd);

        const int BatchDays = 30;  // Batch size: ~252 days / 30 = 9 calls (vs 252)
        var bars = new List<OhlcvBar>();
        var barLock = new object();

        // Fetch all tickers in parallel (5 concurrent) to maximize throughput
        await Parallel.ForEachAsync(tickers, new ParallelOptions { MaxDegreeOfParallelism = 5, CancellationToken = cancellationToken },
            async (ticker, ct) =>
            {
                var tickerBars = new List<OhlcvBar>();

                // Fetch in 30-day batches
                for (var batchStart = windowStart; batchStart <= windowEnd; batchStart = batchStart.AddDays(BatchDays))
                {
                    var batchEnd = batchStart.AddDays(BatchDays - 1) > windowEnd
                        ? windowEnd
                        : batchStart.AddDays(BatchDays - 1);

                    // 100ms throttle between batches
                    await Task.Delay(100, ct);

                    var batchBars = await krxData.GetDailyOhlcvAsync(
                        ticker, batchStart, batchEnd, ct);
                    tickerBars.AddRange(batchBars);
                }

                lock (barLock)
                {
                    bars.AddRange(tickerBars);
                }
            });

        logger.LogInformation("Backfilled {BarCount} OHLCV bars (parallel mode: 5 tickers, 30-day chunks)", bars.Count);
        return bars;
    }

    /// <summary>
    /// Fetch transaction fee schedule for window.
    /// </summary>
    public async Task<IReadOnlyList<FeeScheduleEntry>> BackfillFeeScheduleAsync(
        DateOnly windowStart,
        DateOnly windowEnd,
        CancellationToken cancellationToken)
    {
        logger.LogInformation(
            "Backfilling fee schedule ({Start:yyyy-MM-dd} to {End:yyyy-MM-dd})",
            windowStart, windowEnd);

        var schedule = await krxData.GetFeeScheduleAsync(windowStart, windowEnd, cancellationToken);

        logger.LogInformation("Backfilled {ScheduleEntries} fee schedule entries", schedule.Count);
        return schedule;
    }

    /// <summary>
    /// Validate data completeness: no gaps, all tickers present, fee schedule continuous.
    /// </summary>
    public async Task<DataBackfillValidationResult> ValidateAsync(
        IReadOnlyList<OhlcvBar> bars,
        IReadOnlyList<FeeScheduleEntry> fees,
        IReadOnlyList<string> expectedTickers,
        DateOnly windowStart,
        DateOnly windowEnd,
        CancellationToken cancellationToken)
    {
        var tradingSessions = await marketCalendar.GetTradingSessionsAsync(
            windowStart, windowEnd, cancellationToken);

        var result = new DataBackfillValidationResult(
            IsValid: true,
            TradingDaysProcessed: 0,
            MissingTickers: new List<string>(),
            DataGaps: new List<string>());

        // Check OHLCV completeness
        var tickersBars = bars.GroupBy(b => b.Ticker).ToDictionary(g => g.Key, g => g.ToList());
        var missingTickers = expectedTickers.Where(t => !tickersBars.ContainsKey(t)).ToList();

        if (missingTickers.Any())
        {
            result = result with { MissingTickers = missingTickers };
        }

        // Check for gaps in each ticker
        foreach (var (ticker, tickerBars) in tickersBars)
        {
            var tickerDates = tickerBars.Select(b => b.Date).OrderBy(d => d).ToList();
            var sessionDates = tradingSessions.ToList();

            var gaps = sessionDates.Where(s => !tickerDates.Contains(s)).ToList();
            if (gaps.Any())
            {
                var updatedGaps = (result.DataGaps ?? new List<string>()).Concat(
                    gaps.Select(g => $"{ticker}:{g:yyyy-MM-dd}")).ToList();
                result = result with { DataGaps = updatedGaps };
            }
        }

        // Check fee schedule continuity
        var feesByDate = fees.GroupBy(f => f.EffectiveDate).ToDictionary(g => g.Key);
        var feeDates = feesByDate.Keys.OrderBy(d => d).ToList();

        if (!feeDates.Any())
        {
            result = result with { IsValid = false };
        }

        result = result with { TradingDaysProcessed = tradingSessions.Count };
        return result;
    }
}

public sealed record DataBackfillValidationResult(
    bool IsValid = true,
    int TradingDaysProcessed = 0,
    List<string>? MissingTickers = null,
    List<string>? DataGaps = null)
{
    public bool HasIssues => !IsValid || (MissingTickers?.Any() ?? false) || (DataGaps?.Any() ?? false);
}

/// <summary>
/// Market calendar service: trading sessions, holidays, special sessions.
/// </summary>
public interface IMarketCalendarService
{
    Task<IReadOnlyList<DateOnly>> GetTradingSessionsAsync(
        DateOnly startDate,
        DateOnly endDate,
        CancellationToken cancellationToken);
}

/// <summary>
/// KRX data service: OHLCV, fee schedule.
/// </summary>
public interface IKrxDataService
{
    Task<IReadOnlyList<DataBackfiller.OhlcvBar>> GetDailyOhlcvAsync(
        string ticker,
        DateOnly startDate,
        DateOnly endDate,
        CancellationToken cancellationToken);

    Task<IReadOnlyList<DataBackfiller.FeeScheduleEntry>> GetFeeScheduleAsync(
        DateOnly startDate,
        DateOnly endDate,
        CancellationToken cancellationToken);
}