KArtSell.Aegis/src/KArtSell.Host/Jobs/ShadowRunJob.cs

using Hangfire;
using KArtSell.BuildingBlocks.Data;
using KArtSell.BuildingBlocks.Reliability;
using KArtSell.BuildingBlocks.Time;
using KArtSell.Modules.ModelOperations.ShadowRun;
using Microsoft.Extensions.Logging;

namespace KArtSell.Host.Jobs;

/// <summary>
/// Orchestrates 252+ trading-day shadow run for model validation.
///
/// Workflow:
/// 1. DataBackfill: Fetch OHLCV, FeeSchedule, MarketCalendar
/// 2. Replay: Simulate model signals, orders, fills across window
/// 3. EvaluationMetrics: Calculate Sharpe, PBO, DSR, etc.
/// 4. Validation: Check all gates (PBO ≤ 20%, DSR ≥ 95%, Cost 2x positive)
/// 5. Persist: Store result in database
///
/// Idempotency: IdempotencyKey + CorrelationId allow safe replay.
/// Queue: q-research (non-critical, can wait for market data)
/// Retry: Transient failures (network) trigger retry; permanent (bad model) logged.
/// </summary>
public sealed class ShadowRunJob(
    DataBackfiller backfiller,
    ReplayEngine replay,
    MetricsCalculator calculator,
    ShadowRunQueries queries,
    IDbConnectionFactory connectionFactory,
    IOutboxWriter outboxWriter,
    IClock clock,
    ILogger<ShadowRunJob> logger)
{
    private const int MaxAttempts = 3;

    private static readonly Action<ILogger, Guid, Exception?> LogStarted =
        LoggerMessage.Define<Guid>(
            LogLevel.Information,
            new EventId(1, nameof(LogStarted)),
            "Shadow run {RunId} started");

    private static readonly Action<ILogger, Guid, Exception?> LogPhase1Complete =
        LoggerMessage.Define<Guid>(
            LogLevel.Information,
            new EventId(2, nameof(LogPhase1Complete)),
            "Shadow run {RunId} phase 1 (backfill) complete");

    private static readonly Action<ILogger, Guid, Exception?> LogPhase2Complete =
        LoggerMessage.Define<Guid>(
            LogLevel.Information,
            new EventId(3, nameof(LogPhase2Complete)),
            "Shadow run {RunId} phase 2 (replay) complete");

    private static readonly Action<ILogger, Guid, Exception?> LogPhase3Complete =
        LoggerMessage.Define<Guid>(
            LogLevel.Information,
            new EventId(4, nameof(LogPhase3Complete)),
            "Shadow run {RunId} phase 3 (evaluation) complete");

    private static readonly Action<ILogger, Guid, bool, Exception?> LogComplete =
        LoggerMessage.Define<Guid, bool>(
            LogLevel.Information,
            new EventId(5, nameof(LogComplete)),
            "Shadow run {RunId} complete; all gates passed: {AllGatesPassed}");

    private static readonly Action<ILogger, Guid, string, Exception?> LogError =
        LoggerMessage.Define<Guid, string>(
            LogLevel.Error,
            new EventId(6, nameof(LogError)),
            "Shadow run {RunId} failed: {ErrorMessage}");

    private static readonly Action<ILogger, Guid, Exception?> LogPhase4Complete =
        LoggerMessage.Define<Guid>(
            LogLevel.Information,
            new EventId(7, nameof(LogPhase4Complete)),
            "Shadow run {RunId} phase 4 (phase segmentation) complete");

    [Queue("q-evaluation")]
    // [DisableConcurrentExecution(timeoutInSeconds: 1800)]  // REMOVED: Allows internal parallel operations (Parallel.ForEachAsync)
    [AutomaticRetry(Attempts = MaxAttempts, OnAttemptsExceeded = AttemptsExceededAction.Fail)]
    public async Task ExecuteAsync(ShadowRunCommand command, CancellationToken cancellationToken = default)
    {
        LogStarted(logger, command.RunId, null);

        try
        {
            // Phase 1: Backfill data
            var ohlcvBars = await backfiller.BackfillOhlcvAsync(
                command.WindowStartDate, command.WindowEndDate,
                new[] { "KOSPI", "KOSDAQ" }.ToList(),  // Simplified: hardcoded tickers
                cancellationToken);

            var feeSchedule = await backfiller.BackfillFeeScheduleAsync(
                command.WindowStartDate, command.WindowEndDate, cancellationToken);

            LogPhase1Complete(logger, command.RunId, null);

            // Phase 2: Replay model
            var tradingSessions = ohlcvBars
                .Select(b => b.Date)
                .Distinct()
                .OrderBy(d => d)
                .ToList();

            var replayResult = await replay.ReplayAsync(
                command.ModelId, ohlcvBars, feeSchedule,
                initialCashBalance: 10_000_000m,  // 10M starting cash
                tradingSessions, cancellationToken);

            LogPhase2Complete(logger, command.RunId, null);

            // Phase 3: Calculate metrics
            var metrics = await calculator.CalculateAsync(
                replayResult, ohlcvBars, feeSchedule, cancellationToken);

            // CRITICAL: Validate metrics are not null (prevents silent failures)
            if (metrics == null)
            {
                logger.LogError("Shadow run {RunId} metrics calculation returned NULL", command.RunId);
                throw new InvalidOperationException($"Metrics cannot be null for run {command.RunId}");
            }

            LogPhase3Complete(logger, command.RunId, null);

            // Phase 4: Phase segmentation
            var phaseBreakdownDto = PhaseSegmentation.Segment(
                replayResult.DailyReturns.ToList());

            var phaseBreakdown = new PhaseBreakdown(
                BullMarket: ConvertPhaseMetrics(phaseBreakdownDto.BullMarket),
                BearMarket: ConvertPhaseMetrics(phaseBreakdownDto.BearMarket),
                Sideways: ConvertPhaseMetrics(phaseBreakdownDto.Sideways),
                HighVolatility: ConvertPhaseMetrics(phaseBreakdownDto.HighVolatility));

            LogPhase4Complete(logger, command.RunId, null);

            // Cost 2x scenario: simulate with double transaction fees
            var actualTotalCost = CalculateTotalCostsFromOrders(replayResult.Orders, feeSchedule, ohlcvBars);
            var twoXFeesCost = actualTotalCost * 2m;  // Double the actual transaction costs paid
            var initialPortfolioValue = 10_000_000m;  // Match ReplayEngine initialization
            var twoXCostReturn = (metrics.TotalReturn * initialPortfolioValue - twoXFeesCost) / initialPortfolioValue;

            var costAnalysis = new CostAnalysis(
                BaseScenarioReturn: metrics.TotalReturn,
                TwoXCostReturn: twoXCostReturn,  // Actual 2x fee impact
                PassesTwoXPositive: twoXCostReturn > 0);

            // Analyze false exits and re-entry profitability
            var falseExitMetrics = FalseExitAnalyzer.Analyze(
                replayResult.Orders, replayResult.Signals, replayResult.PortfolioHistory);

            var falseExitAnalysis = new FalseExitAnalysis(
                FalseExitCount: falseExitMetrics.FalseExitCount,
                ReentrySuccessCount: falseExitMetrics.ReentrySuccessCount,
                ReentrySuccessRate: falseExitMetrics.ReentrySuccessRate,
                AverageDaysOutOfPosition: falseExitMetrics.AverageDaysOutOfPosition);

            var validationGates = new ValidationGates(
                PboUnder20: metrics.ProbOfBacktestOverfit <= 0.20m,
                DsrAbove95: metrics.DailySharePercentile >= 0.95m,
                CostTwoXPositive: costAnalysis.PassesTwoXPositive,
                AllGatesPassed: metrics.ProbOfBacktestOverfit <= 0.20m
                    && metrics.DailySharePercentile >= 0.95m
                    && costAnalysis.PassesTwoXPositive);

            var result = new ShadowRunResult(
                RunId: command.RunId,
                ModelId: command.ModelId,
                WindowStartDate: command.WindowStartDate,
                WindowEndDate: command.WindowEndDate,
                Status: validationGates.AllGatesPassed
                    ? ShadowRunStatus.EvaluationComplete
                    : ShadowRunStatus.EvaluationComplete,
                Metrics: metrics,
                PhaseAnalysis: phaseBreakdown,
                CostAnalysis: costAnalysis,
                FalseExitAnalysis: falseExitAnalysis,
                ValidationGates: validationGates,
                CreatedAt: clock.UtcNow);

            // Phase 5-6: Persist shadow run result + emit completion event (transactional)
            await EmitShadowRunCompletedEventAsync(
                result, command.CorrelationId, validationGates.AllGatesPassed, cancellationToken);

            LogComplete(logger, command.RunId, validationGates.AllGatesPassed, null);
        }
        catch (Exception ex)
        {
            LogError(logger, command.RunId, ex.Message, ex);
            throw;  // Hangfire will classify as transient/permanent based on exception type
        }
    }

    private async Task EmitShadowRunCompletedEventAsync(
        ShadowRunResult result,
        Guid correlationId,
        bool allGatesPassed,
        CancellationToken cancellationToken)
    {
        try
        {
            // CRITICAL: InsertShadowRunAsync MUST be called to persist metrics
            logger.LogDebug("Inserting shadow run {RunId} with metrics to database", result.RunId);
            await queries.InsertShadowRunAsync(result, cancellationToken);
            logger.LogDebug("Shadow run {RunId} metrics persisted successfully to database", result.RunId);

            var eventMessage = new OutboxMessage(
                MessageId: Guid.NewGuid(),
                EventType: "ShadowRunCompleted",
                SchemaVersion: 1,
                PayloadJson: System.Text.Json.JsonSerializer.Serialize(new
                {
                    result.RunId,
                    result.ModelId,
                    CorrelationId = correlationId,
                    AllGatesPassed = allGatesPassed,
                    result.Metrics.TotalReturn,
                    result.Metrics.SharpeRatio,
                    result.Metrics.ProbOfBacktestOverfit,
                    result.Metrics.DailySharePercentile,
                    CompletedAt = clock.UtcNow
                }),
                CorrelationId: correlationId.ToString(),
                OccurredAt: clock.UtcNow,
                PayloadHash: GeneratePayloadHash(result.RunId.ToString()));

            await using var connection = await connectionFactory.OpenAsync(cancellationToken);
            await using var transaction = await connection.BeginTransactionAsync(cancellationToken);

            try
            {
                await outboxWriter.AddAsync(connection, transaction, eventMessage, cancellationToken);
                await transaction.CommitAsync(cancellationToken);

                logger.LogInformation(
                    "Shadow run {RunId} completed; event emitted to outbox for async consumers",
                    result.RunId);
            }
            catch
            {
                await transaction.RollbackAsync(cancellationToken);
                throw;
            }
        }
        catch (Exception ex)
        {
            logger.LogError(ex, "Failed to emit ShadowRunCompleted event for {RunId}", result.RunId);
            throw;
        }
    }

    private static string GeneratePayloadHash(string payload)
    {
        var hash = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(payload));
        return System.Convert.ToBase64String(hash);
    }

    private static PhaseMetrics ConvertPhaseMetrics(PhaseMetricsDto dto)
        => new PhaseMetrics(
            TradingDays: dto.TradingDays,
            Return: dto.Return,
            Sharpe: dto.Sharpe,
            WinRate: dto.WinRate,
            MaxDrawdown: dto.MaxDrawdown);

    private static decimal CalculateTotalCostsFromOrders(
        IReadOnlyList<ReplayEngine.Order> orders,
        IReadOnlyList<DataBackfiller.FeeScheduleEntry> feeSchedule,
        IReadOnlyList<DataBackfiller.OhlcvBar> ohlcvBars)
    {
        decimal totalCosts = 0m;

        foreach (var order in orders.Where(o => o.FilledPrice.HasValue))
        {
            var filledPrice = order.FilledPrice!.Value;
            var cost = order.Quantity * filledPrice;

            // Get fee schedule for this order's date
            var fee = feeSchedule.FirstOrDefault(f => f.EffectiveDate <= order.FilledDate);
            var feePercent = fee?.TransactionFeePercent ?? 0.001m;

            totalCosts += cost * feePercent;
        }

        return totalCosts;
    }
}