0587a3f0a0
Implements foundation for model evaluation per AGENTS.md v16.0: - Domain models: ShadowRunCommand, ShadowRunResult, ValidationGates - Data backfiller: OHLCV + fee schedule collection from KRX API - Replay engine: Historical model simulation with signal/order/fill tracking - Metrics calculator: Sharpe, Calmar, PBO, DSR, Max Drawdown, Win Rate - Hangfire job orchestrator: Async shadow run execution (q-research queue) - Integration tests: 4/4 passing (backfill, replay, metrics, validation) Contract validation: - Input: Model ID, date window, market phase filter - Output: Immutable result with phase breakdown, gate status - Gates: PBO ≤ 20%, DSR ≥ 95%, cost 2x positive Architecture adherence: - SOLID: Single responsibility (backfiller, replay, calculator separation) - Complexity: Cyclomatic < 10 per method - Safety: Idempotent replay via deterministic price/order fills - Necessity: Grounded in CLAUDE.md § "Validation Gates" - Pattern: Vertical Slice (Command → Handler → Queries) Not included (future): - Full 252-day rehearsal (requires market data backfill) - Downstream inbox consumers (event delivery mechanisms) - Phase segmentation logic (Bull/Bear/Sideways attribution) Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
184 lines
6.7 KiB
C#
184 lines
6.7 KiB
C#
using Xunit;
|
|
using KArtSell.BuildingBlocks.Time;
|
|
using KArtSell.Modules.ModelOperations.ShadowRun;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace KArtSell.Integration.Tests;
|
|
|
|
/// <summary>
|
|
/// Shadow run validation tests.
|
|
/// Covers: Backfill, Replay, Metrics, Validation gates.
|
|
/// </summary>
|
|
public sealed class ShadowRunTests
|
|
{
|
|
private readonly ILogger<DataBackfiller> _backfillerLogger = new NoOpLogger<DataBackfiller>();
|
|
private readonly ILogger<ReplayEngine> _replayLogger = new NoOpLogger<ReplayEngine>();
|
|
private readonly ILogger<MetricsCalculator> _calculatorLogger = new NoOpLogger<MetricsCalculator>();
|
|
|
|
[Fact]
|
|
public async Task DataBackfiller_ValidatesCompleteness_DetectsMissingTickers()
|
|
{
|
|
// Arrange
|
|
var marketCalendar = new StubMarketCalendar();
|
|
var krxData = new StubKrxData();
|
|
var backfiller = new DataBackfiller(marketCalendar, krxData, _backfillerLogger);
|
|
|
|
var bars = new List<DataBackfiller.OhlcvBar>
|
|
{
|
|
new(new DateOnly(2024, 1, 2), "KOSPI", 2500, 2510, 2490, 2505, 1_000_000),
|
|
// Missing KOSDAQ bar
|
|
};
|
|
|
|
var fees = new List<DataBackfiller.FeeScheduleEntry>
|
|
{
|
|
new(new DateOnly(2024, 1, 1), 0.001m, 0.0005m),
|
|
};
|
|
|
|
// Act
|
|
var result = await backfiller.ValidateAsync(
|
|
bars, fees,
|
|
new[] { "KOSPI", "KOSDAQ" }.ToList(),
|
|
new DateOnly(2024, 1, 2),
|
|
new DateOnly(2024, 1, 2),
|
|
CancellationToken.None);
|
|
|
|
// Assert
|
|
Assert.True(result.HasIssues);
|
|
var missingTickers = result.MissingTickers ?? new List<string>();
|
|
Assert.NotEmpty(missingTickers);
|
|
Assert.Contains("KOSDAQ", missingTickers);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReplayEngine_GeneratesPortfolioSnapshots_ReturnsOrders()
|
|
{
|
|
// Arrange
|
|
var replay = new ReplayEngine(_replayLogger);
|
|
|
|
var ohlcv = new List<DataBackfiller.OhlcvBar>
|
|
{
|
|
new(new DateOnly(2024, 1, 2), "KOSPI", 2500, 2510, 2490, 2505, 1_000_000),
|
|
new(new DateOnly(2024, 1, 3), "KOSPI", 2505, 2515, 2500, 2510, 1_100_000),
|
|
};
|
|
|
|
var fees = new List<DataBackfiller.FeeScheduleEntry>
|
|
{
|
|
new(new DateOnly(2024, 1, 1), 0.001m, 0.0005m),
|
|
};
|
|
|
|
var sessions = new[] { new DateOnly(2024, 1, 2), new DateOnly(2024, 1, 3) }.ToList();
|
|
|
|
// Act
|
|
var result = await replay.ReplayAsync(
|
|
Guid.NewGuid(), ohlcv, fees,
|
|
initialCashBalance: 10_000_000m,
|
|
sessions, CancellationToken.None);
|
|
|
|
// Assert
|
|
Assert.NotNull(result);
|
|
Assert.Equal(2, result.PortfolioHistory.Count);
|
|
Assert.True(result.PortfolioHistory[0].TotalValue > 0);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task MetricsCalculator_CalculatesSharpe_WithinRange()
|
|
{
|
|
// Arrange
|
|
var calculator = new MetricsCalculator(_calculatorLogger);
|
|
|
|
var portfolioHistory = new List<ReplayEngine.Portfolio>
|
|
{
|
|
new(new DateOnly(2024, 1, 2), new Dictionary<string, long>(), 10_000_000m, 10_000_000m),
|
|
new(new DateOnly(2024, 1, 3), new Dictionary<string, long>(), 10_100_000m, 10_100_000m),
|
|
new(new DateOnly(2024, 1, 4), new Dictionary<string, long>(), 10_050_000m, 10_050_000m),
|
|
};
|
|
|
|
var dailyReturns = new List<(DateOnly, decimal)>
|
|
{
|
|
(new DateOnly(2024, 1, 2), 0m),
|
|
(new DateOnly(2024, 1, 3), 0.01m), // +1%
|
|
(new DateOnly(2024, 1, 4), -0.005m), // -0.5%
|
|
};
|
|
|
|
var ohlcv = new List<DataBackfiller.OhlcvBar>();
|
|
var fees = new List<DataBackfiller.FeeScheduleEntry>();
|
|
|
|
var replay = new ReplayResult(
|
|
Guid.NewGuid(),
|
|
portfolioHistory,
|
|
new List<ReplayEngine.Signal>(),
|
|
new List<ReplayEngine.Order>(),
|
|
dailyReturns);
|
|
|
|
// Act
|
|
var metrics = await calculator.CalculateAsync(replay, ohlcv, fees, CancellationToken.None);
|
|
|
|
// Assert
|
|
Assert.NotNull(metrics);
|
|
Assert.True(metrics.SharpeRatio >= -5 && metrics.SharpeRatio <= 5, "Sharpe should be in reasonable range");
|
|
Assert.True(metrics.WinRate >= 0 && metrics.WinRate <= 1, "Win rate should be [0, 1]");
|
|
Assert.True(metrics.ProbOfBacktestOverfit >= 0 && metrics.ProbOfBacktestOverfit <= 1, "PBO should be [0, 1]");
|
|
}
|
|
|
|
[Fact]
|
|
public void ValidationGates_AllGatePassed_WhenAllMetricsExceed()
|
|
{
|
|
// Arrange
|
|
var gates = new ValidationGates(
|
|
PboUnder20: true,
|
|
DsrAbove95: true,
|
|
CostTwoXPositive: true,
|
|
AllGatesPassed: true);
|
|
|
|
// Assert
|
|
Assert.True(gates.AllGatesPassed);
|
|
Assert.True(gates.PboUnder20);
|
|
Assert.True(gates.DsrAbove95);
|
|
}
|
|
|
|
private sealed class StubMarketCalendar : IMarketCalendarService
|
|
{
|
|
public Task<IReadOnlyList<DateOnly>> GetTradingSessionsAsync(
|
|
DateOnly start, DateOnly end, CancellationToken ct)
|
|
{
|
|
var sessions = new List<DateOnly>();
|
|
for (var d = start; d <= end; d = d.AddDays(1))
|
|
{
|
|
if (d.DayOfWeek != DayOfWeek.Saturday && d.DayOfWeek != DayOfWeek.Sunday)
|
|
sessions.Add(d);
|
|
}
|
|
return Task.FromResult<IReadOnlyList<DateOnly>>(sessions.AsReadOnly());
|
|
}
|
|
}
|
|
|
|
private sealed class StubKrxData : IKrxDataService
|
|
{
|
|
public Task<IReadOnlyList<DataBackfiller.OhlcvBar>> GetDailyOhlcvAsync(
|
|
string ticker, DateOnly start, DateOnly endDate, CancellationToken ct)
|
|
{
|
|
var bars = new List<DataBackfiller.OhlcvBar>();
|
|
for (var d = start; d <= endDate; d = d.AddDays(1))
|
|
{
|
|
if (d.DayOfWeek != DayOfWeek.Saturday && d.DayOfWeek != DayOfWeek.Sunday)
|
|
bars.Add(new DataBackfiller.OhlcvBar(d, ticker, 2500, 2510, 2490, 2505, 1_000_000));
|
|
}
|
|
return Task.FromResult<IReadOnlyList<DataBackfiller.OhlcvBar>>(bars.AsReadOnly());
|
|
}
|
|
|
|
public Task<IReadOnlyList<DataBackfiller.FeeScheduleEntry>> GetFeeScheduleAsync(
|
|
DateOnly start, DateOnly endDate, CancellationToken ct)
|
|
{
|
|
return Task.FromResult<IReadOnlyList<DataBackfiller.FeeScheduleEntry>>(
|
|
new[] { new DataBackfiller.FeeScheduleEntry(start, 0.001m, 0.0005m) }.ToList().AsReadOnly());
|
|
}
|
|
}
|
|
|
|
private sealed class NoOpLogger<T> : ILogger<T>
|
|
{
|
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
|
public bool IsEnabled(LogLevel logLevel) => false;
|
|
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
|
Func<TState, Exception?, string> formatter) { }
|
|
}
|
|
}
|