using Dapper; using KArtSell.BuildingBlocks.Data; using KArtSell.BuildingBlocks.Time; using KArtSell.Modules.ModelOperations.Observability; using Npgsql; using Xunit; namespace KArtSell.Integration.Tests; /// /// Observability Metrics Tests /// Covers: Batch SLA, Data Quality, Duplicates, Reconciliation, Model Drift /// Following AGENTS.md v16.0: Evidence-based monitoring, constraint validation /// public sealed class ObservabilityMetricsTests : IAsyncLifetime { private readonly string _connectionString; private readonly NpgsqlDataSource _dataSource; private readonly IDbConnectionFactory _connectionFactory; private readonly IClock _clock = new SystemClock(); public ObservabilityMetricsTests() { _connectionString = Environment.GetEnvironmentVariable("KARTSELL_POSTGRES") ?? "Host=localhost;Port=5432;Database=kartselldb;Username=kartsell;Password=kartsell4321@!"; _dataSource = new NpgsqlDataSourceBuilder(_connectionString).Build(); _connectionFactory = new NpgsqlConnectionFactory(_dataSource); } public async Task InitializeAsync() { await using var connection = await _dataSource.OpenConnectionAsync(); await using var cmd = connection.CreateCommand(); cmd.CommandText = "SELECT 1"; await cmd.ExecuteScalarAsync(); } public async Task DisposeAsync() { await _dataSource.DisposeAsync(); } /// /// Gate 5.1: Batch SLA Metrics - Job completion tracking /// [Fact] public async Task ObservabilityMetrics_BatchSLA_ReturnsCompletionMetrics() { // Arrange: Service with clock var service = new ObservabilityService(_connectionFactory, _clock); // Act: Get batch SLA metrics var metrics = await service.GetBatchSlaMetricsAsync(CancellationToken.None); // Assert: Metrics have expected structure Assert.NotNull(metrics); Assert.True(metrics.QueueDepth >= 0); Assert.True(metrics.AverageCompletionTimeMs >= 0); Assert.True(metrics.TotalJobsCompleted >= 0); Assert.True(metrics.RetryCount >= 0); Assert.True(metrics.MeasuredAt <= _clock.UtcNow.DateTime); } /// /// Gate 5.2: Data Quality Metrics - Quarantine monitoring /// [Fact] public async Task ObservabilityMetrics_DataQuality_ReturnsQuarantineData() { // Arrange: Service var service = new ObservabilityService(_connectionFactory, _clock); // Act: Get data quality metrics var metrics = await service.GetDataQualityMetricsAsync(CancellationToken.None); // Assert: Metrics structure valid Assert.NotNull(metrics); Assert.True(metrics.QuarantinedJobCount >= 0); Assert.NotNull(metrics.TopQuarantineReasons); Assert.True(metrics.AverageQuarantineAgeHours >= 0); } /// /// Gate 5.3: Duplicate Detection - Constraint violation tracking /// [Fact] public async Task ObservabilityMetrics_DuplicateDetection_IdentifiesDuplicates() { // Arrange: Create outbox message and duplicate inbox records var outboxId = Guid.NewGuid(); var now = _clock.UtcNow.DateTime; await using var connection = await _connectionFactory.OpenAsync(CancellationToken.None); // Create outbox message await connection.ExecuteAsync(""" INSERT INTO building_blocks.outbox_message (message_id, event_type, schema_version, payload_json, correlation_id, occurred_at, payload_hash, published_at) VALUES (@Id, 'TestEvent', 1, '{"test":"data"}'::jsonb, @CorrId, @Now, 'hash123', @Now) """, new { Id = outboxId, CorrId = Guid.NewGuid().ToString(), Now = now }); // Note: Cannot insert actual duplicates due to UNIQUE constraint, but can query for structure // Act: Get duplicate detection metrics var service = new ObservabilityService(_connectionFactory, _clock); var metrics = await service.GetDuplicateDetectionMetricsAsync(CancellationToken.None); // Assert: Metrics structure valid Assert.NotNull(metrics); Assert.True(metrics.DuplicateViolationCount >= 0); Assert.True(metrics.AffectedMessageCount >= 0); } /// /// Gate 5.4: Reconciliation - Outbox/Inbox matching /// [Fact] public async Task ObservabilityMetrics_Reconciliation_CalculatesCompleteness() { // Arrange: Service var service = new ObservabilityService(_connectionFactory, _clock); // Act: Get reconciliation metrics var metrics = await service.GetReconciliationMetricsAsync(CancellationToken.None); // Assert: Completeness is between 0-100% Assert.NotNull(metrics); Assert.True(metrics.AuditTrailCompleteness >= 0 && metrics.AuditTrailCompleteness <= 100); Assert.True(metrics.OutboxMessageCount >= 0); Assert.True(metrics.InboxProcessedCount >= 0); Assert.True(metrics.MismatchCount >= 0); } /// /// Gate 5.5: Model Drift - OOS performance tracking /// [Fact] public async Task ObservabilityMetrics_ModelDrift_TracksOutOfSamplePerformance() { // Arrange: Create shadow_run with validation metrics var runId = Guid.NewGuid(); var modelId = Guid.NewGuid(); var now = _clock.UtcNow.DateTime; await using var connection = await _connectionFactory.OpenAsync(CancellationToken.None); await connection.ExecuteAsync(""" INSERT INTO model_operations.shadow_run (run_id, model_id, window_start, window_end, status, published_at, validation_gates_json) VALUES (@RunId, @ModelId, @Start, @End, 'EvaluationComplete', @Now, @Gates) """, new { RunId = runId, ModelId = modelId, Start = new DateOnly(2024, 1, 2), End = new DateOnly(2024, 8, 31), Now = now, Gates = """{"dsr_above_95":0.96,"sharpe":1.5,"pbo":0.15,"cost_2x_positive":true}""" }); // Act: Get model drift metrics var service = new ObservabilityService(_connectionFactory, _clock); var metrics = await service.GetModelDriftMetricsAsync(CancellationToken.None); // Assert: Metrics track OOS performance Assert.NotNull(metrics); Assert.True(metrics.ModelsUnderMonitoring > 0); Assert.True(metrics.AverageOosPerformance >= 0); Assert.True(metrics.BaselineSharpeRatio >= 0); } /// /// Gate 5.6: Alert Thresholds - Conditions for alerts defined /// [Fact] public async Task ObservabilityMetrics_AlertThresholds_IdentifiesCriticalConditions() { // Arrange: Service var service = new ObservabilityService(_connectionFactory, _clock); // Act: Get all metrics var batchSla = await service.GetBatchSlaMetricsAsync(CancellationToken.None); var dataQuality = await service.GetDataQualityMetricsAsync(CancellationToken.None); var duplicates = await service.GetDuplicateDetectionMetricsAsync(CancellationToken.None); var reconciliation = await service.GetReconciliationMetricsAsync(CancellationToken.None); var modelDrift = await service.GetModelDriftMetricsAsync(CancellationToken.None); // Assert: Define alert thresholds (AGENTS.md v16.0 constraint enforcement) // Critical alerts: var criticalAlerts = new List(); if (duplicates.DuplicateViolationCount > 0) criticalAlerts.Add("CRITICAL: Duplicate inbox messages detected"); if (reconciliation.AuditTrailCompleteness < 95) criticalAlerts.Add("WARNING: Audit trail completeness < 95%"); if (dataQuality.QuarantinedJobCount > 10) criticalAlerts.Add("WARNING: > 10 jobs in quarantine"); if (modelDrift.PerformanceDegradedCount > 0) criticalAlerts.Add("WARNING: Model performance degradation detected"); // Assert: All metrics successfully retrieved (alert mechanism can use these) Assert.NotNull(batchSla); Assert.NotNull(dataQuality); Assert.NotNull(duplicates); Assert.NotNull(reconciliation); Assert.NotNull(modelDrift); } // ========== Helper Class ========== private sealed class NpgsqlConnectionFactory : IDbConnectionFactory { private readonly NpgsqlDataSource _dataSource; public NpgsqlConnectionFactory(NpgsqlDataSource dataSource) { _dataSource = dataSource; } public async ValueTask OpenAsync(CancellationToken cancellationToken) => await _dataSource.OpenConnectionAsync(cancellationToken); } }