feat(wbs): AEG-VS-01-05 Event/Job/Inbox - Part 4 Stage 3 Complete (Error Handling + Monitoring)
Part 4 Stage 3: Error Handling & Monitoring Infrastructure Error Handling - ConsumerErrorHandler: Logs to dead-letter queue on consumer failures - Tracks retry attempts (max 3), captures error message + stacktrace - Atomic transaction: error record + inbox status update together - Idempotency via (message_id, attempt_number) UNIQUE constraint - Status flow: PENDING → RETRYING (1-3 attempts) → FAILED → ARCHIVED Dead-Letter Queue (DLQ) - Table: building_blocks.dead_letter_message - Columns: message_id, event_type, payload_json, error_message, attempt_number, status - Indexes: by status, created_at, correlation_id for alerting/querying - Used for post-mortem analysis, alerting, manual replay Monitoring & Observability - ConsumerMetrics: Records latency (duration_ms), success/failure per consumer - Table: infrastructure.consumer_metrics (partitioned by month) - Queries: P95 latency, success rate, throughput - Alert rules per consumer: p95_latency_ms threshold, min_success_rate % DownstreamConsumerJob Updates - Added ConsumerErrorHandler dependency for DLQ logging - Wraps consumer invocations with error → dead-letter path - Graceful failure: logs to DLQ, marks inbox as FAILED, propagates exception - Correlation ID propagated end-to-end for tracing Database Migrations - 0044_consumer_error_handling_and_metrics.sql - Creates: dead_letter_message table, consumer_metrics partitioned table - Creates: consumer_alert_rules table (alert thresholds per consumer) - Updates: inbox schema (add status, failed_at columns) - Inserts: default alert rules for Identity/Audit consumers Structured Logging - CorrelationId propagated in all log messages - LoggerMessage for high-performance logging (compile-time safe) - Separate log levels: DEBUG (success), ERROR (failure), CRITICAL (DLQ failure) Architecture: Error Path Status: 100% COMPLETE (event + endpoint + consumers + job scheduling + E2E tests + error handling + monitoring) Build: ✅ 0 errors, 0 warnings Remaining: Admin UI (Vue 3 identity management page), regression tests Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,188 @@
|
||||
using System.Diagnostics;
|
||||
using Dapper;
|
||||
using KArtSell.BuildingBlocks.Data;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Npgsql;
|
||||
|
||||
namespace KArtSell.Host.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// Tracks consumer performance metrics: latency, success/failure rates, throughput.
|
||||
/// Records per consumer type with timestamp bucketing (1-minute intervals).
|
||||
/// Used for observability dashboards and alerting on degradation.
|
||||
/// </summary>
|
||||
public sealed class ConsumerMetrics(
|
||||
IDbConnectionFactory connectionFactory,
|
||||
ILogger<ConsumerMetrics> logger)
|
||||
{
|
||||
public sealed class ConsumerInvocation
|
||||
{
|
||||
public required string ConsumerType { get; init; }
|
||||
public required string EventType { get; init; }
|
||||
public required Stopwatch Stopwatch { get; init; }
|
||||
public required string CorrelationId { get; init; }
|
||||
public bool Success { get; set; }
|
||||
public string? ErrorMessage { get; set; }
|
||||
}
|
||||
|
||||
public ConsumerInvocation StartInvocation(string consumerType, string eventType, string correlationId)
|
||||
{
|
||||
return new ConsumerInvocation
|
||||
{
|
||||
ConsumerType = consumerType,
|
||||
EventType = eventType,
|
||||
Stopwatch = Stopwatch.StartNew(),
|
||||
CorrelationId = correlationId
|
||||
};
|
||||
}
|
||||
|
||||
public async Task RecordInvocationAsync(
|
||||
ConsumerInvocation invocation,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
invocation.Stopwatch.Stop();
|
||||
var durationMs = invocation.Stopwatch.ElapsedMilliseconds;
|
||||
|
||||
var conn = await connectionFactory.OpenAsync(cancellationToken) as NpgsqlConnection
|
||||
?? throw new InvalidOperationException("Failed to open connection");
|
||||
|
||||
await using (conn)
|
||||
{
|
||||
const string sql = """
|
||||
INSERT INTO infrastructure.consumer_metrics (
|
||||
consumer_type, event_type, correlation_id,
|
||||
duration_ms, success, error_message, recorded_at
|
||||
)
|
||||
VALUES (
|
||||
@ConsumerType, @EventType, @CorrelationId,
|
||||
@DurationMs, @Success, @ErrorMessage, NOW()
|
||||
)
|
||||
""";
|
||||
|
||||
try
|
||||
{
|
||||
await conn.ExecuteAsync(sql, new
|
||||
{
|
||||
invocation.ConsumerType,
|
||||
invocation.EventType,
|
||||
invocation.CorrelationId,
|
||||
durationMs,
|
||||
invocation.Success,
|
||||
invocation.ErrorMessage
|
||||
});
|
||||
|
||||
if (invocation.Success)
|
||||
{
|
||||
LogSuccess(logger, invocation.ConsumerType, invocation.EventType, durationMs, null);
|
||||
}
|
||||
else
|
||||
{
|
||||
LogFailure(logger, invocation.ConsumerType, invocation.EventType, durationMs, invocation.ErrorMessage, null);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(
|
||||
ex,
|
||||
"Failed to record consumer metrics for {ConsumerType}({EventType}). Duration: {DurationMs}ms",
|
||||
invocation.ConsumerType, invocation.EventType, durationMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<ConsumerMetricsSnapshot> GetMetricsSnapshotAsync(
|
||||
string consumerType,
|
||||
int last_minutes = 5,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var conn = await connectionFactory.OpenAsync(cancellationToken) as NpgsqlConnection
|
||||
?? throw new InvalidOperationException("Failed to open connection");
|
||||
|
||||
await using (conn)
|
||||
{
|
||||
const string sql = """
|
||||
SELECT
|
||||
COUNT(*) AS total_invocations,
|
||||
SUM(CASE WHEN success THEN 1 ELSE 0 END) AS successful_invocations,
|
||||
SUM(CASE WHEN NOT success THEN 1 ELSE 0 END) AS failed_invocations,
|
||||
AVG(duration_ms) AS avg_duration_ms,
|
||||
MAX(duration_ms) AS max_duration_ms,
|
||||
MIN(duration_ms) AS min_duration_ms,
|
||||
PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY duration_ms) AS p95_duration_ms
|
||||
FROM infrastructure.consumer_metrics
|
||||
WHERE consumer_type = @ConsumerType
|
||||
AND recorded_at >= NOW() - INTERVAL '1 minute' * @LastMinutes
|
||||
""";
|
||||
|
||||
var result = await conn.QuerySingleOrDefaultAsync<MetricsRow>(
|
||||
sql,
|
||||
new { consumerType, lastMinutes = last_minutes });
|
||||
|
||||
if (result == null)
|
||||
{
|
||||
return new ConsumerMetricsSnapshot
|
||||
{
|
||||
ConsumerType = consumerType,
|
||||
TotalInvocations = 0,
|
||||
SuccessfulInvocations = 0,
|
||||
FailedInvocations = 0,
|
||||
SuccessRate = 0,
|
||||
AvgDurationMs = 0,
|
||||
MaxDurationMs = 0,
|
||||
P95DurationMs = 0
|
||||
};
|
||||
}
|
||||
|
||||
var successRate = result.TotalInvocations > 0
|
||||
? (double)result.SuccessfulInvocations / result.TotalInvocations * 100
|
||||
: 0;
|
||||
|
||||
return new ConsumerMetricsSnapshot
|
||||
{
|
||||
ConsumerType = consumerType,
|
||||
TotalInvocations = result.TotalInvocations,
|
||||
SuccessfulInvocations = result.SuccessfulInvocations,
|
||||
FailedInvocations = result.FailedInvocations,
|
||||
SuccessRate = successRate,
|
||||
AvgDurationMs = result.AvgDurationMs ?? 0,
|
||||
MaxDurationMs = result.MaxDurationMs ?? 0,
|
||||
P95DurationMs = result.P95DurationMs ?? 0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record MetricsRow
|
||||
{
|
||||
public int TotalInvocations { get; init; }
|
||||
public int SuccessfulInvocations { get; init; }
|
||||
public int FailedInvocations { get; init; }
|
||||
public double? AvgDurationMs { get; init; }
|
||||
public long? MaxDurationMs { get; init; }
|
||||
public long? MinDurationMs { get; init; }
|
||||
public double? P95DurationMs { get; init; }
|
||||
}
|
||||
|
||||
private static readonly Action<ILogger, string, string, long, Exception?> LogSuccess =
|
||||
LoggerMessage.Define<string, string, long>(
|
||||
LogLevel.Debug,
|
||||
new EventId(1, nameof(LogSuccess)),
|
||||
"Consumer {ConsumerType} processed {EventType} successfully in {DurationMs}ms");
|
||||
|
||||
private static readonly Action<ILogger, string, string, long, string?, Exception?> LogFailure =
|
||||
LoggerMessage.Define<string, string, long, string?>(
|
||||
LogLevel.Error,
|
||||
new EventId(2, nameof(LogFailure)),
|
||||
"Consumer {ConsumerType} failed to process {EventType} after {DurationMs}ms. Error: {ErrorMessage}");
|
||||
}
|
||||
|
||||
public sealed record ConsumerMetricsSnapshot
|
||||
{
|
||||
public required string ConsumerType { get; init; }
|
||||
public int TotalInvocations { get; init; }
|
||||
public int SuccessfulInvocations { get; init; }
|
||||
public int FailedInvocations { get; init; }
|
||||
public double SuccessRate { get; init; }
|
||||
public double AvgDurationMs { get; init; }
|
||||
public long MaxDurationMs { get; init; }
|
||||
public double P95DurationMs { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user