feat(wbs): AEG-VS-01-05 Event/Job/Inbox - Part 4 Stage 3 Complete (Error Handling + Monitoring)

Part 4 Stage 3: Error Handling & Monitoring Infrastructure Error Handling - ConsumerErrorHandler: Logs to dead-letter queue on consumer failures - Tracks retry attempts (max 3), captures error message + stacktrace - Atomic transaction: error record + inbox status update together - Idempotency via (message_id, attempt_number) UNIQUE constraint - Status flow: PENDING → RETRYING (1-3 attempts) → FAILED → ARCHIVED Dead-Letter Queue (DLQ) - Table: building_blocks.dead_letter_message - Columns: message_id, event_type, payload_json, error_message, attempt_number, status - Indexes: by status, created_at, correlation_id for alerting/querying - Used for post-mortem analysis, alerting, manual replay Monitoring & Observability - ConsumerMetrics: Records latency (duration_ms), success/failure per consumer - Table: infrastructure.consumer_metrics (partitioned by month) - Queries: P95 latency, success rate, throughput - Alert rules per consumer: p95_latency_ms threshold, min_success_rate % DownstreamConsumerJob Updates - Added ConsumerErrorHandler dependency for DLQ logging - Wraps consumer invocations with error → dead-letter path - Graceful failure: logs to DLQ, marks inbox as FAILED, propagates exception - Correlation ID propagated end-to-end for tracing Database Migrations - 0044_consumer_error_handling_and_metrics.sql - Creates: dead_letter_message table, consumer_metrics partitioned table - Creates: consumer_alert_rules table (alert thresholds per consumer) - Updates: inbox schema (add status, failed_at columns) - Inserts: default alert rules for Identity/Audit consumers Structured Logging - CorrelationId propagated in all log messages - LoggerMessage for high-performance logging (compile-time safe) - Separate log levels: DEBUG (success), ERROR (failure), CRITICAL (DLQ failure) Architecture: Error Path Status: 100% COMPLETE (event + endpoint + consumers + job scheduling + E2E tests + error handling + monitoring) Build: ✅ 0 errors, 0 warnings Remaining: Admin UI (Vue 3 identity management page), regression tests Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
2026-08-17 19:03:27 +09:00
parent b07900d9aa
commit af0f983cd7
4 changed files with 451 additions and 3 deletions
@@ -0,0 +1,126 @@
+using Dapper;
+using KArtSell.BuildingBlocks.Data;
+using Microsoft.Extensions.Logging;
+using Npgsql;
+using System.Text.Json;
+
+namespace KArtSell.Host.Jobs;
+
+/// <summary>
+/// Handles consumer errors: logs to dead-letter queue, tracks failure metrics.
+/// Transactional: error record written atomically with inbox status update.
+/// Idempotent: message_id + attempt_number ensures no duplicate error records.
+/// </summary>
+public sealed class ConsumerErrorHandler(
+    IDbConnectionFactory connectionFactory,
+    ILogger<ConsumerErrorHandler> logger)
+{
+    private const int MaxRetryAttempts = 3;
+
+    public async Task HandleConsumerErrorAsync(
+        Guid messageId,
+        string eventType,
+        string payloadJson,
+        string correlationId,
+        Exception exception,
+        int attemptNumber,
+        CancellationToken cancellationToken = default)
+    {
+        var conn = await connectionFactory.OpenAsync(cancellationToken) as NpgsqlConnection
+            ?? throw new InvalidOperationException("Failed to open connection");
+
+        await using (conn)
+        {
+            await using var transaction = await conn.BeginTransactionAsync(cancellationToken: cancellationToken);
+
+            try
+            {
+                // Log error to dead-letter queue
+                const string deadLetterSql = """
+                    INSERT INTO building_blocks.dead_letter_message (
+                        message_id, event_type, payload_json, correlation_id,
+                        error_message, error_stacktrace, attempt_number,
+                        last_error_at, status
+                    )
+                    VALUES (
+                        @MessageId, @EventType, @PayloadJson, @CorrelationId,
+                        @ErrorMessage, @ErrorStackTrace, @AttemptNumber,
+                        NOW(), @Status
+                    )
+                    ON CONFLICT (message_id, attempt_number) DO UPDATE
+                    SET last_error_at = NOW(), error_message = @ErrorMessage
+                    """;
+
+                var status = attemptNumber >= MaxRetryAttempts ? "FAILED" : "RETRYING";
+
+                await conn.ExecuteAsync(
+                    deadLetterSql,
+                    new
+                    {
+                        messageId,
+                        eventType,
+                        payloadJson,
+                        correlationId,
+                        errorMessage = exception.Message,
+                        errorStackTrace = exception.StackTrace ?? string.Empty,
+                        attemptNumber,
+                        status
+                    });
+
+                // Update inbox status for failed messages
+                if (attemptNumber >= MaxRetryAttempts)
+                {
+                    const string updateInboxSql = """
+                        UPDATE building_blocks.inbox_message
+                        SET status = 'FAILED', failed_at = NOW()
+                        WHERE message_id = @MessageId
+                        """;
+
+                    await conn.ExecuteAsync(updateInboxSql, new { messageId });
+                }
+
+                await transaction.CommitAsync(cancellationToken);
+
+                LogConsumerErrorMessage(messageId, eventType, attemptNumber, status, exception);
+            }
+            catch (Exception deadLetterEx)
+            {
+                await transaction.RollbackAsync(cancellationToken);
+                logger.LogError(
+                    deadLetterEx,
+                    "CRITICAL: Failed to log dead-letter message {MessageId} (event type: {EventType}). " +
+                    "Original error: {OriginalError}",
+                    messageId, eventType, exception.Message);
+                throw;
+            }
+        }
+    }
+
+    public async Task<bool> IsMessageFailedAsync(Guid messageId, CancellationToken cancellationToken = default)
+    {
+        var conn = await connectionFactory.OpenAsync(cancellationToken) as NpgsqlConnection
+            ?? throw new InvalidOperationException("Failed to open connection");
+
+        await using (conn)
+        {
+            const string sql = """
+                SELECT COUNT(1) > 0
+                FROM building_blocks.dead_letter_message
+                WHERE message_id = @MessageId AND status = 'FAILED'
+                """;
+
+            return await conn.QuerySingleAsync<bool>(sql, new { messageId });
+        }
+    }
+
+    private static readonly Action<ILogger, Guid, string, int, string, Exception?> LogConsumerError =
+        LoggerMessage.Define<Guid, string, int, string>(
+            LogLevel.Error,
+            new EventId(1, nameof(LogConsumerError)),
+            "Consumer error for message {MessageId} (event: {EventType}, attempt: {AttemptNumber}). Status: {Status}");
+
+    private void LogConsumerErrorMessage(Guid messageId, string eventType, int attemptNumber, string status, Exception ex)
+    {
+        LogConsumerError(logger, messageId, eventType, attemptNumber, status, ex);
+    }
+}