diff --git a/tests/KArtSell.Integration.Tests/OutboxInboxCrashRecoveryTests.cs b/tests/KArtSell.Integration.Tests/OutboxInboxCrashRecoveryTests.cs new file mode 100644 index 00000000..d78a3ad7 --- /dev/null +++ b/tests/KArtSell.Integration.Tests/OutboxInboxCrashRecoveryTests.cs @@ -0,0 +1,410 @@ +using System.Data; +using System.Data.Common; +using Dapper; +using KArtSell.BuildingBlocks.Data; +using KArtSell.BuildingBlocks.Hashing; +using KArtSell.BuildingBlocks.Time; +using KArtSell.Host.Jobs; +using Microsoft.Extensions.Logging; +using Npgsql; +using Xunit; + +namespace KArtSell.Integration.Tests; + +/// +/// Outbox/Inbox Crash-Recovery & Audit Reconciliation Tests +/// Covers: Process crashes, idempotency, consumer retries, audit trails +/// Following AGENTS.md v16.0: Failure modes, recovery validation, evidence preservation +/// +public sealed class OutboxInboxCrashRecoveryTests : IAsyncLifetime +{ + private readonly string _connectionString; + private readonly NpgsqlDataSource _dataSource; + private readonly IDbConnectionFactory _connectionFactory; + private readonly IClock _clock = new SystemClock(); + private readonly ILogger _logger = new NoOpLogger(); + + public OutboxInboxCrashRecoveryTests() + { + _connectionString = Environment.GetEnvironmentVariable("KARTSELL_POSTGRES") + ?? "Host=localhost;Port=5432;Database=kartselldb;Username=kartsell;Password=kartsell4321@!"; + _dataSource = new NpgsqlDataSourceBuilder(_connectionString).Build(); + _connectionFactory = new NpgsqlConnectionFactory(_dataSource); + } + + public async Task InitializeAsync() + { + // Verify database is accessible + await using var connection = await _dataSource.OpenConnectionAsync(); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT 1"; + await cmd.ExecuteScalarAsync(); + } + + public async Task DisposeAsync() + { + await _dataSource.DisposeAsync(); + } + + /// + /// Gate 2.1: Outbox Durability - Messages survive process crash + /// Scenario: Process crashes before marking message published. + /// Recovery: Restart retrieves the unpublished message. + /// + [Fact] + public async Task CrashRecovery_OutboxMessage_SurvivesProcessCrash() + { + // Arrange: Insert unpublished message simulating pre-crash state + var messageId = Guid.NewGuid(); + var now = _clock.UtcNow; + const string payload = """{"decision_id":"crash-test"}"""; + var hash = ContentHasher.Sha256(payload); + var correlationId = Guid.NewGuid().ToString(); + + await using (var connection = await _connectionFactory.OpenAsync(CancellationToken.None)) + { + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.outbox_message + (message_id, event_type, schema_version, payload_json, correlation_id, occurred_at, payload_hash) + VALUES (@MessageId, @EventType, 1, CAST(@PayloadJson AS jsonb), @CorrelationId, @OccurredAt, @Hash) + """, + new + { + MessageId = messageId, + EventType = "CrashTestEvent", + PayloadJson = payload, + CorrelationId = correlationId, + OccurredAt = now.AddMinutes(-2), + Hash = hash + }); + } + + // Simulate process crash: Verify message is still unpublished + await using (var connection = await _connectionFactory.OpenAsync(CancellationToken.None)) + { + var published = await connection.QuerySingleOrDefaultAsync( + "SELECT published_at FROM building_blocks.outbox_message WHERE message_id = @MessageId", + new { MessageId = messageId }); + + Assert.Null(published); + } + + // Act: Restart process - query for unpublished messages (what job would retrieve) + await using (var connection = await _connectionFactory.OpenAsync(CancellationToken.None)) + { + var unpublished = await connection.QueryAsync(""" + SELECT message_id FROM building_blocks.outbox_message + WHERE published_at IS NULL + ORDER BY occurred_at ASC + """); + + // Assert: Message is still retrievable for processing + Assert.Contains(messageId, unpublished); + } + } + + /// + /// Gate 2.2: Inbox Idempotency - Duplicate message handling via UNIQUE constraint + /// Scenario: Same outbox message processed by multiple consumers (concurrent). + /// Expected: UNIQUE(message_id, consumer) prevents duplicates. + /// + [Fact] + public async Task CrashRecovery_InboxIdempotency_PreventsDuplicatesByConsumer() + { + // Arrange: Create outbox message + var messageId = Guid.NewGuid(); + var now = _clock.UtcNow; + const string payload = """{"idempotency":"test"}"""; + var hash = ContentHasher.Sha256(payload); + + await using var connection = await _connectionFactory.OpenAsync(CancellationToken.None); + + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.outbox_message + (message_id, event_type, schema_version, payload_json, correlation_id, occurred_at, payload_hash, published_at) + VALUES (@MessageId, @EventType, 1, CAST(@PayloadJson AS jsonb), @CorrelationId, @OccurredAt, @Hash, @Now) + """, + new + { + MessageId = messageId, + EventType = "IdempotencyTestEvent", + PayloadJson = payload, + CorrelationId = Guid.NewGuid().ToString(), + OccurredAt = now.AddMinutes(-2), + Hash = hash, + Now = now.DateTime + }); + + // Insert first inbox record (simulating first consumer) + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.inbox_message (message_id, consumer, status) + VALUES (@MessageId, @Consumer1, 'Pending') + """, + new { MessageId = messageId, Consumer1 = "Consumer1" }); + + // Act: Try to insert duplicate (same message_id, same consumer) + var exception = await Assert.ThrowsAsync(async () => + { + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.inbox_message (message_id, consumer, status) + VALUES (@MessageId, @Consumer1, 'Pending') + """, + new { MessageId = messageId, Consumer1 = "Consumer1" }); + }); + + // Assert: UNIQUE constraint violation (error code 23505 = unique_violation) + Assert.Contains("23505", exception.SqlState); + } + + /// + /// Gate 2.3: Inbox Status Transitions - Invalid transitions blocked + /// Scenario: Process marks message as Processed before processed_at is set. + /// Expected: Trigger enforces processed_at must be set when status = Processed. + /// + [Fact] + public async Task CrashRecovery_InboxStatus_EnforcesProcessedAtTimestamp() + { + // Arrange: Create outbox message + var messageId = Guid.NewGuid(); + var now = _clock.UtcNow; + const string payload = """{"status":"test"}"""; + var hash = ContentHasher.Sha256(payload); + + await using var connection = await _connectionFactory.OpenAsync(CancellationToken.None); + + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.outbox_message + (message_id, event_type, schema_version, payload_json, correlation_id, occurred_at, payload_hash, published_at) + VALUES (@MessageId, @EventType, 1, CAST(@PayloadJson AS jsonb), @CorrelationId, @OccurredAt, @Hash, @Now) + """, + new + { + MessageId = messageId, + EventType = "StatusTestEvent", + PayloadJson = payload, + CorrelationId = Guid.NewGuid().ToString(), + OccurredAt = now.AddMinutes(-2), + Hash = hash, + Now = now.DateTime + }); + + // Insert inbox message in Pending state + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.inbox_message (message_id, consumer, status) + VALUES (@MessageId, @Consumer, 'Pending') + """, + new { MessageId = messageId, Consumer = "TestConsumer" }); + + // Act: Try to mark as Processed without setting processed_at + var exception = await Assert.ThrowsAsync(async () => + { + await connection.ExecuteAsync(""" + UPDATE building_blocks.inbox_message + SET status = 'Processed' + WHERE message_id = @MessageId + """, + new { MessageId = messageId }); + }); + + // Assert: Trigger violation (processed_at must be set) + Assert.Contains("processed_at must be set", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Gate 2.4: Consumer Failure Retry - Failed messages are retrievable for retry + /// Scenario: Consumer process crashes mid-processing, message marked Failed. + /// Recovery: Retry job queries Failed messages and retries them. + /// + [Fact] + public async Task CrashRecovery_ConsumerFailure_FailedMessagesRetrieval() + { + // Arrange: Create outbox message + var messageId = Guid.NewGuid(); + var now = _clock.UtcNow; + const string payload = """{"retry":"test"}"""; + var hash = ContentHasher.Sha256(payload); + var errorMsg = "Consumer process crashed before completion"; + + await using var connection = await _connectionFactory.OpenAsync(CancellationToken.None); + + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.outbox_message + (message_id, event_type, schema_version, payload_json, correlation_id, occurred_at, payload_hash, published_at) + VALUES (@MessageId, @EventType, 1, CAST(@PayloadJson AS jsonb), @CorrelationId, @OccurredAt, @Hash, @Now) + """, + new + { + MessageId = messageId, + EventType = "RetryTestEvent", + PayloadJson = payload, + CorrelationId = Guid.NewGuid().ToString(), + OccurredAt = now.AddMinutes(-2), + Hash = hash, + Now = now.DateTime + }); + + // Insert inbox message with Failed status + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.inbox_message (message_id, consumer, status, error_message, attempted_at) + VALUES (@MessageId, @Consumer, 'Failed', @ErrorMsg, @AttemptedAt) + """, + new + { + MessageId = messageId, + Consumer = "RetryConsumer", + ErrorMsg = errorMsg, + AttemptedAt = now.DateTime + }); + + // Act: Query for failed messages (to retry) + var failedMessages = await connection.QueryAsync<(Guid MessageId, string Consumer, string ErrorMessage)>(""" + SELECT message_id, consumer, error_message + FROM building_blocks.inbox_message + WHERE status = 'Failed' + ORDER BY attempted_at DESC + LIMIT 10 + """); + + // Assert: Failed message is retrievable + Assert.NotEmpty(failedMessages); + var failed = failedMessages.First(); + Assert.Equal(messageId, failed.MessageId); + Assert.Equal("RetryConsumer", failed.Consumer); + Assert.Contains("crashed", failed.ErrorMessage, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Gate 2.5: Audit Reconciliation - All events are tracked with correlation ID + /// Scenario: Process completes event handling. Audit log should have end-to-end trace. + /// Expected: Every inbox message has correlation_id linking to outbox message. + /// + [Fact] + public async Task CrashRecovery_AuditReconciliation_CorrelationIdTracing() + { + // Arrange: Create complete event flow + var messageId = Guid.NewGuid(); + var correlationId = Guid.NewGuid().ToString(); + var now = _clock.UtcNow; + const string payload = """{"audit":"trace"}"""; + var hash = ContentHasher.Sha256(payload); + + await using var connection = await _connectionFactory.OpenAsync(CancellationToken.None); + + // Insert outbox message + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.outbox_message + (message_id, event_type, schema_version, payload_json, correlation_id, occurred_at, payload_hash, published_at) + VALUES (@MessageId, @EventType, 1, CAST(@PayloadJson AS jsonb), @CorrelationId, @OccurredAt, @Hash, @Now) + """, + new + { + MessageId = messageId, + EventType = "AuditTraceEvent", + PayloadJson = payload, + CorrelationId = correlationId, + OccurredAt = now.AddMinutes(-2), + Hash = hash, + Now = now.DateTime + }); + + // Insert inbox message (simulating successful processing) + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.inbox_message + (message_id, consumer, status, processed_at) + VALUES (@MessageId, @Consumer, 'Processed', @Now) + """, + new + { + MessageId = messageId, + Consumer = "AuditConsumer", + Now = now.DateTime + }); + + // Act: Reconcile - Verify inbox message links back to outbox via message_id + var reconciliation = await connection.QuerySingleOrDefaultAsync<(string OutboxCorrelationId, string InboxConsumer, string InboxStatus)?>(""" + SELECT + o.correlation_id, + i.consumer, + i.status + FROM building_blocks.outbox_message o + INNER JOIN building_blocks.inbox_message i ON o.message_id = i.message_id + WHERE o.message_id = @MessageId + """, + new { MessageId = messageId }); + + // Assert: End-to-end traceability + Assert.NotNull(reconciliation); + Assert.Equal(correlationId, reconciliation!.Value.OutboxCorrelationId); + Assert.Equal("AuditConsumer", reconciliation!.Value.InboxConsumer); + Assert.Equal("Processed", reconciliation!.Value.InboxStatus); + } + + /// + /// Gate 2.6: Multiple Consumers - Same outbox event routed to multiple consumers + /// Scenario: One outbox message should create N inbox records (one per consumer). + /// Expected: Each consumer processes independently (idempotent dedup per consumer). + /// + [Fact] + public async Task CrashRecovery_MultipleConsumers_IndependentProcessing() + { + // Arrange: Create outbox message for multiple consumers + var messageId = Guid.NewGuid(); + var now = _clock.UtcNow; + const string payload = """{"broadcast":"multi-consumer"}"""; + var hash = ContentHasher.Sha256(payload); + + await using var connection = await _connectionFactory.OpenAsync(CancellationToken.None); + + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.outbox_message + (message_id, event_type, schema_version, payload_json, correlation_id, occurred_at, payload_hash, published_at) + VALUES (@MessageId, @EventType, 1, CAST(@PayloadJson AS jsonb), @CorrelationId, @OccurredAt, @Hash, @Now) + """, + new + { + MessageId = messageId, + EventType = "MultiConsumerEvent", + PayloadJson = payload, + CorrelationId = Guid.NewGuid().ToString(), + OccurredAt = now.AddMinutes(-2), + Hash = hash, + Now = now.DateTime + }); + + // Insert inbox records for three consumers + var consumers = new[] { "Consumer1", "Consumer2", "Consumer3" }; + foreach (var consumer in consumers) + { + await connection.ExecuteAsync(""" + INSERT INTO building_blocks.inbox_message (message_id, consumer, status) + VALUES (@MessageId, @Consumer, 'Pending') + """, + new { MessageId = messageId, Consumer = consumer }); + } + + // Act: Query inbox records for this message + var inboxRecords = await connection.QueryAsync<(string Consumer, string Status)>(""" + SELECT consumer, status + FROM building_blocks.inbox_message + WHERE message_id = @MessageId + ORDER BY consumer + """, + new { MessageId = messageId }); + + // Assert: All three consumers have independent records + Assert.Equal(3, inboxRecords.Count()); + Assert.Contains("Consumer1", inboxRecords.Select(r => r.Consumer)); + Assert.Contains("Consumer2", inboxRecords.Select(r => r.Consumer)); + Assert.Contains("Consumer3", inboxRecords.Select(r => r.Consumer)); + Assert.All(inboxRecords, r => Assert.Equal("Pending", r.Status)); + } + + // ========== Helper Classes ========== + + private sealed class NoOpLogger : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => false; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { } + } +}