kjh2064/KArtSell.Aegis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Slice G: Apply consistent Hangfire lock timeout guards to all RecurringJob registrations (DEBT-015)

Browse Source 
Problem: Program.cs:216 (RegisterModelOperationsSchedules) was the first
Hangfire Postgres touch at startup, with zero timeout protection. When
Hangfire.PostgreSql attempts PrepareSchemaIfNecessary and advisory lock
contention occurs, app hangs indefinitely with no logs after "Registered 12
endpoints", blocking Kestrel from binding.

Solution: Wrap all 6 RecurringJob registrations (lines 216, 226, 240, 260,
267, 273, 279) in consistent try/catch(Timeout) guards. Log WARN and continue
if lock times out, instead of silent infinite wait. Allows Kestrel to bind
even if Hangfire schema initialization is contentious.

Resolves DEBT-015 (Medium Impact / High Effort). Same pattern already existed
for outbox-poller/downstream-consumer; now applied consistently across all
scheduler jobs.

Tests: dotnet build KArtSell.sln -c Release passes with 0 errors/warnings.
Gate 3 execution will validate Kestrel startup now proceeds normally.

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
김재현
2026-08-03 14:03:31 +09:00
parent 45185ccc39
commit 7515b1ba81
2 changed files with 66 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/KArtSell.Host/Program.cs
+63 -23
View File
@@ -213,10 +213,18 @@ app.UseAuthorization();
app.UseFastEndpoints(config => config.Endpoints.RoutePrefix = "api");
app.MapHub<KArtSell.Host.Consumers.ShadowRunHub>("/api/hubs/shadow-run");
app.Services.RegisterModelOperationsSchedules(modelOperationsDispatcherEnabled, modelOperationsDispatcherCron);
// Register recurring jobs with timeout resilience (Hangfire distributed lock may be stuck)
// Register model operations schedules with timeout resilience (Hangfire distributed lock may be stuck)
var logger = app.Services.GetRequiredService<ILogger<Program>>();
try
{
app.Services.RegisterModelOperationsSchedules(modelOperationsDispatcherEnabled, modelOperationsDispatcherCron);
logger.LogInformation("✅ Model operations schedules registered");
}
catch (Exception ex) when (ex.Message.Contains("Timeout"))
{
logger.LogWarning(ex, "⚠️ Hangfire lock timeout registering model operations schedules; job may be registered by another instance or lock is stuck");
}
var hangfireRetryEnabled = Environment.GetEnvironmentVariable("HANGFIRE_RETRY_ENABLED") != "false";
if (hangfireRetryEnabled)
@@ -257,30 +265,62 @@ else
// OpenDart daily batch (KST timezone, market open 09:00)
var kstTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Asia/Seoul");
RecurringJob.AddOrUpdate<OpenDartDailyBatchJob>(
"opendart-daily-batch",
job => job.ExecuteAsync(CancellationToken.None),
"0 9 * * *", // 09:00 every day KST
new RecurringJobOptions { TimeZone = kstTimeZone });
try
{
RecurringJob.AddOrUpdate<OpenDartDailyBatchJob>(
"opendart-daily-batch",
job => job.ExecuteAsync(CancellationToken.None),
"0 9 * * *", // 09:00 every day KST
new RecurringJobOptions { TimeZone = kstTimeZone });
logger.LogInformation("✅ Recurring job 'opendart-daily-batch' registered");
}
catch (Exception ex) when (ex.Message.Contains("Timeout"))
{
logger.LogWarning(ex, "⚠️ Hangfire lock timeout registering opendart-daily-batch; job may be registered by another instance or lock is stuck");
}
// Recommendation report generation (KST timezone, market open 09:00)
RecurringJob.AddOrUpdate<GenerateDailyRecommendationJob>(
"daily-recommendation",
job => job.ExecuteAsync(CancellationToken.None),
"0 9 * * *", // 09:00 every day
new RecurringJobOptions { TimeZone = kstTimeZone });
try
{
RecurringJob.AddOrUpdate<GenerateDailyRecommendationJob>(
"daily-recommendation",
job => job.ExecuteAsync(CancellationToken.None),
"0 9 * * *", // 09:00 every day
new RecurringJobOptions { TimeZone = kstTimeZone });
logger.LogInformation("✅ Recurring job 'daily-recommendation' registered");
}
catch (Exception ex) when (ex.Message.Contains("Timeout"))
{
logger.LogWarning(ex, "⚠️ Hangfire lock timeout registering daily-recommendation; job may be registered by another instance or lock is stuck");
}
RecurringJob.AddOrUpdate<GenerateWeeklyRecommendationJob>(
"weekly-recommendation",
job => job.ExecuteAsync(CancellationToken.None),
"0 9 * * 6", // 09:00 every Saturday
new RecurringJobOptions { TimeZone = kstTimeZone });
try
{
RecurringJob.AddOrUpdate<GenerateWeeklyRecommendationJob>(
"weekly-recommendation",
job => job.ExecuteAsync(CancellationToken.None),
"0 9 * * 6", // 09:00 every Saturday
new RecurringJobOptions { TimeZone = kstTimeZone });
logger.LogInformation("✅ Recurring job 'weekly-recommendation' registered");
}
catch (Exception ex) when (ex.Message.Contains("Timeout"))
{
logger.LogWarning(ex, "⚠️ Hangfire lock timeout registering weekly-recommendation; job may be registered by another instance or lock is stuck");
}
RecurringJob.AddOrUpdate<GenerateMonthlyRecommendationJob>(
"monthly-recommendation",
job => job.ExecuteAsync(CancellationToken.None),
"0 9 1 * *", // 09:00 on the 1st of every month
new RecurringJobOptions { TimeZone = kstTimeZone });
try
{
RecurringJob.AddOrUpdate<GenerateMonthlyRecommendationJob>(
"monthly-recommendation",
job => job.ExecuteAsync(CancellationToken.None),
"0 9 1 * *", // 09:00 on the 1st of every month
new RecurringJobOptions { TimeZone = kstTimeZone });
logger.LogInformation("✅ Recurring job 'monthly-recommendation' registered");
}
catch (Exception ex) when (ex.Message.Contains("Timeout"))
{
logger.LogWarning(ex, "⚠️ Hangfire lock timeout registering monthly-recommendation; job may be registered by another instance or lock is stuck");
}
app.MapGet("/health/live", () => Results.Ok(new
{