diff --git a/gate3_rehearsal.ps1 b/gate3_rehearsal.ps1 new file mode 100644 index 00000000..df8aef49 --- /dev/null +++ b/gate3_rehearsal.ps1 @@ -0,0 +1,116 @@ +# Gate 3 Shadow Run Rehearsal Script +# Prerequisites: SSH tunnel must be open in separate terminal +# ssh -L 5432:127.0.0.1:5432 kjh2064@178.104.200.7 + +param( + [string]$ModelId = "00000000-0000-0000-0000-000000000001", + [string]$WindowStart = "2024-01-02", + [string]$WindowEnd = "2024-08-31", + [int]$PollIntervalSeconds = 15, + [int]$MaxWaitMinutes = 60 +) + +# API configuration +$ApiUrl = "http://127.0.0.1:5002/api/shadow-runs" +$User = "gate3-rehearsal" +$Role = "researcher" + +# Headers for authentication +$headers = @{ + "X-KArtSell-User" = $User + "X-KArtSell-Role" = $Role + "Content-Type" = "application/json" +} + +# Shadow Run request body +$body = @{ + modelId = $ModelId + windowStartDate = $WindowStart + windowEndDate = $WindowEnd +} | ConvertTo-Json + +Write-Host "🎯 Gate 3 Shadow Run Rehearsal" -ForegroundColor Cyan +Write-Host "================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "📋 Configuration:" -ForegroundColor Yellow +Write-Host " API URL: $ApiUrl" +Write-Host " Model ID: $ModelId" +Write-Host " Window: $WindowStart to $WindowEnd" +Write-Host " Poll Interval: ${PollIntervalSeconds}s" +Write-Host " Max Wait: ${MaxWaitMinutes}m" +Write-Host "" + +# Step 1: Initiate Shadow Run +Write-Host "1️⃣ Initiating Shadow Run..." -ForegroundColor Green +try { + $response = Invoke-WebRequest -Uri $ApiUrl ` + -Method POST ` + -Headers $headers ` + -Body $body ` + -ContentType "application/json" ` + -ErrorAction Stop + + $result = $response.Content | ConvertFrom-Json + $runId = $result.runId + + Write-Host " ✅ Shadow Run created: $runId" -ForegroundColor Green + Write-Host " Status: $($result.status)" -ForegroundColor Green + Write-Host "" +} catch { + Write-Host " ❌ Failed to initiate Shadow Run" -ForegroundColor Red + Write-Host " Error: $($_.Exception.Message)" -ForegroundColor Red + exit 1 +} + +# Step 2: Poll for completion +Write-Host "2️⃣ Polling for completion..." -ForegroundColor Green +$pollUrl = "$ApiUrl/$runId" +$startTime = Get-Date +$maxWaitMs = $MaxWaitMinutes * 60 * 1000 +$pollCount = 0 + +while ($true) { + $elapsed = (Get-Date) - $startTime + $elapsedMs = $elapsed.TotalMilliseconds + + if ($elapsedMs -gt $maxWaitMs) { + Write-Host " ❌ Timeout after ${MaxWaitMinutes} minutes" -ForegroundColor Red + exit 1 + } + + $pollCount++ + Write-Host " [$pollCount] Polling... (${elapsed:mm\:ss} elapsed)" -ForegroundColor Cyan + + try { + $response = Invoke-WebRequest -Uri $pollUrl ` + -Method GET ` + -Headers $headers ` + -ContentType "application/json" ` + -ErrorAction Stop + + $result = $response.Content | ConvertFrom-Json + $status = $result.status + + Write-Host " Status: $status" -ForegroundColor Cyan + + if ($status -eq "completed") { + Write-Host " ✅ Shadow Run completed!" -ForegroundColor Green + Write-Host "" + Write-Host "3️⃣ Results:" -ForegroundColor Green + Write-Host " Runtime: $($result.runtimeSeconds) seconds" + Write-Host " Message: $($result.message)" + Write-Host "" + Write-Host "✨ Gate 3 Rehearsal COMPLETE" -ForegroundColor Green + exit 0 + } elseif ($status -eq "failed") { + Write-Host " ❌ Shadow Run failed" -ForegroundColor Red + Write-Host " Error: $($result.message)" -ForegroundColor Red + exit 1 + } + } catch { + Write-Host " ⚠️ Poll error: $($_.Exception.Message)" -ForegroundColor Yellow + } + + # Wait before next poll + Start-Sleep -Seconds $PollIntervalSeconds +} diff --git a/src/KArtSell.Host/Program.cs b/src/KArtSell.Host/Program.cs index 0af83a3f..f15c9f14 100644 --- a/src/KArtSell.Host/Program.cs +++ b/src/KArtSell.Host/Program.cs @@ -215,17 +215,35 @@ app.MapHub("/api/hubs/shadow-run"); app.Services.RegisterModelOperationsSchedules(modelOperationsDispatcherEnabled, modelOperationsDispatcherCron); -RecurringJob.AddOrUpdate( - "outbox-poller", - job => job.ExecuteAsync(CancellationToken.None), - "* * * * *", - new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc }); +// Register recurring jobs with timeout resilience (Hangfire distributed lock may be stuck) +var logger = app.Services.GetRequiredService>(); +try +{ + RecurringJob.AddOrUpdate( + "outbox-poller", + job => job.ExecuteAsync(CancellationToken.None), + "* * * * *", + new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc }); + logger.LogInformation("✅ Recurring job 'outbox-poller' registered"); +} +catch (Exception ex) when (ex.Message.Contains("Timeout")) +{ + logger.LogWarning(ex, "⚠️ Hangfire lock timeout registering outbox-poller; job may be registered by another instance or lock is stuck"); +} -RecurringJob.AddOrUpdate( - "downstream-consumer", - job => job.ExecuteAsync(CancellationToken.None), - "* * * * *", - new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc }); +try +{ + RecurringJob.AddOrUpdate( + "downstream-consumer", + job => job.ExecuteAsync(CancellationToken.None), + "* * * * *", + new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc }); + logger.LogInformation("✅ Recurring job 'downstream-consumer' registered"); +} +catch (Exception ex) when (ex.Message.Contains("Timeout")) +{ + logger.LogWarning(ex, "⚠️ Hangfire lock timeout registering downstream-consumer; job may be registered by another instance or lock is stuck"); +} // OpenDart daily batch (KST timezone, market open 09:00) var kstTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Asia/Seoul");