using FastEndpoints; using Hangfire; using Hangfire.PostgreSql; using KArtSell.BuildingBlocks.Capabilities; using Microsoft.Extensions.Caching.Memory; using KArtSell.Host.Jobs; using KArtSell.Host.Configuration; using KArtSell.Host.Infrastructure; using KArtSell.Host.Observability; using KArtSell.Host.Features.Observability; using KArtSell.BuildingBlocks.Data; using KArtSell.BuildingBlocks.Reliability; using KArtSell.BuildingBlocks.Time; using KArtSell.Host.Security; using KArtSell.Modules.ModelOperations; using KArtSell.Modules.ModelOperations.Scheduling; using KArtSell.Modules.SignalEngine; using Microsoft.AspNetCore.Authentication; using Npgsql; using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; using Serilog; using Serilog.Events; var builder = WebApplication.CreateBuilder(args); // Load Telegram secrets for Serilog notifications var telegramBotToken = Environment.GetEnvironmentVariable("TELEGRAM_BOT") ?? string.Empty; var telegramChatId = Environment.GetEnvironmentVariable("CHAT_ID") ?? string.Empty; builder.Host.UseSerilog((context, services, logger) => { var config = logger .ReadFrom.Configuration(context.Configuration) .ReadFrom.Services(services) .Enrich.FromLogContext() .WriteTo.Console(); // Add async Telegram sink for ERROR and FATAL logs (non-blocking queue) if (!string.IsNullOrEmpty(telegramBotToken) && !string.IsNullOrEmpty(telegramChatId)) { config = config.WriteTo.Sink(new TelegramSinkAsync(telegramBotToken, telegramChatId), LogEventLevel.Error); } }); // Load secrets from environment variables (set by CI/CD or user-secrets in dev) var connectionString = ResolveSecret( builder.Configuration.GetConnectionString("Postgres"), "KARTSELL_POSTGRES") ?? throw new InvalidOperationException("ConnectionStrings:Postgres is required. Set via environment variable KARTSELL_POSTGRES or user-secrets."); var krxApiKey = ResolveSecret( builder.Configuration["ExternalApis:KrxOpenApi:ApiKey"], "KRX_OPENAPI") ?? throw new InvalidOperationException("KRX_OPENAPI is required. Set via Gitea Actions Secrets or environment."); var modelOperationsDispatcherEnabled = builder.Configuration.GetValue("ModelOperations:DispatcherEnabled"); var modelOperationsDispatcherCron = builder.Configuration["ModelOperations:DispatcherCron"] ?? "*/15 * * * *"; // Register external API options with resolved secrets builder.Services.AddOptions() .Bind(builder.Configuration.GetSection(ExternalApiOptions.SectionName)) .Configure(opts => opts.KrxOpenApi.ApiKey = krxApiKey) .ValidateOnStart(); builder.Services.AddOptions() .Bind(builder.Configuration.GetSection(CapabilityOptions.SectionName)) .Validate(x => !x.AutomaticOrder, "AutomaticOrder must remain OFF in this package.") .Validate(x => !x.KisOrderAdapter, "KisOrderAdapter must remain OFF until a separately approved release.") .Validate(x => !modelOperationsDispatcherEnabled || x.ShadowEvaluation, "ModelOperations dispatcher requires ShadowEvaluation capability and remains evidence-only.") .ValidateOnStart(); var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build(); builder.Services.AddSingleton(dataSource); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); // Shadow Run Services builder.Services.AddMemoryCache(); builder.Services.AddHttpClient(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); // Consumer Services (for downstream job processing) builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); // Recommendation Report Services builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); // OpenDart Services builder.Services.AddScoped(); builder.Services.AddScoped(); // KIS Connection Pool builder.Services.AddSingleton(); // Rate Limiter builder.Services.AddSingleton(); // Circuit Breaker builder.Services.AddSingleton(); builder.Services.AddHttpClient(); // KRX Data Service (real API, with KRX_API_KEY; fallback to stub data if key missing) builder.Services.AddHttpClient(); builder.Services.AddScoped(sp => sp.GetRequiredService()); // Observability Metrics builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(sp => new KArtSell.Modules.ModelOperations.Observability.ObservabilityService( sp.GetRequiredService())); // API Metrics builder.Services.AddSingleton(); builder.Services.AddProblemDetails(); const string authenticationScheme = "KArtSell"; var authenticationMode = builder.Configuration["Authentication:Mode"] ?? "FailClosed"; var authenticationBuilder = builder.Services .AddAuthentication(options => { options.DefaultAuthenticateScheme = authenticationScheme; options.DefaultChallengeScheme = authenticationScheme; }); if (builder.Environment.IsDevelopment() && authenticationMode.Equals("DevelopmentHeader", StringComparison.OrdinalIgnoreCase)) { authenticationBuilder.AddScheme( authenticationScheme, _ => { }); } else { authenticationBuilder.AddScheme( authenticationScheme, _ => { }); } builder.Services.AddAuthorization(); builder.Services.AddSignalR(); builder.Services.AddSignalEngineModule(); builder.Services.AddModelOperationsModule(); builder.Services.AddFastEndpoints(); // AFTER modules registered (so their endpoints are included) builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); builder.Services.AddHangfire(config => config.UsePostgreSqlStorage(options => options.UseNpgsqlConnection(connectionString))); builder.Services.AddHangfireServer(options => { options.Queues = [ "q-control", "q-market-data", "q-fundamentals", "q-feature-risk", "q-recommendation", "q-evaluation", "q-reconciliation", "q-research", "q-backfill" ]; options.WorkerCount = Math.Max(2, Environment.ProcessorCount / 2); }); builder.Services.AddOpenTelemetry() .ConfigureResource(resource => resource.AddService("KArtSell.Host")) .WithTracing(tracing => tracing .AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation() .AddOtlpExporter()) .WithMetrics(metrics => metrics .AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation() .AddRuntimeInstrumentation() .AddOtlpExporter()); var app = builder.Build(); app.UseExceptionHandler(); app.UseStatusCodePages(); app.UseSerilogRequestLogging(); app.UseMiddleware(); // Rate limiting middleware if (app.Environment.IsDevelopment()) { app.UseSwagger(); app.UseSwaggerUI(); } app.UseAuthentication(); app.UseAuthorization(); app.UseFastEndpoints(config => config.Endpoints.RoutePrefix = "api"); app.MapHub("/api/hubs/shadow-run"); app.MapGet("/health/live", () => Results.Ok(new { status = "ok", automaticOrderCapability = "OFF", algorithmStatus = "RESEARCH_CANDIDATE_NOT_PRODUCTION" })); app.MapGet("/health/ready", async (NpgsqlDataSource source, CancellationToken ct) => { await using var connection = await source.OpenConnectionAsync(ct); await using var command = connection.CreateCommand(); command.CommandText = "select 1"; await command.ExecuteScalarAsync(ct); return Results.Ok(new { status = "ready", database = "reachable" }); }); // Start app in background and register Hangfire jobs after Kestrel binds var logger = app.Services.GetRequiredService>(); var runTask = app.RunAsync(); // Give Kestrel time to bind (typically < 1 second) await Task.Delay(2000); logger.LogInformation("📡 Kestrel binding complete, now registering Hangfire jobs in background..."); // Register model operations schedules with timeout (Hangfire distributed lock may be stuck) try { var scheduleTask = Task.Run(() => { app.Services.RegisterModelOperationsSchedules(modelOperationsDispatcherEnabled, modelOperationsDispatcherCron); }); if (!scheduleTask.Wait(TimeSpan.FromSeconds(5))) { logger.LogWarning("⚠️ Hangfire lock timeout registering model operations schedules; continuing anyway"); } else { logger.LogInformation("✅ Model operations schedules registered"); } } catch (Exception ex) { logger.LogWarning(ex, "⚠️ Error registering model operations schedules; continuing anyway"); } // Register recurring jobs (with general exception handling) var hangfireRetryEnabled = Environment.GetEnvironmentVariable("HANGFIRE_RETRY_ENABLED") != "false"; if (hangfireRetryEnabled) { try { RecurringJob.AddOrUpdate( "outbox-poller", job => job.ExecuteAsync(CancellationToken.None), "* * * * *", new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc }); logger.LogInformation("✅ Recurring job 'outbox-poller' registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ Hangfire error registering outbox-poller; continuing anyway"); } try { RecurringJob.AddOrUpdate( "downstream-consumer", job => job.ExecuteAsync(CancellationToken.None), "* * * * *", new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc }); logger.LogInformation("✅ Recurring job 'downstream-consumer' registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ Hangfire error registering downstream-consumer; continuing anyway"); } } else { logger.LogInformation("⏭️ Hangfire recurring jobs skipped"); } // Register other Hangfire jobs var kstTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Asia/Seoul"); try { RecurringJob.AddOrUpdate("opendart-daily-batch", job => job.ExecuteAsync(CancellationToken.None), "0 9 * * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ opendart-daily-batch registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ opendart-daily-batch error"); } try { RecurringJob.AddOrUpdate("daily-recommendation", job => job.ExecuteAsync(CancellationToken.None), "0 9 * * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ daily-recommendation registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ daily-recommendation error"); } try { RecurringJob.AddOrUpdate("weekly-recommendation", job => job.ExecuteAsync(CancellationToken.None), "0 9 * * 6", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ weekly-recommendation registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ weekly-recommendation error"); } try { RecurringJob.AddOrUpdate("monthly-recommendation", job => job.ExecuteAsync(CancellationToken.None), "0 9 1 * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ monthly-recommendation registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ monthly-recommendation error"); } logger.LogInformation("🎯 Hangfire background jobs initialization complete"); // Wait for app to run await runTask; /// /// Resolve secrets from environment variables, handling placeholders like ${VAR_NAME}. /// Priority: environment variable → config value (if not a placeholder) → null /// static string? ResolveSecret(string? configValue, string environmentVariable) { // 1. Check if environment variable is set (highest priority) var envValue = Environment.GetEnvironmentVariable(environmentVariable); if (!string.IsNullOrEmpty(envValue)) return envValue; // 2. Check if config has a placeholder (e.g., "${VAR_NAME}") if (!string.IsNullOrEmpty(configValue)) { if (configValue.StartsWith("${", StringComparison.Ordinal) && configValue.EndsWith('}')) { // This is a placeholder, try to resolve from environment return Environment.GetEnvironmentVariable(environmentVariable); } // Config has actual value (local dev) return configValue; } // 3. No value found return null; } public partial class Program;