KArtSell.Aegis/src/KArtSell.Host/Program.cs

using FastEndpoints;
using Hangfire;
using Hangfire.PostgreSql;
using KArtSell.BuildingBlocks.Capabilities;
using Microsoft.Extensions.Caching.Memory;
using KArtSell.Host.Jobs;
using KArtSell.Host.Configuration;
using KArtSell.Host.Infrastructure;
using KArtSell.Host.Observability;
using KArtSell.Host.Features.Observability;
using KArtSell.BuildingBlocks.Data;
using KArtSell.BuildingBlocks.Reliability;
using KArtSell.BuildingBlocks.Time;
using KArtSell.Host.Security;
using KArtSell.Modules.ModelOperations;
using KArtSell.Modules.ModelOperations.Scheduling;
using KArtSell.Modules.SignalEngine;
using Microsoft.AspNetCore.Authentication;
using Npgsql;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using Serilog;
using Serilog.Events;

var builder = WebApplication.CreateBuilder(args);

// Load Telegram secrets for Serilog notifications
var telegramBotToken = Environment.GetEnvironmentVariable("TELEGRAM_BOT") ?? string.Empty;
var telegramChatId = Environment.GetEnvironmentVariable("CHAT_ID") ?? string.Empty;

builder.Host.UseSerilog((context, services, logger) =>
{
    var config = logger
        .ReadFrom.Configuration(context.Configuration)
        .ReadFrom.Services(services)
        .Enrich.FromLogContext()
        .WriteTo.Console();

    // Add async Telegram sink for ERROR and FATAL logs (non-blocking queue)
    if (!string.IsNullOrEmpty(telegramBotToken) && !string.IsNullOrEmpty(telegramChatId))
    {
        config = config.WriteTo.Sink(new TelegramSinkAsync(telegramBotToken, telegramChatId), LogEventLevel.Error);
    }
});

// Load secrets from environment variables (set by CI/CD or user-secrets in dev)
var connectionString = ResolveSecret(
    builder.Configuration.GetConnectionString("Postgres"),
    "KARTSELL_POSTGRES")
    ?? throw new InvalidOperationException("ConnectionStrings:Postgres is required. Set via environment variable KARTSELL_POSTGRES or user-secrets.");

var krxApiKey = ResolveSecret(
    builder.Configuration["ExternalApis:KrxOpenApi:ApiKey"],
    "KRX_OPENAPI")
    ?? throw new InvalidOperationException("KRX_OPENAPI is required. Set via Gitea Actions Secrets or environment.");

var modelOperationsDispatcherEnabled = builder.Configuration.GetValue<bool>("ModelOperations:DispatcherEnabled");
var modelOperationsDispatcherCron = builder.Configuration["ModelOperations:DispatcherCron"] ?? "*/15 * * * *";

// Register external API options with resolved secrets
builder.Services.AddOptions<ExternalApiOptions>()
    .Bind(builder.Configuration.GetSection(ExternalApiOptions.SectionName))
    .Configure(opts => opts.KrxOpenApi.ApiKey = krxApiKey)
    .ValidateOnStart();

builder.Services.AddOptions<CapabilityOptions>()
    .Bind(builder.Configuration.GetSection(CapabilityOptions.SectionName))
    .Validate(x => !x.AutomaticOrder, "AutomaticOrder must remain OFF in this package.")
    .Validate(x => !x.KisOrderAdapter, "KisOrderAdapter must remain OFF until a separately approved release.")
    .Validate(x => !modelOperationsDispatcherEnabled || x.ShadowEvaluation,
        "ModelOperations dispatcher requires ShadowEvaluation capability and remains evidence-only.")
    .ValidateOnStart();

var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
builder.Services.AddSingleton(dataSource);
builder.Services.AddSingleton<IDbConnectionFactory, NpgsqlConnectionFactory>();
builder.Services.AddSingleton<IOutboxWriter, DapperOutboxWriter>();
builder.Services.AddSingleton<IInboxStore, DapperInboxStore>();
builder.Services.AddSingleton<IJobRunRepository, DapperJobRunRepository>();
builder.Services.AddSingleton<DapperOutboxMessageReader>();
builder.Services.AddSingleton<IClock, KArtSell.BuildingBlocks.Time.SystemClock>();

// Shadow Run Services
builder.Services.AddMemoryCache();
builder.Services.AddHttpClient();

builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.DataBackfiller>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.ReplayEngine>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.MetricsCalculator>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.ShadowRunQueries>();
builder.Services.AddScoped<KArtSell.Host.Features.ShadowRun.InitiateShadowRunHandler>();
builder.Services.AddScoped<KArtSell.Host.Features.ShadowRun.GetShadowRunQuery>();

// Consumer Services (for downstream job processing)
builder.Services.AddScoped<KArtSell.Host.Consumers.ShadowRunCompletedConsumer>();
builder.Services.AddScoped<KArtSell.Host.Consumers.ApprovalQueueConsumer>();
builder.Services.AddScoped<KArtSell.Host.Consumers.AuditLogConsumer>();

// Recommendation Report Services
builder.Services.AddScoped<RecommendationReportGenerator>();
builder.Services.AddScoped<GenerateDailyRecommendationJob>();
builder.Services.AddScoped<GenerateWeeklyRecommendationJob>();
builder.Services.AddScoped<GenerateMonthlyRecommendationJob>();

// OpenDart Services
builder.Services.AddScoped<OpenDartService>();
builder.Services.AddScoped<OpenDartDailyBatchJob>();

// KIS Connection Pool
builder.Services.AddSingleton<KisConnectionPool>();

// Rate Limiter
builder.Services.AddSingleton<RateLimiterService>();

// Circuit Breaker
builder.Services.AddSingleton<CircuitBreakerPolicyFactory>();
builder.Services.AddHttpClient<ResilientHttpClient>();

// KRX Data Service (real API, with KRX_API_KEY; fallback to stub data if key missing)
builder.Services.AddHttpClient<KArtSell.Modules.ModelOperations.ShadowRun.Services.KrxDataService>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.IKrxDataService>(sp =>
    sp.GetRequiredService<KArtSell.Modules.ModelOperations.ShadowRun.Services.KrxDataService>());

// Observability Metrics
builder.Services.AddScoped<MetricsPolicy>();
builder.Services.AddScoped<KArtSell.BuildingBlocks.Observability.MetricsSql>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Observability.IObservabilityService>(sp =>
    new KArtSell.Modules.ModelOperations.Observability.ObservabilityService(
        sp.GetRequiredService<KArtSell.BuildingBlocks.Observability.MetricsSql>()));

// API Metrics
builder.Services.AddSingleton<KArtSell.Host.Observability.ApiCallMetricsService>();

builder.Services.AddProblemDetails();

const string authenticationScheme = "KArtSell";
var authenticationMode = builder.Configuration["Authentication:Mode"] ?? "FailClosed";
var authenticationBuilder = builder.Services
    .AddAuthentication(options =>
    {
        options.DefaultAuthenticateScheme = authenticationScheme;
        options.DefaultChallengeScheme = authenticationScheme;
    });

if (builder.Environment.IsDevelopment()
    && authenticationMode.Equals("DevelopmentHeader", StringComparison.OrdinalIgnoreCase))
{
    authenticationBuilder.AddScheme<AuthenticationSchemeOptions, DevelopmentHeaderAuthenticationHandler>(
        authenticationScheme,
        _ => { });
}
else
{
    authenticationBuilder.AddScheme<AuthenticationSchemeOptions, FailClosedAuthenticationHandler>(
        authenticationScheme,
        _ => { });
}

builder.Services.AddAuthorization();
builder.Services.AddSignalR();
builder.Services.AddSignalEngineModule();
builder.Services.AddModelOperationsModule();
builder.Services.AddFastEndpoints();  // AFTER modules registered (so their endpoints are included)
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();

builder.Services.AddHangfire(config => config.UsePostgreSqlStorage(options =>
    options.UseNpgsqlConnection(connectionString)));
builder.Services.AddHangfireServer(options =>
{
    options.Queues =
    [
        "q-control",
        "q-market-data",
        "q-fundamentals",
        "q-feature-risk",
        "q-recommendation",
        "q-evaluation",
        "q-reconciliation",
        "q-research",
        "q-backfill"
    ];
    options.WorkerCount = Math.Max(2, Environment.ProcessorCount / 2);
});

builder.Services.AddOpenTelemetry()
    .ConfigureResource(resource => resource.AddService("KArtSell.Host"))
    .WithTracing(tracing => tracing
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddOtlpExporter())
    .WithMetrics(metrics => metrics
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddRuntimeInstrumentation()
        .AddOtlpExporter());

var app = builder.Build();

app.UseExceptionHandler();
app.UseStatusCodePages();
app.UseSerilogRequestLogging();
app.UseMiddleware<RateLimiterMiddleware>();  // Rate limiting middleware
if (app.Environment.IsDevelopment())
{
    app.UseSwagger();
    app.UseSwaggerUI();
}

app.UseAuthentication();
app.UseAuthorization();
app.UseFastEndpoints(config => config.Endpoints.RoutePrefix = "api");
app.MapHub<KArtSell.Host.Consumers.ShadowRunHub>("/api/hubs/shadow-run");

app.MapGet("/health/live", () => Results.Ok(new
{
    status = "ok",
    automaticOrderCapability = "OFF",
    algorithmStatus = "RESEARCH_CANDIDATE_NOT_PRODUCTION"
}));

app.MapGet("/health/ready", async (NpgsqlDataSource source, CancellationToken ct) =>
{
    await using var connection = await source.OpenConnectionAsync(ct);
    await using var command = connection.CreateCommand();
    command.CommandText = "select 1";
    await command.ExecuteScalarAsync(ct);
    return Results.Ok(new { status = "ready", database = "reachable" });
});

// Start app in background and register Hangfire jobs after Kestrel binds
var logger = app.Services.GetRequiredService<ILogger<Program>>();
var runTask = app.RunAsync();

// Give Kestrel time to bind (typically < 1 second)
await Task.Delay(2000);
logger.LogInformation("📡 Kestrel binding complete, now registering Hangfire jobs in background...");

// Register model operations schedules with timeout (Hangfire distributed lock may be stuck)
try
{
    var scheduleTask = Task.Run(() =>
    {
        app.Services.RegisterModelOperationsSchedules(modelOperationsDispatcherEnabled, modelOperationsDispatcherCron);
    });

    if (!scheduleTask.Wait(TimeSpan.FromSeconds(5)))
    {
        logger.LogWarning("⚠️ Hangfire lock timeout registering model operations schedules; continuing anyway");
    }
    else
    {
        logger.LogInformation("✅ Model operations schedules registered");
    }
}
catch (Exception ex)
{
    logger.LogWarning(ex, "⚠️ Error registering model operations schedules; continuing anyway");
}

// Register recurring jobs (with general exception handling)
var hangfireRetryEnabled = Environment.GetEnvironmentVariable("HANGFIRE_RETRY_ENABLED") != "false";
if (hangfireRetryEnabled)
{
    try
    {
        RecurringJob.AddOrUpdate<OutboxPollerJob>(
            "outbox-poller",
            job => job.ExecuteAsync(CancellationToken.None),
            "* * * * *",
            new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc });
        logger.LogInformation("✅ Recurring job 'outbox-poller' registered");
    }
    catch (Exception ex)
    {
        logger.LogWarning(ex, "⚠️ Hangfire error registering outbox-poller; continuing anyway");
    }

    try
    {
        RecurringJob.AddOrUpdate<DownstreamConsumerJob>(
            "downstream-consumer",
            job => job.ExecuteAsync(CancellationToken.None),
            "* * * * *",
            new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc });
        logger.LogInformation("✅ Recurring job 'downstream-consumer' registered");
    }
    catch (Exception ex)
    {
        logger.LogWarning(ex, "⚠️ Hangfire error registering downstream-consumer; continuing anyway");
    }
}
else
{
    logger.LogInformation("⏭️ Hangfire recurring jobs skipped");
}

// Register other Hangfire jobs
var kstTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Asia/Seoul");
try { RecurringJob.AddOrUpdate<OpenDartDailyBatchJob>("opendart-daily-batch", job => job.ExecuteAsync(CancellationToken.None), "0 9 * * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ opendart-daily-batch registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ opendart-daily-batch error"); }
try { RecurringJob.AddOrUpdate<GenerateDailyRecommendationJob>("daily-recommendation", job => job.ExecuteAsync(CancellationToken.None), "0 9 * * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ daily-recommendation registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ daily-recommendation error"); }
try { RecurringJob.AddOrUpdate<GenerateWeeklyRecommendationJob>("weekly-recommendation", job => job.ExecuteAsync(CancellationToken.None), "0 9 * * 6", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ weekly-recommendation registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ weekly-recommendation error"); }
try { RecurringJob.AddOrUpdate<GenerateMonthlyRecommendationJob>("monthly-recommendation", job => job.ExecuteAsync(CancellationToken.None), "0 9 1 * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ monthly-recommendation registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ monthly-recommendation error"); }

logger.LogInformation("🎯 Hangfire background jobs initialization complete");

// Wait for app to run
await runTask;

/// <summary>
/// Resolve secrets from environment variables, handling placeholders like ${VAR_NAME}.
/// Priority: environment variable → config value (if not a placeholder) → null
/// </summary>
static string? ResolveSecret(string? configValue, string environmentVariable)
{
    // 1. Check if environment variable is set (highest priority)
    var envValue = Environment.GetEnvironmentVariable(environmentVariable);
    if (!string.IsNullOrEmpty(envValue))
        return envValue;

    // 2. Check if config has a placeholder (e.g., "${VAR_NAME}")
    if (!string.IsNullOrEmpty(configValue))
    {
        if (configValue.StartsWith("${", StringComparison.Ordinal) && configValue.EndsWith('}'))
        {
            // This is a placeholder, try to resolve from environment
            return Environment.GetEnvironmentVariable(environmentVariable);
        }

        // Config has actual value (local dev)
        return configValue;
    }

    // 3. No value found
    return null;
}

public partial class Program;