KArtSell.Aegis/src/KArtSell.Host/Program.cs

using FastEndpoints;
using Hangfire;
using Hangfire.PostgreSql;
using KArtSell.BuildingBlocks.Capabilities;
using Microsoft.Extensions.Caching.Memory;
using KArtSell.Host.Jobs;
using KArtSell.Host.Configuration;
using KArtSell.Host.Infrastructure;
using KArtSell.Host.Observability;
using KArtSell.Host.Features.Observability;
using KArtSell.BuildingBlocks.Data;
using KArtSell.BuildingBlocks.Reliability;
using KArtSell.BuildingBlocks.Time;
using KArtSell.Host.Security;
using KArtSell.Modules.ModelOperations;
using KArtSell.Modules.ModelOperations.Scheduling;
using KArtSell.Modules.SignalEngine;
using Microsoft.AspNetCore.Authentication;
using Npgsql;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;
using Serilog;
using Serilog.Events;

var builder = WebApplication.CreateBuilder(args);

// Load Telegram secrets for Serilog notifications
var telegramBotToken = Environment.GetEnvironmentVariable("TELEGRAM_BOT") ?? string.Empty;
var telegramChatId = Environment.GetEnvironmentVariable("CHAT_ID") ?? string.Empty;

builder.Host.UseSerilog((context, services, logger) =>
{
    var config = logger
        .ReadFrom.Configuration(context.Configuration)
        .ReadFrom.Services(services)
        .Enrich.FromLogContext()
        .WriteTo.Console();

    // Add async Telegram sink for ERROR and FATAL logs (non-blocking queue)
    if (!string.IsNullOrEmpty(telegramBotToken) && !string.IsNullOrEmpty(telegramChatId))
    {
        config = config.WriteTo.Sink(new TelegramSinkAsync(telegramBotToken, telegramChatId), LogEventLevel.Error);
    }
});

// Load secrets from environment variables (set by CI/CD or user-secrets in dev)
var connectionString = ResolveSecret(
    builder.Configuration.GetConnectionString("Postgres"),
    "KARTSELL_POSTGRES")
    ?? throw new InvalidOperationException("ConnectionStrings:Postgres is required. Set via environment variable KARTSELL_POSTGRES or user-secrets.");

var krxApiKey = ResolveSecret(
    builder.Configuration["ExternalApis:KrxOpenApi:ApiKey"],
    "KRX_OPENAPI");
// API key is optional; KrxDataService falls back to stub data if missing (AGENTS.md Gate 3 testing)

var modelOperationsDispatcherEnabled = builder.Configuration.GetValue<bool>("ModelOperations:DispatcherEnabled");
var modelOperationsDispatcherCron = builder.Configuration["ModelOperations:DispatcherCron"] ?? "*/15 * * * *";

// Register external API options with resolved secrets
builder.Services.AddOptions<ExternalApiOptions>()
    .Bind(builder.Configuration.GetSection(ExternalApiOptions.SectionName))
    .Configure(opts => opts.KrxOpenApi.ApiKey = krxApiKey ?? string.Empty)
    .ValidateOnStart();

builder.Services.AddOptions<CapabilityOptions>()
    .Bind(builder.Configuration.GetSection(CapabilityOptions.SectionName))
    .Validate(x => !x.AutomaticOrder, "AutomaticOrder must remain OFF in this package.")
    .Validate(x => !x.KisOrderAdapter, "KisOrderAdapter must remain OFF until a separately approved release.")
    .Validate(x => !modelOperationsDispatcherEnabled || x.ShadowEvaluation,
        "ModelOperations dispatcher requires ShadowEvaluation capability and remains evidence-only.")
    .ValidateOnStart();

var dataSource = new NpgsqlDataSourceBuilder(connectionString).Build();
builder.Services.AddSingleton(dataSource);
builder.Services.AddSingleton<IDbConnectionFactory, NpgsqlConnectionFactory>();
builder.Services.AddSingleton<IOutboxWriter, DapperOutboxWriter>();
builder.Services.AddSingleton<IInboxStore, DapperInboxStore>();
builder.Services.AddSingleton<IJobRunRepository, DapperJobRunRepository>();
builder.Services.AddSingleton<DapperOutboxMessageReader>();
builder.Services.AddSingleton<IClock, KArtSell.BuildingBlocks.Time.SystemClock>();

// Shadow Run Services
builder.Services.AddMemoryCache();
builder.Services.AddHttpClient();

builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.DataBackfiller>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.ReplayEngine>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.MetricsCalculator>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.ShadowRunQueries>();
builder.Services.AddScoped<KArtSell.Host.Features.ShadowRun.InitiateShadowRunHandler>();
builder.Services.AddScoped<KArtSell.Host.Features.ShadowRun.GetShadowRunQuery>();

// Consumer Services (for downstream job processing)
builder.Services.AddScoped<KArtSell.Host.Consumers.ShadowRunCompletedConsumer>();
builder.Services.AddScoped<KArtSell.Host.Consumers.ApprovalQueueConsumer>();
builder.Services.AddScoped<KArtSell.Host.Consumers.AuditLogConsumer>();
builder.Services.AddScoped<KArtSell.Host.Consumers.AuditTrailConsumer>();

// Recommendation Report Services
builder.Services.AddScoped<RecommendationReportGenerator>();
builder.Services.AddScoped<GenerateDailyRecommendationJob>();
builder.Services.AddScoped<GenerateWeeklyRecommendationJob>();
builder.Services.AddScoped<GenerateMonthlyRecommendationJob>();

// OpenDart Services
builder.Services.AddScoped<OpenDartService>();
builder.Services.AddScoped<OpenDartDailyBatchJob>();

// KIS Connection Pool
builder.Services.AddSingleton<KisConnectionPool>();

// Rate Limiter
builder.Services.AddSingleton<RateLimiterService>();

// Circuit Breaker
builder.Services.AddSingleton<CircuitBreakerPolicyFactory>();
builder.Services.AddHttpClient<ResilientHttpClient>();

// KRX Data Service (real API, with KRX_API_KEY; fallback to stub data if key missing)
builder.Services.AddHttpClient<KArtSell.Modules.ModelOperations.ShadowRun.Services.KrxDataService>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.ShadowRun.IKrxDataService>(sp =>
    sp.GetRequiredService<KArtSell.Modules.ModelOperations.ShadowRun.Services.KrxDataService>());

// Observability Metrics
builder.Services.AddScoped<MetricsPolicy>();
builder.Services.AddScoped<KArtSell.BuildingBlocks.Observability.MetricsSql>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Observability.IObservabilityService>(sp =>
    new KArtSell.Modules.ModelOperations.Observability.ObservabilityService(
        sp.GetRequiredService<KArtSell.BuildingBlocks.Observability.MetricsSql>()));

// API Metrics
builder.Services.AddSingleton<KArtSell.Host.Observability.ApiCallMetricsService>();

// Feature Services (DI for Endpoints)
// Market Data (VS-03)
builder.Services.AddScoped<KArtSell.Host.Features.MarketData.IMarketDataIngestionService>(sp =>
    new KArtSell.Host.Features.MarketData.MarketDataIngestionService(
        sp.GetRequiredService<NpgsqlDataSource>(),
        sp.GetRequiredService<IBackgroundJobClient>()));

// Portfolio (VS-04~05)
builder.Services.AddScoped<KArtSell.Host.Features.Portfolio.IPortfolioRebalanceService>(sp =>
    new KArtSell.Host.Features.Portfolio.PortfolioRebalanceService(
        sp.GetRequiredService<NpgsqlDataSource>(),
        sp.GetRequiredService<IBackgroundJobClient>(),
        sp.GetRequiredService<IClock>()));

builder.Services.AddScoped<KArtSell.Host.Features.Portfolio.IRiskMetricsService>(sp =>
    new KArtSell.Host.Features.Portfolio.RiskMetricsService(
        sp.GetRequiredService<NpgsqlDataSource>(),
        sp.GetRequiredService<IClock>()));

// Risk & Stress (VS-06~07)
builder.Services.AddScoped<KArtSell.Host.Features.Portfolio.IStressTestService>(sp =>
    new KArtSell.Host.Features.Portfolio.StressTestService(
        sp.GetRequiredService<NpgsqlDataSource>(),
        sp.GetRequiredService<IBackgroundJobClient>()));

builder.Services.AddScoped<KArtSell.Host.Features.Portfolio.IAlertService>(sp =>
    new KArtSell.Host.Features.Portfolio.AlertService(
        sp.GetRequiredService<NpgsqlDataSource>()));

// Dashboard (VS-08)
builder.Services.AddScoped<KArtSell.Host.Features.Portfolio.IDashboardService>(sp =>
    new KArtSell.Host.Features.Portfolio.DashboardService(
        sp.GetRequiredService<NpgsqlDataSource>(),
        sp.GetRequiredService<IClock>()));

// Security Master (VS-02) - Temporarily disabled: ISecurityMasterRulesStore implementation pending
// builder.Services.AddScoped<KArtSell.Host.Features.SecurityMaster.ISecurityMasterSyncHandler>(sp =>
//     new KArtSell.Host.Features.SecurityMaster.SecurityMasterSyncHandler(
//         sp.GetRequiredService<NpgsqlDataSource>(),
//         sp.GetRequiredService<KArtSell.Host.Features.SecurityMaster.IRemoteSecurityMasterClient>(),
//         sp.GetRequiredService<KArtSell.Host.Features.SecurityMaster.ISecurityMasterRulesStore>(),
//         sp.GetRequiredService<IClock>()));

// Shared IDbConnection (per-scope, opened from the pooled data source) for slices using raw Dapper/IDbConnection
builder.Services.AddScoped<System.Data.IDbConnection>(sp => sp.GetRequiredService<NpgsqlDataSource>().OpenConnection());

// Sell Decision Engine (VS-10)
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.SellDecision.ISellDecisionSql, KArtSell.Modules.ModelOperations.SellDecision.SellDecisionSql>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.SellDecision.IPboValidator, KArtSell.Modules.ModelOperations.SellDecision.PboValidator>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.SellDecision.IDsrValidator, KArtSell.Modules.ModelOperations.SellDecision.DsrValidator>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.SellDecision.IOosValidator, KArtSell.Modules.ModelOperations.SellDecision.OosValidator>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.SellDecision.ISellPriorityRanker, KArtSell.Modules.ModelOperations.SellDecision.SellPriorityRanker>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.SellDecision.IGenerateSellDecisionHandler>(sp =>
    new KArtSell.Modules.ModelOperations.SellDecision.GenerateSellDecisionHandler(
        connectionString,
        sp.GetRequiredService<KArtSell.Modules.ModelOperations.SellDecision.ISellDecisionSql>(),
        sp.GetRequiredService<KArtSell.Modules.ModelOperations.SellDecision.IPboValidator>(),
        sp.GetRequiredService<KArtSell.Modules.ModelOperations.SellDecision.IDsrValidator>(),
        sp.GetRequiredService<KArtSell.Modules.ModelOperations.SellDecision.IOosValidator>(),
        sp.GetRequiredService<KArtSell.Modules.ModelOperations.SellDecision.ISellPriorityRanker>(),
        sp.GetRequiredService<IClock>()));

// Trade Execution (VS-12)
builder.Services.AddHttpClient<KArtSell.Modules.ModelOperations.TradeExecution.IKisTradeExecutionService, KArtSell.Modules.ModelOperations.TradeExecution.KisTradeExecutionService>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.TradeExecution.ITradeSql, KArtSell.Modules.ModelOperations.TradeExecution.TradeSql>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.TradeExecution.SubmitTradeHandler>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.TradeExecution.PollTradeStatusHandler>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.TradeExecution.ConfirmSettlementHandler>();
builder.Services.AddScoped<KArtSell.Host.Jobs.TradeStatusPollingJob>();

// Portfolio Reconciliation (VS-14)
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.PortfolioReconciliation.IReconciliationRepository, KArtSell.Modules.ModelOperations.PortfolioReconciliation.ReconciliationSql>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.PortfolioReconciliation.CostBasisCalculator>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.PortfolioReconciliation.MismatchDetector>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.PortfolioReconciliation.ReconciliationEngine>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.PortfolioReconciliation.ReconcileTradeHandler>();

// Approval Workflow (VS-26, formerly VS-03; maker-checker — see docs/DECISIONS/ADR-WBS-001-slice-renumbering.md)
builder.Services.AddScoped(sp => new KArtSell.Modules.ModelOperations.Features.ApprovalWorkflow.ApprovalWorkflowSql(connectionString));
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Features.ApprovalWorkflow.CreateApprovalProposalHandler>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Features.ApprovalWorkflow.ProposeForReviewHandler>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Features.ApprovalWorkflow.ApproveApprovalHandler>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Features.ApprovalWorkflow.ActivateModelHandler>();

// Compliance / Audit Trail / GDPR (VS-04)
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Compliance.AuditSql>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Compliance.LogAuditEventCommandHandler>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Compliance.ProcessGdprRequestHandler>();
builder.Services.AddScoped<KArtSell.Modules.ModelOperations.Compliance.GdprRedactionJob>();

builder.Services.AddProblemDetails();

const string authenticationScheme = "KArtSell";
var authenticationMode = builder.Configuration["Authentication:Mode"] ?? "FailClosed";
var authenticationBuilder = builder.Services
    .AddAuthentication(options =>
    {
        options.DefaultAuthenticateScheme = authenticationScheme;
        options.DefaultChallengeScheme = authenticationScheme;
    });

if (builder.Environment.IsDevelopment()
    && authenticationMode.Equals("DevelopmentHeader", StringComparison.OrdinalIgnoreCase))
{
    authenticationBuilder.AddScheme<AuthenticationSchemeOptions, DevelopmentHeaderAuthenticationHandler>(
        authenticationScheme,
        _ => { });
}
else
{
    authenticationBuilder.AddScheme<AuthenticationSchemeOptions, FailClosedAuthenticationHandler>(
        authenticationScheme,
        _ => { });
}

builder.Services.AddAuthorization();
builder.Services.AddSignalR();
builder.Services.AddSignalEngineModule();
builder.Services.AddModelOperationsModule();
builder.Services.AddFastEndpoints();  // AFTER modules registered (so their endpoints are included)
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();

builder.Services.AddHangfire(config => config.UsePostgreSqlStorage(options =>
    options.UseNpgsqlConnection(connectionString)));

// Hangfire server can be disabled via HANGFIRE_SERVER_ENABLED=false (useful for testing/debugging port binding)
var hangfireServerEnabled = Environment.GetEnvironmentVariable("HANGFIRE_SERVER_ENABLED") != "false";
if (hangfireServerEnabled)
{
    builder.Services.AddHangfireServer(options =>
    {
        options.Queues =
        [
            "q-control",
            "q-market-data",
            "q-fundamentals",
            "q-feature-risk",
            "q-recommendation",
            "q-evaluation",
            "q-reconciliation",
            "q-research",
            "q-backfill"
        ];
        options.WorkerCount = Math.Max(2, Environment.ProcessorCount / 2);
    });
}

builder.Services.AddOpenTelemetry()
    .ConfigureResource(resource => resource.AddService("KArtSell.Host"))
    .WithTracing(tracing => tracing
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddOtlpExporter())
    .WithMetrics(metrics => metrics
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddRuntimeInstrumentation()
        .AddOtlpExporter());

var app = builder.Build();

app.UseExceptionHandler();
app.UseStatusCodePages();
app.UseSerilogRequestLogging();
app.UseMiddleware<RateLimiterMiddleware>();  // Rate limiting middleware
if (app.Environment.IsDevelopment())
{
    app.UseSwagger();
    app.UseSwaggerUI();
}

app.UseAuthentication();
app.UseAuthorization();
app.UseFastEndpoints(config => config.Endpoints.RoutePrefix = "api");
app.MapHub<KArtSell.Host.Consumers.ShadowRunHub>("/api/hubs/shadow-run");

app.MapGet("/health/live", () => Results.Ok(new
{
    status = "ok",
    automaticOrderCapability = "OFF",
    algorithmStatus = "RESEARCH_CANDIDATE_NOT_PRODUCTION"
}));

app.MapGet("/health/ready", async (NpgsqlDataSource source, CancellationToken ct) =>
{
    await using var connection = await source.OpenConnectionAsync(ct);
    await using var command = connection.CreateCommand();
    command.CommandText = "select 1";
    await command.ExecuteScalarAsync(ct);
    return Results.Ok(new { status = "ready", database = "reachable" });
});

// Start app in background and register Hangfire jobs after Kestrel binds
var logger = app.Services.GetRequiredService<ILogger<Program>>();
var runTask = app.RunAsync();

// Give Kestrel time to bind (typically < 1 second)
await Task.Delay(2000);
logger.LogInformation("📡 Kestrel binding complete, now registering Hangfire jobs in background...");

// Register model operations schedules with timeout (Hangfire distributed lock may be stuck)
try
{
    var scheduleTask = Task.Run(() =>
    {
        app.Services.RegisterModelOperationsSchedules(modelOperationsDispatcherEnabled, modelOperationsDispatcherCron);
    });

    if (!scheduleTask.Wait(TimeSpan.FromSeconds(5)))
    {
        logger.LogWarning("⚠️ Hangfire lock timeout registering model operations schedules; continuing anyway");
    }
    else
    {
        logger.LogInformation("✅ Model operations schedules registered");
    }
}
catch (Exception ex)
{
    logger.LogWarning(ex, "⚠️ Error registering model operations schedules; continuing anyway");
}

// Register recurring jobs (with general exception handling)
var hangfireRetryEnabled = Environment.GetEnvironmentVariable("HANGFIRE_RETRY_ENABLED") != "false";
if (hangfireRetryEnabled)
{
    try
    {
        RecurringJob.AddOrUpdate<OutboxPollerJob>(
            "outbox-poller",
            job => job.ExecuteAsync(CancellationToken.None),
            "* * * * *",
            new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc });
        logger.LogInformation("✅ Recurring job 'outbox-poller' registered");
    }
    catch (Exception ex)
    {
        logger.LogWarning(ex, "⚠️ Hangfire error registering outbox-poller; continuing anyway");
    }

    try
    {
        RecurringJob.AddOrUpdate<DownstreamConsumerJob>(
            "downstream-consumer",
            job => job.ExecuteAsync(CancellationToken.None),
            "* * * * *",
            new RecurringJobOptions { TimeZone = TimeZoneInfo.Utc });
        logger.LogInformation("✅ Recurring job 'downstream-consumer' registered");
    }
    catch (Exception ex)
    {
        logger.LogWarning(ex, "⚠️ Hangfire error registering downstream-consumer; continuing anyway");
    }
}
else
{
    logger.LogInformation("⏭️ Hangfire recurring jobs skipped");
}

// Register other Hangfire jobs
var kstTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Asia/Seoul");
try { RecurringJob.AddOrUpdate<HistoricalBatchShadowRunJob>("historical-batch-shadow-run", job => job.ExecuteAsync(null, CancellationToken.None), "0 21 * * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ historical-batch-shadow-run registered (daily 21:00 KST)"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ historical-batch-shadow-run error"); }
try { RecurringJob.AddOrUpdate<OpenDartDailyBatchJob>("opendart-daily-batch", job => job.ExecuteAsync(CancellationToken.None), "0 9 * * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ opendart-daily-batch registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ opendart-daily-batch error"); }
try { RecurringJob.AddOrUpdate<GenerateDailyRecommendationJob>("daily-recommendation", job => job.ExecuteAsync(CancellationToken.None), "0 9 * * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ daily-recommendation registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ daily-recommendation error"); }
try { RecurringJob.AddOrUpdate<GenerateWeeklyRecommendationJob>("weekly-recommendation", job => job.ExecuteAsync(CancellationToken.None), "0 9 * * 6", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ weekly-recommendation registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ weekly-recommendation error"); }
try { RecurringJob.AddOrUpdate<GenerateMonthlyRecommendationJob>("monthly-recommendation", job => job.ExecuteAsync(CancellationToken.None), "0 9 1 * *", new RecurringJobOptions { TimeZone = kstTimeZone }); logger.LogInformation("✅ monthly-recommendation registered"); } catch (Exception ex) { logger.LogWarning(ex, "⚠️ monthly-recommendation error"); }
RecurringJob.RemoveIfExists("trade-status-polling");
logger.LogWarning("KIS trade-status-polling removed: KIS trading is hard-disabled.");

logger.LogInformation("🎯 Hangfire background jobs initialization complete");

// Wait for app to run
await runTask;

/// <summary>
/// Resolve secrets from environment variables, handling placeholders like ${VAR_NAME}.
/// Priority: environment variable → config value (if not a placeholder) → null
/// </summary>
static string? ResolveSecret(string? configValue, string environmentVariable)
{
    // 1. Check if environment variable is set (highest priority)
    var envValue = Environment.GetEnvironmentVariable(environmentVariable);
    if (!string.IsNullOrEmpty(envValue))
        return envValue;

    // 2. Check if config has a placeholder (e.g., "${VAR_NAME}")
    if (!string.IsNullOrEmpty(configValue))
    {
        if (configValue.StartsWith("${", StringComparison.Ordinal) && configValue.EndsWith('}'))
        {
            // This is a placeholder, try to resolve from environment
            return Environment.GetEnvironmentVariable(environmentVariable);
        }

        // Config has actual value (local dev)
        return configValue;
    }

    // 3. No value found
    return null;
}

public partial class Program;