Files
QuantEngineByItz/src/dotnet/QuantEngine.Web/Program.cs
T
kjh2064 5589a0432b
Validators (Pushes and Pull Requests) / validate-ui-and-storage (push) Successful in 16s
Validators (Pushes and Pull Requests) / validate-core (push) Failing after 56s
feat(collection): wire KIS collection end-to-end, add price-history pipeline (WBS QE-M0/M1/M2)
Critical re-review of the QuantEngine WBS evidence system found several
regressions of the "no fake gates" discipline established by M0, plus a
still-unwired M1 collection path. This closes 10 more WBS tasks
(QE-M1-01..06, QE-M2-01/02/04/05/06 — see spec/60_quant_engine_wbs.yaml)
with real, gate-verified evidence (18/34 total).

M1 — real KIS data now lands in PostgreSQL end-to-end:
- SchedulerService: load ticker universe from GatherTradingData.json instead
  of a hardcoded array; fix a Hangfire scoped-service resolution bug.
- KisDataCollectionOrchestrator: restore logging on the lineage-event write
  path (was a bare `catch {}` swallowing all failures silently); persist
  daily OHLCV bars into quantengine.price_history_daily per run.
- Verified live: POST /api/collection/run -> Hangfire -> orchestrator ->
  KIS mock API -> PostgreSQL, with Playwright DOM/API parity evidence.

M2 — historical price-history pipeline:
- CollectionRepository: SavePriceHistoryDailyAsync (idempotent upsert),
  GetPriceHistorySummaryAsync (per-ticker aggregation) + a new
  DateOnlyTypeHandler registered globally, since Dapper has no built-in
  System.DateOnly support in either direction (write threw
  NotSupportedException, read threw a constructor-mismatch
  InvalidOperationException — found by exercising both paths live).
- tools/validate_price_history_integrity_v1.py: gap-freeness (vs KIS
  trading calendar) + price-sanity gate over collected history.
- Admin Collection page: new "히스토리 현황" summary table +
  GET /api/collection/history-summary, with Playwright evidence.

Governance/gate fixes:
- validate_market_time_series_schema_v1.py mislabeled its own output
  "runtime_database_query": "DATA_GATED" despite never opening a DB
  connection (pure file/regex check) — relabeled "check_scope":
  "STATIC_STRUCTURAL_ONLY" and wired the node into the release DAG so it
  isn't only reachable from ci.yml, matching every other validator.
  Live-data authority for the same claim stays with QE-M2-01's pg_query
  gate (spec/60), documented in spec/64.
- Fixed a WBS log_pattern check (QE-M1-06) that couldn't match its own
  multi-line target; loosened two depends_on edges (QE-M1-05/06,
  QE-M2-04/05) that encoded "needs X verified" when the real requirement
  was only "needs X's code merged."
- Discovered and fixed admin-pages.spec.ts logging in with the wrong
  seeded password (admin/admin instead of admin/quant123!, per CLAUDE.md)
  — every test in that suite had been silently failing at the login step.

Deferred: QE-M2-03 (2-year backfill) — the KIS mock/VTS token endpoint
started returning 403 after the first successful call this session; looks
like a token-issuance rate limit or credential issue on KIS's side, not a
code defect. Backfilling at scale right now would just generate more 403s,
so left QE-M2-03 PENDING pending KIS account/console verification.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-12 21:07:53 +09:00

220 lines
8.1 KiB
C#

using Microsoft.AspNetCore.DataProtection;
using QuantEngine.Infrastructure.Data;
using QuantEngine.Infrastructure.Repositories;
using QuantEngine.Infrastructure.Services;
using QuantEngine.Core.Interfaces;
using QuantEngine.Application.Services;
using QuantEngine.Application.Interfaces;
using Serilog;
using QuantEngine.Web.Services;
using Hangfire;
using Npgsql;
using FastEndpoints;
using FluentValidation;
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Information()
.WriteTo.Console()
.WriteTo.File("logs/quantengine-.log", rollingInterval: RollingInterval.Day)
.CreateLogger();
// Dapper has no built-in handler for System.DateOnly (params or result mapping) — register once globally.
Dapper.SqlMapper.AddTypeHandler(new DateOnlyTypeHandler());
try
{
var builder = WebApplication.CreateBuilder(args);
builder.Host.UseSerilog();
// Data Protection: without this, ASP.NET Core derives its key-ring
// discriminator from the app's physical content root path. Every
// deployment lands in a brand-new directory
// (~/deployments/quantengine_{tag}_{hash}/), so the discriminator
// changed on every single deploy and every previously-issued auth
// cookie became undecryptable -- forcing all users to log in again
// after each release. SetApplicationName pins a stable discriminator;
// PersistKeysToFileSystem points at a location outside the versioned
// deployment folders so the actual key material also survives restarts.
var dataProtectionKeysPath = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
"quantengine-keys");
builder.Services.AddDataProtection()
.SetApplicationName("QuantEngine")
.PersistKeysToFileSystem(new DirectoryInfo(dataProtectionKeysPath));
// Authentication & Authorization
builder.Services.AddAuthentication(opts =>
{
opts.DefaultAuthenticateScheme = AdminAuthDefaults.Scheme;
opts.DefaultChallengeScheme = AdminAuthDefaults.Scheme;
})
.AddCookie(AdminAuthDefaults.Scheme, opts =>
{
opts.Cookie.Name = AdminAuthDefaults.CookieName;
opts.Cookie.HttpOnly = true;
opts.Cookie.SameSite = SameSiteMode.Lax;
opts.Cookie.SecurePolicy = builder.Environment.IsProduction() ? CookieSecurePolicy.Always : CookieSecurePolicy.SameAsRequest;
opts.LoginPath = "/Account/Login";
opts.AccessDeniedPath = "/Account/AccessDenied";
opts.SlidingExpiration = true;
opts.ExpireTimeSpan = TimeSpan.FromHours(12);
});
builder.Services.AddAuthorization(options =>
{
options.AddPolicy(AdminAuthDefaults.Scheme, policy =>
{
policy.RequireAuthenticatedUser();
});
});
builder.Services.AddAntiforgery();
// Razor Pages with authorization conventions
builder.Services.AddRazorPages(options =>
{
options.Conventions.AuthorizeFolder("/Admin", AdminAuthDefaults.Scheme);
options.Conventions.AllowAnonymousToPage("/Account/Login");
options.Conventions.AllowAnonymousToPage("/Account/AccessDenied");
});
// Authentication Services
builder.Services.AddScoped<IIpLockoutService, IpLockoutService>();
builder.Services.AddScoped<AuthService>();
// PostgreSQL Dapper Setup
var connectionString = builder.Configuration.GetConnectionString("DefaultConnection")
?? throw new InvalidOperationException("Connection string 'DefaultConnection' is required.");
var configuredDatabase = new NpgsqlConnectionStringBuilder(connectionString).Database;
if (!string.Equals(configuredDatabase, "quantenginedb", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("QuantEngine must use the quantenginedb PostgreSQL database.");
}
var dataSource = NpgsqlDataSource.Create(connectionString);
builder.Services.AddSingleton(dataSource);
builder.Services.AddSingleton<IDbConnectionFactory>(new DbConnectionFactory(dataSource));
builder.Services.AddSingleton(sp => new DbMigrator(connectionString, sp.GetRequiredService<ILogger<DbMigrator>>()));
// Repository Services
builder.Services.AddScoped<IWorkspaceRepository, WorkspaceRepository>();
builder.Services.AddScoped<IPostgresqlHistoryStore, PostgresqlHistoryStore>();
builder.Services.AddScoped<INormalizedLearningStore, NormalizedLearningStore>();
builder.Services.AddScoped<ILearningDatasetReader, LearningDatasetReader>();
builder.Services.AddScoped<DecisionLearningService>();
builder.Services.AddScoped<LearningDatasetService>();
builder.Services.AddSingleton<GatherTradingDataParser>();
builder.Services.AddScoped<JsonSeedIngestionService>();
builder.Services.AddScoped<IPostgresqlHistorySnapshotReader, PostgresqlHistorySnapshotReader>();
builder.Services.AddScoped<HistoryIngestionService>();
builder.Services.AddScoped<ICollectionRepository, CollectionRepository>();
builder.Services.AddScoped<ITokenCache, PostgresTokenCache>();
builder.Services.AddHttpClient<IKisApiClient, KisApiClient>();
// Collection Pipeline Services
builder.Services.AddScoped<SourcePriorityResolver>();
builder.Services.AddScoped<PriceDataNormalizer>();
builder.Services.AddScoped<ICollectionOrchestrator, KisDataCollectionOrchestrator>();
// Hangfire Background Jobs
try
{
var hangfireConnectionString = builder.Configuration.GetConnectionString("HangfireConnection") ?? connectionString;
builder.Services.AddHangfireServices(hangfireConnectionString);
}
catch (Exception ex)
{
Log.Warning("Hangfire initialization failed: {Message}", ex.Message);
}
// FluentValidation
builder.Services.AddValidatorsFromAssemblyContaining<QuantEngine.Infrastructure.Repositories.WorkspaceRepository>();
// FastEndpoints (for API endpoints)
builder.Services.AddFastEndpoints();
var app = builder.Build();
app.UseSerilogRequestLogging();
app.UseFastEndpoints();
// Database migration on startup
using (var scope = app.Services.CreateScope())
{
var migrator = scope.ServiceProvider.GetRequiredService<DbMigrator>();
var workspaceRepo = scope.ServiceProvider.GetRequiredService<IWorkspaceRepository>();
var collectionRepo = scope.ServiceProvider.GetRequiredService<ICollectionRepository>();
var tokenCache = scope.ServiceProvider.GetRequiredService<ITokenCache>();
try
{
migrator.Migrate();
await workspaceRepo.GetAccountsAsync();
await collectionRepo.GetDashboardStateAsync();
await tokenCache.GetCachedTokenAsync("_init_test_");
Log.Information("Database migration and initialization successful");
}
catch (Exception ex)
{
if (!app.Environment.IsDevelopment())
throw;
Log.Warning("Database initialization warning (development only): {Message}", ex.Message);
}
}
// Error handling & HSTS
if (!app.Environment.IsDevelopment())
{
app.UseExceptionHandler("/Error", createScopeForErrors: true);
app.UseHsts();
}
app.UseHttpsRedirection();
app.UseStaticFiles();
app.UseRouting();
app.UseAntiforgery();
app.UseAuthentication();
app.UseAuthorization();
// Hangfire Dashboard
try
{
app.UseHangfireSetup(app.Services);
}
catch (Exception ex)
{
Log.Warning("Hangfire setup failed: {Message}", ex.Message);
}
// Root redirect: unauthenticated → /Account/Login, authenticated → /Admin/Dashboard
app.MapGet("/", context =>
{
if (context.User?.Identity?.IsAuthenticated ?? false)
context.Response.Redirect("/Admin/Dashboard");
else
context.Response.Redirect("/Account/Login");
return Task.CompletedTask;
});
// Login redirect convenience route
app.MapGet("/login", context =>
{
context.Response.Redirect("/Account/Login", permanent: false);
return Task.CompletedTask;
});
app.MapRazorPages();
app.Run();
}
catch (Exception ex)
{
Log.Fatal(ex, "Application terminated unexpectedly");
throw;
}
finally
{
Log.CloseAndFlush();
}