5589a0432b
Critical re-review of the QuantEngine WBS evidence system found several
regressions of the "no fake gates" discipline established by M0, plus a
still-unwired M1 collection path. This closes 10 more WBS tasks
(QE-M1-01..06, QE-M2-01/02/04/05/06 — see spec/60_quant_engine_wbs.yaml)
with real, gate-verified evidence (18/34 total).
M1 — real KIS data now lands in PostgreSQL end-to-end:
- SchedulerService: load ticker universe from GatherTradingData.json instead
of a hardcoded array; fix a Hangfire scoped-service resolution bug.
- KisDataCollectionOrchestrator: restore logging on the lineage-event write
path (was a bare `catch {}` swallowing all failures silently); persist
daily OHLCV bars into quantengine.price_history_daily per run.
- Verified live: POST /api/collection/run -> Hangfire -> orchestrator ->
KIS mock API -> PostgreSQL, with Playwright DOM/API parity evidence.
M2 — historical price-history pipeline:
- CollectionRepository: SavePriceHistoryDailyAsync (idempotent upsert),
GetPriceHistorySummaryAsync (per-ticker aggregation) + a new
DateOnlyTypeHandler registered globally, since Dapper has no built-in
System.DateOnly support in either direction (write threw
NotSupportedException, read threw a constructor-mismatch
InvalidOperationException — found by exercising both paths live).
- tools/validate_price_history_integrity_v1.py: gap-freeness (vs KIS
trading calendar) + price-sanity gate over collected history.
- Admin Collection page: new "히스토리 현황" summary table +
GET /api/collection/history-summary, with Playwright evidence.
Governance/gate fixes:
- validate_market_time_series_schema_v1.py mislabeled its own output
"runtime_database_query": "DATA_GATED" despite never opening a DB
connection (pure file/regex check) — relabeled "check_scope":
"STATIC_STRUCTURAL_ONLY" and wired the node into the release DAG so it
isn't only reachable from ci.yml, matching every other validator.
Live-data authority for the same claim stays with QE-M2-01's pg_query
gate (spec/60), documented in spec/64.
- Fixed a WBS log_pattern check (QE-M1-06) that couldn't match its own
multi-line target; loosened two depends_on edges (QE-M1-05/06,
QE-M2-04/05) that encoded "needs X verified" when the real requirement
was only "needs X's code merged."
- Discovered and fixed admin-pages.spec.ts logging in with the wrong
seeded password (admin/admin instead of admin/quant123!, per CLAUDE.md)
— every test in that suite had been silently failing at the login step.
Deferred: QE-M2-03 (2-year backfill) — the KIS mock/VTS token endpoint
started returning 403 after the first successful call this session; looks
like a token-issuance rate limit or credential issue on KIS's side, not a
code defect. Backfilling at scale right now would just generate more 403s,
so left QE-M2-03 PENDING pending KIS account/console verification.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
220 lines
8.1 KiB
C#
220 lines
8.1 KiB
C#
using Microsoft.AspNetCore.DataProtection;
|
|
using QuantEngine.Infrastructure.Data;
|
|
using QuantEngine.Infrastructure.Repositories;
|
|
using QuantEngine.Infrastructure.Services;
|
|
using QuantEngine.Core.Interfaces;
|
|
using QuantEngine.Application.Services;
|
|
using QuantEngine.Application.Interfaces;
|
|
using Serilog;
|
|
using QuantEngine.Web.Services;
|
|
using Hangfire;
|
|
using Npgsql;
|
|
using FastEndpoints;
|
|
using FluentValidation;
|
|
|
|
Log.Logger = new LoggerConfiguration()
|
|
.MinimumLevel.Information()
|
|
.WriteTo.Console()
|
|
.WriteTo.File("logs/quantengine-.log", rollingInterval: RollingInterval.Day)
|
|
.CreateLogger();
|
|
|
|
// Dapper has no built-in handler for System.DateOnly (params or result mapping) — register once globally.
|
|
Dapper.SqlMapper.AddTypeHandler(new DateOnlyTypeHandler());
|
|
|
|
try
|
|
{
|
|
var builder = WebApplication.CreateBuilder(args);
|
|
builder.Host.UseSerilog();
|
|
|
|
// Data Protection: without this, ASP.NET Core derives its key-ring
|
|
// discriminator from the app's physical content root path. Every
|
|
// deployment lands in a brand-new directory
|
|
// (~/deployments/quantengine_{tag}_{hash}/), so the discriminator
|
|
// changed on every single deploy and every previously-issued auth
|
|
// cookie became undecryptable -- forcing all users to log in again
|
|
// after each release. SetApplicationName pins a stable discriminator;
|
|
// PersistKeysToFileSystem points at a location outside the versioned
|
|
// deployment folders so the actual key material also survives restarts.
|
|
var dataProtectionKeysPath = Path.Combine(
|
|
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
|
|
"quantengine-keys");
|
|
builder.Services.AddDataProtection()
|
|
.SetApplicationName("QuantEngine")
|
|
.PersistKeysToFileSystem(new DirectoryInfo(dataProtectionKeysPath));
|
|
|
|
// Authentication & Authorization
|
|
builder.Services.AddAuthentication(opts =>
|
|
{
|
|
opts.DefaultAuthenticateScheme = AdminAuthDefaults.Scheme;
|
|
opts.DefaultChallengeScheme = AdminAuthDefaults.Scheme;
|
|
})
|
|
.AddCookie(AdminAuthDefaults.Scheme, opts =>
|
|
{
|
|
opts.Cookie.Name = AdminAuthDefaults.CookieName;
|
|
opts.Cookie.HttpOnly = true;
|
|
opts.Cookie.SameSite = SameSiteMode.Lax;
|
|
opts.Cookie.SecurePolicy = builder.Environment.IsProduction() ? CookieSecurePolicy.Always : CookieSecurePolicy.SameAsRequest;
|
|
opts.LoginPath = "/Account/Login";
|
|
opts.AccessDeniedPath = "/Account/AccessDenied";
|
|
opts.SlidingExpiration = true;
|
|
opts.ExpireTimeSpan = TimeSpan.FromHours(12);
|
|
});
|
|
|
|
builder.Services.AddAuthorization(options =>
|
|
{
|
|
options.AddPolicy(AdminAuthDefaults.Scheme, policy =>
|
|
{
|
|
policy.RequireAuthenticatedUser();
|
|
});
|
|
});
|
|
builder.Services.AddAntiforgery();
|
|
|
|
// Razor Pages with authorization conventions
|
|
builder.Services.AddRazorPages(options =>
|
|
{
|
|
options.Conventions.AuthorizeFolder("/Admin", AdminAuthDefaults.Scheme);
|
|
options.Conventions.AllowAnonymousToPage("/Account/Login");
|
|
options.Conventions.AllowAnonymousToPage("/Account/AccessDenied");
|
|
});
|
|
|
|
// Authentication Services
|
|
builder.Services.AddScoped<IIpLockoutService, IpLockoutService>();
|
|
builder.Services.AddScoped<AuthService>();
|
|
|
|
// PostgreSQL Dapper Setup
|
|
var connectionString = builder.Configuration.GetConnectionString("DefaultConnection")
|
|
?? throw new InvalidOperationException("Connection string 'DefaultConnection' is required.");
|
|
|
|
var configuredDatabase = new NpgsqlConnectionStringBuilder(connectionString).Database;
|
|
if (!string.Equals(configuredDatabase, "quantenginedb", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
throw new InvalidOperationException("QuantEngine must use the quantenginedb PostgreSQL database.");
|
|
}
|
|
|
|
var dataSource = NpgsqlDataSource.Create(connectionString);
|
|
builder.Services.AddSingleton(dataSource);
|
|
builder.Services.AddSingleton<IDbConnectionFactory>(new DbConnectionFactory(dataSource));
|
|
builder.Services.AddSingleton(sp => new DbMigrator(connectionString, sp.GetRequiredService<ILogger<DbMigrator>>()));
|
|
|
|
// Repository Services
|
|
builder.Services.AddScoped<IWorkspaceRepository, WorkspaceRepository>();
|
|
builder.Services.AddScoped<IPostgresqlHistoryStore, PostgresqlHistoryStore>();
|
|
builder.Services.AddScoped<INormalizedLearningStore, NormalizedLearningStore>();
|
|
builder.Services.AddScoped<ILearningDatasetReader, LearningDatasetReader>();
|
|
builder.Services.AddScoped<DecisionLearningService>();
|
|
builder.Services.AddScoped<LearningDatasetService>();
|
|
builder.Services.AddSingleton<GatherTradingDataParser>();
|
|
builder.Services.AddScoped<JsonSeedIngestionService>();
|
|
builder.Services.AddScoped<IPostgresqlHistorySnapshotReader, PostgresqlHistorySnapshotReader>();
|
|
builder.Services.AddScoped<HistoryIngestionService>();
|
|
builder.Services.AddScoped<ICollectionRepository, CollectionRepository>();
|
|
builder.Services.AddScoped<ITokenCache, PostgresTokenCache>();
|
|
builder.Services.AddHttpClient<IKisApiClient, KisApiClient>();
|
|
|
|
// Collection Pipeline Services
|
|
builder.Services.AddScoped<SourcePriorityResolver>();
|
|
builder.Services.AddScoped<PriceDataNormalizer>();
|
|
builder.Services.AddScoped<ICollectionOrchestrator, KisDataCollectionOrchestrator>();
|
|
|
|
// Hangfire Background Jobs
|
|
try
|
|
{
|
|
var hangfireConnectionString = builder.Configuration.GetConnectionString("HangfireConnection") ?? connectionString;
|
|
builder.Services.AddHangfireServices(hangfireConnectionString);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning("Hangfire initialization failed: {Message}", ex.Message);
|
|
}
|
|
|
|
// FluentValidation
|
|
builder.Services.AddValidatorsFromAssemblyContaining<QuantEngine.Infrastructure.Repositories.WorkspaceRepository>();
|
|
|
|
// FastEndpoints (for API endpoints)
|
|
builder.Services.AddFastEndpoints();
|
|
|
|
var app = builder.Build();
|
|
|
|
app.UseSerilogRequestLogging();
|
|
app.UseFastEndpoints();
|
|
|
|
// Database migration on startup
|
|
using (var scope = app.Services.CreateScope())
|
|
{
|
|
var migrator = scope.ServiceProvider.GetRequiredService<DbMigrator>();
|
|
var workspaceRepo = scope.ServiceProvider.GetRequiredService<IWorkspaceRepository>();
|
|
var collectionRepo = scope.ServiceProvider.GetRequiredService<ICollectionRepository>();
|
|
var tokenCache = scope.ServiceProvider.GetRequiredService<ITokenCache>();
|
|
|
|
try
|
|
{
|
|
migrator.Migrate();
|
|
await workspaceRepo.GetAccountsAsync();
|
|
await collectionRepo.GetDashboardStateAsync();
|
|
await tokenCache.GetCachedTokenAsync("_init_test_");
|
|
Log.Information("Database migration and initialization successful");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (!app.Environment.IsDevelopment())
|
|
throw;
|
|
Log.Warning("Database initialization warning (development only): {Message}", ex.Message);
|
|
}
|
|
}
|
|
|
|
// Error handling & HSTS
|
|
if (!app.Environment.IsDevelopment())
|
|
{
|
|
app.UseExceptionHandler("/Error", createScopeForErrors: true);
|
|
app.UseHsts();
|
|
}
|
|
|
|
app.UseHttpsRedirection();
|
|
app.UseStaticFiles();
|
|
app.UseRouting();
|
|
app.UseAntiforgery();
|
|
|
|
app.UseAuthentication();
|
|
app.UseAuthorization();
|
|
|
|
// Hangfire Dashboard
|
|
try
|
|
{
|
|
app.UseHangfireSetup(app.Services);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning("Hangfire setup failed: {Message}", ex.Message);
|
|
}
|
|
|
|
// Root redirect: unauthenticated → /Account/Login, authenticated → /Admin/Dashboard
|
|
app.MapGet("/", context =>
|
|
{
|
|
if (context.User?.Identity?.IsAuthenticated ?? false)
|
|
context.Response.Redirect("/Admin/Dashboard");
|
|
else
|
|
context.Response.Redirect("/Account/Login");
|
|
return Task.CompletedTask;
|
|
});
|
|
|
|
// Login redirect convenience route
|
|
app.MapGet("/login", context =>
|
|
{
|
|
context.Response.Redirect("/Account/Login", permanent: false);
|
|
return Task.CompletedTask;
|
|
});
|
|
|
|
app.MapRazorPages();
|
|
|
|
app.Run();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Fatal(ex, "Application terminated unexpectedly");
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
Log.CloseAndFlush();
|
|
}
|