using System; using System.IO; using System.Threading; using System.Threading.Tasks; using FastEndpoints; using Npgsql; using ExcelDataReader; using Microsoft.Extensions.Configuration; namespace QuantEngine.Web.Endpoints; /// /// BFF FastEndpoints for streaming Excel file upload and importing to PostgreSQL using COPY binary protocol. /// SOLID: Single Responsibility for streaming large files to prevent OOM. /// public class BulkInsertMarketExcelEndpoint : EndpointWithoutRequest { private readonly string _connectionString; public BulkInsertMarketExcelEndpoint(IConfiguration config) { _connectionString = config.GetConnectionString("DefaultConnection") ?? throw new ArgumentNullException("ConnectionStrings__DefaultConnection"); System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); } public override void Configure() { Post("/api/admin/market/upload-excel-stream"); AllowFileUploads(); AllowAnonymous(); } public override async Task HandleAsync(CancellationToken ct) { if (Files.Count == 0) { await SendAsync(new { success = false, message = "업로드된 파일이 없습니다." }, 400, ct); return; } var file = Files[0]; using var fileStream = file.OpenReadStream(); using var reader = ExcelReaderFactory.CreateReader(fileStream); using var conn = new NpgsqlConnection(_connectionString); await conn.OpenAsync(ct); using var writer = await conn.BeginBinaryImportAsync( "COPY quantengine.market_raw_history (ticker, as_of_date, close_price, nav_price, disparate_ratio, raw_payload, provenance) FROM STDIN (FORMAT BINARY)", ct ); bool isHeader = true; int processedRows = 0; while (reader.Read()) { if (isHeader) { isHeader = false; continue; } string ticker = reader.GetValue(0)?.ToString() ?? string.Empty; string asOfDate = reader.GetValue(1)?.ToString() ?? string.Empty; decimal closePrice = Convert.ToDecimal(reader.GetValue(2) ?? 0); decimal navPrice = Convert.ToDecimal(reader.GetValue(3) ?? 0); decimal disparateRatio = navPrice > 0 ? (closePrice - navPrice) / navPrice : 0; if (string.IsNullOrWhiteSpace(ticker) || ticker.Length < 6) continue; await writer.StartRowAsync(ct); await writer.WriteAsync(ticker, ct); await writer.WriteAsync(asOfDate, ct); await writer.WriteAsync(closePrice, ct); await writer.WriteAsync(navPrice, ct); await writer.WriteAsync(disparateRatio, ct); await writer.WriteAsync("{}", ct); await writer.WriteAsync("{\"source\": \"excel_stream_uploader\"}", ct); processedRows++; } await writer.CompleteAsync(ct); await SendAsync(new { success = true, count = processedRows, message = "성공적으로 스트리밍 적재 완료되었습니다." }, cancellation: ct); } }