89 lines
3.1 KiB
C#
89 lines
3.1 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using FastEndpoints;
|
|
using Npgsql;
|
|
using ExcelDataReader;
|
|
using Microsoft.Extensions.Configuration;
|
|
|
|
namespace QuantEngine.Web.Endpoints;
|
|
|
|
/// <summary>
|
|
/// BFF FastEndpoints for streaming Excel file upload and importing to PostgreSQL using COPY binary protocol.
|
|
/// SOLID: Single Responsibility for streaming large files to prevent OOM.
|
|
/// </summary>
|
|
public class BulkInsertMarketExcelEndpoint : EndpointWithoutRequest
|
|
{
|
|
private readonly string _connectionString;
|
|
|
|
public BulkInsertMarketExcelEndpoint(IConfiguration config)
|
|
{
|
|
_connectionString = config.GetConnectionString("DefaultConnection")
|
|
?? throw new ArgumentNullException("ConnectionStrings__DefaultConnection");
|
|
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
|
|
}
|
|
|
|
public override void Configure()
|
|
{
|
|
Post("/api/admin/market/upload-excel-stream");
|
|
AllowFileUploads();
|
|
AllowAnonymous();
|
|
}
|
|
|
|
public override async Task HandleAsync(CancellationToken ct)
|
|
{
|
|
if (Files.Count == 0)
|
|
{
|
|
await SendAsync(new { success = false, message = "업로드된 파일이 없습니다." }, 400, ct);
|
|
return;
|
|
}
|
|
|
|
var file = Files[0];
|
|
using var fileStream = file.OpenReadStream();
|
|
using var reader = ExcelReaderFactory.CreateReader(fileStream);
|
|
|
|
using var conn = new NpgsqlConnection(_connectionString);
|
|
await conn.OpenAsync(ct);
|
|
|
|
using var writer = await conn.BeginBinaryImportAsync(
|
|
"COPY quantengine.market_raw_history (ticker, as_of_date, close_price, nav_price, disparate_ratio, raw_payload, provenance) FROM STDIN (FORMAT BINARY)",
|
|
ct
|
|
);
|
|
|
|
bool isHeader = true;
|
|
int processedRows = 0;
|
|
|
|
while (reader.Read())
|
|
{
|
|
if (isHeader)
|
|
{
|
|
isHeader = false;
|
|
continue;
|
|
}
|
|
|
|
string ticker = reader.GetValue(0)?.ToString() ?? string.Empty;
|
|
string asOfDate = reader.GetValue(1)?.ToString() ?? string.Empty;
|
|
decimal closePrice = Convert.ToDecimal(reader.GetValue(2) ?? 0);
|
|
decimal navPrice = Convert.ToDecimal(reader.GetValue(3) ?? 0);
|
|
decimal disparateRatio = navPrice > 0 ? (closePrice - navPrice) / navPrice : 0;
|
|
|
|
if (string.IsNullOrWhiteSpace(ticker) || ticker.Length < 6) continue;
|
|
|
|
await writer.StartRowAsync(ct);
|
|
await writer.WriteAsync(ticker, ct);
|
|
await writer.WriteAsync(asOfDate, ct);
|
|
await writer.WriteAsync(closePrice, ct);
|
|
await writer.WriteAsync(navPrice, ct);
|
|
await writer.WriteAsync(disparateRatio, ct);
|
|
await writer.WriteAsync("{}", ct);
|
|
await writer.WriteAsync("{\"source\": \"excel_stream_uploader\"}", ct);
|
|
|
|
processedRows++;
|
|
}
|
|
|
|
await writer.CompleteAsync(ct);
|
|
await SendAsync(new { success = true, count = processedRows, message = "성공적으로 스트리밍 적재 완료되었습니다." }, cancellation: ct);
|
|
}
|
|
}
|