V13-FE-011: finalize search list layout slice
This commit is contained in:
@@ -0,0 +1,113 @@
|
||||
using ClosedXML.Excel;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
public sealed class ClosedXmlWorkbookService
|
||||
{
|
||||
public IReadOnlyList<string> ReadHeaders(Stream stream)
|
||||
{
|
||||
using var workbook = new XLWorkbook(stream);
|
||||
var sheet = workbook.Worksheets.First();
|
||||
var last = sheet.Row(1).LastCellUsed()?.Address.ColumnNumber ?? 0;
|
||||
if (last == 0) return [];
|
||||
|
||||
return Enumerable.Range(1, last)
|
||||
.Select(i => sheet.Cell(1, i).GetFormattedString().Trim())
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
public IEnumerable<ImportRawRow> ReadRows(Stream stream, int maxRows)
|
||||
{
|
||||
using var workbook = new XLWorkbook(stream);
|
||||
var sheet = workbook.Worksheets.First();
|
||||
var lastColumn = sheet.Row(1).LastCellUsed()?.Address.ColumnNumber ?? 0;
|
||||
var lastRow = sheet.LastRowUsed()?.RowNumber() ?? 1;
|
||||
var headers = Enumerable.Range(1, lastColumn)
|
||||
.Select(i => sheet.Cell(1, i).GetFormattedString().Trim())
|
||||
.ToArray();
|
||||
|
||||
if (lastRow - 1 > maxRows)
|
||||
throw new InvalidOperationException($"최대 {maxRows:N0}행까지 업로드할 수 있습니다.");
|
||||
|
||||
for (var rowNo = 2; rowNo <= lastRow; rowNo++)
|
||||
{
|
||||
var values = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
var hasValue = false;
|
||||
for (var column = 1; column <= lastColumn; column++)
|
||||
{
|
||||
var header = headers[column - 1];
|
||||
if (string.IsNullOrWhiteSpace(header)) continue;
|
||||
var value = sheet.Cell(rowNo, column).GetFormattedString().Trim();
|
||||
if (!string.IsNullOrWhiteSpace(value)) hasValue = true;
|
||||
values[header] = string.IsNullOrWhiteSpace(value) ? null : value;
|
||||
}
|
||||
if (hasValue) yield return new ImportRawRow(rowNo, values);
|
||||
}
|
||||
}
|
||||
|
||||
public MemoryStream CreateTemplate(ImportDefinition definition)
|
||||
{
|
||||
var workbook = new XLWorkbook();
|
||||
var sheet = workbook.AddWorksheet("업로드");
|
||||
var fields = definition.Fields.Where(x => x.Importable).ToArray();
|
||||
|
||||
for (var i = 0; i < fields.Length; i++)
|
||||
{
|
||||
var cell = sheet.Cell(1, i + 1);
|
||||
cell.Value = fields[i].Label;
|
||||
cell.Style.Font.Bold = true;
|
||||
cell.Style.Fill.BackgroundColor = XLColor.LightGray;
|
||||
cell.Comment.AddText(fields[i].Required ? "필수 입력" : "선택 입력");
|
||||
}
|
||||
|
||||
sheet.SheetView.FreezeRows(1);
|
||||
sheet.Columns(1, fields.Length).Width = 18;
|
||||
sheet.Row(1).Height = 22;
|
||||
sheet.Range(1, 1, 1, fields.Length).SetAutoFilter();
|
||||
|
||||
var help = workbook.AddWorksheet("필드설명");
|
||||
help.Cell("A1").Value = "필드";
|
||||
help.Cell("B1").Value = "필수";
|
||||
help.Cell("C1").Value = "형식";
|
||||
help.Cell("D1").Value = "허용 별칭";
|
||||
help.Range("A1:D1").Style.Font.Bold = true;
|
||||
for (var i = 0; i < fields.Length; i++)
|
||||
{
|
||||
var row = i + 2;
|
||||
help.Cell(row, 1).Value = fields[i].Label;
|
||||
help.Cell(row, 2).Value = fields[i].Required ? "Y" : "N";
|
||||
help.Cell(row, 3).Value = fields[i].DataType;
|
||||
help.Cell(row, 4).Value = string.Join(", ", fields[i].Aliases);
|
||||
}
|
||||
help.Columns().AdjustToContents(8, 35);
|
||||
|
||||
var output = new MemoryStream();
|
||||
workbook.SaveAs(output);
|
||||
output.Position = 0;
|
||||
workbook.Dispose();
|
||||
return output;
|
||||
}
|
||||
|
||||
public MemoryStream CreateErrorWorkbook(
|
||||
Stream source,
|
||||
IReadOnlyDictionary<int, IReadOnlyList<ImportIssue>> issues)
|
||||
{
|
||||
using var workbook = new XLWorkbook(source);
|
||||
var sheet = workbook.Worksheets.First();
|
||||
var errorColumn = (sheet.Row(1).LastCellUsed()?.Address.ColumnNumber ?? 0) + 1;
|
||||
sheet.Cell(1, errorColumn).Value = "오류사유";
|
||||
sheet.Cell(1, errorColumn).Style.Font.Bold = true;
|
||||
|
||||
foreach (var (rowNo, rowIssues) in issues)
|
||||
{
|
||||
sheet.Cell(rowNo, errorColumn).Value = string.Join(" | ", rowIssues.Select(x => x.Message));
|
||||
}
|
||||
sheet.Column(errorColumn).Width = 45;
|
||||
|
||||
var output = new MemoryStream();
|
||||
workbook.SaveAs(output);
|
||||
output.Position = 0;
|
||||
return output;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
using Npgsql;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
public interface IImportDefinition
|
||||
{
|
||||
ImportDefinition Definition { get; }
|
||||
|
||||
Task<IReadOnlyList<ImportRowValidation>> ValidateAsync(
|
||||
IReadOnlyList<ImportRawRow> rows,
|
||||
IReadOnlyDictionary<string, string> targetToSource,
|
||||
NpgsqlConnection connection,
|
||||
CancellationToken ct);
|
||||
|
||||
/// <summary>
|
||||
/// Commit valid staged rows. Implementations own domain grouping and transaction boundaries.
|
||||
/// They must be idempotent for a session retry.
|
||||
/// </summary>
|
||||
Task<ImportCommitResult> CommitAsync(Guid sessionId, string actor, CancellationToken ct);
|
||||
}
|
||||
|
||||
public sealed class ImportDefinitionRegistry(IEnumerable<IImportDefinition> definitions)
|
||||
{
|
||||
private readonly IReadOnlyDictionary<string, IImportDefinition> _definitions =
|
||||
definitions.ToDictionary(x => x.Definition.Id, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public IImportDefinition Get(string importType) =>
|
||||
_definitions.TryGetValue(importType, out var value)
|
||||
? value
|
||||
: throw new KeyNotFoundException($"Unknown import type: {importType}");
|
||||
|
||||
public bool TryGet(string importType, out IImportDefinition? definition) =>
|
||||
_definitions.TryGetValue(importType, out definition);
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
public static class ImportStatuses
|
||||
{
|
||||
public const string Created = "created";
|
||||
public const string Uploaded = "uploaded";
|
||||
public const string MappingRequired = "mapping-required";
|
||||
public const string Validating = "validating";
|
||||
public const string Validated = "validated";
|
||||
public const string Committing = "committing";
|
||||
public const string Completed = "completed";
|
||||
public const string PartiallyCompleted = "partially-completed";
|
||||
public const string Failed = "failed";
|
||||
public const string Cancelled = "cancelled";
|
||||
}
|
||||
|
||||
public sealed record ImportFieldDefinition(
|
||||
string Key,
|
||||
string Label,
|
||||
IReadOnlyList<string> Aliases,
|
||||
string DataType,
|
||||
bool Required = false,
|
||||
int? MaxLength = null,
|
||||
int? Precision = null,
|
||||
int? Scale = null,
|
||||
string? LookupEntity = null,
|
||||
bool Importable = true,
|
||||
bool Exportable = true);
|
||||
|
||||
public sealed record ImportDefinition(
|
||||
string Id,
|
||||
string ScreenId,
|
||||
string Entity,
|
||||
string Title,
|
||||
IReadOnlyList<ImportFieldDefinition> Fields,
|
||||
bool AllowCreate,
|
||||
bool AllowUpdate,
|
||||
long MaxFileSizeBytes = 20 * 1024 * 1024,
|
||||
int MaxRows = 100_000);
|
||||
|
||||
public sealed record ImportMapping(
|
||||
string SourceColumn,
|
||||
string? TargetField,
|
||||
string Source,
|
||||
decimal? Confidence = null,
|
||||
string? Reason = null);
|
||||
|
||||
public sealed record ImportIssue(
|
||||
int RowNumber,
|
||||
string? Field,
|
||||
string? SourceColumn,
|
||||
string Code,
|
||||
string Message,
|
||||
string Severity);
|
||||
|
||||
public sealed record ImportProgressEvent(
|
||||
Guid SessionId,
|
||||
string Status,
|
||||
int ProgressPercent,
|
||||
int TotalRows,
|
||||
int ProcessedRows,
|
||||
int ValidRows,
|
||||
int InvalidRows,
|
||||
int WarningRows,
|
||||
string? Message = null);
|
||||
|
||||
public sealed record ImportSessionDto(
|
||||
Guid Id,
|
||||
string ImportType,
|
||||
string ScreenId,
|
||||
string FileName,
|
||||
string Status,
|
||||
int TotalRows,
|
||||
int ValidRows,
|
||||
int InvalidRows,
|
||||
int WarningRows,
|
||||
int CreatedRows,
|
||||
int UpdatedRows,
|
||||
int ProgressPercent,
|
||||
IReadOnlyList<string> SourceColumns,
|
||||
IReadOnlyList<ImportMapping> Mapping,
|
||||
IReadOnlyList<ImportIssue>? Errors,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset? CompletedAt);
|
||||
|
||||
public sealed record ImportRawRow(int RowNumber, IReadOnlyDictionary<string, string?> Values);
|
||||
public sealed record ImportNormalizedRow(int RowNumber, JsonDocument Data, string? DomainKey);
|
||||
|
||||
public sealed record ImportRowValidation(
|
||||
int RowNumber,
|
||||
IReadOnlyDictionary<string, string?> RawData,
|
||||
string? DomainKey,
|
||||
JsonDocument? NormalizedData,
|
||||
IReadOnlyList<ImportIssue> Errors,
|
||||
IReadOnlyList<ImportIssue> Warnings)
|
||||
{
|
||||
public bool IsValid => Errors.Count == 0;
|
||||
}
|
||||
|
||||
public sealed record ImportCommitResult(int Created, int Updated, int Failed);
|
||||
@@ -0,0 +1,26 @@
|
||||
using Kbx.Contracts.Generated;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
public static class ImportFieldDefinitionFactory
|
||||
{
|
||||
public static ImportFieldDefinition FromKbxField(string key, bool? required = null, string? label = null)
|
||||
{
|
||||
var field = KbxFieldCatalog.Get(key);
|
||||
if (!field.Importable)
|
||||
throw new InvalidOperationException($"KBX field '{key}' is not importable.");
|
||||
|
||||
return new ImportFieldDefinition(
|
||||
Key: field.Key,
|
||||
Label: label ?? field.Label,
|
||||
Aliases: field.Aliases,
|
||||
DataType: field.DataType,
|
||||
Required: required ?? field.Required,
|
||||
MaxLength: field.MaxLength,
|
||||
Precision: field.Precision,
|
||||
Scale: field.Scale,
|
||||
LookupEntity: field.LookupEntity,
|
||||
Importable: field.Importable,
|
||||
Exportable: field.Exportable);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using System.Security.Claims;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
public static class ImportIdentity
|
||||
{
|
||||
public static string TenantId(ClaimsPrincipal user) =>
|
||||
user.FindFirst("tenant_id")?.Value
|
||||
?? user.FindFirst("tenant")?.Value
|
||||
?? "default";
|
||||
|
||||
public static string UserId(ClaimsPrincipal user) =>
|
||||
user.FindFirst(ClaimTypes.NameIdentifier)?.Value
|
||||
?? user.Identity?.Name
|
||||
?? "unknown";
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
namespace Shared.Excel;
|
||||
|
||||
public sealed class ImportMappingEngine
|
||||
{
|
||||
public IReadOnlyList<ImportMapping> Map(
|
||||
IReadOnlyList<string> sourceColumns,
|
||||
ImportDefinition definition,
|
||||
IReadOnlyList<ImportMapping>? saved = null)
|
||||
{
|
||||
var savedBySource = (saved ?? [])
|
||||
.Where(x => x.TargetField is not null)
|
||||
.ToDictionary(x => Normalize(x.SourceColumn), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var result = new List<ImportMapping>(sourceColumns.Count);
|
||||
foreach (var source in sourceColumns)
|
||||
{
|
||||
var normalized = Normalize(source);
|
||||
if (savedBySource.TryGetValue(normalized, out var savedMapping) &&
|
||||
definition.Fields.Any(x => x.Key.Equals(savedMapping.TargetField, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
result.Add(savedMapping with { SourceColumn = source, Source = "saved", Confidence = 1m, Reason = "저장된 매핑" });
|
||||
continue;
|
||||
}
|
||||
|
||||
var exact = definition.Fields.FirstOrDefault(x => Normalize(x.Label) == normalized || Normalize(x.Key) == normalized);
|
||||
if (exact is not null)
|
||||
{
|
||||
result.Add(new(source, exact.Key, "exact", 1m, "필드명 일치"));
|
||||
continue;
|
||||
}
|
||||
|
||||
var alias = definition.Fields.FirstOrDefault(x => x.Aliases.Any(a => Normalize(a) == normalized));
|
||||
if (alias is not null)
|
||||
{
|
||||
result.Add(new(source, alias.Key, "alias", 1m, $"별칭 '{source}' 일치"));
|
||||
continue;
|
||||
}
|
||||
|
||||
result.Add(new(source, null, "manual", null, null));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string Normalize(string value) =>
|
||||
new(value.Where(char.IsLetterOrDigit).Select(char.ToLowerInvariant).ToArray());
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
public static class ImportSourceSignature
|
||||
{
|
||||
public static string Create(IEnumerable<string> columns)
|
||||
{
|
||||
var canonical = string.Join("|", columns.Select(Normalize).Order(StringComparer.Ordinal));
|
||||
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(canonical))).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string Normalize(string value) =>
|
||||
new(value.Where(char.IsLetterOrDigit).Select(char.ToLowerInvariant).ToArray());
|
||||
}
|
||||
|
||||
public interface IImportMappingSuggester
|
||||
{
|
||||
Task<IReadOnlyList<ImportMapping>> SuggestAsync(
|
||||
IReadOnlyList<string> unmappedColumns,
|
||||
ImportDefinition definition,
|
||||
CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Safe default. A production AI adapter can replace this registration. The adapter must return
|
||||
/// only target fields present in ImportDefinition and never mutate data or persist a mapping by itself.
|
||||
/// </summary>
|
||||
public sealed class NoopImportMappingSuggester : IImportMappingSuggester
|
||||
{
|
||||
public Task<IReadOnlyList<ImportMapping>> SuggestAsync(
|
||||
IReadOnlyList<string> unmappedColumns,
|
||||
ImportDefinition definition,
|
||||
CancellationToken ct) => Task.FromResult<IReadOnlyList<ImportMapping>>([]);
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
[Authorize]
|
||||
public sealed class ImportProgressHub : Hub
|
||||
{
|
||||
public Task Subscribe(string sessionId) =>
|
||||
Groups.AddToGroupAsync(Context.ConnectionId, $"import:{sessionId}");
|
||||
|
||||
public Task Unsubscribe(string sessionId) =>
|
||||
Groups.RemoveFromGroupAsync(Context.ConnectionId, $"import:{sessionId}");
|
||||
}
|
||||
|
||||
public sealed class ImportProgressPublisher(IHubContext<ImportProgressHub> hub)
|
||||
{
|
||||
public Task PublishAsync(ImportProgressEvent value, CancellationToken ct) =>
|
||||
hub.Clients.Group($"import:{value.SessionId}")
|
||||
.SendAsync("ImportProgress", value, ct);
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Dapper;
|
||||
using Npgsql;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
public sealed class ImportRepository(NpgsqlDataSource dataSource)
|
||||
{
|
||||
private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public async Task<Guid> CreateSessionAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
ImportDefinition definition,
|
||||
string fileName,
|
||||
string contentType,
|
||||
byte[] data,
|
||||
IReadOnlyList<string> sourceColumns,
|
||||
IReadOnlyList<ImportMapping> mapping,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var id = Guid.NewGuid();
|
||||
var hash = Convert.ToHexString(SHA256.HashData(data)).ToLowerInvariant();
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
await using var tx = await connection.BeginTransactionAsync(ct);
|
||||
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
insert into kbx.import_sessions(
|
||||
id, tenant_id, user_id, screen_id, import_type, original_file_name, status,
|
||||
source_columns, mapping, created_at, updated_at)
|
||||
values(
|
||||
@Id, @TenantId, @UserId, @ScreenId, @ImportType, @FileName, @Status,
|
||||
cast(@SourceColumns as jsonb), cast(@Mapping as jsonb), now(), now());
|
||||
""", new {
|
||||
Id = id, TenantId = tenantId, UserId = userId,
|
||||
definition.ScreenId, ImportType = definition.Id, FileName = fileName,
|
||||
Status = ImportStatuses.MappingRequired,
|
||||
SourceColumns = JsonSerializer.Serialize(sourceColumns, Json),
|
||||
Mapping = JsonSerializer.Serialize(mapping, Json),
|
||||
}, tx, cancellationToken: ct));
|
||||
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
insert into kbx.import_files(session_id, content_type, file_size, sha256, data)
|
||||
values(@SessionId, @ContentType, @FileSize, @Sha256, @Data);
|
||||
""", new { SessionId = id, ContentType = contentType, FileSize = data.LongLength, Sha256 = hash, Data = data }, tx, cancellationToken: ct));
|
||||
|
||||
await tx.CommitAsync(ct);
|
||||
return id;
|
||||
}
|
||||
|
||||
public async Task<(string ContentType, byte[] Data)> GetFileAsync(Guid sessionId, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
return await connection.QuerySingleAsync<(string, byte[])>(new CommandDefinition(
|
||||
"select content_type, data from kbx.import_files where session_id=@SessionId",
|
||||
new { SessionId = sessionId }, cancellationToken: ct));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<ImportMapping>?> GetSavedMappingAsync(
|
||||
string tenantId, string userId, string importType, IReadOnlyList<string> sourceColumns, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
var signature = ImportSourceSignature.Create(sourceColumns);
|
||||
var json = await connection.QueryFirstOrDefaultAsync<string>(new CommandDefinition("""
|
||||
select mapping::text from kbx.saved_import_mappings
|
||||
where tenant_id=@TenantId and user_id=@UserId and import_type=@ImportType and source_signature=@Signature
|
||||
order by updated_at desc limit 1;
|
||||
""", new { TenantId = tenantId, UserId = userId, ImportType = importType, Signature = signature }, cancellationToken: ct));
|
||||
return json is null ? null : JsonSerializer.Deserialize<ImportMapping[]>(json, Json);
|
||||
}
|
||||
|
||||
public async Task SaveNamedMappingAsync(
|
||||
string tenantId, string userId, string importType, string name, IReadOnlyList<string> sourceColumns, IReadOnlyList<ImportMapping> mapping, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
insert into kbx.saved_import_mappings(id,tenant_id,user_id,import_type,name,source_signature,mapping,created_at,updated_at)
|
||||
values(@Id,@TenantId,@UserId,@ImportType,@Name,@Signature,cast(@Mapping as jsonb),now(),now())
|
||||
on conflict(tenant_id,user_id,import_type,name) do update
|
||||
set source_signature=excluded.source_signature,mapping=excluded.mapping,updated_at=now();
|
||||
""", new {
|
||||
Id = Guid.NewGuid(), TenantId=tenantId, UserId=userId, ImportType=importType, Name=name,
|
||||
Signature=ImportSourceSignature.Create(sourceColumns), Mapping=JsonSerializer.Serialize(mapping, Json)
|
||||
}, cancellationToken: ct));
|
||||
}
|
||||
|
||||
public async Task SaveMappingAsync(Guid sessionId, IReadOnlyList<ImportMapping> mapping, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
update kbx.import_sessions set mapping=cast(@Mapping as jsonb), status=@Status, updated_at=now()
|
||||
where id=@SessionId;
|
||||
""", new { SessionId = sessionId, Mapping = JsonSerializer.Serialize(mapping, Json), Status = ImportStatuses.MappingRequired }, cancellationToken: ct));
|
||||
}
|
||||
|
||||
public async Task<ImportSessionDto?> GetAsync(Guid sessionId, string tenantId, string userId, bool includeErrors, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
var row = await connection.QuerySingleOrDefaultAsync<SessionRow>(new CommandDefinition("""
|
||||
select id, import_type as "ImportType", screen_id as "ScreenId", original_file_name as "FileName", status,
|
||||
total_rows as "TotalRows", valid_rows as "ValidRows", invalid_rows as "InvalidRows", warning_rows as "WarningRows", created_rows as "CreatedRows", updated_rows as "UpdatedRows",
|
||||
progress_percent as "ProgressPercent", source_columns::text as "SourceColumnsJson",
|
||||
mapping::text as "MappingJson", created_at as "CreatedAt", completed_at as "CompletedAt"
|
||||
from kbx.import_sessions
|
||||
where id=@SessionId and tenant_id=@TenantId and user_id=@UserId;
|
||||
""", new { SessionId = sessionId, TenantId = tenantId, UserId = userId }, cancellationToken: ct));
|
||||
if (row is null) return null;
|
||||
|
||||
IReadOnlyList<ImportIssue>? errors = null;
|
||||
if (includeErrors)
|
||||
{
|
||||
var issueJson = await connection.QueryAsync<string>(new CommandDefinition("""
|
||||
select errors::text from kbx.import_rows
|
||||
where session_id=@SessionId and jsonb_array_length(errors) > 0
|
||||
order by row_number limit 100;
|
||||
""", new { SessionId = sessionId }, cancellationToken: ct));
|
||||
errors = issueJson.SelectMany(x => JsonSerializer.Deserialize<ImportIssue[]>(x, Json) ?? []).ToArray();
|
||||
}
|
||||
|
||||
return new ImportSessionDto(
|
||||
row.Id, row.ImportType, row.ScreenId, row.FileName, row.Status,
|
||||
row.TotalRows, row.ValidRows, row.InvalidRows, row.WarningRows,
|
||||
row.CreatedRows, row.UpdatedRows, row.ProgressPercent,
|
||||
JsonSerializer.Deserialize<string[]>(row.SourceColumnsJson, Json) ?? [],
|
||||
JsonSerializer.Deserialize<ImportMapping[]>(row.MappingJson, Json) ?? [],
|
||||
errors, row.CreatedAt, row.CompletedAt);
|
||||
}
|
||||
|
||||
public async Task<(string ImportType, IReadOnlyList<ImportMapping> Mapping)> GetWorkDefinitionAsync(Guid sessionId, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
var row = await connection.QuerySingleAsync<(string ImportType, string MappingJson)>(new CommandDefinition(
|
||||
"select import_type as "ImportType", mapping::text as "MappingJson" from kbx.import_sessions where id=@SessionId",
|
||||
new { SessionId = sessionId }, cancellationToken: ct));
|
||||
return (row.ImportType, JsonSerializer.Deserialize<ImportMapping[]>(row.MappingJson, Json) ?? []);
|
||||
}
|
||||
|
||||
|
||||
public async Task<bool> TryTransitionAsync(Guid sessionId, IReadOnlyCollection<string> from, string to, int progress, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
var affected = await connection.ExecuteAsync(new CommandDefinition("""
|
||||
update kbx.import_sessions
|
||||
set status=@To, progress_percent=@Progress, updated_at=now()
|
||||
where id=@SessionId and status = any(@From);
|
||||
""", new { SessionId = sessionId, From = from.ToArray(), To = to, Progress = progress }, cancellationToken: ct));
|
||||
return affected == 1;
|
||||
}
|
||||
|
||||
public async Task SetStatusAsync(Guid sessionId, string status, int progress, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
await connection.ExecuteAsync(new CommandDefinition(
|
||||
"update kbx.import_sessions set status=@Status, progress_percent=@Progress, updated_at=now() where id=@SessionId",
|
||||
new { SessionId = sessionId, Status = status, Progress = progress }, cancellationToken: ct));
|
||||
}
|
||||
|
||||
public async Task ReplaceRowsAsync(Guid sessionId, IReadOnlyList<ImportRowValidation> rows, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
await using var tx = await connection.BeginTransactionAsync(ct);
|
||||
await connection.ExecuteAsync(new CommandDefinition("delete from kbx.import_rows where session_id=@SessionId", new { SessionId = sessionId }, tx, cancellationToken: ct));
|
||||
foreach (var row in rows)
|
||||
{
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
insert into kbx.import_rows(session_id,row_number,raw_data,normalized_data,status,errors,warnings,domain_key)
|
||||
values(@SessionId,@RowNumber,cast(@RawData as jsonb),cast(@NormalizedData as jsonb),@Status,cast(@Errors as jsonb),cast(@Warnings as jsonb),@DomainKey);
|
||||
""", new {
|
||||
SessionId = sessionId, row.RowNumber,
|
||||
RawData = JsonSerializer.Serialize(row.RawData, Json),
|
||||
NormalizedData = row.NormalizedData?.RootElement.GetRawText(),
|
||||
Status = row.IsValid ? "VALID" : "INVALID",
|
||||
Errors = JsonSerializer.Serialize(row.Errors, Json),
|
||||
Warnings = JsonSerializer.Serialize(row.Warnings, Json),
|
||||
row.DomainKey,
|
||||
}, tx, cancellationToken: ct));
|
||||
}
|
||||
|
||||
var total = rows.Count;
|
||||
var valid = rows.Count(x => x.IsValid);
|
||||
var invalid = total - valid;
|
||||
var warningRows = rows.Count(x => x.Warnings.Count > 0);
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
update kbx.import_sessions
|
||||
set status=@Status,total_rows=@Total,valid_rows=@Valid,invalid_rows=@Invalid,
|
||||
warning_rows=@Warnings,progress_percent=100,updated_at=now()
|
||||
where id=@SessionId;
|
||||
""", new { SessionId = sessionId, Status = ImportStatuses.Validated, Total = total, Valid = valid, Invalid = invalid, Warnings = warningRows }, tx, cancellationToken: ct));
|
||||
await tx.CommitAsync(ct);
|
||||
}
|
||||
|
||||
public async Task CompleteAsync(Guid sessionId, ImportCommitResult result, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
var status = result.Failed > 0 ? ImportStatuses.PartiallyCompleted : ImportStatuses.Completed;
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
update kbx.import_sessions
|
||||
set status=@Status, created_rows=@Created, updated_rows=@Updated,
|
||||
invalid_rows=invalid_rows + @Failed, progress_percent=100,
|
||||
completed_at=now(), updated_at=now()
|
||||
where id=@SessionId;
|
||||
""", new { SessionId = sessionId, Status = status, result.Created, result.Updated, result.Failed }, cancellationToken: ct));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyDictionary<int, IReadOnlyList<ImportIssue>>> GetErrorsAsync(Guid sessionId, CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
var rows = await connection.QueryAsync<(int RowNumber, string ErrorsJson)>(new CommandDefinition("""
|
||||
select row_number as "RowNumber", errors::text as "ErrorsJson" from kbx.import_rows
|
||||
where session_id=@SessionId and jsonb_array_length(errors) > 0 order by row_number;
|
||||
""", new { SessionId = sessionId }, cancellationToken: ct));
|
||||
return rows.ToDictionary(x => x.RowNumber, x => (IReadOnlyList<ImportIssue>)(JsonSerializer.Deserialize<ImportIssue[]>(x.ErrorsJson, Json) ?? []));
|
||||
}
|
||||
|
||||
private sealed record SessionRow(
|
||||
Guid Id, string ImportType, string ScreenId, string FileName, string Status,
|
||||
int TotalRows, int ValidRows, int InvalidRows, int WarningRows,
|
||||
int CreatedRows, int UpdatedRows, int ProgressPercent,
|
||||
string SourceColumnsJson, string MappingJson,
|
||||
DateTimeOffset CreatedAt, DateTimeOffset? CompletedAt);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
using Modules.Common.Imports.Jobs;
|
||||
using Modules.OMS.Orders.Import;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
public static class KbxExcelImportRegistration
|
||||
{
|
||||
public static IServiceCollection AddKbxExcelImport(this IServiceCollection services)
|
||||
{
|
||||
services.AddSignalR();
|
||||
services.AddSingleton<XlsxSafetyInspector>();
|
||||
services.AddSingleton<ClosedXmlWorkbookService>();
|
||||
services.AddSingleton<ImportMappingEngine>();
|
||||
services.AddSingleton<ImportRepository>();
|
||||
services.AddSingleton<IImportMappingSuggester, NoopImportMappingSuggester>();
|
||||
services.AddSingleton<IImportDefinition, OrderImportDefinition>();
|
||||
services.AddSingleton<ImportDefinitionRegistry>();
|
||||
services.AddSingleton<ImportProgressPublisher>();
|
||||
services.AddTransient<ValidateImportJob>();
|
||||
services.AddTransient<CommitImportJob>();
|
||||
services.AddTransient<PurgeExpiredImportsJob>();
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IEndpointRouteBuilder MapKbxExcelImport(this IEndpointRouteBuilder endpoints)
|
||||
{
|
||||
endpoints.MapHub<ImportProgressHub>("/hubs/import-progress");
|
||||
return endpoints;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using Dapper;
|
||||
using Npgsql;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
/// <summary>
|
||||
/// Run daily from Hangfire. The default reference retention is 14 days.
|
||||
/// Audit/domain records survive; raw uploaded files and staging rows do not.
|
||||
/// </summary>
|
||||
public sealed class PurgeExpiredImportsJob(NpgsqlDataSource dataSource)
|
||||
{
|
||||
public async Task<int> RunAsync(CancellationToken ct)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync(ct);
|
||||
return await connection.ExecuteAsync(new CommandDefinition("""
|
||||
delete from kbx.import_sessions
|
||||
where expires_at < now()
|
||||
and status in ('completed','partially-completed','failed','cancelled');
|
||||
""", cancellationToken: ct));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
using Xunit;
|
||||
|
||||
namespace Shared.Excel.Tests;
|
||||
|
||||
public sealed class ImportMappingEngineTests
|
||||
{
|
||||
private static readonly ImportDefinition Definition = new(
|
||||
"test.orders", "TEST-ORD-001", "order", "주문 Import",
|
||||
[
|
||||
new("itemCode", "품목코드", ["상품코드", "SKU"], "lookup", true),
|
||||
new("quantity", "수량", ["주문수량", "Qty"], "quantity", true),
|
||||
], true, false);
|
||||
|
||||
[Fact]
|
||||
public void Exact_label_has_priority()
|
||||
{
|
||||
var mapping = new ImportMappingEngine().Map(["품목코드", "수량"], Definition);
|
||||
Assert.Equal("itemCode", mapping[0].TargetField);
|
||||
Assert.Equal("exact", mapping[0].Source);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Alias_is_resolved_without_ai()
|
||||
{
|
||||
var mapping = new ImportMappingEngine().Map(["SKU", "주문수량"], Definition);
|
||||
Assert.Equal("itemCode", mapping[0].TargetField);
|
||||
Assert.Equal("alias", mapping[0].Source);
|
||||
Assert.Equal("quantity", mapping[1].TargetField);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Saved_mapping_wins_before_new_inference()
|
||||
{
|
||||
var saved = new[] { new ImportMapping("MySku", "itemCode", "manual") };
|
||||
var mapping = new ImportMappingEngine().Map(["MySku"], Definition, saved);
|
||||
Assert.Equal("itemCode", mapping[0].TargetField);
|
||||
Assert.Equal("saved", mapping[0].Source);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using System.IO.Compression;
|
||||
using Xunit;
|
||||
|
||||
namespace Shared.Excel.Tests;
|
||||
|
||||
public sealed class XlsxSafetyInspectorTests
|
||||
{
|
||||
[Fact]
|
||||
public void Rejects_zip_without_xlsx_structure()
|
||||
{
|
||||
using var stream = new MemoryStream();
|
||||
using (var zip = new ZipArchive(stream, ZipArchiveMode.Create, true))
|
||||
{
|
||||
var entry = zip.CreateEntry("hello.txt");
|
||||
using var writer = new StreamWriter(entry.Open());
|
||||
writer.Write("not an xlsx");
|
||||
}
|
||||
Assert.Throws<InvalidDataException>(() => new XlsxSafetyInspector().EnsureSafe(stream.ToArray()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using System.IO.Compression;
|
||||
|
||||
namespace Shared.Excel;
|
||||
|
||||
public sealed class XlsxSafetyInspector
|
||||
{
|
||||
public const long DefaultMaxUncompressedBytes = 250L * 1024 * 1024;
|
||||
public const int DefaultMaxEntries = 5_000;
|
||||
|
||||
public void EnsureSafe(byte[] bytes, long maxUncompressedBytes = DefaultMaxUncompressedBytes, int maxEntries = DefaultMaxEntries)
|
||||
{
|
||||
using var stream = new MemoryStream(bytes, writable: false);
|
||||
using var archive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: false);
|
||||
if (archive.Entries.Count == 0 || archive.Entries.Count > maxEntries)
|
||||
throw new InvalidDataException("Excel 파일 내부 구조가 비정상적입니다.");
|
||||
|
||||
if (!archive.Entries.Any(x => x.FullName.Equals("[Content_Types].xml", StringComparison.OrdinalIgnoreCase)) ||
|
||||
!archive.Entries.Any(x => x.FullName.Equals("xl/workbook.xml", StringComparison.OrdinalIgnoreCase)))
|
||||
throw new InvalidDataException("유효한 .xlsx 통합문서가 아닙니다.");
|
||||
|
||||
long total = 0;
|
||||
foreach (var entry in archive.Entries)
|
||||
{
|
||||
total = checked(total + entry.Length);
|
||||
if (total > maxUncompressedBytes)
|
||||
throw new InvalidDataException("압축 해제된 Excel 데이터가 허용 크기를 초과합니다.");
|
||||
|
||||
// A very small compressed entry expanding to an extreme size is suspicious.
|
||||
if (entry.CompressedLength > 0 && entry.Length > 10L * 1024 * 1024 && entry.Length / entry.CompressedLength > 200)
|
||||
throw new InvalidDataException("비정상적인 압축률의 Excel 항목이 감지되었습니다.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user