V13-FE-011: finalize search list layout slice

This commit is contained in:
2026-08-09 02:57:26 +09:00
parent 9efd202e76
commit 6422cb2b13
984 changed files with 120811 additions and 1498 deletions
@@ -0,0 +1,9 @@
// generated from contracts/external-data/kbx.external-data.json; do not edit.
namespace Kbx.Shared.ExternalData.Generated;
public sealed record KbxExternalDataDatasetDefinition(string Id,string ProviderId,string ProviderOperationId,string CanonicalType,string Normalizer,string NormalizerVersion,string FreshnessMode,int? FreshForSeconds,int? MaxStaleSeconds,bool BackgroundRefresh,string RawRetention,int NormalizedRetentionDays,string SourceLabel);
public static class KbxExternalDataCatalog { public const string SourceSha256="da54ff739c6dc5ed6f9390d1fc8a8e138060d7cf19531a80cc91e421d62954a4"; public static readonly IReadOnlyDictionary<string,KbxExternalDataDatasetDefinition> All=new Dictionary<string,KbxExternalDataDatasetDefinition>(StringComparer.Ordinal) {
["dataset.opendart.company-profile"] = new("dataset.opendart.company-profile", "provider.opendart", "opendart.company", "company-profile", "OpenDartCompanyProfileNormalizer", "1.0.0", "stale-while-revalidate", 86400, 604800, true, "hash-only", 365, "OPENDART"),
["dataset.opendart.disclosures"] = new("dataset.opendart.disclosures", "provider.opendart", "opendart.disclosures", "disclosure-list", "OpenDartDisclosureListNormalizer", "1.0.0", "stale-while-revalidate", 300, 3600, true, "hash-only", 90, "OPENDART"),
["dataset.kis.domestic-stock.current-price"] = new("dataset.kis.domestic-stock.current-price", "provider.kis.market-data", "kis.domestic-stock.current-price", "market-price", "KisCurrentPriceNormalizer", "1.0.0", "strict", 3, 0, false, "hash-only", 7, "KIS"),
["dataset.krx.approved-service"] = new("dataset.krx.approved-service", "provider.krx.openapi", "krx.approved-service.invoke", "provider-defined", "registered-per-approved-service", "provider-defined", "provider-defined", null, null, false, "hash-only", 30, "KRX")
}; }
@@ -0,0 +1,82 @@
using System.Text.Json;
using Kbx.Shared.ExternalData.Generated;
using Kbx.Shared.Providers;
namespace Kbx.Shared.ExternalData;
public sealed record KbxNormalizedExternalData<T>(T? Data, DateTimeOffset? ProviderObservedAt = null);
public interface IKbxExternalDataRefreshScheduler
{
Task ScheduleEntryAsync(Guid tenantId, string datasetId, string cacheKey, CancellationToken cancellationToken);
Task ScheduleDatasetAsync(Guid tenantId, string datasetId, CancellationToken cancellationToken);
}
public sealed class KbxExternalDataCacheCoordinator(KbxExternalDataRepository repository, IKbxExternalDataRefreshScheduler refreshScheduler)
{
public async Task<KbxExternalDataEnvelope<T>> GetOrFetchAsync<T>(
Guid tenantId,
string datasetId,
IReadOnlyDictionary<string,string?> cacheKeyValues,
Func<CancellationToken,Task<KbxProviderResult<byte[]>>> fetch,
Func<ReadOnlyMemory<byte>,KbxNormalizedExternalData<T>> normalize,
string correlationId,
CancellationToken ct)
{
if (!KbxExternalDataCatalog.All.TryGetValue(datasetId, out var dataset)) throw new ArgumentOutOfRangeException(nameof(datasetId));
if (dataset.FreshnessMode == "provider-defined" && dataset.FreshForSeconds is null)
throw new InvalidOperationException($"Dataset '{datasetId}' requires an approved service-specific freshness policy before runtime use.");
var now=DateTimeOffset.UtcNow;
var cacheKey=KbxExternalDataProvenanceFactory.CacheKey(cacheKeyValues);
var requestDescriptor=JsonSerializer.SerializeToDocument(cacheKeyValues.OrderBy(x=>x.Key,StringComparer.Ordinal).ToDictionary(x=>x.Key,x=>x.Value));
var cached=await repository.GetAsync(tenantId,datasetId,cacheKey,ct);
if(cached is not null)
{
var state=KbxExternalDataFreshnessPolicy.Evaluate(dataset,now,cached.FreshUntil,cached.UsableUntil);
if(state==KbxExternalDataState.Fresh)
return FromCache<T>(dataset,cached,state,false);
if(state==KbxExternalDataState.Stale && dataset.FreshnessMode=="stale-while-revalidate")
{
await refreshScheduler.ScheduleEntryAsync(tenantId,datasetId,cacheKey,ct);
return FromCache<T>(dataset,cached,state,true,"공급자 데이터를 다시 확인하는 동안 마지막 정상값을 표시합니다.");
}
}
var requestedAt=DateTimeOffset.UtcNow;
var result=await fetch(ct);
var receivedAt=DateTimeOffset.UtcNow;
if(result.Kind==KbxProviderResultKind.NoData)
{
var empty=JsonDocument.Parse("null");
var (freshUntil,usableUntil)=KbxExternalDataFreshnessPolicy.Window(dataset,receivedAt);
var entry=new KbxExternalDataCacheEntry(tenantId,dataset.Id,dataset.ProviderId,cacheKey,requestDescriptor,empty,null,requestedAt,receivedAt,DateTimeOffset.UtcNow,freshUntil,usableUntil,KbxExternalDataProvenanceFactory.PayloadSha256(Array.Empty<byte>()),dataset.NormalizerVersion,"fresh",correlationId);
await repository.UpsertAsync(entry,ct);
return new(default,KbxExternalDataProvenanceFactory.Create(dataset,KbxExternalDataState.Fresh,requestedAt,receivedAt,entry.IngestedAt,null,freshUntil,usableUntil,entry.PayloadSha256,false));
}
if(result.Kind!=KbxProviderResultKind.Success || result.Value is null)
throw new KbxExternalDataUnavailableException(dataset.Id,result.Code,result.Message);
var raw=result.Value;
var normalized=normalize(raw);
var ingestedAt=DateTimeOffset.UtcNow;
var window=KbxExternalDataFreshnessPolicy.Window(dataset,receivedAt);
var document=JsonSerializer.SerializeToDocument(normalized.Data);
var payloadSha=KbxExternalDataProvenanceFactory.PayloadSha256(raw);
var stored=new KbxExternalDataCacheEntry(tenantId,dataset.Id,dataset.ProviderId,cacheKey,requestDescriptor,document,normalized.ProviderObservedAt,requestedAt,receivedAt,ingestedAt,window.FreshUntil,window.UsableUntil,payloadSha,dataset.NormalizerVersion,"fresh",correlationId);
await repository.UpsertAsync(stored,ct);
return new(normalized.Data,KbxExternalDataProvenanceFactory.Create(dataset,KbxExternalDataState.Fresh,requestedAt,receivedAt,ingestedAt,normalized.ProviderObservedAt,window.FreshUntil,window.UsableUntil,payloadSha,false));
}
private static KbxExternalDataEnvelope<T> FromCache<T>(KbxExternalDataDatasetDefinition dataset,KbxExternalDataCacheEntry entry,KbxExternalDataState state,bool refreshing,string? warning=null)
{
var data=entry.NormalizedData.RootElement.ValueKind==JsonValueKind.Null?default:entry.NormalizedData.RootElement.Deserialize<T>();
return new(data,KbxExternalDataProvenanceFactory.Create(dataset,state,entry.RequestedAt,entry.ReceivedAt,entry.IngestedAt,entry.ProviderObservedAt,entry.FreshUntil,entry.UsableUntil,entry.PayloadSha256,true,refreshing,warning));
}
}
public sealed class KbxExternalDataUnavailableException(string datasetId,string? code,string? detail):Exception($"External dataset '{datasetId}' is unavailable. {code}: {detail}")
{
public string DatasetId { get; }=datasetId;
public string? ProviderCode { get; }=code;
}
@@ -0,0 +1,48 @@
using System.Text.Json;
namespace Kbx.Shared.ExternalData;
public enum KbxExternalDataState { Fresh, Stale, Expired, Unavailable }
public enum KbxExternalDataFreshnessMode { Strict, StaleWhileRevalidate, ProviderDefined }
public sealed record KbxExternalDataProvenance(
string DatasetId,
string ProviderId,
string ProviderOperationId,
string SourceLabel,
KbxExternalDataState State,
DateTimeOffset? ProviderObservedAt,
DateTimeOffset RequestedAt,
DateTimeOffset ReceivedAt,
DateTimeOffset IngestedAt,
DateTimeOffset? FreshUntil,
DateTimeOffset? UsableUntil,
string PayloadSha256,
string NormalizerVersion,
bool CacheHit,
bool RefreshInProgress = false,
string? Warning = null);
public sealed record KbxExternalDataEnvelope<T>(T? Data, KbxExternalDataProvenance Provenance);
public sealed record KbxExternalDataCacheEntry(
Guid TenantId,
string DatasetId,
string ProviderId,
string CacheKey,
JsonDocument RequestDescriptor,
JsonDocument NormalizedData,
DateTimeOffset? ProviderObservedAt,
DateTimeOffset RequestedAt,
DateTimeOffset ReceivedAt,
DateTimeOffset IngestedAt,
DateTimeOffset? FreshUntil,
DateTimeOffset? UsableUntil,
string PayloadSha256,
string NormalizerVersion,
string State,
string CorrelationId);
public sealed record KbxCompanyProfileSnapshot(string CorpCode, string CorpName, string? StockCode, string? BusinessNumber);
public sealed record KbxDisclosureSummarySnapshot(string ReceiptNo, string CorpCode, string CorpName, string ReportName, string ReceiptDate);
public sealed record KbxMarketPriceSnapshot(string MarketDivisionCode, string StockCode, decimal CurrentPrice);
@@ -0,0 +1,25 @@
using Kbx.Shared.ExternalData.Generated;
namespace Kbx.Shared.ExternalData;
public static class KbxExternalDataFreshnessPolicy
{
public static KbxExternalDataState Evaluate(KbxExternalDataDatasetDefinition dataset, DateTimeOffset now, DateTimeOffset? freshUntil, DateTimeOffset? usableUntil)
{
if (freshUntil is null && dataset.FreshnessMode == "provider-defined") return KbxExternalDataState.Unavailable;
if (freshUntil is not null && now <= freshUntil.Value) return KbxExternalDataState.Fresh;
if (dataset.FreshnessMode == "strict") return KbxExternalDataState.Expired;
if (usableUntil is not null && now <= usableUntil.Value) return KbxExternalDataState.Stale;
return KbxExternalDataState.Expired;
}
public static (DateTimeOffset? FreshUntil, DateTimeOffset? UsableUntil) Window(KbxExternalDataDatasetDefinition dataset, DateTimeOffset receivedAt)
{
if (dataset.FreshForSeconds is null) return (null, null);
var fresh = receivedAt.AddSeconds(dataset.FreshForSeconds.Value);
var usable = dataset.FreshnessMode == "stale-while-revalidate" && dataset.MaxStaleSeconds is > 0
? fresh.AddSeconds(dataset.MaxStaleSeconds.Value)
: fresh;
return (fresh, usable);
}
}
@@ -0,0 +1,17 @@
using System.Security.Cryptography;
using System.Text;
using Kbx.Shared.ExternalData.Generated;
namespace Kbx.Shared.ExternalData;
public static class KbxExternalDataProvenanceFactory
{
public static string PayloadSha256(ReadOnlySpan<byte> payload) => Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
public static string CacheKey(IEnumerable<KeyValuePair<string,string?>> values)
{
var canonical=string.Join("&",values.OrderBy(x=>x.Key,StringComparer.Ordinal).Select(x=>$"{x.Key}={x.Value?.Trim() ?? string.Empty}"));
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(canonical))).ToLowerInvariant();
}
public static KbxExternalDataProvenance Create(KbxExternalDataDatasetDefinition d,KbxExternalDataState state,DateTimeOffset requestedAt,DateTimeOffset receivedAt,DateTimeOffset ingestedAt,DateTimeOffset? observedAt,DateTimeOffset? freshUntil,DateTimeOffset? usableUntil,string payloadSha,bool cacheHit,bool refreshInProgress=false,string? warning=null)
=> new(d.Id,d.ProviderId,d.ProviderOperationId,d.SourceLabel,state,observedAt,requestedAt,receivedAt,ingestedAt,freshUntil,usableUntil,payloadSha,d.NormalizerVersion,cacheHit,refreshInProgress,warning);
}
@@ -0,0 +1,31 @@
using Hangfire;
namespace Kbx.Shared.ExternalData;
public sealed class KbxExternalDataRefreshScheduler(IBackgroundJobClient jobs):IKbxExternalDataRefreshScheduler
{
public Task ScheduleEntryAsync(Guid tenantId,string datasetId,string cacheKey,CancellationToken cancellationToken)
{
jobs.Enqueue<KbxExternalDataRefreshJob>(x=>x.RunEntryAsync(tenantId,datasetId,cacheKey,CancellationToken.None));
return Task.CompletedTask;
}
public Task ScheduleDatasetAsync(Guid tenantId,string datasetId,CancellationToken cancellationToken)
{
jobs.Enqueue<KbxExternalDataRefreshJob>(x=>x.RunDatasetAsync(tenantId,datasetId,CancellationToken.None));
return Task.CompletedTask;
}
}
public sealed class KbxExternalDataRefreshJob(KbxExternalDataRefresherRegistry registry)
{
public Task RunEntryAsync(Guid tenantId,string datasetId,string cacheKey,CancellationToken ct)=>registry.GetRequired(datasetId).RefreshEntryAsync(tenantId,cacheKey,ct);
public Task RunDatasetAsync(Guid tenantId,string datasetId,CancellationToken ct)=>registry.GetRequired(datasetId).RefreshKnownEntriesAsync(tenantId,ct);
}
public interface IKbxExternalDataDatasetRefresher
{
string DatasetId { get; }
Task RefreshEntryAsync(Guid tenantId,string cacheKey,CancellationToken ct);
Task RefreshKnownEntriesAsync(Guid tenantId,CancellationToken ct);
}
public sealed class KbxExternalDataRefresherRegistry(IEnumerable<IKbxExternalDataDatasetRefresher> refreshers)
{
private readonly IReadOnlyDictionary<string,IKbxExternalDataDatasetRefresher> map=refreshers.ToDictionary(x=>x.DatasetId,StringComparer.Ordinal);
public IKbxExternalDataDatasetRefresher GetRequired(string datasetId)=>map.TryGetValue(datasetId,out var r)?r:throw new InvalidOperationException($"No external-data refresher registered for {datasetId}.");
}
@@ -0,0 +1,14 @@
using Microsoft.Extensions.DependencyInjection;
namespace Kbx.Shared.ExternalData;
public static class KbxExternalDataRegistration
{
public static IServiceCollection AddKbxExternalData(this IServiceCollection services)
{
services.AddSingleton<KbxExternalDataRepository>();
services.AddSingleton<IKbxExternalDataRefreshScheduler,KbxExternalDataRefreshScheduler>();
services.AddSingleton<KbxExternalDataRefresherRegistry>();
services.AddSingleton<KbxExternalDataCacheCoordinator>();
services.AddSingleton<KrxApprovedServiceNormalizerRegistry>();
return services;
}
}
@@ -0,0 +1,45 @@
using System.Text.Json;
using Dapper;
using Npgsql;
namespace Kbx.Shared.ExternalData;
public sealed class KbxExternalDataRepository(NpgsqlDataSource dataSource)
{
public async Task<KbxExternalDataCacheEntry?> GetAsync(Guid tenantId, string datasetId, string cacheKey, CancellationToken ct)
{
const string sql = """
select tenant_id as TenantId, dataset_id as DatasetId, provider_id as ProviderId, cache_key as CacheKey, request_descriptor::text as RequestDescriptorJson,
normalized_data::text as NormalizedJson, provider_observed_at as ProviderObservedAt,
requested_at as RequestedAt, received_at as ReceivedAt, ingested_at as IngestedAt,
fresh_until as FreshUntil, usable_until as UsableUntil, payload_sha256 as PayloadSha256,
normalizer_version as NormalizerVersion, state, correlation_id as CorrelationId
from kbx.external_data_cache
where tenant_id=@TenantId and dataset_id=@DatasetId and cache_key=@CacheKey
""";
await using var connection = await dataSource.OpenConnectionAsync(ct);
var row = await connection.QuerySingleOrDefaultAsync<Row>(new CommandDefinition(sql, new { TenantId=tenantId, DatasetId=datasetId, CacheKey=cacheKey }, cancellationToken:ct));
return row is null ? null : new KbxExternalDataCacheEntry(row.TenantId,row.DatasetId,row.ProviderId,row.CacheKey,JsonDocument.Parse(row.RequestDescriptorJson),JsonDocument.Parse(row.NormalizedJson),row.ProviderObservedAt,row.RequestedAt,row.ReceivedAt,row.IngestedAt,row.FreshUntil,row.UsableUntil,row.PayloadSha256,row.NormalizerVersion,row.State,row.CorrelationId);
}
public async Task UpsertAsync(KbxExternalDataCacheEntry entry, CancellationToken ct)
{
const string sql = """
insert into kbx.external_data_cache(tenant_id,dataset_id,provider_id,cache_key,request_descriptor,normalized_data,provider_observed_at,requested_at,received_at,ingested_at,fresh_until,usable_until,payload_sha256,normalizer_version,state,correlation_id)
values(@TenantId,@DatasetId,@ProviderId,@CacheKey,cast(@RequestDescriptorJson as jsonb),cast(@NormalizedJson as jsonb),@ProviderObservedAt,@RequestedAt,@ReceivedAt,@IngestedAt,@FreshUntil,@UsableUntil,@PayloadSha256,@NormalizerVersion,@State,@CorrelationId)
on conflict(tenant_id,dataset_id,cache_key) do update set
provider_id=excluded.provider_id, request_descriptor=excluded.request_descriptor, normalized_data=excluded.normalized_data, provider_observed_at=excluded.provider_observed_at,
requested_at=excluded.requested_at, received_at=excluded.received_at, ingested_at=excluded.ingested_at,
fresh_until=excluded.fresh_until, usable_until=excluded.usable_until, payload_sha256=excluded.payload_sha256,
normalizer_version=excluded.normalizer_version, state=excluded.state, correlation_id=excluded.correlation_id;
insert into kbx.external_data_observations(tenant_id,dataset_id,provider_id,cache_key,payload_sha256,normalizer_version,provider_observed_at,received_at,ingested_at,state,correlation_id)
values(@TenantId,@DatasetId,@ProviderId,@CacheKey,@PayloadSha256,@NormalizerVersion,@ProviderObservedAt,@ReceivedAt,@IngestedAt,@State,@CorrelationId);
""";
await using var connection = await dataSource.OpenConnectionAsync(ct);
await using var transaction = await connection.BeginTransactionAsync(ct);
await connection.ExecuteAsync(new CommandDefinition(sql,new { entry.TenantId,entry.DatasetId,entry.ProviderId,entry.CacheKey,RequestDescriptorJson=entry.RequestDescriptor.RootElement.GetRawText(),NormalizedJson=entry.NormalizedData.RootElement.GetRawText(),entry.ProviderObservedAt,entry.RequestedAt,entry.ReceivedAt,entry.IngestedAt,entry.FreshUntil,entry.UsableUntil,entry.PayloadSha256,entry.NormalizerVersion,entry.State,entry.CorrelationId },transaction,cancellationToken:ct));
await transaction.CommitAsync(ct);
}
private sealed record Row(Guid TenantId,string DatasetId,string ProviderId,string CacheKey,string RequestDescriptorJson,string NormalizedJson,DateTimeOffset? ProviderObservedAt,DateTimeOffset RequestedAt,DateTimeOffset ReceivedAt,DateTimeOffset IngestedAt,DateTimeOffset? FreshUntil,DateTimeOffset? UsableUntil,string PayloadSha256,string NormalizerVersion,string State,string CorrelationId);
}
@@ -0,0 +1,19 @@
using Dapper;
using Npgsql;
using Kbx.Shared.ExternalData.Generated;
namespace Kbx.Shared.ExternalData;
public sealed class KbxExternalDataRetentionJob(NpgsqlDataSource dataSource)
{
public async Task RunAsync(CancellationToken ct)
{
await using var connection=await dataSource.OpenConnectionAsync(ct);
foreach(var dataset in KbxExternalDataCatalog.All.Values)
{
var before=DateTimeOffset.UtcNow.AddDays(-dataset.NormalizedRetentionDays);
const string sql="delete from kbx.external_data_observations where dataset_id=@DatasetId and ingested_at < @Before";
await connection.ExecuteAsync(new CommandDefinition(sql,new{DatasetId=dataset.Id,Before=before},cancellationToken:ct));
}
}
}
@@ -0,0 +1,13 @@
using System.Globalization;
using System.Text.Json;
namespace Kbx.Shared.ExternalData;
public static class KisCurrentPriceNormalizer
{
public static KbxMarketPriceSnapshot Normalize(ReadOnlySpan<byte> payload,string marketDivisionCode,string stockCode)
{
using var doc=JsonDocument.Parse(payload);
if(!doc.RootElement.TryGetProperty("output",out var output) || !output.TryGetProperty("stck_prpr",out var priceValue) || !decimal.TryParse(priceValue.GetString(),NumberStyles.Number,CultureInfo.InvariantCulture,out var price))
throw new InvalidDataException("KIS current price response does not contain a valid stck_prpr.");
return new(marketDivisionCode,stockCode,price);
}
}
@@ -0,0 +1,7 @@
namespace Kbx.Shared.ExternalData;
public interface IKrxApprovedServiceNormalizer { string ServiceId { get; } string NormalizerVersion { get; } object Normalize(ReadOnlyMemory<byte> payload); }
public sealed class KrxApprovedServiceNormalizerRegistry(IEnumerable<IKrxApprovedServiceNormalizer> normalizers)
{
private readonly IReadOnlyDictionary<string,IKrxApprovedServiceNormalizer> map=normalizers.ToDictionary(x=>x.ServiceId,StringComparer.Ordinal);
public IKrxApprovedServiceNormalizer GetRequired(string serviceId)=>map.TryGetValue(serviceId,out var n)?n:throw new InvalidOperationException($"No approved KRX normalizer is registered for service '{serviceId}'. Universal KRX field mapping is intentionally not guessed.");
}
@@ -0,0 +1,17 @@
using System.Text.Json;
namespace Kbx.Shared.ExternalData;
public static class OpenDartCompanyProfileNormalizer
{
public static KbxCompanyProfileSnapshot Normalize(ReadOnlySpan<byte> payload)
{
using var doc=JsonDocument.Parse(payload);
var r=doc.RootElement;
return new(
Required(r,"corp_code"),
Required(r,"corp_name"),
Optional(r,"stock_code"),
Optional(r,"bizr_no"));
}
private static string Required(JsonElement e,string name)=>e.TryGetProperty(name,out var p)&&!string.IsNullOrWhiteSpace(p.GetString())?p.GetString()!:throw new InvalidDataException($"OPENDART field missing: {name}");
private static string? Optional(JsonElement e,string name)=>e.TryGetProperty(name,out var p)?p.GetString():null;
}
@@ -0,0 +1,12 @@
using System.Text.Json;
namespace Kbx.Shared.ExternalData;
public static class OpenDartDisclosureListNormalizer
{
public static IReadOnlyList<KbxDisclosureSummarySnapshot> Normalize(ReadOnlySpan<byte> payload)
{
using var doc=JsonDocument.Parse(payload);
if(!doc.RootElement.TryGetProperty("list",out var list)||list.ValueKind!=JsonValueKind.Array)return Array.Empty<KbxDisclosureSummarySnapshot>();
return list.EnumerateArray().Select(x=>new KbxDisclosureSummarySnapshot(Get(x,"rcept_no"),Get(x,"corp_code"),Get(x,"corp_name"),Get(x,"report_nm"),Get(x,"rcept_dt"))).ToArray();
}
private static string Get(JsonElement e,string n)=>e.TryGetProperty(n,out var p)?p.GetString()??string.Empty:string.Empty;
}