diff --git a/src/KArtSell.Host/Infrastructure/TelegramSinkAsync.cs b/src/KArtSell.Host/Infrastructure/TelegramSinkAsync.cs
new file mode 100644
index 00000000..a3c99879
--- /dev/null
+++ b/src/KArtSell.Host/Infrastructure/TelegramSinkAsync.cs
@@ -0,0 +1,127 @@
+using System.Net;
+using System.Net.Http;
+using System.Threading.Channels;
+using Microsoft.Extensions.Logging;
+using Serilog;
+using Serilog.Core;
+using Serilog.Events;
+
+namespace KArtSell.Host.Infrastructure;
+
+///
+/// Async Telegram sink for Serilog: non-blocking queue + exponential backoff
+/// Processes ERROR/FATAL logs via background channel, prevents logging from blocking
+///
+public sealed class TelegramSinkAsync : ILogEventSink, IAsyncDisposable
+{
+ private readonly string _telegramBotToken;
+ private readonly string _telegramChatId;
+ private readonly HttpClient _httpClient;
+ private readonly Channel _queue;
+ private readonly Task _backgroundTask;
+ private readonly CancellationTokenSource _cts;
+
+ public TelegramSinkAsync(string telegramBotToken, string telegramChatId, HttpClient? httpClient = null)
+ {
+ _telegramBotToken = telegramBotToken;
+ _telegramChatId = telegramChatId;
+ _httpClient = httpClient ?? new HttpClient();
+ _queue = Channel.CreateUnbounded();
+ _cts = new CancellationTokenSource();
+ _backgroundTask = ProcessQueueAsync(_cts.Token);
+ }
+
+ public void Emit(LogEvent logEvent)
+ {
+ // Only queue ERROR and FATAL
+ if (logEvent.Level < LogEventLevel.Error)
+ return;
+
+ // Non-blocking: enqueue only
+ _queue.Writer.TryWrite(logEvent);
+ }
+
+ private async Task ProcessQueueAsync(CancellationToken cancellationToken)
+ {
+ try
+ {
+ await foreach (var logEvent in _queue.Reader.ReadAllAsync(cancellationToken))
+ {
+ // Rate limit: 100ms spacer between messages
+ await Task.Delay(100, cancellationToken);
+
+ // Retry: 3x with exponential backoff
+ var backoffMs = 100;
+ for (int attempt = 0; attempt < 3; attempt++)
+ {
+ try
+ {
+ await SendTelegramMessageAsync(logEvent, cancellationToken);
+ break;
+ }
+ catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests)
+ {
+ if (attempt < 2)
+ {
+ backoffMs *= 2;
+ await Task.Delay(backoffMs, cancellationToken);
+ }
+ }
+ catch
+ {
+ // Silently fail to prevent logging loops
+ if (attempt == 2) break;
+ }
+ }
+ }
+ }
+ catch (OperationCanceledException)
+ {
+ // Expected during shutdown
+ }
+ }
+
+ private async Task SendTelegramMessageAsync(LogEvent logEvent, CancellationToken cancellationToken)
+ {
+ var emoji = logEvent.Level == LogEventLevel.Fatal ? "🔴" : "⚠️";
+ var levelName = logEvent.Level.ToString().ToUpperInvariant();
+
+ var message = $@"{emoji} *{levelName}* - K-ArtSell Aegis
+
+{logEvent.MessageTemplate.Render(logEvent.Properties)}
+
+_Timestamp: {logEvent.Timestamp:O}_";
+
+ if (logEvent.Exception != null)
+ {
+ message += $@"
+
+```
+{logEvent.Exception.GetType().Name}: {logEvent.Exception.Message}
+```";
+ }
+
+ var url = $"https://api.telegram.org/bot{_telegramBotToken}/sendMessage";
+ var content = new FormUrlEncodedContent(new Dictionary
+ {
+ { "chat_id", _telegramChatId },
+ { "text", message },
+ { "parse_mode", "Markdown" }
+ });
+
+ var response = await _httpClient.PostAsync(url, content, cancellationToken);
+ response.EnsureSuccessStatusCode();
+ }
+
+ public async ValueTask DisposeAsync()
+ {
+ _queue.Writer.Complete();
+ _cts.Cancel();
+ try
+ {
+ await _backgroundTask;
+ }
+ catch (OperationCanceledException) { }
+ _cts.Dispose();
+ }
+}
diff --git a/src/KArtSell.Host/Observability/ApiCallMetricsService.cs b/src/KArtSell.Host/Observability/ApiCallMetricsService.cs
new file mode 100644
index 00000000..3ff180b7
--- /dev/null
+++ b/src/KArtSell.Host/Observability/ApiCallMetricsService.cs
@@ -0,0 +1,130 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Logging;
+
+namespace KArtSell.Host.Observability;
+
+///
+/// In-memory API call metrics tracking (24h retention).
+/// Records: success/failure, latency, retry count, rate limiting, quota remaining.
+///
+public sealed class ApiCallMetricsService : IDisposable
+{
+ private readonly ConcurrentDictionary _metrics = new();
+ private readonly ILogger _logger;
+ private readonly Timer _cleanupTimer;
+
+ public ApiCallMetricsService(ILogger logger)
+ {
+ _logger = logger;
+ // Cleanup old entries every hour
+ _cleanupTimer = new Timer(CleanupOldEntries, null, TimeSpan.FromHours(1), TimeSpan.FromHours(1));
+ }
+
+ public void Dispose()
+ {
+ _cleanupTimer?.Dispose();
+ }
+
+ public void RecordApiCall(
+ string apiName,
+ bool success,
+ int latencyMs = 0,
+ int retryCount = 0,
+ bool rateLimited = false,
+ int? remainingQuota = null)
+ {
+ var key = $"{apiName}:{DateTimeOffset.UtcNow:yyyy-MM-dd HH:mm}";
+
+ _metrics.AddOrUpdate(key, _ =>
+ new ApiMetric
+ {
+ ApiName = apiName,
+ Timestamp = DateTimeOffset.UtcNow,
+ Success = success,
+ LatencyMs = latencyMs,
+ RetryCount = retryCount,
+ RateLimited = rateLimited,
+ RemainingQuota = remainingQuota
+ },
+ (_, existing) => existing); // Keep first entry per minute
+
+ if (rateLimited)
+ {
+ _logger.LogWarning(
+ "API rate limited: {ApiName}, remaining: {Quota}, retry: {RetryCount}",
+ apiName, remainingQuota, retryCount);
+ }
+ }
+
+ public IReadOnlyList GetMetrics(string? apiNameFilter = null)
+ {
+ var results = _metrics.Values.AsEnumerable();
+
+ if (!string.IsNullOrEmpty(apiNameFilter))
+ {
+ results = results.Where(m => m.ApiName.Contains(apiNameFilter, StringComparison.OrdinalIgnoreCase));
+ }
+
+ return results.OrderByDescending(m => m.Timestamp).ToList().AsReadOnly();
+ }
+
+ public Dictionary GetSummary()
+ {
+ var summary = new Dictionary();
+
+ foreach (var group in _metrics.Values.GroupBy(m => m.ApiName))
+ {
+ var metrics = group.ToList();
+ summary[group.Key] = new ApiSummary
+ {
+ TotalCalls = metrics.Count,
+ SuccessCount = metrics.Count(m => m.Success),
+ FailureCount = metrics.Count(m => !m.Success),
+ RateLimitCount = metrics.Count(m => m.RateLimited),
+ AverageLatencyMs = metrics.Average(m => m.LatencyMs),
+ MinRemainingQuota = metrics.Where(m => m.RemainingQuota.HasValue).Min(m => m.RemainingQuota),
+ LastUpdated = metrics.Max(m => m.Timestamp)
+ };
+ }
+
+ return summary;
+ }
+
+ private void CleanupOldEntries(object? state)
+ {
+ var cutoff = DateTimeOffset.UtcNow.AddHours(-24);
+ var oldKeys = _metrics.Where(kvp => kvp.Value.Timestamp < cutoff).Select(kvp => kvp.Key).ToList();
+
+ foreach (var key in oldKeys)
+ {
+ _metrics.TryRemove(key, out _);
+ }
+
+ if (oldKeys.Count > 0)
+ {
+ _logger.LogDebug("Cleaned up {Count} old API metrics", oldKeys.Count);
+ }
+ }
+}
+
+public sealed record ApiMetric
+{
+ public required string ApiName { get; init; }
+ public required DateTimeOffset Timestamp { get; init; }
+ public required bool Success { get; init; }
+ public required int LatencyMs { get; init; }
+ public required int RetryCount { get; init; }
+ public required bool RateLimited { get; init; }
+ public required int? RemainingQuota { get; init; }
+}
+
+public sealed record ApiSummary
+{
+ public required int TotalCalls { get; init; }
+ public required int SuccessCount { get; init; }
+ public required int FailureCount { get; init; }
+ public required int RateLimitCount { get; init; }
+ public required double AverageLatencyMs { get; init; }
+ public required int? MinRemainingQuota { get; init; }
+ public required DateTimeOffset LastUpdated { get; init; }
+}
diff --git a/src/KArtSell.Host/Program.cs b/src/KArtSell.Host/Program.cs
index 23e98415..7403f01d 100644
--- a/src/KArtSell.Host/Program.cs
+++ b/src/KArtSell.Host/Program.cs
@@ -35,10 +35,10 @@ builder.Host.UseSerilog((context, services, logger) =>
.Enrich.FromLogContext()
.WriteTo.Console();
- // Add Telegram sink for ERROR and FATAL logs
+ // Add async Telegram sink for ERROR and FATAL logs (non-blocking queue)
if (!string.IsNullOrEmpty(telegramBotToken) && !string.IsNullOrEmpty(telegramChatId))
{
- config = config.WriteTo.Sink(new TelegramSink(telegramBotToken, telegramChatId), LogEventLevel.Error);
+ config = config.WriteTo.Sink(new TelegramSinkAsync(telegramBotToken, telegramChatId), LogEventLevel.Error);
}
});
@@ -96,6 +96,9 @@ builder.Services.AddScoped();
builder.Services.AddScoped();
builder.Services.AddScoped();
+// API Metrics
+builder.Services.AddSingleton();
+
builder.Services.AddProblemDetails();
builder.Services.AddFastEndpoints();
diff --git a/src/KArtSell.Modules.ModelOperations/ShadowRun/DataBackfiller.cs b/src/KArtSell.Modules.ModelOperations/ShadowRun/DataBackfiller.cs
index 1cee52e0..077a962f 100644
--- a/src/KArtSell.Modules.ModelOperations/ShadowRun/DataBackfiller.cs
+++ b/src/KArtSell.Modules.ModelOperations/ShadowRun/DataBackfiller.cs
@@ -43,16 +43,32 @@ public sealed class DataBackfiller(
"Backfilling OHLCV: {TickerCount} tickers, {TradingDays} trading days ({Start:yyyy-MM-dd} to {End:yyyy-MM-dd})",
tickers.Count, tradingSessions.Count, windowStart, windowEnd);
+ const int BatchDays = 30; // Batch size: ~252 days / 30 = 9 calls (vs 252)
var bars = new List();
foreach (var ticker in tickers)
{
- var tickerBars = await krxData.GetDailyOhlcvAsync(
- ticker, windowStart, windowEnd, cancellationToken);
+ var tickerBars = new List();
+
+ // Fetch in 30-day batches
+ for (var batchStart = windowStart; batchStart <= windowEnd; batchStart = batchStart.AddDays(BatchDays))
+ {
+ var batchEnd = batchStart.AddDays(BatchDays - 1) > windowEnd
+ ? windowEnd
+ : batchStart.AddDays(BatchDays - 1);
+
+ // 100ms throttle between batches
+ await Task.Delay(100, cancellationToken);
+
+ var batchBars = await krxData.GetDailyOhlcvAsync(
+ ticker, batchStart, batchEnd, cancellationToken);
+ tickerBars.AddRange(batchBars);
+ }
+
bars.AddRange(tickerBars);
}
- logger.LogInformation("Backfilled {BarCount} OHLCV bars", bars.Count);
+ logger.LogInformation("Backfilled {BarCount} OHLCV bars (batch mode: 30-day chunks)", bars.Count);
return bars;
}
diff --git a/src/KArtSell.Modules.ModelOperations/ShadowRun/Services/KrxDataService.cs b/src/KArtSell.Modules.ModelOperations/ShadowRun/Services/KrxDataService.cs
index b12b802b..d0a0fce9 100644
--- a/src/KArtSell.Modules.ModelOperations/ShadowRun/Services/KrxDataService.cs
+++ b/src/KArtSell.Modules.ModelOperations/ShadowRun/Services/KrxDataService.cs
@@ -17,7 +17,8 @@ public sealed class KrxDataService : IKrxDataService
private const int CacheDurationMinutes = 1440; // 24 hours
private const int MaxRetries = 3;
- private const int RetryDelayMs = 1000;
+ private const int InitialBackoffMs = 100;
+ private const int MaxBackoffMs = 30000;
private const string KrxApiBaseUrl = "https://openapi.krx.co.kr";
private static readonly Action LogFetchingOhlcv =
@@ -73,9 +74,10 @@ public sealed class KrxDataService : IKrxDataService
return cached!;
}
- // Fetch with retry
+ // Fetch with exponential backoff retry
var bars = new List();
int attempt = 0;
+ int backoffMs = InitialBackoffMs;
while (attempt < MaxRetries)
{
@@ -85,10 +87,19 @@ public sealed class KrxDataService : IKrxDataService
bars = ParseOhlcvResponse(ticker, response);
break;
}
+ catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests && attempt < MaxRetries - 1)
+ {
+ // 429: Rate limit hit → exponential backoff
+ backoffMs = Math.Min(backoffMs * 2, MaxBackoffMs);
+ LogRetryError(_logger, $"Rate limited (429), backoff {backoffMs}ms (attempt {attempt + 1}/{MaxRetries})", ex);
+ await Task.Delay(backoffMs, cancellationToken);
+ attempt++;
+ }
catch (HttpRequestException ex) when (IsTransientError(ex) && attempt < MaxRetries - 1)
{
+ // Other transient errors → fixed 1s delay
LogRetryError(_logger, $"{ex.Message} (attempt {attempt + 1}/{MaxRetries})", ex);
- await Task.Delay(RetryDelayMs, cancellationToken);
+ await Task.Delay(1000, cancellationToken);
attempt++;
}
catch (HttpRequestException ex) when (!IsTransientError(ex))
@@ -174,6 +185,17 @@ public sealed class KrxDataService : IKrxDataService
$"&isuCd={ticker}";
var response = await _httpClient.GetAsync(endpoint, cancellationToken);
+
+ // Check rate limit header
+ if (response.Headers.TryGetValues("X-RateLimit-Remaining", out var remaining))
+ {
+ if (int.TryParse(remaining.First(), out var limit) && limit < 10)
+ {
+ _logger.LogWarning("KRX rate limit low: {Remaining} requests remaining", limit);
+ await Task.Delay(5000, cancellationToken); // 5s pause
+ }
+ }
+
response.EnsureSuccessStatusCode();
var json = await response.Content.ReadAsStringAsync(cancellationToken);