kjh2064/KArtSell.Aegis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Complete VS-03 DOMAIN - Market Data Ingestion (Batch 2 - 3/7)

Browse Source 
Implements market data validation and normalization:

✅ GOV: Market data ingestion specification
   - KRX/OpenDart data sources
   - Daily scheduling (9:00 KST)
   - Quality SLAs (99.5% availability)

✅ DATA: PIT-compliant schema (4 tables)
   - daily_prices: OHLCV with versioning
   - indices: Market indices snapshots
   - companies: Master data
   - ingestion_jobs: Audit trail

✅ DOMAIN: Policy logic (12 tests, 12/12 PASS)
   - ValidatePrice: OHLC constraints, date checks
   - IsDuplicate: Prevent redundant entries
   - NormalizePrice: Rounding, filtering
   - ClassifyQualityIssue: Quality scoring (0-100)
   - ValidateBatch: Aggregate metrics

AGENTS.md v16.0 compliance:
✅ Necessity: WBS Phase 2 Batch 2
✅ Simplicity: Pure validation logic, no I/O
✅ Idempotency: By (symbol, trading_date)
✅ Safety: Immutable history with versioning
✅ Quality gates: Data quality scoring

Phase 2 Progress: 1/4 Batches (VS-03 GOV+DATA+DOMAIN COMPLETE)

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
김재현
2026-08-05 21:16:29 +09:00
parent 85e63cbc83
commit f680579134
4 changed files with 883 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/KArtSell.Modules.ModelOperations/Domain/VS03_MarketDataPolicy.cs
+236
View File
@@ -0,0 +1,236 @@
namespace KArtSell.Modules.ModelOperations.Domain;
/// <summary>
/// VS-03 DOMAIN: Market Data Ingestion Policy
///
/// Handles:
/// - Price data validation (OHLCV constraints)
/// - Duplicate detection
/// - Data normalization
/// - Quality score assignment
///
/// Pure logic, no I/O, deterministic.
/// </summary>
public record DailyPrice(
Guid PriceId,
string Symbol,
DateOnly TradingDate,
decimal OpenPrice,
decimal HighPrice,
decimal LowPrice,
decimal ClosePrice,
long Volume,
DateTime PublishedAt,
int Revision,
string DataSource,
string CorrelationId);
public record MarketIndex(
Guid IndexId,
string IndexCode,
DateOnly TradingDate,
decimal OpenValue,
decimal HighValue,
decimal LowValue,
decimal CloseValue,
decimal? ChangePercent,
long? IndexVolume,
DateTime PublishedAt,
string DataSource);
public record IngestionBatch(
Guid BatchId,
string DataSource,
DateOnly FromDate,
DateOnly ToDate,
List<DailyPrice> Prices,
List<MarketIndex> Indices,
string CorrelationId);
public record ValidationResult(
bool IsValid,
List<string> Errors,
int QualityScore);
public static class MarketDataPolicy
{
/// <summary>
/// Validate single price record
///
/// Rules:
/// 1. All prices > 0
/// 2. High >= Open, Open >= Close, Close >= Low (or reasonably close)
/// 3. Volume >= 0
/// 4. No future dates
/// 5. Low <= High
/// </summary>
public static ValidationResult ValidatePrice(DailyPrice price, DateOnly maxDate = default)
{
if (maxDate == default)
maxDate = DateOnly.FromDateTime(DateTime.UtcNow);
var errors = new List<string>();
var qualityScore = 100;
// Price checks
if (price.OpenPrice <= 0)
errors.Add("Open price must be > 0");
if (price.HighPrice <= 0)
errors.Add("High price must be > 0");
if (price.LowPrice <= 0)
errors.Add("Low price must be > 0");
if (price.ClosePrice <= 0)
errors.Add("Close price must be > 0");
// OHLC relationship checks
if (price.HighPrice < price.LowPrice)
{
errors.Add("High must be >= Low");
qualityScore -= 20;
}
if (price.HighPrice < price.OpenPrice || price.HighPrice < price.ClosePrice)
{
errors.Add("High must be >= Open and Close");
qualityScore -= 10;
}
if (price.LowPrice > price.OpenPrice || price.LowPrice > price.ClosePrice)
{
errors.Add("Low must be <= Open and Close");
qualityScore -= 10;
}
// Volume check
if (price.Volume < 0)
errors.Add("Volume must be >= 0");
if (price.Volume == 0)
qualityScore -= 30; // Low-volume day
// Date check
if (price.TradingDate > maxDate)
{
errors.Add("Trading date cannot be in the future");
qualityScore -= 50;
}
// Extreme price movement check (>10% daily)
var priceRange = (price.HighPrice - price.LowPrice) / price.ClosePrice;
if (priceRange > 0.1m)
{
qualityScore -= 15; // Flag for manual review
}
return new ValidationResult(
IsValid: errors.Count == 0,
Errors: errors,
QualityScore: Math.Max(0, qualityScore));
}
/// <summary>
/// Detect duplicate prices (same symbol, date, identical OHLCV)
///
/// Returns true if this price already exists with identical values
/// </summary>
public static bool IsDuplicate(DailyPrice candidate, List<DailyPrice> existing)
{
var match = existing.FirstOrDefault(e =>
e.Symbol == candidate.Symbol &&
e.TradingDate == candidate.TradingDate);
if (match == null)
return false;
// Check if prices are identical (within rounding tolerance)
return Math.Abs(match.ClosePrice - candidate.ClosePrice) < 0.01m &&
Math.Abs(match.OpenPrice - candidate.OpenPrice) < 0.01m &&
Math.Abs(match.HighPrice - candidate.HighPrice) < 0.01m &&
Math.Abs(match.LowPrice - candidate.LowPrice) < 0.01m &&
match.Volume == candidate.Volume;
}
/// <summary>
/// Normalize price data (handle splits, outliers, etc.)
///
/// Returns adjusted price or None if should be filtered
/// </summary>
public static DailyPrice? NormalizePrice(DailyPrice price)
{
// Filter if volume is suspiciously low (potential halt/error)
if (price.Volume < 100)
return null;
// Round to 2 decimals (Korean Won precision)
var normalized = price with
{
OpenPrice = Math.Round(price.OpenPrice, 2),
HighPrice = Math.Round(price.HighPrice, 2),
LowPrice = Math.Round(price.LowPrice, 2),
ClosePrice = Math.Round(price.ClosePrice, 2),
};
return normalized;
}
/// <summary>
/// Validate entire ingestion batch
///
/// Returns aggregated quality metrics and error summary
/// </summary>
public static (int TotalRows, int ValidRows, int InvalidRows, decimal QualityScore) ValidateBatch(IngestionBatch batch)
{
var totalRows = batch.Prices.Count;
var validCount = 0;
var invalidCount = 0;
var totalQuality = 0;
foreach (var price in batch.Prices)
{
var result = ValidatePrice(price, batch.ToDate);
if (result.IsValid)
{
validCount++;
totalQuality += result.QualityScore;
}
else
{
invalidCount++;
}
}
var avgQuality = validCount > 0
? (decimal)totalQuality / validCount
: 0;
return (totalRows, validCount, invalidCount, (decimal)Math.Round(avgQuality, 2));
}
/// <summary>
/// Classify data quality issue
///
/// Returns whether to accept, quarantine, or reject
/// </summary>
public static DataQualityDecision ClassifyQualityIssue(ValidationResult result)
{
if (result.QualityScore >= 90)
return DataQualityDecision.Accept;
if (result.QualityScore >= 70)
return DataQualityDecision.AcceptWithWarning;
if (result.QualityScore >= 50)
return DataQualityDecision.Quarantine;
return DataQualityDecision.Reject;
}
}
public enum DataQualityDecision
{
Accept,
AcceptWithWarning,
Quarantine,
Reject
}