feat(core): 데이터 수집 파이프라인 Core 인터페이스 추가 및 Makefile 생성
Quant Engine CI/CD Pipeline / validate-core (push) Failing after 11s
WBS-9.3 - NULL Policy CI Gate / NULL Policy Validation (push) Failing after 8s
Quant Engine CI/CD Pipeline / validate-ui-and-storage (push) Has been skipped
Deploy to Production / Build & Deploy to Production (push) Successful in 1m25s
Snapshot Admin Deployment / build-and-deploy (push) Failing after 1m3s
Quant Engine CI/CD Pipeline / validate-core (push) Failing after 11s
WBS-9.3 - NULL Policy CI Gate / NULL Policy Validation (push) Failing after 8s
Quant Engine CI/CD Pipeline / validate-ui-and-storage (push) Has been skipped
Deploy to Production / Build & Deploy to Production (push) Successful in 1m25s
Snapshot Admin Deployment / build-and-deploy (push) Failing after 1m3s
**Stage 2 (Python → .NET) 진행:** - ITokenCache.cs: KIS API 토큰 캐싱 추상화 - 기존 Python sqlite3 로직 → PostgreSQL 기반으로 마이그레이션 - GetCachedTokenAsync(), SaveTokenAsync(), ClearExpiredTokensAsync() - IDataCollectionStore.cs: 데이터 수집 저장소 추상화 계약 - Python data_collection_store_v1.py 계약 매핑 - UpsertRun/Snapshot/Error, Fetch 메서드 - CollectionRunRecord, CollectionSnapshotRecord, CollectionErrorRecord DTO - CollectionDashboardStateRecord 대시보드 상태 모델 - ICollectionRepository.cs: 웹 API용 데이터 수집 저장소 인터페이스 - 높은 수준의 추상화 (Dapper + PostgreSQL) - SaveRun, UpdateRunStatus, SaveSnapshot, SaveError - GetRecentRuns, GetRunSnapshots, GetRunErrors, GetDashboardState - GetLatestSnapshotsForTicker **Stage 3 (Node.js → .NET) 완료:** - Makefile: npm scripts를 make 타겟으로 변환 - ops:prepare, ops:validate, ops:data-collect 등 주요 작업 - dotnet:build, dotnet:run, dotnet:watch 개발 명령어 - 단계: Python 도구 호출 유지 (Phase 2 완료까지) **다음 단계:** - CollectionRepository PostgreSQL 구현체 (Dapper) - TokenCache PostgreSQL 구현체 - DataCollectionStore PostgreSQL 구현체 (필요시) - Program.cs DI 등록 - Web API Collection 엔드포인트 추가 Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
.PHONY: help ops:prepare ops:validate ops:build ops:data-collect ops:render ops:release ops:package full-gate
|
||||
|
||||
help:
|
||||
@echo "QuantEngine v0.1 — Operations CLI"
|
||||
@echo ""
|
||||
@echo "Core operations:"
|
||||
@echo " make ops:render — Render operational report from packet"
|
||||
@echo " make ops:validate — Validate release pipeline"
|
||||
@echo " make ops:release — Full release DAG"
|
||||
@echo " make ops:package — Package for deployment"
|
||||
@echo " make full-gate — Strict validation (all gates must PASS)"
|
||||
@echo ""
|
||||
@echo "Data operations:"
|
||||
@echo " make ops:prepare — Convert XLSX → JSON"
|
||||
@echo " make ops:data-collect — KIS data collection"
|
||||
@echo ""
|
||||
@echo "Development:"
|
||||
@echo " make dotnet:build — Build .NET projects"
|
||||
@echo " make dotnet:run — Run Web API (port 8788)"
|
||||
@echo " make dotnet:watch — Hot-reload API server"
|
||||
|
||||
ops:prepare:
|
||||
python tools/convert_xlsx_to_json.py
|
||||
|
||||
ops:validate:
|
||||
python tools/run_release_dag_v3.py --mode release
|
||||
|
||||
ops:build:
|
||||
python tools/build_bundle.py
|
||||
|
||||
ops:data-collect:
|
||||
python tools/run_kis_data_collection_v1.py --input-json GatherTradingData.json --sqlite-db src/quant_engine/kis_data_collection.db --output-json Temp/kis_data_collection_v1.json --kis-account real
|
||||
|
||||
ops:render:
|
||||
dotnet run --project src/dotnet/QuantEngine.Tools/QuantEngine.Tools.csproj -- report --packet=Temp/final_decision_packet_active.json --out=Temp/operational_report.json
|
||||
|
||||
ops:release:
|
||||
python tools/run_release_dag_v3.py --mode full
|
||||
|
||||
ops:package:
|
||||
python tools/refresh_trading_calendar.py && python tools/prepare_upload_zip.py --validation-mode release
|
||||
|
||||
full-gate:
|
||||
python tools/run_release_dag_v3.py --mode release --strict
|
||||
|
||||
dotnet:build:
|
||||
cd src/dotnet && dotnet build
|
||||
|
||||
dotnet:run:
|
||||
cd src/dotnet && dotnet run --project src/DataFeed.Api/QuantEngine.Web/QuantEngine.Web.csproj
|
||||
|
||||
dotnet:watch:
|
||||
cd src/dotnet && dotnet watch run --project src/QuantEngine.Web/QuantEngine.Web.csproj
|
||||
|
||||
dotnet:test:
|
||||
cd src/dotnet && dotnet test
|
||||
@@ -0,0 +1,56 @@
|
||||
namespace QuantEngine.Core.Interfaces;
|
||||
|
||||
/// <summary>
|
||||
/// Data collection repository (Dapper + PostgreSQL).
|
||||
/// Higher-level abstraction over IDataCollectionStore for Web API consumers.
|
||||
/// </summary>
|
||||
public interface ICollectionRepository
|
||||
{
|
||||
/// <summary>
|
||||
/// Save new collection run.
|
||||
/// </summary>
|
||||
Task SaveRunAsync(CollectionRunRecord run);
|
||||
|
||||
/// <summary>
|
||||
/// Update run with completion status.
|
||||
/// </summary>
|
||||
Task UpdateRunStatusAsync(string runId, string status, string? finishedAt = null, int? totalSnapshots = null, int? totalErrors = null);
|
||||
|
||||
/// <summary>
|
||||
/// Save collection snapshot.
|
||||
/// </summary>
|
||||
Task SaveSnapshotAsync(CollectionSnapshotRecord snapshot);
|
||||
|
||||
/// <summary>
|
||||
/// Save collection error.
|
||||
/// </summary>
|
||||
Task SaveErrorAsync(CollectionErrorRecord error);
|
||||
|
||||
/// <summary>
|
||||
/// Fetch recent collection runs for UI dashboard.
|
||||
/// </summary>
|
||||
/// <param name="limit">Number of runs to return (default: 20)</param>
|
||||
Task<List<CollectionRunRecord>> GetRecentRunsAsync(int limit = 20);
|
||||
|
||||
/// <summary>
|
||||
/// Fetch snapshots for a specific run.
|
||||
/// </summary>
|
||||
Task<List<CollectionSnapshotRecord>> GetRunSnapshotsAsync(string runId);
|
||||
|
||||
/// <summary>
|
||||
/// Fetch errors for a specific run.
|
||||
/// </summary>
|
||||
/// <param name="runId">Run ID</param>
|
||||
/// <param name="limit">Max errors to return (default: 50)</param>
|
||||
Task<List<CollectionErrorRecord>> GetRunErrorsAsync(string runId, int limit = 50);
|
||||
|
||||
/// <summary>
|
||||
/// Get collection pipeline dashboard state for Web UI.
|
||||
/// </summary>
|
||||
Task<CollectionDashboardStateRecord> GetDashboardStateAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Fetch latest snapshots for a ticker across all datasets.
|
||||
/// </summary>
|
||||
Task<List<CollectionSnapshotRecord>> GetLatestSnapshotsForTickerAsync(string ticker, int limit = 10);
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
namespace QuantEngine.Core.Interfaces;
|
||||
|
||||
/// <summary>
|
||||
/// Data collection storage abstraction layer.
|
||||
/// Maps Python data_collection_store_v1.py contracts to .NET.
|
||||
/// Supports PostgreSQL backend for production.
|
||||
/// </summary>
|
||||
public interface IDataCollectionStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Initialize storage tables and schema (idempotent).
|
||||
/// </summary>
|
||||
Task InitializeAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Insert or update collection run record.
|
||||
/// </summary>
|
||||
Task UpsertRunAsync(CollectionRunRecord run);
|
||||
|
||||
/// <summary>
|
||||
/// Insert collection snapshot record.
|
||||
/// </summary>
|
||||
Task UpsertSnapshotAsync(CollectionSnapshotRecord snapshot);
|
||||
|
||||
/// <summary>
|
||||
/// Append error record from collection attempt.
|
||||
/// </summary>
|
||||
Task AppendErrorAsync(CollectionErrorRecord error);
|
||||
|
||||
/// <summary>
|
||||
/// Fetch recent collection runs for dashboard.
|
||||
/// </summary>
|
||||
/// <param name="limit">Max number of runs to return</param>
|
||||
Task<List<CollectionRunRecord>> FetchRecentRunsAsync(int limit = 20);
|
||||
|
||||
/// <summary>
|
||||
/// Fetch latest snapshots for a ticker/dataset combination.
|
||||
/// </summary>
|
||||
Task<List<CollectionSnapshotRecord>> FetchLatestSnapshotsAsync(string ticker, string? datasetName = null, int limit = 10);
|
||||
|
||||
/// <summary>
|
||||
/// Get collection pipeline dashboard state.
|
||||
/// </summary>
|
||||
Task<CollectionDashboardStateRecord> GetDashboardStateAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Collection run record (maps Python CollectionRun).
|
||||
/// </summary>
|
||||
public record CollectionRunRecord(
|
||||
string RunId,
|
||||
string Status,
|
||||
string StartedAt,
|
||||
string? FinishedAt = null,
|
||||
int? TotalSnapshots = null,
|
||||
int? TotalErrors = null,
|
||||
string UpdatedAt = ""
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Collection snapshot record (maps Python CollectionSnapshot).
|
||||
/// </summary>
|
||||
public record CollectionSnapshotRecord(
|
||||
string RunId,
|
||||
string DatasetName,
|
||||
string Ticker,
|
||||
string SourceName,
|
||||
string PayloadJson,
|
||||
string CapturedAt,
|
||||
string CreatedAt = ""
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Collection error record (maps Python CollectionSourceError).
|
||||
/// </summary>
|
||||
public record CollectionErrorRecord(
|
||||
string RunId,
|
||||
string SourceName,
|
||||
string ErrorKind,
|
||||
string ErrorMessage,
|
||||
string Ticker = "",
|
||||
string CreatedAt = ""
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Dashboard state summary.
|
||||
/// </summary>
|
||||
public record CollectionDashboardStateRecord(
|
||||
string? LastRunId,
|
||||
string? LastRunStatus,
|
||||
string? LastFinishedAt,
|
||||
int TotalSnapshots,
|
||||
int TotalErrors,
|
||||
List<CollectionErrorRecord> RecentErrors
|
||||
);
|
||||
@@ -0,0 +1,28 @@
|
||||
namespace QuantEngine.Core.Interfaces;
|
||||
|
||||
/// <summary>
|
||||
/// Token caching for KIS API authentication.
|
||||
/// Replaces Python's sqlite3-based token storage with PostgreSQL.
|
||||
/// </summary>
|
||||
public interface ITokenCache
|
||||
{
|
||||
/// <summary>
|
||||
/// Retrieve cached access token if valid and not near expiration.
|
||||
/// </summary>
|
||||
/// <param name="account">Account type: "real" or "mock"</param>
|
||||
/// <returns>Access token if cached and valid; null if expired or missing</returns>
|
||||
Task<string?> GetCachedTokenAsync(string account);
|
||||
|
||||
/// <summary>
|
||||
/// Store access token with expiration time.
|
||||
/// </summary>
|
||||
/// <param name="account">Account type: "real" or "mock"</param>
|
||||
/// <param name="token">Access token</param>
|
||||
/// <param name="expiresAt">Token expiration time (UTC)</param>
|
||||
Task SaveTokenAsync(string account, string token, DateTime expiresAt);
|
||||
|
||||
/// <summary>
|
||||
/// Clear expired tokens and stale entries.
|
||||
/// </summary>
|
||||
Task ClearExpiredTokensAsync();
|
||||
}
|
||||
Reference in New Issue
Block a user