Workstream G: Implement AEG-X-009 P1-P6 (KRX/OpenDart/KIS API integration)
- P1: KRX OpenAPI service (indices, stocks, OHLCV data) - P2: OpenDart API service (company disclosures, quarterly financials) - P3: KIS API service (trading orders, portfolio holdings) - P4-P6: Daily scheduling, error classification, SLA tracking, LKG fallback - Schema: market_data schema with append-only import logs - Error handling: transient/permanent classification + exponential backoff - Idempotency: correlation_id deduplication for safe replay - Services: 3 independent data services with caching, retry logic - Handler: Centralized import orchestration with logging - Job: Hangfire daily scheduler (q-evaluation queue, 16:30-20:30 KST window) - Tests: Unit & integration scenarios for import execution - AGENTS.md v16.0 13/13 compliance ✅ Closes workstream G (Phase 2 preparation). Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
-- Migration 0033: Market Data Import Logs (KRX, OpenDart, KIS)
|
||||
-- Purpose: Append-only audit trail for external API data imports with PIT tracking
|
||||
|
||||
-- ============================================================================
|
||||
-- MARKET_DATA SCHEMA: Import Audit & Evidence
|
||||
-- ============================================================================
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS market_data;
|
||||
|
||||
-- KRX OpenAPI import log (indices, stocks, sectors)
|
||||
CREATE TABLE IF NOT EXISTS market_data.krx_imports (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
import_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
row_count INT NOT NULL,
|
||||
checksum VARCHAR(256), -- SHA256 of imported data for deduplication
|
||||
status VARCHAR(50) NOT NULL, -- 'SUCCESS', 'FAILURE', 'PARTIAL'
|
||||
error_message TEXT,
|
||||
details JSONB, -- Event-specific metadata (endpoint, records_skipped, api_latency_ms)
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL,
|
||||
revision INT NOT NULL DEFAULT 1,
|
||||
CONSTRAINT krx_imports_status_check CHECK (status IN ('SUCCESS', 'FAILURE', 'PARTIAL'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_krx_imports_import_at ON market_data.krx_imports(import_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_krx_imports_status ON market_data.krx_imports(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_krx_imports_correlation_id ON market_data.krx_imports(correlation_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_krx_imports_published_at ON market_data.krx_imports(published_at);
|
||||
|
||||
-- OpenDart API import log (company disclosures, quarterly financials)
|
||||
CREATE TABLE IF NOT EXISTS market_data.opendart_imports (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
import_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
row_count INT NOT NULL,
|
||||
checksum VARCHAR(256), -- SHA256 of imported data for deduplication
|
||||
status VARCHAR(50) NOT NULL, -- 'SUCCESS', 'FAILURE', 'PARTIAL'
|
||||
error_message TEXT,
|
||||
details JSONB, -- Event-specific metadata (api_endpoint, query_params, quota_used)
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL,
|
||||
revision INT NOT NULL DEFAULT 1,
|
||||
CONSTRAINT opendart_imports_status_check CHECK (status IN ('SUCCESS', 'FAILURE', 'PARTIAL'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_opendart_imports_import_at ON market_data.opendart_imports(import_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_opendart_imports_status ON market_data.opendart_imports(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_opendart_imports_correlation_id ON market_data.opendart_imports(correlation_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_opendart_imports_published_at ON market_data.opendart_imports(published_at);
|
||||
|
||||
-- KIS API import log (trading orders, portfolio reconciliation)
|
||||
CREATE TABLE IF NOT EXISTS market_data.kis_imports (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
import_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
row_count INT NOT NULL,
|
||||
checksum VARCHAR(256), -- SHA256 of imported data for deduplication
|
||||
status VARCHAR(50) NOT NULL, -- 'SUCCESS', 'FAILURE', 'PARTIAL'
|
||||
error_message TEXT,
|
||||
details JSONB, -- Event-specific metadata (order_count, execution_latency_ms, token_refresh_required)
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL,
|
||||
revision INT NOT NULL DEFAULT 1,
|
||||
CONSTRAINT kis_imports_status_check CHECK (status IN ('SUCCESS', 'FAILURE', 'PARTIAL'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_imports_import_at ON market_data.kis_imports(import_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_imports_status ON market_data.kis_imports(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_imports_correlation_id ON market_data.kis_imports(correlation_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_imports_published_at ON market_data.kis_imports(published_at);
|
||||
|
||||
-- ============================================================================
|
||||
-- IMPORT ERROR CLASSIFICATION (for DQ quarantine & retry logic)
|
||||
-- ============================================================================
|
||||
|
||||
-- Error classification for transient vs permanent failures
|
||||
CREATE TABLE IF NOT EXISTS market_data.import_error_classification (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
import_id UUID NOT NULL, -- References one of krx/opendart/kis_imports
|
||||
api_name VARCHAR(50) NOT NULL, -- 'krx', 'opendart', 'kis'
|
||||
error_type VARCHAR(100) NOT NULL, -- e.g., 'TIMEOUT', 'RATE_LIMIT', 'INVALID_SCHEMA', 'AUTHENTICATION_FAILED'
|
||||
classification VARCHAR(50) NOT NULL, -- 'TRANSIENT', 'PERMANENT', 'DATA_QUALITY'
|
||||
retry_eligible BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
escalation_required BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_import_error_classification_api ON market_data.import_error_classification(api_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_import_error_classification_error_type ON market_data.import_error_classification(error_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_import_error_classification_retry_eligible ON market_data.import_error_classification(retry_eligible);
|
||||
|
||||
-- ============================================================================
|
||||
-- IMPORT SLA TRACKING (for compliance & monitoring)
|
||||
-- ============================================================================
|
||||
|
||||
-- Daily SLA target: import should complete within 4 hours of market close (16:30 KST)
|
||||
-- Target window: 16:30-20:30 KST
|
||||
CREATE TABLE IF NOT EXISTS market_data.import_sla_tracking (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
api_name VARCHAR(50) NOT NULL, -- 'krx', 'opendart', 'kis'
|
||||
import_date DATE NOT NULL,
|
||||
scheduled_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
started_at TIMESTAMP WITH TIME ZONE,
|
||||
completed_at TIMESTAMP WITH TIME ZONE,
|
||||
duration_seconds INT,
|
||||
sla_met BOOLEAN, -- True if completed within 4 hours of market close
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL,
|
||||
UNIQUE(api_name, import_date)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_import_sla_tracking_api ON market_data.import_sla_tracking(api_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_import_sla_tracking_import_date ON market_data.import_sla_tracking(import_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_import_sla_tracking_sla_met ON market_data.import_sla_tracking(sla_met);
|
||||
|
||||
-- Last Known Good (LKG) cache for fallback
|
||||
CREATE TABLE IF NOT EXISTS market_data.lkg_cache (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
api_name VARCHAR(50) NOT NULL, -- 'krx', 'opendart', 'kis'
|
||||
cache_date DATE NOT NULL,
|
||||
data_snapshot JSONB NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(api_name, cache_date)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_lkg_cache_api ON market_data.lkg_cache(api_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_lkg_cache_date ON market_data.lkg_cache(cache_date);
|
||||
|
||||
-- Permissions: schema owned by executing role
|
||||
-- In production, add explicit GRANT via separate admin script after schema creation
|
||||
@@ -0,0 +1,57 @@
|
||||
-- Approval Workflow Schema (VS-03)
|
||||
-- Maker-Checker approval gates for model activation
|
||||
-- INSERT-only, PIT-tracked with correlation_id
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS model_operations;
|
||||
|
||||
-- Approval proposals (DRAFT → PROPOSED → APPROVED → ACTIVE)
|
||||
CREATE TABLE IF NOT EXISTS model_operations.approval_proposals (
|
||||
id UUID PRIMARY KEY,
|
||||
model_id UUID NOT NULL REFERENCES model_operations.models(id),
|
||||
status VARCHAR(50) NOT NULL,
|
||||
created_by VARCHAR(255) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
justification TEXT NOT NULL,
|
||||
effective_at DATE NOT NULL,
|
||||
proposed_at TIMESTAMPTZ,
|
||||
approved_by VARCHAR(255),
|
||||
approved_at TIMESTAMPTZ,
|
||||
approval_notes TEXT,
|
||||
activated_by VARCHAR(255),
|
||||
activated_at TIMESTAMPTZ,
|
||||
published_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
revision INT NOT NULL DEFAULT 1,
|
||||
correlation_id UUID NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_proposals_model_id ON model_operations.approval_proposals(model_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_proposals_status ON model_operations.approval_proposals(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_proposals_created_by ON model_operations.approval_proposals(created_by);
|
||||
|
||||
-- Approval evidence (PBO/DSR/OOS artifacts)
|
||||
CREATE TABLE IF NOT EXISTS model_operations.approval_evidence (
|
||||
id UUID PRIMARY KEY,
|
||||
approval_proposal_id UUID NOT NULL REFERENCES model_operations.approval_proposals(id),
|
||||
evidence_type VARCHAR(50) NOT NULL,
|
||||
evidence_url TEXT NOT NULL,
|
||||
reviewer_comment TEXT,
|
||||
published_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
correlation_id UUID NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_evidence_proposal_id ON model_operations.approval_evidence(approval_proposal_id);
|
||||
|
||||
-- Approval events (audit trail)
|
||||
CREATE TABLE IF NOT EXISTS model_operations.approval_events (
|
||||
id UUID PRIMARY KEY,
|
||||
approval_proposal_id UUID NOT NULL REFERENCES model_operations.approval_proposals(id),
|
||||
event_type VARCHAR(50) NOT NULL,
|
||||
actor_email VARCHAR(255) NOT NULL,
|
||||
event_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
details JSONB,
|
||||
published_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
correlation_id UUID NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_events_proposal_id ON model_operations.approval_events(approval_proposal_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_approval_events_type ON model_operations.approval_events(event_type);
|
||||
Reference in New Issue
Block a user