Merge pull request 'Workstream I: Implement VS-04 Audit Trail + GDPR' (#24) from feat/I-vs04-audit-trail into main
Reviewed-on: #24
This commit was merged in pull request #24.
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
-- Migration 0033: Market Data Import Logs (KRX, OpenDart, KIS)
|
||||
-- Purpose: Append-only audit trail for external API data imports with PIT tracking
|
||||
|
||||
-- ============================================================================
|
||||
-- MARKET_DATA SCHEMA: Import Audit & Evidence
|
||||
-- ============================================================================
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS market_data;
|
||||
|
||||
-- KRX OpenAPI import log (indices, stocks, sectors)
|
||||
CREATE TABLE IF NOT EXISTS market_data.krx_imports (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
import_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
row_count INT NOT NULL,
|
||||
checksum VARCHAR(256), -- SHA256 of imported data for deduplication
|
||||
status VARCHAR(50) NOT NULL, -- 'SUCCESS', 'FAILURE', 'PARTIAL'
|
||||
error_message TEXT,
|
||||
details JSONB, -- Event-specific metadata (endpoint, records_skipped, api_latency_ms)
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL,
|
||||
revision INT NOT NULL DEFAULT 1,
|
||||
CONSTRAINT krx_imports_status_check CHECK (status IN ('SUCCESS', 'FAILURE', 'PARTIAL'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_krx_imports_import_at ON market_data.krx_imports(import_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_krx_imports_status ON market_data.krx_imports(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_krx_imports_correlation_id ON market_data.krx_imports(correlation_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_krx_imports_published_at ON market_data.krx_imports(published_at);
|
||||
|
||||
-- OpenDart API import log (company disclosures, quarterly financials)
|
||||
CREATE TABLE IF NOT EXISTS market_data.opendart_imports (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
import_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
row_count INT NOT NULL,
|
||||
checksum VARCHAR(256), -- SHA256 of imported data for deduplication
|
||||
status VARCHAR(50) NOT NULL, -- 'SUCCESS', 'FAILURE', 'PARTIAL'
|
||||
error_message TEXT,
|
||||
details JSONB, -- Event-specific metadata (api_endpoint, query_params, quota_used)
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL,
|
||||
revision INT NOT NULL DEFAULT 1,
|
||||
CONSTRAINT opendart_imports_status_check CHECK (status IN ('SUCCESS', 'FAILURE', 'PARTIAL'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_opendart_imports_import_at ON market_data.opendart_imports(import_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_opendart_imports_status ON market_data.opendart_imports(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_opendart_imports_correlation_id ON market_data.opendart_imports(correlation_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_opendart_imports_published_at ON market_data.opendart_imports(published_at);
|
||||
|
||||
-- KIS API import log (trading orders, portfolio reconciliation)
|
||||
CREATE TABLE IF NOT EXISTS market_data.kis_imports (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
import_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
row_count INT NOT NULL,
|
||||
checksum VARCHAR(256), -- SHA256 of imported data for deduplication
|
||||
status VARCHAR(50) NOT NULL, -- 'SUCCESS', 'FAILURE', 'PARTIAL'
|
||||
error_message TEXT,
|
||||
details JSONB, -- Event-specific metadata (order_count, execution_latency_ms, token_refresh_required)
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL,
|
||||
revision INT NOT NULL DEFAULT 1,
|
||||
CONSTRAINT kis_imports_status_check CHECK (status IN ('SUCCESS', 'FAILURE', 'PARTIAL'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_imports_import_at ON market_data.kis_imports(import_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_imports_status ON market_data.kis_imports(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_imports_correlation_id ON market_data.kis_imports(correlation_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_imports_published_at ON market_data.kis_imports(published_at);
|
||||
|
||||
-- ============================================================================
|
||||
-- IMPORT ERROR CLASSIFICATION (for DQ quarantine & retry logic)
|
||||
-- ============================================================================
|
||||
|
||||
-- Error classification for transient vs permanent failures
|
||||
CREATE TABLE IF NOT EXISTS market_data.import_error_classification (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
import_id UUID NOT NULL, -- References one of krx/opendart/kis_imports
|
||||
api_name VARCHAR(50) NOT NULL, -- 'krx', 'opendart', 'kis'
|
||||
error_type VARCHAR(100) NOT NULL, -- e.g., 'TIMEOUT', 'RATE_LIMIT', 'INVALID_SCHEMA', 'AUTHENTICATION_FAILED'
|
||||
classification VARCHAR(50) NOT NULL, -- 'TRANSIENT', 'PERMANENT', 'DATA_QUALITY'
|
||||
retry_eligible BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
escalation_required BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_import_error_classification_api ON market_data.import_error_classification(api_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_import_error_classification_error_type ON market_data.import_error_classification(error_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_import_error_classification_retry_eligible ON market_data.import_error_classification(retry_eligible);
|
||||
|
||||
-- ============================================================================
|
||||
-- IMPORT SLA TRACKING (for compliance & monitoring)
|
||||
-- ============================================================================
|
||||
|
||||
-- Daily SLA target: import should complete within 4 hours of market close (16:30 KST)
|
||||
-- Target window: 16:30-20:30 KST
|
||||
CREATE TABLE IF NOT EXISTS market_data.import_sla_tracking (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
api_name VARCHAR(50) NOT NULL, -- 'krx', 'opendart', 'kis'
|
||||
import_date DATE NOT NULL,
|
||||
scheduled_at TIMESTAMP WITH TIME ZONE NOT NULL,
|
||||
started_at TIMESTAMP WITH TIME ZONE,
|
||||
completed_at TIMESTAMP WITH TIME ZONE,
|
||||
duration_seconds INT,
|
||||
sla_met BOOLEAN, -- True if completed within 4 hours of market close
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
correlation_id UUID NOT NULL,
|
||||
UNIQUE(api_name, import_date)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_import_sla_tracking_api ON market_data.import_sla_tracking(api_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_import_sla_tracking_import_date ON market_data.import_sla_tracking(import_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_import_sla_tracking_sla_met ON market_data.import_sla_tracking(sla_met);
|
||||
|
||||
-- Last Known Good (LKG) cache for fallback
|
||||
CREATE TABLE IF NOT EXISTS market_data.lkg_cache (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
api_name VARCHAR(50) NOT NULL, -- 'krx', 'opendart', 'kis'
|
||||
cache_date DATE NOT NULL,
|
||||
data_snapshot JSONB NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
published_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(api_name, cache_date)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_lkg_cache_api ON market_data.lkg_cache(api_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_lkg_cache_date ON market_data.lkg_cache(cache_date);
|
||||
|
||||
-- Permissions: schema owned by executing role
|
||||
-- In production, add explicit GRANT via separate admin script after schema creation
|
||||
@@ -0,0 +1,84 @@
|
||||
-- Workstream I: VS-04 Audit Trail (Immutable events + GDPR compliance)
|
||||
-- Creates compliance audit trail for model operations, regulatory reporting, and GDPR redaction
|
||||
|
||||
-- Audit events (immutable, INSERT-only)
|
||||
CREATE TABLE IF NOT EXISTS compliance.audit_events (
|
||||
id UUID PRIMARY KEY,
|
||||
event_type VARCHAR(100) NOT NULL, -- MODEL_CREATED, APPROVAL_PROPOSED, APPROVAL_APPROVED, MODEL_ACTIVATED, SELL_DECISION_MADE, SELL_EXECUTED, BACKTEST_COMPLETED, DATA_CORRECTION, etc.
|
||||
entity_type VARCHAR(50) NOT NULL, -- MODEL, APPROVAL, SELL_DECISION, TRADE_EXECUTION
|
||||
entity_id UUID NOT NULL,
|
||||
actor_email VARCHAR(255) NOT NULL,
|
||||
actor_role VARCHAR(50), -- MAKER, CHECKER, SRE, SYSTEM
|
||||
event_at TIMESTAMPTZ NOT NULL,
|
||||
result VARCHAR(50) NOT NULL, -- SUCCESS, FAILURE, PARTIAL
|
||||
error_message TEXT,
|
||||
details JSONB, -- Event-specific metadata
|
||||
evidence_links TEXT[], -- S3 artifact URLs (PBO scores, OOS returns, backtest reports)
|
||||
ip_address INET, -- Source IP for forensics
|
||||
user_agent TEXT, -- Client identifier
|
||||
published_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
correlation_id UUID NOT NULL, -- Links related events
|
||||
revision INT NOT NULL DEFAULT 1
|
||||
);
|
||||
|
||||
-- Indexes for compliance querying
|
||||
CREATE INDEX idx_audit_events_entity_id ON compliance.audit_events(entity_id);
|
||||
CREATE INDEX idx_audit_events_event_type ON compliance.audit_events(event_type);
|
||||
CREATE INDEX idx_audit_events_actor_email ON compliance.audit_events(actor_email);
|
||||
CREATE INDEX idx_audit_events_event_at ON compliance.audit_events(event_at);
|
||||
CREATE INDEX idx_audit_events_correlation_id ON compliance.audit_events(correlation_id);
|
||||
|
||||
-- GDPR retention tracking (personal data retention policy)
|
||||
CREATE TABLE IF NOT EXISTS compliance.gdpr_retention (
|
||||
id UUID PRIMARY KEY,
|
||||
event_id UUID NOT NULL REFERENCES compliance.audit_events(id),
|
||||
customer_id UUID, -- Links to personal data
|
||||
data_categories VARCHAR(50)[], -- PII, EMAIL, TRADING_HISTORY, PORTFOLIO_DATA, etc.
|
||||
retention_ends_at DATE, -- When to purge
|
||||
purge_status VARCHAR(50) NOT NULL DEFAULT 'PENDING', -- PENDING, PURGED, EXCEPTION
|
||||
purged_at TIMESTAMPTZ,
|
||||
exception_reason TEXT,
|
||||
published_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
revision INT NOT NULL DEFAULT 1
|
||||
);
|
||||
|
||||
-- Indexes for GDPR processing
|
||||
CREATE INDEX idx_gdpr_retention_customer_id ON compliance.gdpr_retention(customer_id);
|
||||
CREATE INDEX idx_gdpr_retention_purge_status ON compliance.gdpr_retention(purge_status);
|
||||
|
||||
-- Event types enumeration (reference, not enforced at DB level)
|
||||
CREATE TABLE IF NOT EXISTS compliance.audit_event_types (
|
||||
event_type VARCHAR(100) PRIMARY KEY,
|
||||
description TEXT,
|
||||
entity_type VARCHAR(50), -- MODEL, APPROVAL, SELL_DECISION, TRADE_EXECUTION
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Seed event types
|
||||
INSERT INTO compliance.audit_event_types (event_type, description, entity_type) VALUES
|
||||
('MODEL_CREATED', 'New model version created', 'MODEL'),
|
||||
('MODEL_ARCHIVED', 'Model retired from use', 'MODEL'),
|
||||
('APPROVAL_PROPOSED', 'Maker submitted activation proposal', 'APPROVAL'),
|
||||
('APPROVAL_APPROVED', 'Checker approved proposal', 'APPROVAL'),
|
||||
('APPROVAL_REJECTED', 'Checker rejected proposal', 'APPROVAL'),
|
||||
('MODEL_ACTIVATED', 'SRE activated model in production', 'MODEL'),
|
||||
('MODEL_DEACTIVATED', 'SRE deactivated model', 'MODEL'),
|
||||
('SELL_DECISION_MADE', 'Signal engine generated sell signal', 'SELL_DECISION'),
|
||||
('SELL_EXECUTED', 'Trade executed based on signal', 'TRADE_EXECUTION'),
|
||||
('BACKTEST_COMPLETED', 'Shadow run/backtest finished', 'MODEL'),
|
||||
('DATA_CORRECTION', 'Source data corrected retroactively', 'MODEL'),
|
||||
('COMPLIANCE_AUDIT', 'Auditor reviewed trail', 'MODEL')
|
||||
ON CONFLICT (event_type) DO NOTHING;
|
||||
|
||||
-- Schema ownership
|
||||
ALTER TABLE compliance.audit_events OWNER TO kartsell;
|
||||
ALTER TABLE compliance.gdpr_retention OWNER TO kartsell;
|
||||
ALTER TABLE compliance.audit_event_types OWNER TO kartsell;
|
||||
|
||||
-- Immutability constraints (enforced via code, not DB triggers)
|
||||
-- INSERT-only: no UPDATE, no DELETE permitted on audit_events
|
||||
-- Timestamps: immutable after insertion (enforced in application layer)
|
||||
-- Correlation_id: immutable for traceability
|
||||
|
||||
-- 7-year retention policy (FSS requirement)
|
||||
-- retention_ends_at defaults to now() + 7 years (enforced in application)
|
||||
Reference in New Issue
Block a user