fix: migration ordering bug that could block fresh-DB deploys
V004_normalize_snapshots_schema.sql had an unguarded FK to a table V2
creates. Under DbUp's default ordinal filename sort, the zero-padded
"V003_"/"V004_" migrations sorted before "V1__", so on a brand-new
database V004 would hard-fail on that FK and abort every migration
after it - V1 through V8 would never run. Confirmed via production
that neither V003 nor V004 had ever actually applied.
Fix: renamed them to V9__/V10__ and added MigrationScriptNameComparer,
which sorts DbUp scripts by numeric V{n} value instead of raw string
order, so double-digit versions can never again sort ahead of earlier
single-digit ones. Added regression tests for both the fixed case and
the original bug shape.
Also updates docs/db/quantengine.dbml (renamed migrations, and closes
out the engine_history/quantengine table-name-collision question -
both schemas are live, backing different code paths, not duplicates)
and CLAUDE.md (migration ordering fix, KIS/OpenDART/KRX credential
env-var-name reference, including a CI secret/env-var name mismatch
found for OpenDART that still needs a decision).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,288 @@
|
||||
-- Migration: V004_normalize_snapshots_schema.sql
|
||||
-- Purpose: Implement 3NF normalization for kis_collection_snapshots
|
||||
-- Phase: Phase 1 (Normalization & SOLID Refactoring)
|
||||
-- Status: APPROVED for Sep 2026 implementation
|
||||
-- Safety: Parallel operation with existing schema via Adapter pattern
|
||||
|
||||
-- ============================================================================
|
||||
-- DIMENSION TABLES (Star Schema)
|
||||
-- ============================================================================
|
||||
|
||||
-- Dimension: Stocks (Reference data)
|
||||
CREATE TABLE IF NOT EXISTS quantengine.stocks (
|
||||
id SERIAL PRIMARY KEY,
|
||||
ticker VARCHAR(10) UNIQUE NOT NULL,
|
||||
name VARCHAR(255),
|
||||
sector VARCHAR(50),
|
||||
market VARCHAR(20), -- 'KOSPI', 'KOSDAQ', etc.
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_stocks_ticker ON quantengine.stocks(ticker);
|
||||
CREATE INDEX IF NOT EXISTS idx_stocks_sector ON quantengine.stocks(sector);
|
||||
|
||||
-- Dimension: Sources (Data provider priority)
|
||||
CREATE TABLE IF NOT EXISTS quantengine.sources (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(50) UNIQUE NOT NULL,
|
||||
priority INT NOT NULL, -- 1=highest (primary), 2=secondary (fallback), etc.
|
||||
fallback_to_id INT REFERENCES quantengine.sources(id), -- Next source if this fails
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Bootstrap sources (KIS collection pipeline fallback chain)
|
||||
INSERT INTO quantengine.sources (name, priority, fallback_to_id) VALUES
|
||||
('KIS', 1, NULL), -- KIS is primary, no fallback
|
||||
('Naver', 2, NULL), -- Fallback 1: Naver Finance
|
||||
('Yahoo', 3, NULL), -- Fallback 2: Yahoo Finance
|
||||
('OpenDART', 4, NULL) -- Fallback 3: OpenDART (Korea FSS)
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- ============================================================================
|
||||
-- FACT TABLE (Normalized Market Data)
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS quantengine.market_data (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
stock_id INT NOT NULL REFERENCES quantengine.stocks(id),
|
||||
source_id INT NOT NULL REFERENCES quantengine.sources(id),
|
||||
|
||||
-- Price data
|
||||
price DECIMAL NOT NULL,
|
||||
bid DECIMAL,
|
||||
ask DECIMAL,
|
||||
volume BIGINT,
|
||||
|
||||
-- Metadata
|
||||
collected_at TIMESTAMPTZ NOT NULL, -- When data was collected (from KIS)
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
-- Audit
|
||||
collection_run_id UUID, -- Link to kis_collection_runs for traceability
|
||||
|
||||
CONSTRAINT chk_price_range CHECK (price > 0),
|
||||
CONSTRAINT chk_bid_ask CHECK (bid IS NULL OR ask IS NULL OR bid <= ask),
|
||||
CONSTRAINT chk_bid_ask_price CHECK (
|
||||
(bid IS NULL AND ask IS NULL) OR
|
||||
(bid IS NOT NULL AND ask IS NOT NULL AND bid <= price AND price <= ask)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_data_stock_collected
|
||||
ON quantengine.market_data(stock_id, collected_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_data_collected
|
||||
ON quantengine.market_data(collected_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_data_source
|
||||
ON quantengine.market_data(source_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_data_run_id
|
||||
ON quantengine.market_data(collection_run_id);
|
||||
|
||||
-- ============================================================================
|
||||
-- NORMALIZED kis_collection_snapshots (Restructured)
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS quantengine.kis_collection_snapshots_v2 (
|
||||
id UUID PRIMARY KEY,
|
||||
run_id UUID NOT NULL REFERENCES quantengine.kis_collection_runs(id) ON DELETE CASCADE,
|
||||
stock_id INT NOT NULL REFERENCES quantengine.stocks(id),
|
||||
market_data_id BIGINT REFERENCES quantengine.market_data(id), -- Denormalized for query perf
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_snapshots_v2_run_id
|
||||
ON quantengine.kis_collection_snapshots_v2(run_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_snapshots_v2_stock_id
|
||||
ON quantengine.kis_collection_snapshots_v2(stock_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_snapshots_v2_created_at
|
||||
ON quantengine.kis_collection_snapshots_v2(created_at DESC);
|
||||
|
||||
-- ============================================================================
|
||||
-- DATA MIGRATION VIEW (for validation)
|
||||
-- ============================================================================
|
||||
|
||||
-- View to compare old vs new schema during migration
|
||||
CREATE OR REPLACE VIEW quantengine.v_snapshot_migration_comparison AS
|
||||
SELECT
|
||||
-- Old schema
|
||||
old_snap.id as old_id,
|
||||
old_snap.ticker as old_ticker,
|
||||
old_snap.price as old_price,
|
||||
old_snap.bid as old_bid,
|
||||
old_snap.ask as old_ask,
|
||||
old_snap.volume as old_volume,
|
||||
|
||||
-- New schema
|
||||
new_snap.id as new_id,
|
||||
stocks.ticker as new_ticker,
|
||||
md.price as new_price,
|
||||
md.bid as new_bid,
|
||||
md.ask as new_ask,
|
||||
md.volume as new_volume,
|
||||
|
||||
-- Comparison
|
||||
CASE
|
||||
WHEN old_snap.ticker IS NULL THEN 'MISSING_IN_OLD'
|
||||
WHEN new_snap.id IS NULL THEN 'MISSING_IN_NEW'
|
||||
WHEN old_snap.price <> md.price OR
|
||||
COALESCE(old_snap.bid, 0) <> COALESCE(md.bid, 0) OR
|
||||
COALESCE(old_snap.ask, 0) <> COALESCE(md.ask, 0) THEN 'DATA_MISMATCH'
|
||||
ELSE 'OK'
|
||||
END as migration_status
|
||||
FROM quantengine.kis_collection_snapshots old_snap
|
||||
FULL OUTER JOIN quantengine.kis_collection_snapshots_v2 new_snap
|
||||
ON old_snap.id = new_snap.id
|
||||
LEFT JOIN quantengine.stocks stocks ON new_snap.stock_id = stocks.id
|
||||
LEFT JOIN quantengine.market_data md ON new_snap.market_data_id = md.id;
|
||||
|
||||
-- ============================================================================
|
||||
-- MIGRATION AUDIT VIEW
|
||||
-- ============================================================================
|
||||
|
||||
CREATE OR REPLACE VIEW quantengine.v_migration_statistics AS
|
||||
SELECT
|
||||
COUNT(*) as total_old_snapshots,
|
||||
COUNT(new_snap.id) as total_new_snapshots,
|
||||
COUNT(CASE WHEN migration_status = 'OK' THEN 1 END) as verified_records,
|
||||
COUNT(CASE WHEN migration_status = 'DATA_MISMATCH' THEN 1 END) as mismatches,
|
||||
COUNT(CASE WHEN migration_status = 'MISSING_IN_NEW' THEN 1 END) as missing_new,
|
||||
ROUND(100.0 * COUNT(CASE WHEN migration_status = 'OK' THEN 1 END) /
|
||||
NULLIF(COUNT(*), 0), 2) as verification_pct
|
||||
FROM quantengine.v_snapshot_migration_comparison;
|
||||
|
||||
-- ============================================================================
|
||||
-- MIGRATION VALIDATION QUERIES (Post-Deployment)
|
||||
-- ============================================================================
|
||||
|
||||
-- 1. Verify table creation
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema='quantengine' AND table_name='stocks') THEN
|
||||
RAISE EXCEPTION 'stocks table not created';
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema='quantengine' AND table_name='sources') THEN
|
||||
RAISE EXCEPTION 'sources table not created';
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema='quantengine' AND table_name='market_data') THEN
|
||||
RAISE EXCEPTION 'market_data table not created';
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema='quantengine' AND table_name='kis_collection_snapshots_v2') THEN
|
||||
RAISE EXCEPTION 'kis_collection_snapshots_v2 table not created';
|
||||
END IF;
|
||||
|
||||
RAISE NOTICE 'All normalization tables created successfully';
|
||||
END $$;
|
||||
|
||||
-- 2. Verify indexes
|
||||
DO $$
|
||||
DECLARE
|
||||
v_index_count INT;
|
||||
BEGIN
|
||||
SELECT COUNT(*) INTO v_index_count
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = 'quantengine'
|
||||
AND tablename IN ('stocks', 'market_data', 'kis_collection_snapshots_v2');
|
||||
|
||||
IF v_index_count < 6 THEN
|
||||
RAISE WARNING 'Expected 6+ indexes on normalization tables, found %', v_index_count;
|
||||
ELSE
|
||||
RAISE NOTICE 'All normalization indexes created successfully (count: %)', v_index_count;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- 3. Verify constraints
|
||||
DO $$
|
||||
DECLARE
|
||||
v_constraint_count INT;
|
||||
BEGIN
|
||||
SELECT COUNT(*) INTO v_constraint_count
|
||||
FROM information_schema.table_constraints
|
||||
WHERE table_schema = 'quantengine'
|
||||
AND table_name IN ('stocks', 'market_data', 'kis_collection_snapshots_v2')
|
||||
AND constraint_type IN ('PRIMARY KEY', 'FOREIGN KEY', 'UNIQUE', 'CHECK');
|
||||
|
||||
RAISE NOTICE 'Normalization constraints created (count: %)', v_constraint_count;
|
||||
END $$;
|
||||
|
||||
-- ============================================================================
|
||||
-- ROLLBACK SCRIPT (if migration must be reversed)
|
||||
-- ============================================================================
|
||||
|
||||
/*
|
||||
-- To rollback this migration:
|
||||
|
||||
-- 1. Drop views
|
||||
DROP VIEW IF EXISTS quantengine.v_migration_statistics;
|
||||
DROP VIEW IF EXISTS quantengine.v_snapshot_migration_comparison;
|
||||
|
||||
-- 2. Drop new tables (preserves data in backup)
|
||||
ALTER TABLE quantengine.kis_collection_snapshots_v2 DROP CONSTRAINT
|
||||
IF EXISTS fk_kis_snapshots_v2_run_id;
|
||||
DROP TABLE IF EXISTS quantengine.kis_collection_snapshots_v2;
|
||||
DROP TABLE IF EXISTS quantengine.market_data;
|
||||
|
||||
-- 3. Drop dimension tables
|
||||
DELETE FROM quantengine.sources WHERE name IN ('KIS', 'Naver', 'Yahoo', 'OpenDART');
|
||||
DROP TABLE IF EXISTS quantengine.sources;
|
||||
DROP TABLE IF EXISTS quantengine.stocks;
|
||||
|
||||
-- 4. Restore Adapter to use legacy schema
|
||||
-- Update Program.cs: builder.AddScoped<ISnapshotRepository, LegacySnapshotRepository>();
|
||||
|
||||
-- Estimated time: 2-3 minutes (depends on data volume)
|
||||
*/
|
||||
|
||||
-- ============================================================================
|
||||
-- MIGRATION NOTES
|
||||
-- ============================================================================
|
||||
|
||||
/*
|
||||
OBJECTIVES:
|
||||
1. Normalize kis_collection_snapshots to 3NF
|
||||
2. Separate concerns: stocks (dimension), market_data (fact), sources (dimension)
|
||||
3. Maintain backward compatibility via Adapter pattern
|
||||
|
||||
NORMALIZATION RATIONALE:
|
||||
- OLD: kis_collection_snapshots contains ticker (denormalized)
|
||||
Problem: ticker appears in many rows → data redundancy
|
||||
|
||||
- NEW: Separate stocks dimension table
|
||||
Benefit: Single source of truth for ticker metadata
|
||||
Cost: One JOIN per query
|
||||
|
||||
DENORMALIZATION:
|
||||
- kis_collection_snapshots_v2 includes market_data_id reference
|
||||
Rationale: Avoid full table scan when reading snapshots
|
||||
Trade-off: +3% storage for -40% query time
|
||||
|
||||
PERFORMANCE EXPECTATIONS:
|
||||
- Query old schema: ~45ms (sequential scan, 100k rows)
|
||||
- Query new schema: ~38ms (index scan, joins optimized)
|
||||
- Improvement: +16% faster
|
||||
|
||||
AUDIT TRAIL:
|
||||
- kis_collection_runs_audit (existing, unchanged)
|
||||
- kis_collection_snapshots_audit (existing, unchanged)
|
||||
- market_data has no separate audit (joins with snapshots_audit)
|
||||
- All changes tracked via kis_collection_snapshots_v2 creation
|
||||
|
||||
ADAPTER PATTERN:
|
||||
- ISnapshotRepository interface (unchanged)
|
||||
- LegacySnapshotRepository: SELECT * FROM kis_collection_snapshots
|
||||
- NormalizedSnapshotRepository: JOIN stocks, market_data FROM kis_collection_snapshots_v2
|
||||
- DI: builder.AddScoped<ISnapshotRepository, NormalizedSnapshotRepository>();
|
||||
- Runtime switch: Easy rollback if performance regresses
|
||||
*/
|
||||
Reference in New Issue
Block a user