fix: migration ordering bug that could block fresh-DB deploys
V004_normalize_snapshots_schema.sql had an unguarded FK to a table V2
creates. Under DbUp's default ordinal filename sort, the zero-padded
"V003_"/"V004_" migrations sorted before "V1__", so on a brand-new
database V004 would hard-fail on that FK and abort every migration
after it - V1 through V8 would never run. Confirmed via production
that neither V003 nor V004 had ever actually applied.
Fix: renamed them to V9__/V10__ and added MigrationScriptNameComparer,
which sorts DbUp scripts by numeric V{n} value instead of raw string
order, so double-digit versions can never again sort ahead of earlier
single-digit ones. Added regression tests for both the fixed case and
the original bug shape.
Also updates docs/db/quantengine.dbml (renamed migrations, and closes
out the engine_history/quantengine table-name-collision question -
both schemas are live, backing different code paths, not duplicates)
and CLAUDE.md (migration ordering fix, KIS/OpenDART/KRX credential
env-var-name reference, including a CI secret/env-var name mismatch
found for OpenDART that still needs a decision).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,328 @@
|
||||
-- Migration: V003_add_audit_trail_tables.sql
|
||||
-- Purpose: Add audit trail tables for tracking all data changes
|
||||
-- Date: 2026-07-24
|
||||
-- Status: APPROVED for Phase 0 implementation
|
||||
|
||||
-- ============================================================================
|
||||
-- kis_collection_runs_audit: Audit trail for collection runs
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS quantengine.kis_collection_runs_audit (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
run_id UUID NOT NULL,
|
||||
|
||||
-- Change metadata
|
||||
action VARCHAR(10) NOT NULL CHECK (action IN ('INSERT', 'UPDATE', 'DELETE')),
|
||||
changed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
changed_by VARCHAR(256) DEFAULT CURRENT_USER,
|
||||
change_reason TEXT,
|
||||
|
||||
-- Data snapshots (before/after)
|
||||
old_values JSONB,
|
||||
new_values JSONB,
|
||||
|
||||
-- Audit trail indexing
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
-- Foreign key constraint (optional - don't enforce if kis_collection_runs might be deleted)
|
||||
-- CONSTRAINT fk_kis_collection_runs_audit FOREIGN KEY (run_id)
|
||||
-- REFERENCES quantengine.kis_collection_runs(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_collection_runs_audit_run_id
|
||||
ON quantengine.kis_collection_runs_audit (run_id, changed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_collection_runs_audit_changed_by
|
||||
ON quantengine.kis_collection_runs_audit (changed_by, changed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_collection_runs_audit_timestamp
|
||||
ON quantengine.kis_collection_runs_audit (changed_at DESC);
|
||||
|
||||
-- ============================================================================
|
||||
-- kis_collection_snapshots_audit: Audit trail for snapshots
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS quantengine.kis_collection_snapshots_audit (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
snapshot_id UUID NOT NULL,
|
||||
|
||||
-- Change metadata
|
||||
action VARCHAR(10) NOT NULL CHECK (action IN ('INSERT', 'UPDATE', 'DELETE')),
|
||||
changed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
changed_by VARCHAR(256) DEFAULT CURRENT_USER,
|
||||
change_reason TEXT,
|
||||
|
||||
-- Data snapshots (before/after)
|
||||
old_values JSONB,
|
||||
new_values JSONB,
|
||||
|
||||
-- Audit trail indexing
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
-- Foreign key constraint (optional)
|
||||
-- CONSTRAINT fk_kis_collection_snapshots_audit FOREIGN KEY (snapshot_id)
|
||||
-- REFERENCES quantengine.kis_collection_snapshots(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_collection_snapshots_audit_snapshot_id
|
||||
ON quantengine.kis_collection_snapshots_audit (snapshot_id, changed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_collection_snapshots_audit_changed_by
|
||||
ON quantengine.kis_collection_snapshots_audit (changed_by, changed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_collection_snapshots_audit_timestamp
|
||||
ON quantengine.kis_collection_snapshots_audit (changed_at DESC);
|
||||
|
||||
-- ============================================================================
|
||||
-- kis_collection_errors_audit: Audit trail for error records
|
||||
-- ============================================================================
|
||||
CREATE TABLE IF NOT EXISTS quantengine.kis_collection_errors_audit (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
error_id UUID NOT NULL,
|
||||
|
||||
-- Change metadata
|
||||
action VARCHAR(10) NOT NULL CHECK (action IN ('INSERT', 'UPDATE', 'DELETE')),
|
||||
changed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
changed_by VARCHAR(256) DEFAULT CURRENT_USER,
|
||||
change_reason TEXT,
|
||||
|
||||
-- Data snapshots
|
||||
old_values JSONB,
|
||||
new_values JSONB,
|
||||
|
||||
-- Audit trail indexing
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_collection_errors_audit_error_id
|
||||
ON quantengine.kis_collection_errors_audit (error_id, changed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_collection_errors_audit_changed_by
|
||||
ON quantengine.kis_collection_errors_audit (changed_by, changed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_kis_collection_errors_audit_timestamp
|
||||
ON quantengine.kis_collection_errors_audit (changed_at DESC);
|
||||
|
||||
-- ============================================================================
|
||||
-- Trigger Functions: Auto-log changes to kis_collection_runs
|
||||
-- ============================================================================
|
||||
CREATE OR REPLACE FUNCTION quantengine.kis_collection_runs_audit_trigger()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
INSERT INTO quantengine.kis_collection_runs_audit (
|
||||
run_id, action, changed_by, new_values, change_reason
|
||||
) VALUES (
|
||||
NEW.id, 'INSERT', CURRENT_USER,
|
||||
jsonb_build_object(
|
||||
'id', NEW.id,
|
||||
'status', NEW.status,
|
||||
'total_snapshots', NEW.total_snapshots,
|
||||
'total_errors', NEW.total_errors,
|
||||
'started_at', NEW.started_at
|
||||
),
|
||||
'Automatic INSERT trigger'
|
||||
);
|
||||
ELSIF TG_OP = 'UPDATE' THEN
|
||||
INSERT INTO quantengine.kis_collection_runs_audit (
|
||||
run_id, action, changed_by, old_values, new_values, change_reason
|
||||
) VALUES (
|
||||
NEW.id, 'UPDATE', CURRENT_USER,
|
||||
jsonb_build_object(
|
||||
'status', OLD.status,
|
||||
'total_snapshots', OLD.total_snapshots,
|
||||
'total_errors', OLD.total_errors
|
||||
),
|
||||
jsonb_build_object(
|
||||
'status', NEW.status,
|
||||
'total_snapshots', NEW.total_snapshots,
|
||||
'total_errors', NEW.total_errors
|
||||
),
|
||||
'Automatic UPDATE trigger'
|
||||
);
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
INSERT INTO quantengine.kis_collection_runs_audit (
|
||||
run_id, action, changed_by, old_values, change_reason
|
||||
) VALUES (
|
||||
OLD.id, 'DELETE', CURRENT_USER,
|
||||
jsonb_build_object(
|
||||
'id', OLD.id,
|
||||
'status', OLD.status
|
||||
),
|
||||
'Automatic DELETE trigger'
|
||||
);
|
||||
END IF;
|
||||
|
||||
RETURN COALESCE(NEW, OLD);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- ============================================================================
|
||||
-- Trigger Functions: Auto-log changes to kis_collection_snapshots
|
||||
-- ============================================================================
|
||||
CREATE OR REPLACE FUNCTION quantengine.kis_collection_snapshots_audit_trigger()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
INSERT INTO quantengine.kis_collection_snapshots_audit (
|
||||
snapshot_id, action, changed_by, new_values, change_reason
|
||||
) VALUES (
|
||||
NEW.id, 'INSERT', CURRENT_USER,
|
||||
jsonb_build_object(
|
||||
'id', NEW.id,
|
||||
'ticker', NEW.ticker,
|
||||
'price', NEW.price,
|
||||
'volume', NEW.volume,
|
||||
'source', NEW.source
|
||||
),
|
||||
'Automatic INSERT trigger'
|
||||
);
|
||||
ELSIF TG_OP = 'UPDATE' THEN
|
||||
INSERT INTO quantengine.kis_collection_snapshots_audit (
|
||||
snapshot_id, action, changed_by, old_values, new_values, change_reason
|
||||
) VALUES (
|
||||
NEW.id, 'UPDATE', CURRENT_USER,
|
||||
jsonb_build_object(
|
||||
'ticker', OLD.ticker,
|
||||
'price', OLD.price,
|
||||
'volume', OLD.volume
|
||||
),
|
||||
jsonb_build_object(
|
||||
'ticker', NEW.ticker,
|
||||
'price', NEW.price,
|
||||
'volume', NEW.volume
|
||||
),
|
||||
'Automatic UPDATE trigger'
|
||||
);
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
INSERT INTO quantengine.kis_collection_snapshots_audit (
|
||||
snapshot_id, action, changed_by, old_values, change_reason
|
||||
) VALUES (
|
||||
OLD.id, 'DELETE', CURRENT_USER,
|
||||
jsonb_build_object(
|
||||
'id', OLD.id,
|
||||
'ticker', OLD.ticker
|
||||
),
|
||||
'Automatic DELETE trigger'
|
||||
);
|
||||
END IF;
|
||||
|
||||
RETURN COALESCE(NEW, OLD);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- ============================================================================
|
||||
-- Create Triggers (activate audit logging)
|
||||
-- ============================================================================
|
||||
|
||||
-- Note: These assume kis_collection_runs and kis_collection_snapshots tables exist
|
||||
-- If tables don't exist yet, create them first, then create triggers
|
||||
|
||||
-- Trigger for kis_collection_runs (if table exists)
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'quantengine'
|
||||
AND table_name = 'kis_collection_runs') THEN
|
||||
DROP TRIGGER IF EXISTS kis_collection_runs_audit_trigger
|
||||
ON quantengine.kis_collection_runs;
|
||||
CREATE TRIGGER kis_collection_runs_audit_trigger
|
||||
AFTER INSERT OR UPDATE OR DELETE
|
||||
ON quantengine.kis_collection_runs
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION quantengine.kis_collection_runs_audit_trigger();
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Trigger for kis_collection_snapshots (if table exists)
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'quantengine'
|
||||
AND table_name = 'kis_collection_snapshots') THEN
|
||||
DROP TRIGGER IF EXISTS kis_collection_snapshots_audit_trigger
|
||||
ON quantengine.kis_collection_snapshots;
|
||||
CREATE TRIGGER kis_collection_snapshots_audit_trigger
|
||||
AFTER INSERT OR UPDATE OR DELETE
|
||||
ON quantengine.kis_collection_snapshots
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION quantengine.kis_collection_snapshots_audit_trigger();
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- ============================================================================
|
||||
-- Validation Views (for querying audit trail)
|
||||
-- ============================================================================
|
||||
|
||||
-- View: Recent changes to collection runs
|
||||
CREATE OR REPLACE VIEW quantengine.v_kis_collection_runs_recent_changes AS
|
||||
SELECT
|
||||
run_id,
|
||||
action,
|
||||
changed_at,
|
||||
changed_by,
|
||||
change_reason,
|
||||
jsonb_pretty(old_values) as old_values,
|
||||
jsonb_pretty(new_values) as new_values
|
||||
FROM quantengine.kis_collection_runs_audit
|
||||
WHERE changed_at > NOW() - INTERVAL '7 days'
|
||||
ORDER BY changed_at DESC;
|
||||
|
||||
-- View: Recent changes to snapshots
|
||||
CREATE OR REPLACE VIEW quantengine.v_kis_collection_snapshots_recent_changes AS
|
||||
SELECT
|
||||
snapshot_id,
|
||||
action,
|
||||
changed_at,
|
||||
changed_by,
|
||||
change_reason,
|
||||
jsonb_pretty(old_values) as old_values,
|
||||
jsonb_pretty(new_values) as new_values
|
||||
FROM quantengine.kis_collection_snapshots_audit
|
||||
WHERE changed_at > NOW() - INTERVAL '7 days'
|
||||
ORDER BY changed_at DESC;
|
||||
|
||||
-- ============================================================================
|
||||
-- Audit Trail Statistics
|
||||
-- ============================================================================
|
||||
|
||||
-- View: Daily audit statistics
|
||||
CREATE OR REPLACE VIEW quantengine.v_audit_statistics_daily AS
|
||||
SELECT
|
||||
DATE(changed_at) as date,
|
||||
COUNT(*) as total_changes,
|
||||
COUNT(DISTINCT changed_by) as unique_users,
|
||||
COUNT(*) FILTER (WHERE action = 'INSERT') as inserts,
|
||||
COUNT(*) FILTER (WHERE action = 'UPDATE') as updates,
|
||||
COUNT(*) FILTER (WHERE action = 'DELETE') as deletes
|
||||
FROM quantengine.kis_collection_runs_audit
|
||||
GROUP BY DATE(changed_at)
|
||||
ORDER BY date DESC;
|
||||
|
||||
-- ============================================================================
|
||||
-- Rollback Script (if needed)
|
||||
-- ============================================================================
|
||||
-- To rollback this migration, run:
|
||||
/*
|
||||
DROP TRIGGER IF EXISTS kis_collection_snapshots_audit_trigger ON quantengine.kis_collection_snapshots;
|
||||
DROP TRIGGER IF EXISTS kis_collection_runs_audit_trigger ON quantengine.kis_collection_runs;
|
||||
DROP FUNCTION IF EXISTS quantengine.kis_collection_snapshots_audit_trigger();
|
||||
DROP FUNCTION IF EXISTS quantengine.kis_collection_runs_audit_trigger();
|
||||
DROP VIEW IF EXISTS quantengine.v_audit_statistics_daily;
|
||||
DROP VIEW IF EXISTS quantengine.v_kis_collection_snapshots_recent_changes;
|
||||
DROP VIEW IF EXISTS quantengine.v_kis_collection_runs_recent_changes;
|
||||
DROP TABLE IF EXISTS quantengine.kis_collection_errors_audit;
|
||||
DROP TABLE IF EXISTS quantengine.kis_collection_snapshots_audit;
|
||||
DROP TABLE IF EXISTS quantengine.kis_collection_runs_audit;
|
||||
*/
|
||||
|
||||
-- ============================================================================
|
||||
-- Migration Validation
|
||||
-- ============================================================================
|
||||
-- Verify audit tables were created successfully
|
||||
SELECT
|
||||
'kis_collection_runs_audit' as table_name,
|
||||
COUNT(*) as row_count
|
||||
FROM quantengine.kis_collection_runs_audit
|
||||
UNION ALL
|
||||
SELECT
|
||||
'kis_collection_snapshots_audit',
|
||||
COUNT(*)
|
||||
FROM quantengine.kis_collection_snapshots_audit
|
||||
UNION ALL
|
||||
SELECT
|
||||
'kis_collection_errors_audit',
|
||||
COUNT(*)
|
||||
FROM quantengine.kis_collection_errors_audit;
|
||||
Reference in New Issue
Block a user