QuantEngineByItz/src/dotnet/QuantEngine.Infrastructure/Migrations/V9__Add_Audit_Trail_Tables.sql

-- Migration: V9__Add_Audit_Trail_Tables.sql (renamed from V003_add_audit_trail_tables.sql
-- on 2026-07-30 — see header note below)
-- Purpose: Add audit trail tables for tracking all data changes
-- Date: 2026-07-24
-- Status: APPROVED for Phase 0 implementation
--
-- 2026-07-30: Originally named V003_add_audit_trail_tables.sql, which alphabetically sorted
-- BEFORE V1__Initial_Schema.sql. Its trigger-creation guards (IF EXISTS checks on
-- kis_collection_runs / kis_collection_snapshots) would have silently no-op'd forever on a
-- fresh database, since those tables (created by V2) wouldn't exist yet when V003 ran first.
-- Renamed to V9 (after DbMigrator.cs was given a numeric-aware script comparer,
-- MigrationScriptNameComparer, so "V9"/"V10" sort correctly relative to "V1".."V8" regardless
-- of digit count) so it now runs after its dependencies exist. Also fixed a MySQL-only inline
-- INDEX syntax that made this script fail outright on PostgreSQL (separate fix, same day).
-- Confirmed via production query that this migration had never actually applied before either fix.

-- ============================================================================
-- kis_collection_runs_audit: Audit trail for collection runs
-- ============================================================================
CREATE TABLE IF NOT EXISTS quantengine.kis_collection_runs_audit (
    id BIGSERIAL PRIMARY KEY,
    run_id UUID NOT NULL,

    -- Change metadata
    action VARCHAR(10) NOT NULL CHECK (action IN ('INSERT', 'UPDATE', 'DELETE')),
    changed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
    changed_by VARCHAR(256) DEFAULT CURRENT_USER,
    change_reason TEXT,

    -- Data snapshots (before/after)
    old_values JSONB,
    new_values JSONB,

    -- Audit trail indexing
    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,

    -- Foreign key constraint (optional - don't enforce if kis_collection_runs might be deleted)
    -- CONSTRAINT fk_kis_collection_runs_audit FOREIGN KEY (run_id)
    --   REFERENCES quantengine.kis_collection_runs(id) ON DELETE CASCADE
);

CREATE INDEX IF NOT EXISTS idx_kis_collection_runs_audit_run_id
    ON quantengine.kis_collection_runs_audit (run_id, changed_at DESC);
CREATE INDEX IF NOT EXISTS idx_kis_collection_runs_audit_changed_by
    ON quantengine.kis_collection_runs_audit (changed_by, changed_at DESC);
CREATE INDEX IF NOT EXISTS idx_kis_collection_runs_audit_timestamp
    ON quantengine.kis_collection_runs_audit (changed_at DESC);

-- ============================================================================
-- kis_collection_snapshots_audit: Audit trail for snapshots
-- ============================================================================
CREATE TABLE IF NOT EXISTS quantengine.kis_collection_snapshots_audit (
    id BIGSERIAL PRIMARY KEY,
    snapshot_id UUID NOT NULL,

    -- Change metadata
    action VARCHAR(10) NOT NULL CHECK (action IN ('INSERT', 'UPDATE', 'DELETE')),
    changed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
    changed_by VARCHAR(256) DEFAULT CURRENT_USER,
    change_reason TEXT,

    -- Data snapshots (before/after)
    old_values JSONB,
    new_values JSONB,

    -- Audit trail indexing
    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,

    -- Foreign key constraint (optional)
    -- CONSTRAINT fk_kis_collection_snapshots_audit FOREIGN KEY (snapshot_id)
    --   REFERENCES quantengine.kis_collection_snapshots(id) ON DELETE CASCADE
);

CREATE INDEX IF NOT EXISTS idx_kis_collection_snapshots_audit_snapshot_id
    ON quantengine.kis_collection_snapshots_audit (snapshot_id, changed_at DESC);
CREATE INDEX IF NOT EXISTS idx_kis_collection_snapshots_audit_changed_by
    ON quantengine.kis_collection_snapshots_audit (changed_by, changed_at DESC);
CREATE INDEX IF NOT EXISTS idx_kis_collection_snapshots_audit_timestamp
    ON quantengine.kis_collection_snapshots_audit (changed_at DESC);

-- ============================================================================
-- kis_collection_errors_audit: Audit trail for error records
-- ============================================================================
CREATE TABLE IF NOT EXISTS quantengine.kis_collection_errors_audit (
    id BIGSERIAL PRIMARY KEY,
    error_id UUID NOT NULL,

    -- Change metadata
    action VARCHAR(10) NOT NULL CHECK (action IN ('INSERT', 'UPDATE', 'DELETE')),
    changed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
    changed_by VARCHAR(256) DEFAULT CURRENT_USER,
    change_reason TEXT,

    -- Data snapshots
    old_values JSONB,
    new_values JSONB,

    -- Audit trail indexing
    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);

CREATE INDEX IF NOT EXISTS idx_kis_collection_errors_audit_error_id
    ON quantengine.kis_collection_errors_audit (error_id, changed_at DESC);
CREATE INDEX IF NOT EXISTS idx_kis_collection_errors_audit_changed_by
    ON quantengine.kis_collection_errors_audit (changed_by, changed_at DESC);
CREATE INDEX IF NOT EXISTS idx_kis_collection_errors_audit_timestamp
    ON quantengine.kis_collection_errors_audit (changed_at DESC);

-- ============================================================================
-- Trigger Functions: Auto-log changes to kis_collection_runs
-- ============================================================================
CREATE OR REPLACE FUNCTION quantengine.kis_collection_runs_audit_trigger()
RETURNS TRIGGER AS $$
BEGIN
    IF TG_OP = 'INSERT' THEN
        INSERT INTO quantengine.kis_collection_runs_audit (
            run_id, action, changed_by, new_values, change_reason
        ) VALUES (
            NEW.id, 'INSERT', CURRENT_USER,
            jsonb_build_object(
                'id', NEW.id,
                'status', NEW.status,
                'total_snapshots', NEW.total_snapshots,
                'total_errors', NEW.total_errors,
                'started_at', NEW.started_at
            ),
            'Automatic INSERT trigger'
        );
    ELSIF TG_OP = 'UPDATE' THEN
        INSERT INTO quantengine.kis_collection_runs_audit (
            run_id, action, changed_by, old_values, new_values, change_reason
        ) VALUES (
            NEW.id, 'UPDATE', CURRENT_USER,
            jsonb_build_object(
                'status', OLD.status,
                'total_snapshots', OLD.total_snapshots,
                'total_errors', OLD.total_errors
            ),
            jsonb_build_object(
                'status', NEW.status,
                'total_snapshots', NEW.total_snapshots,
                'total_errors', NEW.total_errors
            ),
            'Automatic UPDATE trigger'
        );
    ELSIF TG_OP = 'DELETE' THEN
        INSERT INTO quantengine.kis_collection_runs_audit (
            run_id, action, changed_by, old_values, change_reason
        ) VALUES (
            OLD.id, 'DELETE', CURRENT_USER,
            jsonb_build_object(
                'id', OLD.id,
                'status', OLD.status
            ),
            'Automatic DELETE trigger'
        );
    END IF;

    RETURN COALESCE(NEW, OLD);
END;
$$ LANGUAGE plpgsql;

-- ============================================================================
-- Trigger Functions: Auto-log changes to kis_collection_snapshots
-- ============================================================================
CREATE OR REPLACE FUNCTION quantengine.kis_collection_snapshots_audit_trigger()
RETURNS TRIGGER AS $$
BEGIN
    IF TG_OP = 'INSERT' THEN
        INSERT INTO quantengine.kis_collection_snapshots_audit (
            snapshot_id, action, changed_by, new_values, change_reason
        ) VALUES (
            NEW.id, 'INSERT', CURRENT_USER,
            jsonb_build_object(
                'id', NEW.id,
                'ticker', NEW.ticker,
                'price', NEW.price,
                'volume', NEW.volume,
                'source', NEW.source
            ),
            'Automatic INSERT trigger'
        );
    ELSIF TG_OP = 'UPDATE' THEN
        INSERT INTO quantengine.kis_collection_snapshots_audit (
            snapshot_id, action, changed_by, old_values, new_values, change_reason
        ) VALUES (
            NEW.id, 'UPDATE', CURRENT_USER,
            jsonb_build_object(
                'ticker', OLD.ticker,
                'price', OLD.price,
                'volume', OLD.volume
            ),
            jsonb_build_object(
                'ticker', NEW.ticker,
                'price', NEW.price,
                'volume', NEW.volume
            ),
            'Automatic UPDATE trigger'
        );
    ELSIF TG_OP = 'DELETE' THEN
        INSERT INTO quantengine.kis_collection_snapshots_audit (
            snapshot_id, action, changed_by, old_values, change_reason
        ) VALUES (
            OLD.id, 'DELETE', CURRENT_USER,
            jsonb_build_object(
                'id', OLD.id,
                'ticker', OLD.ticker
            ),
            'Automatic DELETE trigger'
        );
    END IF;

    RETURN COALESCE(NEW, OLD);
END;
$$ LANGUAGE plpgsql;

-- ============================================================================
-- Create Triggers (activate audit logging)
-- ============================================================================

-- Note: These assume kis_collection_runs and kis_collection_snapshots tables exist
-- If tables don't exist yet, create them first, then create triggers

-- Trigger for kis_collection_runs (if table exists)
DO $$
BEGIN
    IF EXISTS (SELECT 1 FROM information_schema.tables
               WHERE table_schema = 'quantengine'
               AND table_name = 'kis_collection_runs') THEN
        DROP TRIGGER IF EXISTS kis_collection_runs_audit_trigger
            ON quantengine.kis_collection_runs;
        CREATE TRIGGER kis_collection_runs_audit_trigger
            AFTER INSERT OR UPDATE OR DELETE
            ON quantengine.kis_collection_runs
            FOR EACH ROW
            EXECUTE FUNCTION quantengine.kis_collection_runs_audit_trigger();
    END IF;
END $$;

-- Trigger for kis_collection_snapshots (if table exists)
DO $$
BEGIN
    IF EXISTS (SELECT 1 FROM information_schema.tables
               WHERE table_schema = 'quantengine'
               AND table_name = 'kis_collection_snapshots') THEN
        DROP TRIGGER IF EXISTS kis_collection_snapshots_audit_trigger
            ON quantengine.kis_collection_snapshots;
        CREATE TRIGGER kis_collection_snapshots_audit_trigger
            AFTER INSERT OR UPDATE OR DELETE
            ON quantengine.kis_collection_snapshots
            FOR EACH ROW
            EXECUTE FUNCTION quantengine.kis_collection_snapshots_audit_trigger();
    END IF;
END $$;

-- ============================================================================
-- Validation Views (for querying audit trail)
-- ============================================================================

-- View: Recent changes to collection runs
CREATE OR REPLACE VIEW quantengine.v_kis_collection_runs_recent_changes AS
SELECT
    run_id,
    action,
    changed_at,
    changed_by,
    change_reason,
    jsonb_pretty(old_values) as old_values,
    jsonb_pretty(new_values) as new_values
FROM quantengine.kis_collection_runs_audit
WHERE changed_at > NOW() - INTERVAL '7 days'
ORDER BY changed_at DESC;

-- View: Recent changes to snapshots
CREATE OR REPLACE VIEW quantengine.v_kis_collection_snapshots_recent_changes AS
SELECT
    snapshot_id,
    action,
    changed_at,
    changed_by,
    change_reason,
    jsonb_pretty(old_values) as old_values,
    jsonb_pretty(new_values) as new_values
FROM quantengine.kis_collection_snapshots_audit
WHERE changed_at > NOW() - INTERVAL '7 days'
ORDER BY changed_at DESC;

-- ============================================================================
-- Audit Trail Statistics
-- ============================================================================

-- View: Daily audit statistics
CREATE OR REPLACE VIEW quantengine.v_audit_statistics_daily AS
SELECT
    DATE(changed_at) as date,
    COUNT(*) as total_changes,
    COUNT(DISTINCT changed_by) as unique_users,
    COUNT(*) FILTER (WHERE action = 'INSERT') as inserts,
    COUNT(*) FILTER (WHERE action = 'UPDATE') as updates,
    COUNT(*) FILTER (WHERE action = 'DELETE') as deletes
FROM quantengine.kis_collection_runs_audit
GROUP BY DATE(changed_at)
ORDER BY date DESC;

-- ============================================================================
-- Rollback Script (if needed)
-- ============================================================================
-- To rollback this migration, run:
/*
DROP TRIGGER IF EXISTS kis_collection_snapshots_audit_trigger ON quantengine.kis_collection_snapshots;
DROP TRIGGER IF EXISTS kis_collection_runs_audit_trigger ON quantengine.kis_collection_runs;
DROP FUNCTION IF EXISTS quantengine.kis_collection_snapshots_audit_trigger();
DROP FUNCTION IF EXISTS quantengine.kis_collection_runs_audit_trigger();
DROP VIEW IF EXISTS quantengine.v_audit_statistics_daily;
DROP VIEW IF EXISTS quantengine.v_kis_collection_snapshots_recent_changes;
DROP VIEW IF EXISTS quantengine.v_kis_collection_runs_recent_changes;
DROP TABLE IF EXISTS quantengine.kis_collection_errors_audit;
DROP TABLE IF EXISTS quantengine.kis_collection_snapshots_audit;
DROP TABLE IF EXISTS quantengine.kis_collection_runs_audit;
*/

-- ============================================================================
-- Migration Validation
-- ============================================================================
-- Verify audit tables were created successfully
SELECT
    'kis_collection_runs_audit' as table_name,
    COUNT(*) as row_count
FROM quantengine.kis_collection_runs_audit
UNION ALL
SELECT
    'kis_collection_snapshots_audit',
    COUNT(*)
FROM quantengine.kis_collection_snapshots_audit
UNION ALL
SELECT
    'kis_collection_errors_audit',
    COUNT(*)
FROM quantengine.kis_collection_errors_audit;