KArtSell.Aegis/docs/monitoring-queries.sql

-- K-ArtSell Aegis v16.0 Monitoring Queries
-- AGENTS.md Observability Standards
-- Reference: CLAUDE.md Operational Dashboards

-- ============================================================================
-- PRIORITY 1: BATCH SLA MONITORING
-- ============================================================================

-- 1.1 Current Queue Depths (all queues)
SELECT 
    queue,
    COUNT(*) as pending_jobs,
    MIN(created_at) as oldest_job,
    AVG(EXTRACT(EPOCH FROM (now() - created_at))) as avg_wait_seconds
FROM hangfire.job
WHERE state_name IN ('Enqueued', 'Scheduled')
GROUP BY queue
ORDER BY pending_jobs DESC;

-- 1.2 Job Completion Times (last 24 hours, by queue)
SELECT 
    queue,
    COUNT(*) as completed_jobs,
    PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (ended_at - created_at))) as p50_latency_sec,
    PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (ended_at - created_at))) as p95_latency_sec,
    PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (ended_at - created_at))) as p99_latency_sec
FROM hangfire.job
WHERE state_name = 'Succeeded'
    AND ended_at > now() - interval '24 hours'
GROUP BY queue
ORDER BY p99_latency_sec DESC;

-- 1.3 Failed Jobs (last 24 hours)
SELECT 
    id,
    queue,
    type,
    state_name,
    exception_type,
    exception_message,
    created_at,
    ended_at
FROM hangfire.job
WHERE state_name = 'Failed'
    AND created_at > now() - interval '24 hours'
ORDER BY ended_at DESC
LIMIT 50;

-- ============================================================================
-- PRIORITY 2: DATA QUALITY QUARANTINE
-- ============================================================================

-- 2.1 DQ-classified Jobs (awaiting manual review)
SELECT 
    id,
    queue,
    type,
    state_name,
    created_at,
    retry_classification,
    exception_message
FROM hangfire.job
WHERE state_data LIKE '%retry_classification%dq%'
    AND state_name IN ('Failed', 'Scheduled')
ORDER BY created_at DESC
LIMIT 100;

-- 2.2 DQ Jobs by Type (trend analysis)
SELECT 
    type,
    COUNT(*) as dq_count,
    MAX(created_at) as latest_dq
FROM hangfire.job
WHERE state_data LIKE '%retry_classification%dq%'
    AND created_at > now() - interval '7 days'
GROUP BY type
ORDER BY dq_count DESC;

-- ============================================================================
-- PRIORITY 3: DUPLICATE DETECTION & RECONCILIATION
-- ============================================================================

-- 3.1 Outbox Duplicate Events (same idempotency key, multiple entries)
SELECT 
    idempotency_key,
    COUNT(*) as duplicate_count,
    MIN(published_at) as first_published,
    MAX(published_at) as last_published,
    event_type
FROM outbox.outbox
WHERE idempotency_key IS NOT NULL
GROUP BY idempotency_key, event_type
HAVING COUNT(*) > 1
ORDER BY duplicate_count DESC
LIMIT 50;

-- 3.2 Inbox Processing Status (pending/processed)
SELECT 
    state,
    COUNT(*) as message_count,
    MIN(created_at) as oldest,
    MAX(created_at) as newest
FROM inbox.inbox
GROUP BY state
ORDER BY message_count DESC;

-- 3.3 Outbox to Inbox Gap (unprocessed events)
SELECT 
    o.id as outbox_id,
    o.idempotency_key,
    o.event_type,
    o.published_at,
    CASE WHEN i.id IS NOT NULL THEN 'PROCESSED' ELSE 'PENDING' END as status,
    AGE(now(), o.published_at) as age
FROM outbox.outbox o
LEFT JOIN inbox.inbox i ON o.idempotency_key = i.idempotency_key
WHERE o.published_at > now() - interval '1 hour'
ORDER BY o.published_at DESC;

-- ============================================================================
-- PRIORITY 4: MODEL DRIFT MONITORING
-- ============================================================================

-- 4.1 Shadow Run Completion Status (Gate 5 Progress)
SELECT 
    id,
    model_id,
    created_at,
    started_at,
    completed_at,
    state_name,
    AGE(COALESCE(completed_at, now()), started_at) as duration,
    trading_day_count,
    pbo_score,
    dsr_score
FROM shadow_runs
WHERE created_at > now() - interval '30 days'
ORDER BY created_at DESC
LIMIT 10;

-- 4.2 Model Metrics Trending (OOS performance vs baseline)
SELECT 
    model_id,
    DATE(created_at) as metric_date,
    AVG(backtest_sharpe) as avg_backtest_sharpe,
    AVG(oos_sharpe) as avg_oos_sharpe,
    AVG(oos_sharpe - backtest_sharpe) as sharpe_divergence
FROM model_metrics
WHERE created_at > now() - interval '90 days'
GROUP BY model_id, DATE(created_at)
ORDER BY model_id, metric_date DESC;

-- ============================================================================
-- PRIORITY 5: SYSTEM HEALTH
-- ============================================================================

-- 5.1 Hangfire Server Health (worker counts, CPU)
SELECT 
    name,
    last_heartbeat,
    worker_count,
    queue_count,
    AGE(now(), last_heartbeat) as heartbeat_age
FROM hangfire.server
ORDER BY last_heartbeat DESC;

-- 5.2 Application Error Rates (last 1 hour)
SELECT 
    DATE_TRUNC('minute', created_at) as minute,
    COUNT(*) as error_count,
    COUNT(CASE WHEN state_name = 'Failed' THEN 1 END) as failed_jobs,
    COUNT(CASE WHEN exception_type LIKE '%Timeout%' THEN 1 END) as timeout_errors
FROM hangfire.job
WHERE created_at > now() - interval '1 hour'
GROUP BY DATE_TRUNC('minute', created_at)
ORDER BY minute DESC;

-- 5.3 Database Connection Pool Status (if monitored)
SELECT 
    datname as database,
    usename as user,
    state,
    COUNT(*) as connection_count,
    MAX(EXTRACT(EPOCH FROM (now() - state_change))) as idle_seconds
FROM pg_stat_activity
WHERE datname = 'kartsell'
GROUP BY datname, usename, state
ORDER BY connection_count DESC;