KArtSell.Aegis/db/migrations/0016_continuous_model_operations.sql

-- v12.4 continuous model operations delta.
-- Evaluation and proposal automation only. This migration does not enable automatic model promotion,
-- automatic order submission or KIS submission.

create schema if not exists evaluation;
create schema if not exists governance;

create table if not exists evaluation.dataset_manifest (
    dataset_id text primary key,
    scope_key text not null,
    content_hash text not null unique,
    source_catalog_version text not null,
    lineage_hash text not null,
    status text not null check (status in ('PROPOSED', 'APPROVED', 'QUARANTINED', 'RETIRED')),
    frozen_at timestamptz not null,
    approved_by text null,
    approved_at timestamptz null,
    created_at timestamptz not null default now()
);

create index if not exists ix_dataset_manifest_scope_frozen
    on evaluation.dataset_manifest (scope_key, frozen_at desc)
    where status = 'APPROVED';

create table if not exists governance.model_version_registry (
    model_version text not null,
    scope_key text not null,
    config_version text not null,
    code_sha text not null,
    contract_version text not null,
    lifecycle_state text not null check (lifecycle_state in
        ('RESEARCH', 'CHALLENGER', 'SHADOW', 'CANDIDATE', 'APPROVED', 'RETIRED', 'ROLLED_BACK')),
    effective_at timestamptz not null,
    retired_at timestamptz null,
    model_card_hash text not null,
    approved_by text null,
    approved_at timestamptz null,
    created_at timestamptz not null default now(),
    primary key (model_version, scope_key, effective_at),
    check (lifecycle_state <> 'APPROVED' or (approved_by is not null and approved_at is not null))
);

create index if not exists ix_model_version_registry_active
    on governance.model_version_registry (scope_key, effective_at desc)
    where lifecycle_state in ('RESEARCH', 'CHALLENGER', 'SHADOW', 'CANDIDATE', 'APPROVED');

create table if not exists evaluation.model_operation_schedule (
    schedule_id uuid primary key,
    operation_code text not null,
    operation_name text not null,
    scope_key text not null,
    cadence text not null check (cadence in ('DAILY', 'WEEKLY', 'MONTHLY', 'QUARTERLY', 'EVENT_DRIVEN')),
    automation_mode text not null check (automation_mode in ('EVALUATION_ONLY', 'PROPOSAL_ONLY', 'DRILL_ONLY')),
    queue_name text not null,
    schedule_version integer not null check (schedule_version > 0),
    enabled boolean not null default true,
    next_due_at timestamptz not null,
    last_dispatched_at timestamptz null,
    last_background_job_id text null,
    max_lag interval not null,
    primary_owner text not null,
    secondary_owner text not null,
    lease_owner text null,
    lease_until timestamptz null,
    last_error_code text null,
    created_at timestamptz not null default now(),
    updated_at timestamptz not null default now(),
    unique (operation_code, scope_key, schedule_version)
);

create index if not exists ix_model_operation_schedule_due
    on evaluation.model_operation_schedule (next_due_at, operation_code, scope_key)
    where enabled = true;

create table if not exists evaluation.model_operation_request (
    request_id uuid primary key,
    schedule_id uuid not null references evaluation.model_operation_schedule(schedule_id),
    operation_code text not null,
    scope_key text not null,
    automation_mode text not null check (automation_mode in ('EVALUATION_ONLY', 'PROPOSAL_ONLY', 'DRILL_ONLY')),
    idempotency_key text not null unique,
    dataset_id text not null,
    data_hash text not null,
    model_version text not null,
    config_version text not null,
    code_sha text not null,
    contract_version text not null,
    lifecycle_state text not null,
    correlation_id text not null,
    status text not null check (status in ('REQUESTED', 'RUNNING', 'SUCCEEDED', 'BUSINESS_HOLD', 'FAILED', 'QUARANTINED')),
    output_hash text null,
    error_code text null,
    requested_at timestamptz not null,
    started_at timestamptz null,
    finished_at timestamptz null
);

create index if not exists ix_model_operation_request_scope_time
    on evaluation.model_operation_request (scope_key, operation_code, requested_at desc);

-- Request is a lifecycle aggregate: status/timing fields are mutable under optimistic/transactional control.
-- Every transition is additionally recorded as an immutable status event so operational history is never lost.
create table if not exists evaluation.model_operation_status_event (
    event_id uuid primary key,
    request_id uuid not null references evaluation.model_operation_request(request_id),
    from_status text null,
    to_status text not null check (to_status in
        ('REQUESTED', 'RUNNING', 'SUCCEEDED', 'BUSINESS_HOLD', 'FAILED', 'QUARANTINED')),
    reason_code text null,
    payload_hash text not null,
    actor_type text not null check (actor_type in ('SYSTEM', 'OPERATOR', 'REPLAY')),
    correlation_id text not null,
    occurred_at timestamptz not null,
    created_at timestamptz not null default now(),
    unique (request_id, to_status, occurred_at, payload_hash)
);

create index if not exists ix_model_operation_status_event_request_time
    on evaluation.model_operation_status_event (request_id, occurred_at);

create table if not exists evaluation.model_metric_definition (
    metric_code text not null,
    definition_version integer not null check (definition_version > 0),
    numerator_definition text not null,
    denominator_definition text not null,
    evaluation_window text not null,
    aggregation_method text not null,
    threshold_json jsonb not null,
    status text not null check (status in ('PROPOSED', 'APPROVED', 'RETIRED')),
    content_hash text not null,
    effective_from timestamptz null,
    approved_by text null,
    approved_at timestamptz null,
    created_at timestamptz not null default now(),
    primary key (metric_code, definition_version),
    unique (content_hash)
);

create table if not exists evaluation.model_metric_observation (
    observation_id uuid primary key,
    request_id uuid not null references evaluation.model_operation_request(request_id),
    metric_code text not null,
    definition_version integer not null,
    scope_key text not null,
    cohort_key text not null,
    value_numeric numeric(30, 12) null,
    value_text text null,
    sample_size bigint not null check (sample_size >= 0),
    dataset_id text not null,
    model_version text not null,
    as_of timestamptz not null,
    content_hash text not null,
    created_at timestamptz not null default now(),
    foreign key (metric_code, definition_version)
        references evaluation.model_metric_definition(metric_code, definition_version),
    check ((value_numeric is null) <> (value_text is null)),
    unique (request_id, metric_code, definition_version, cohort_key, content_hash)
);

create table if not exists evaluation.model_evaluation_snapshot (
    snapshot_id uuid primary key,
    request_id uuid not null references evaluation.model_operation_request(request_id),
    scope_key text not null,
    model_version text not null,
    dataset_id text not null,
    gate_contract_version text not null,
    gate_decision text not null check (gate_decision in ('PASS', 'WARN', 'HOLD', 'FAIL')),
    blocking_reasons_json jsonb not null,
    warnings_json jsonb not null,
    evidence_hash text not null,
    as_of timestamptz not null,
    created_at timestamptz not null default now(),
    unique (request_id, evidence_hash)
);

create table if not exists governance.model_improvement_proposal (
    proposal_id uuid primary key,
    source_request_id uuid not null references evaluation.model_operation_request(request_id),
    scope_key text not null,
    base_model_version text not null,
    proposal_type text not null check (proposal_type in
        ('DATA_REMEDIATION', 'FEATURE_REVIEW', 'THRESHOLD_REVIEW', 'POLICY_REFACTORING',
         'CALIBRATION_REVIEW', 'RISK_LIMIT_REVIEW', 'TECH_DEBT_REPAYMENT')),
    problem_statement text not null,
    supporting_evidence_json jsonb not null,
    counter_evidence_json jsonb not null,
    expected_benefit text not null,
    risks text not null,
    required_tests_json jsonb not null,
    status text not null check (status in
        ('DRAFT', 'REVIEW_REQUIRED', 'APPROVED_FOR_RESEARCH', 'REJECTED', 'EXPIRED')),
    created_by text not null,
    created_at timestamptz not null,
    reviewed_by text null,
    reviewed_at timestamptz null,
    expires_at timestamptz not null,
    content_hash text not null unique,
    check (status <> 'APPROVED_FOR_RESEARCH' or (reviewed_by is not null and reviewed_at is not null))
);

create table if not exists governance.model_promotion_review (
    review_id uuid primary key,
    scope_key text not null,
    candidate_model_version text not null,
    evidence_snapshot_id uuid not null references evaluation.model_evaluation_snapshot(snapshot_id),
    independent_validation_hash text not null,
    compliance_review_hash text null,
    security_review_hash text null,
    decision text not null check (decision in ('PENDING', 'APPROVED_FOR_SHADOW', 'APPROVED_FOR_PILOT', 'REJECTED', 'EXPIRED')),
    maker_id text not null,
    checker_id text null,
    created_at timestamptz not null,
    decided_at timestamptz null,
    check (maker_id <> coalesce(checker_id, '')),
    check (decision = 'PENDING' or (checker_id is not null and decided_at is not null))
);

create table if not exists governance.model_rollback_drill (
    drill_id uuid primary key,
    scope_key text not null,
    champion_model_version text not null,
    fallback_model_version text not null,
    runbook_version text not null,
    started_at timestamptz not null,
    finished_at timestamptz null,
    result text not null check (result in ('RUNNING', 'PASS', 'FAIL', 'BUSINESS_HOLD')),
    recovery_time_seconds integer null check (recovery_time_seconds is null or recovery_time_seconds >= 0),
    evidence_hash text null,
    owner text not null,
    secondary_owner text not null
);

-- Append-only evidence. Request status is mutable, but status_event/observation/snapshot history is immutable.
drop trigger if exists model_operation_status_event_append_only on evaluation.model_operation_status_event;
create trigger model_operation_status_event_append_only
before update or delete on evaluation.model_operation_status_event
for each row execute function building_blocks.prevent_append_only_change();

drop trigger if exists model_metric_observation_append_only on evaluation.model_metric_observation;
create trigger model_metric_observation_append_only
before update or delete on evaluation.model_metric_observation
for each row execute function building_blocks.prevent_append_only_change();

drop trigger if exists model_evaluation_snapshot_append_only on evaluation.model_evaluation_snapshot;
create trigger model_evaluation_snapshot_append_only
before update or delete on evaluation.model_evaluation_snapshot
for each row execute function building_blocks.prevent_append_only_change();

-- Schedules intentionally automate evidence collection/proposal generation only.
insert into evaluation.model_operation_schedule
(schedule_id, operation_code, operation_name, scope_key, cadence, automation_mode, queue_name,
 schedule_version, enabled, next_due_at, max_lag, primary_owner, secondary_owner)
values
('e4100000-0000-4000-8000-000000000010', 'J10', 'OutcomeEvaluationRun', 'GLOBAL', 'DAILY', 'EVALUATION_ONLY', 'q-evaluation', 1, true, now() + interval '1 hour', interval '1 day', 'Quant/Ops', 'Data/QA'),
('e4100000-0000-4000-8000-000000000011', 'J11', 'DailyScorecardBuild', 'GLOBAL', 'DAILY', 'EVALUATION_ONLY', 'q-evaluation', 1, true, now() + interval '2 hours', interval '1 day', 'Quant/Ops', 'Risk/SRE'),
('e4100000-0000-4000-8000-000000000017', 'J17', 'DriftDetectionRun', 'GLOBAL', 'DAILY', 'EVALUATION_ONLY', 'q-evaluation', 1, true, now() + interval '3 hours', interval '1 day', 'Quant/Risk', 'Data/SRE'),
('e4100000-0000-4000-8000-000000000018', 'J18', 'ChampionChallengerEvaluation', 'GLOBAL', 'WEEKLY', 'EVALUATION_ONLY', 'q-evaluation', 1, true, now() + interval '1 day', interval '7 days', 'Quant/Risk', 'QA/InvestmentCommittee'),
('e4100000-0000-4000-8000-000000000019', 'J19', 'FrozenOosBacktest', 'GLOBAL', 'MONTHLY', 'EVALUATION_ONLY', 'q-research', 1, true, now() + interval '7 days', interval '31 days', 'Quant/Data', 'QA/Risk'),
('e4100000-0000-4000-8000-000000000020', 'J20', 'RobustnessPboDsrRun', 'GLOBAL', 'QUARTERLY', 'EVALUATION_ONLY', 'q-research', 1, true, now() + interval '30 days', interval '95 days', 'Quant/Risk', 'IndependentValidation'),
('e4100000-0000-4000-8000-000000000021', 'J21', 'ModelImprovementProposalBuild', 'GLOBAL', 'MONTHLY', 'PROPOSAL_ONLY', 'q-research', 1, true, now() + interval '14 days', interval '31 days', 'Quant Lead', 'Risk/Architect'),
('e4100000-0000-4000-8000-000000000022', 'J22', 'PromotionEvidenceReviewBuild', 'GLOBAL', 'MONTHLY', 'PROPOSAL_ONLY', 'q-control', 1, true, now() + interval '21 days', interval '31 days', 'Risk/InvestmentCommittee', 'Compliance/QA'),
('e4100000-0000-4000-8000-000000000023', 'J23', 'ModelRollbackDrill', 'GLOBAL', 'QUARTERLY', 'DRILL_ONLY', 'q-control', 1, true, now() + interval '45 days', interval '95 days', 'SRE/Risk', 'Module Owner/QA'),
('e4100000-0000-4000-8000-000000000024', 'J24', 'DataRevisionRevalidation', 'GLOBAL', 'WEEKLY', 'EVALUATION_ONLY', 'q-backfill', 1, true, now() + interval '2 days', interval '7 days', 'Data/Quant', 'DBA/QA')
on conflict (operation_code, scope_key, schedule_version) do nothing;