QuantEngineByItz/.gitea/workflows/ci.yml

name: Validators (Pushes and Pull Requests)

on:
  pull_request:
    branches: [ main ]
  push:
    branches: [ main ]
  workflow_dispatch:

concurrency:
  group: quantengine-ci-${{ github.ref }}
  cancel-in-progress: true

env:
  DOTNET_VERSION: '9.0.x'
  PYTHONUNBUFFERED: '1'
  PYTHONDONTWRITEBYTECODE: '1'

jobs:
  # ========================================================================
  # Core & Setup Job (Critical validators + database setup)
  # ========================================================================
  core:
    name: "Core Validators & Database Setup"
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    env:
      QE_WBS_PG_DSN: "host=postgres port=5432 dbname=quantenginedb user=quantengine_ci password=quantengine_ci options='-c search_path=quantengine' sslmode=disable"
      PGPASSWORD: quantengine_ci
      PGHOST: postgres
      PGPORT: 5432
      PYTHONPATH: "$HOME/python_deps/core:."
    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_USER: quantengine_ci
          POSTGRES_PASSWORD: quantengine_ci
          POSTGRES_DB: quantenginedb
        options: >-
          --health-cmd pg_isready
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Setup Python (Official)
      uses: actions/setup-python@v4
      with:
        python-version: '3.12'
        cache: 'pip'
        cache-dependency-path: '**/requirements.txt'

    - name: Clear pip cache (CI stability)
      run: pip cache purge

    - name: Configure Runtime Paths
      run: |
        export PATH=/usr/local/bin:$PATH
        echo "/usr/local/bin" >> $GITHUB_PATH

        # Ensure Temp directory exists
        mkdir -p Temp

        echo "=== 런타임 확인 ==="
        /usr/bin/python3 --version
        node --version
        npm --version

    - name: Setup Python Environment
      run: |
        # Install from requirements.txt (cache key from setup-python)
        pip install --disable-pip-version-check --quiet --upgrade pip setuptools wheel
        pip install --disable-pip-version-check --quiet --no-cache-dir -r requirements.txt psycopg2-binary

        # Verify installation
        python3 -c 'import requests, yaml, openpyxl, pytest, psycopg; print("✓ Python dependencies installed")'

    - name: Apply Database Migrations
      env:
        PGPASSWORD: quantengine_ci
        PGHOST: postgres
        PGPORT: 5432
      run: |
        which psql || (sudo apt-get update -qq && sudo apt-get install -y -qq postgresql-client)

        echo "=== Waiting for PostgreSQL to be ready ==="
        ATTEMPT=0
        MAX_ATTEMPTS=30
        while [ $ATTEMPT -lt $MAX_ATTEMPTS ]; do
          if psql -U quantengine_ci -d quantenginedb -c "SELECT version();" 2>/dev/null; then
            echo "✓ PostgreSQL is ready"
            break
          fi
          ATTEMPT=$((ATTEMPT + 1))
          echo "Attempt $ATTEMPT/$MAX_ATTEMPTS: PostgreSQL not ready, waiting..."
          sleep 2
        done

        if [ $ATTEMPT -eq $MAX_ATTEMPTS ]; then
          echo "ERROR: PostgreSQL failed to start after $MAX_ATTEMPTS attempts"
          exit 1
        fi

        echo "=== Applying Migrations ==="
        for f in $(ls src/dotnet/QuantEngine.Infrastructure/Migrations/V*.sql | sort -V); do
          echo "Applying: $f"
          psql -U quantengine_ci -d quantenginedb -v ON_ERROR_STOP=1 -f "$f" || {
            echo "ERROR: Failed to apply $f"
            psql -U quantengine_ci -d quantenginedb -c "SELECT tablename FROM pg_tables WHERE schemaname='quantengine' ORDER BY tablename;"
            exit 1
          }
        done

        echo "=== Verifying Migrations ==="
        AUDIT_COUNT=$(psql -U quantengine_ci -d quantenginedb -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='quantengine' AND table_name LIKE 'kis_%_audit'")
        echo "kis_*_audit tables: $AUDIT_COUNT"

        if [ "$AUDIT_COUNT" -lt 3 ]; then
          echo "ERROR: Expected 3 audit tables, found $AUDIT_COUNT"
          psql -U quantengine_ci -d quantenginedb -c "SELECT tablename FROM pg_tables WHERE schemaname='quantengine' ORDER BY tablename;"
          exit 1
        fi

        echo "✓ Database migrations applied & verified (3 audit tables created)"

    - name: Setup .NET SDK
      uses: actions/setup-dotnet@v4
      with:
        dotnet-version: ${{ env.DOTNET_VERSION }}

    - name: "[CRITICAL] Run .NET Unit Tests"
      run: dotnet test src/dotnet/QuantEngine.Core.Tests/QuantEngine.Core.Tests.csproj -c Release --nologo -p:TreatWarningsAsErrors=true

    - name: "[CRITICAL] No Direct API Trading Gate"
      run: python3 tools/validate_no_direct_api_trading_v1.py

    - name: "[CRITICAL] Validate KIS API Credentials (mock)"
      env:
        KIS_APP_Key_TEST: ${{ vars.KIS_APP_KEY_TEST }}
        KIS_APP_Secret_TEST: ${{ vars.KIS_APP_SECRET_TEST }}
      run: python3 tools/validate_kis_api_credentials_v1.py --account mock --ticker 005930 --dry-run

    - name: Setup Node Dependencies (with cache)
      run: |
        CACHE_BASE="$HOME/gitea_node_cache"
        LOCK_HASH=$(md5sum package-lock.json 2>/dev/null | cut -d' ' -f1 || echo "no-lock")
        CACHE_DIR="$CACHE_BASE/$LOCK_HASH"

        if [ -d "$CACHE_DIR/node_modules" ] && [ -L node_modules ] && [ "$(readlink node_modules)" = "$CACHE_DIR/node_modules" ]; then
          echo "✓ node_modules cache hit: $LOCK_HASH"
        else
          if [ -e node_modules ] || [ -L node_modules ]; then rm -rf node_modules; fi
          if [ ! -d "$CACHE_DIR/node_modules" ]; then
            echo "Installing npm packages..."
            npm ci --quiet
            mkdir -p "$CACHE_DIR"
            cp -r node_modules "$CACHE_DIR/node_modules"
            ls -dt "$CACHE_BASE"/*/  2>/dev/null | tail -n +4 | xargs rm -rf 2>/dev/null || true
          fi
          ln -s "$CACHE_DIR/node_modules" node_modules
        fi
        echo "✓ node_modules ready"

    - name: Validate Specs & Formulas
      run: |
        python3 tools/validate_specs.py
        python3 tools/validate_formula_registry.py
        python3 tools/validate_golden_coverage_100.py
        echo "✓ Spec validations passed"

    - name: Generate WBS Verdicts (CI-Reproducible Tasks)
      run: |
        python3 - <<'PY'
        from pathlib import Path
        import subprocess
        import yaml

        root = Path.cwd()
        spec = yaml.safe_load((root / "spec" / "60_quant_engine_wbs.yaml").read_text(encoding="utf-8"))
        for task_id, task in (spec.get("tasks") or {}).items():
            if task.get("status") != "DONE":
                continue
            mode = ((task.get("execution") or {}).get("mode"))
            if mode in {"not_ci_reproducible", "manual_user_action"}:
                continue
            result = subprocess.run(["python3", "tools/verify_wbs_task_v1.py", "--task", task_id], cwd=root)
            if result.returncode != 0:
                print(f"⚠ verdict skipped for {task_id} (exit={result.returncode})")
        PY

  # ========================================================================
  # WBS & Audit Validation (Depends on core)
  # ========================================================================
  wbs-audit:
    name: "WBS & Audit Validations"
    needs: core
    runs-on: ubuntu-latest

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Setup Python (Official)
      uses: actions/setup-python@v4
      with:
        python-version: '3.12'
        cache: 'pip'
        cache-dependency-path: '**/requirements.txt'

    - name: Clear pip cache (CI stability)
      run: pip cache purge

    - name: Setup Python Environment
      run: |
        pip install --disable-pip-version-check --quiet --upgrade pip
        pip install --disable-pip-version-check --quiet pyyaml
        echo "✓ Python dependencies installed"

    - name: Validate WBS & Audits
      run: |
        python3 tools/validate_platform_transition_wbs_v1.py
        python3 tools/harness_coverage_auditor.py
        python3 tools/validate_market_time_series_schema_v1.py
        python3 tools/validate_quant_engine_wbs_v1.py
        python3 tools/validate_dotnet_migration_roadmap_v1.py
        echo "✓ WBS & audit validations passed"

  # ========================================================================
  # .NET Contracts & Parity Validation (Parallel)
  # ========================================================================
  dotnet-contracts:
    name: ".NET Contracts"
    needs: core
    runs-on: ubuntu-latest

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python (Official)
      uses: actions/setup-python@v4
      with:
        python-version: '3.12'
        cache: 'pip'
        cache-dependency-path: '**/requirements.txt'

    - name: Clear pip cache (CI stability)
      run: pip cache purge

    - name: Setup .NET SDK
      uses: actions/setup-dotnet@v4
      with:
        dotnet-version: ${{ env.DOTNET_VERSION }}

    - name: Setup Python & .NET
      run: |
        pip install --disable-pip-version-check --quiet --upgrade pip
        pip install --disable-pip-version-check --quiet pyyaml
        dotnet tool install -g dotnet-format || dotnet tool update -g dotnet-format
        echo "✓ Tools installed"

    - name: Validate .NET Contracts
      run: |
        python3 tools/validate_dotnet_migration_execution_plan_v1.py
        python3 tools/validate_dotnet_parity_contract_v1.py
        python3 tools/validate_dotnet_provenance_contract_v1.py
        python3 tools/validate_dotnet_scheduler_contract_v1.py
        python3 tools/validate_dotnet_normalization_contract_v1.py
        python3 tools/validate_dotnet_idempotency_contract_v1.py
        python3 tools/validate_dotnet_cicd_chain_contract_v1.py
        python3 tools/validate_dotnet_domain_parity_backlog_v1.py
        python3 tools/validate_dotnet_read_model_contract_v1.py
        python3 tools/validate_dotnet_domain_parity_artifact_v1.py
        echo "✓ .NET contracts validated"

    - name: Run All .NET Unit Tests
      run: dotnet test src/dotnet/QuantEngine.sln --configuration Release

  # ========================================================================
  # UI & Storage Backend Validation (Parallel)
  # ========================================================================
  ui-storage:
    name: "UI & Storage Validation"
    runs-on: ubuntu-latest

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python (Official)
      uses: actions/setup-python@v4
      with:
        python-version: '3.12'
        cache: 'pip'
        cache-dependency-path: '**/requirements.txt'

    - name: Clear pip cache (CI stability)
      run: pip cache purge

    - name: Setup Python Environment
      run: |
        pip install --disable-pip-version-check --quiet --upgrade pip setuptools wheel
        pip install --disable-pip-version-check --quiet -r requirements.txt
        echo "✓ Python dependencies installed"

    - name: Validate UI & Storage
      run: |
        python3 tools/validate_snapshot_admin_web_v1.py
        python3 -m pytest tests/unit/test_storage_backend_v1.py tests/unit/test_validate_kis_api_credentials_v1.py tests/unit/test_qualitative_sell_strategy_store_v1.py tests/unit/test_kis_api_client_v1.py tests/unit/test_snapshot_admin_store_v1.py tests/unit/test_snapshot_admin_web_v1.py -q
        echo "✓ UI & storage validations passed"

  # ========================================================================
  # Database & Schema Validation (Parallel)
  # ========================================================================
  database-schema:
    name: "Database & Schema Validation"
    runs-on: ubuntu-latest

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python (Official)
      uses: actions/setup-python@v4
      with:
        python-version: '3.12'
        cache: 'pip'
        cache-dependency-path: '**/requirements.txt'

    - name: Clear pip cache (CI stability)
      run: pip cache purge

    - name: Setup Python Environment
      run: |
        pip install --disable-pip-version-check --quiet --upgrade pip
        pip install --disable-pip-version-check --quiet pyyaml
        echo "✓ Python dependencies installed"

    - name: Validate Database Pipeline
      run: |
        python3 tools/validate_db_first_pipeline_v1.py
        python3 tools/validate_dotnet_postgresql_json_cutover_v1.py
        python3 tools/generate_postgresql_history_schema_v1.py
        python3 tools/validate_postgresql_history_contract_v1.py
        echo "✓ Database validations passed"

  # ========================================================================
  # Calibration & Performance Pipeline (Depends on core)
  # ========================================================================
  calibration-pipeline:
    name: "Calibration & Performance"
    needs: core
    runs-on: ubuntu-latest

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python (Official)
      uses: actions/setup-python@v4
      with:
        python-version: '3.12'
        cache: 'pip'
        cache-dependency-path: '**/requirements.txt'

    - name: Clear pip cache (CI stability)
      run: pip cache purge

    - name: Setup Python Environment
      run: |
        pip install --disable-pip-version-check --quiet --upgrade pip
        pip install --disable-pip-version-check --quiet pyyaml
        echo "✓ Python dependencies installed"

    - name: Ensure Temp Directory
      run: mkdir -p Temp

    - name: Build Calibration Components
      run: |
        python3 tools/build_calibration_priority_v1.py
        python3 tools/build_calibration_change_ledger_v4.py
        python3 tools/validate_calibration_change_ledger_v1.py
        echo "✓ Calibration components built"

    - name: Validate Qualitative Strategy
      run: |
        python3 tools/validate_qualitative_sell_strategy_pipeline_v1.py
        echo "✓ Qualitative sell strategy validated"

  # ========================================================================
  # Operational Report & Decision Packet (Depends on calibration)
  # ========================================================================
  operational-reporting:
    name: "Operational Report & Decision Packet"
    needs: calibration-pipeline
    runs-on: ubuntu-latest

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python (Official)
      uses: actions/setup-python@v4
      with:
        python-version: '3.12'
        cache: 'pip'
        cache-dependency-path: '**/requirements.txt'

    - name: Clear pip cache (CI stability)
      run: pip cache purge

    - name: Setup .NET SDK
      uses: actions/setup-dotnet@v4
      with:
        dotnet-version: ${{ env.DOTNET_VERSION }}

    - name: Setup Python & .NET
      run: |
        pip install --disable-pip-version-check --quiet --upgrade pip
        pip install --disable-pip-version-check --quiet pyyaml
        echo "✓ Dependencies installed"

    - name: Ensure Temp Directory & Mock Packets
      run: |
        mkdir -p Temp
        python3 -c 'import json; json.dump({"order_blueprint_json":{},"cash_recovery_plan_json":{},"per_ticker":[{"ticker":"DATA_MISSING","gate":"DATA_MISSING"}],"meta":{"formulas_run":[],"source_file":"GatherTradingData.json"}},open("Temp/computed_harness_v1.json","w"),ensure_ascii=False,indent=2)'
        if [ ! -f Temp/final_decision_packet_active.json ]; then
          python3 -c 'import json; json.dump({"formula_id":"FINAL_DECISION_PACKET_V2","meta":{"generated_at":"2026-06-29T00:00:00Z"},"canonical_metrics":{"total_asset_krw":None},"portfolio_snapshot":{},"order_table":[],"pass_100":{"gate":"DATA_MISSING","score_0_100":None},"execution_readiness":{"gate":"DATA_MISSING","min_axis_score":None},"prediction":{"match_rate_pct":None}},open("Temp/final_decision_packet_active.json","w"),ensure_ascii=False,indent=2)'
        fi

    - name: Build Operational Report
      run: |
        python3 tools/update_proposal_evaluation_history.py --json GatherTradingData.json --history Temp/proposal_evaluation_history.json
        python3 tools/build_performance_readiness_replay_bridge_v1.py --hist Temp/proposal_evaluation_history.json --out Temp/performance_readiness_replay_bridge_v1.json
        python3 tools/build_outcome_quality_score_v1.py --json GatherTradingData.json --out Temp/outcome_quality_score_v1.json --policy spec/strategy_execution_lock_policy.yaml
        python3 tools/build_trade_quality_from_t5_v1.py --hist Temp/proposal_evaluation_history.json --out Temp/trade_quality_from_t5_v1.json
        python3 tools/build_operational_alpha_calibration_v2.py --out Temp/operational_alpha_calibration_v2.json
        python3 tools/validate_operational_alpha_calibration_v2.py --input Temp/operational_alpha_calibration_v2.json --out Temp/validate_operational_alpha_calibration_v2.json
        python3 tools/build_operational_t20_outcome_ledger_v1.py --json GatherTradingData.json --out Temp/operational_t20_outcome_ledger_v1.json
        echo "✓ Operational components built"

    - name: Validate & Render Packets
      run: |
        python3 tools/validate_live_data_activation_gate_v1.py
        python3 tools/validate_replay_live_separation_v1.py
        dotnet run --project src/dotnet/QuantEngine.Tools/QuantEngine.Tools.csproj -p:TreatWarningsAsErrors=true -- packet-v4 --packet=Temp/final_decision_packet_active.json --out=Temp/final_decision_packet_v4.json
        dotnet run --project src/dotnet/QuantEngine.Tools/QuantEngine.Tools.csproj -p:TreatWarningsAsErrors=true -- report --packet=Temp/final_decision_packet_active.json --out=Temp/operational_report.json
        python3 tools/validate_report_packet_sync_v1.py --packet Temp/final_decision_packet_active.json --report Temp/operational_report.json | tee Temp/validate_report_packet_sync_v1.json
        python3 tools/validate_report_section_completeness_v1.py
        python3 tools/validate_json_generator_outputs_v1.py
        echo "✓ Operational report validated"

    - name: Package & Upload Artifacts
      if: always()
      uses: actions/upload-artifact@v3
      with:
        name: operational-report-artifacts
        path: |
          Temp/operational_report.json
          Temp/operational_alpha_calibration_v2.json
          Temp/validate_operational_alpha_calibration_v2.json
          Temp/operational_t20_outcome_ledger_v1.json

  # ========================================================================
  # Security & Secrets Validation (Parallel)
  # ========================================================================
  security-validation:
    name: "Security & Secrets"
    runs-on: ubuntu-latest

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python (Official)
      uses: actions/setup-python@v4
      with:
        python-version: '3.12'
        cache: 'pip'
        cache-dependency-path: '**/requirements.txt'

    - name: Clear pip cache (CI stability)
      run: pip cache purge

    - name: Setup Python Environment
      run: |
        pip install --disable-pip-version-check --quiet --upgrade pip
        pip install --disable-pip-version-check --quiet pyyaml
        echo "✓ Python dependencies installed"

    - name: Validate Security Configuration
      run: |
        python3 tools/validate_gitea_secrets_contract_v1.py
        python3 tools/validate_snapshot_admin_workflow_v1.py
        echo "✓ Security validations passed"

  # ========================================================================
  # CI Workflow Lint (Independent)
  # ========================================================================
  workflow-lint:
    name: "CI Workflow Lint"
    runs-on: ubuntu-latest

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python (Official)
      uses: actions/setup-python@v4
      with:
        python-version: '3.12'
        cache: 'pip'
        cache-dependency-path: '**/requirements.txt'

    - name: Clear pip cache (CI stability)
      run: pip cache purge

    - name: Setup Python Environment
      run: |
        pip install --disable-pip-version-check --quiet --upgrade pip
        pip install --disable-pip-version-check --quiet pyyaml
        python3 -c 'import yaml; print("✓ PyYAML installed")'

    - name: Lint CI Workflow
      run: python3 tools/validate_gitea_ci_workflow_lint_v1.py --workflow .gitea/workflows/ci.yml

  # ========================================================================
  # Final Notification (All jobs complete)
  # ========================================================================
  notify-results:
    name: "Notify PR Results"
    if: always() && github.event_name == 'pull_request'
    needs:
      - core
      - wbs-audit
      - dotnet-contracts
      - ui-storage
      - database-schema
      - calibration-pipeline
      - operational-reporting
      - security-validation
      - workflow-lint
    runs-on: ubuntu-latest

    steps:
    - name: Report Validation Status
      run: |
        echo "CI Validation Results:"
        echo "  Core: ${{ needs.core.result }}"
        echo "  WBS/Audit: ${{ needs.wbs-audit.result }}"
        echo "  .NET Contracts: ${{ needs.dotnet-contracts.result }}"
        echo "  UI/Storage: ${{ needs.ui-storage.result }}"
        echo "  Database: ${{ needs.database-schema.result }}"
        echo "  Calibration: ${{ needs.calibration-pipeline.result }}"
        echo "  Reporting: ${{ needs.operational-reporting.result }}"
        echo "  Security: ${{ needs.security-validation.result }}"
        echo "  Workflow Lint: ${{ needs.workflow-lint.result }}"