QuantEngineByItz/.gitea/workflows/kis_data_collection.yml

name: KIS Data Collection & Validation

on:
  schedule:
    - cron: "30 0 * * 1-5"  # Daily 00:30 KST (weekdays only)
  workflow_dispatch:
    inputs:
      dry_run:
        description: "Dry run mode (mock account only)"
        required: false
        default: "true"
        type: choice
        options:
          - "true"
          - "false"

jobs:
  validate-credentials:
    name: "Validate KIS API Credentials"
    runs-on: ubuntu-latest
    outputs:
      mock-valid: ${{ steps.mock.outcome }}
      prod-valid: ${{ steps.prod.outcome }}

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python Environment
      run: |
        PYTHON_DEPS="$HOME/python_deps/kis"
        mkdir -p "$PYTHON_DEPS"
        /usr/bin/python3 -m pip install --disable-pip-version-check --quiet \
          --target "$PYTHON_DEPS" pyyaml
        export PYTHONPATH="$PYTHON_DEPS:${PYTHONPATH:-}"
        echo "PYTHONPATH=$PYTHON_DEPS:${PYTHONPATH:-}" >> "$GITHUB_ENV"

    - name: Validate Mock Credentials
      id: mock
      env:
        KIS_APP_Key_TEST: ${{ vars.KIS_APP_KEY_TEST }}
        KIS_APP_Secret_TEST: ${{ vars.KIS_APP_SECRET_TEST }}
      continue-on-error: true
      run: |
        python3 tools/validate_kis_api_credentials_v1.py \
          --account mock \
          --ticker 005930 \
          --dry-run
        echo "✓ Mock credentials validated"

    - name: Validate Production Credentials (CI-only)
      id: prod
      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'false' }}
      env:
        KIS_APP_Key: ${{ vars.KIS_APP_KEY }}
        KIS_APP_Secret: ${{ vars.KIS_APP_SECRET }}
      continue-on-error: true
      run: |
        python3 tools/validate_kis_api_credentials_v1.py \
          --account real \
          --ticker 005930 \
          --dry-run
        echo "✓ Production credentials validated"

  validate-database-pipeline:
    name: "Validate Database Pipeline"
    runs-on: ubuntu-latest

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python Environment
      run: |
        PYTHON_DEPS="$HOME/python_deps/db_validate"
        mkdir -p "$PYTHON_DEPS"
        /usr/bin/python3 -m pip install --disable-pip-version-check --quiet \
          --target "$PYTHON_DEPS" pyyaml
        export PYTHONPATH="$PYTHON_DEPS:${PYTHONPATH:-}"
        echo "PYTHONPATH=$PYTHON_DEPS:${PYTHONPATH:-}" >> "$GITHUB_ENV"

    - name: Validate PostgreSQL JSON Cutover
      run: |
        python3 tools/validate_dotnet_postgresql_json_cutover_v1.py
        echo "✓ PostgreSQL JSON cutover validated"

    - name: Validate Database Schema
      run: |
        python3 tools/validate_db_first_pipeline_v1.py
        echo "✓ Database schema pipeline validated"

  validate-data-quality:
    name: "Validate Daily Data Consistency"
    runs-on: ubuntu-latest
    env:
      PYTHONPATH: "$HOME/python_deps/quality:."

    steps:
    - name: Checkout Code
      uses: actions/checkout@v3

    - name: Setup Python Environment
      run: |
        PYTHON_DEPS="$HOME/python_deps/quality"
        mkdir -p "$PYTHON_DEPS"
        /usr/bin/python3 -m pip install --disable-pip-version-check --quiet \
          --target "$PYTHON_DEPS" pyyaml
        export PYTHONPATH="$PYTHON_DEPS:${PYTHONPATH:-}"
        echo "PYTHONPATH=$PYTHON_DEPS:${PYTHONPATH:-}" >> "$GITHUB_ENV"

    - name: Run Daily Data Consistency Validation
      run: |
        mkdir -p Temp
        python3 tools/validate_data_consistency_daily_v1.py --mode warn
        echo "✓ Daily data consistency validation completed"
        cat Temp/data_consistency_report.json | python3 -m json.tool

  notify-status:
    name: "Notify Collection Status"
    if: always()
    needs: [validate-credentials, validate-database-pipeline, validate-data-quality]
    runs-on: ubuntu-latest

    steps:
    - name: Report Status
      env:
        CRED_STATUS: ${{ needs.validate-credentials.result }}
        DB_STATUS: ${{ needs.validate-database-pipeline.result }}
        QUALITY_STATUS: ${{ needs.validate-data-quality.result }}
      run: |
        echo "═══════════════════════════════════════════════════════════"
        echo "KIS Data Collection & Validation Report"
        echo "═══════════════════════════════════════════════════════════"
        echo ""
        echo "Credentials Validation: $([ "$CRED_STATUS" = "success" ] && echo "✅ PASS" || echo "❌ FAIL")"
        echo "Database Pipeline: $([ "$DB_STATUS" = "success" ] && echo "✅ PASS" || echo "❌ FAIL")"
        echo "Data Quality: $([ "$QUALITY_STATUS" = "success" ] && echo "✅ PASS" || echo "❌ FAIL")"
        echo ""
        if [ "$CRED_STATUS" = "success" ] && [ "$DB_STATUS" = "success" ] && [ "$QUALITY_STATUS" = "success" ]; then
          echo "✅ All validations passed — KIS API is ready"
          exit 0
        else
          echo "❌ Some validations failed — review logs above"
          exit 1
        fi