245 lines
10 KiB
YAML
245 lines
10 KiB
YAML
name: KIS Data Collection (SQLite Canonical Feed)
|
|
|
|
# ─────────────────────────────────────────────────────────────────
|
|
# [중요] 이 워크플로우는 KIS Open API를 코어로 하는 read-only 데이터 수집만 수행한다.
|
|
# GatherTradingData.json + live read-only APIs를 통해 SQLite canonical store를 갱신한다.
|
|
# xlsx는 이 워크플로우의 직접 입력이 아니며, KIS 실패 시에만 별도 보조 경로에서 사용한다.
|
|
#
|
|
# 스케줄: 영업일(월~금) 08:00~17:00 KST, 2시간 간격(08/10/12/14/16시).
|
|
# Gitea Actions의 schedule cron은 UTC 기준으로 평가된다(서버 타임존이 별도
|
|
# 설정되어 있지 않은 경우의 기본값). 아래 cron은 UTC로 작성했다:
|
|
# KST 08:00 = UTC 전날 23:00 → 요일은 "한국 기준 평일"에 맞춰 UTC 0-4(일~목)로 이동
|
|
# KST 10/12/14/16:00 = UTC 01/03/05/07:00, 같은 날(UTC 월~금, 1-5)
|
|
#
|
|
# [실제 Gitea 서버 타임존이 Asia/Seoul로 설정되어 있다면] 아래 cron을 그대로
|
|
# "0 8,10,12,14,16 * * 1-5" 한 줄로 교체하면 된다 — 첫 실행 후 Actions 실행
|
|
# 기록의 타임스탬프를 확인해 KST 08시 전후로 도는지 검증할 것(추정하지 말고 확인).
|
|
#
|
|
# 스케줄 주기 변경: 아래 schedule 목록의 cron 줄을 추가/삭제/수정하면 된다.
|
|
# 예) 1시간 간격으로 바꾸려면 09,11,13,15시 슬롯을 추가.
|
|
# ─────────────────────────────────────────────────────────────────
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "0 23 * * 0-4" # KST 월~금 08:00 (UTC 일~목 23:00)
|
|
- cron: "0 1 * * 1-5" # KST 월~금 10:00 (UTC 01:00)
|
|
- cron: "0 3 * * 1-5" # KST 월~금 12:00 (UTC 03:00)
|
|
- cron: "0 5 * * 1-5" # KST 월~금 14:00 (UTC 05:00)
|
|
- cron: "0 7 * * 1-5" # KST 월~금 16:00 (UTC 07:00)
|
|
workflow_dispatch: # 수동 실행 — 스케줄 검증/즉시 재시도용
|
|
|
|
jobs:
|
|
validate-kis-config-smoke:
|
|
if: github.event_name == 'workflow_dispatch'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout Code
|
|
run: |
|
|
if [ -d .git ]; then
|
|
git remote set-url origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git
|
|
else
|
|
git init
|
|
git remote add origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git
|
|
fi
|
|
TARGET_REF="${GITHUB_REF_NAME:-main}"
|
|
git fetch origin "$TARGET_REF" --depth=1
|
|
git reset --hard FETCH_HEAD
|
|
|
|
- name: Setup Python Environment
|
|
run: |
|
|
VENV_BASE=/volume1/gitea/python_venv
|
|
REQ_HASH=$(md5sum tools/run_kis_data_collection_v1.py 2>/dev/null | cut -d' ' -f1 || echo "kis-default")
|
|
VENV="$VENV_BASE/$REQ_HASH"
|
|
|
|
if [ ! -f "$VENV/bin/python" ]; then
|
|
mkdir -p "$VENV_BASE"
|
|
/usr/bin/python3 -m venv "$VENV"
|
|
if [ ! -f "$VENV/bin/pip" ]; then
|
|
curl -sS https://bootstrap.pypa.io/pip/3.8/get-pip.py -o get-pip.py
|
|
"$VENV/bin/python" get-pip.py --quiet
|
|
rm get-pip.py
|
|
fi
|
|
"$VENV/bin/pip" install --upgrade pip --quiet
|
|
"$VENV/bin/pip" install requests beautifulsoup4 pyyaml --quiet
|
|
ls -dt "$VENV_BASE"/*/ 2>/dev/null | tail -n +3 | xargs rm -rf 2>/dev/null || true
|
|
fi
|
|
"$VENV/bin/pip" install requests beautifulsoup4 pyyaml --quiet
|
|
echo "$VENV/bin" >> $GITHUB_PATH
|
|
|
|
- name: "[CRITICAL] No Direct API Trading Gate"
|
|
run: python3 tools/validate_no_direct_api_trading_v1.py
|
|
|
|
- name: "[CRITICAL] Validate KIS API Credentials (mock)"
|
|
env:
|
|
# Gitea repository variables are injected here; the Python loader reads these env names.
|
|
KIS_APP_Key_TEST: ${{ vars.KIS_APP_KEY_TEST }}
|
|
KIS_APP_Secret_TEST: ${{ vars.KIS_APP_SECRET_TEST }}
|
|
run: |
|
|
if [ -z "${KIS_APP_Key_TEST:-}" ]; then
|
|
echo "::error::Gitea variable KIS_APP_KEY_TEST is missing or empty"
|
|
exit 1
|
|
fi
|
|
if [ -z "${KIS_APP_Secret_TEST:-}" ]; then
|
|
echo "::error::Gitea variable KIS_APP_SECRET_TEST is missing or empty"
|
|
exit 1
|
|
fi
|
|
python3 tools/validate_kis_api_credentials_v1.py \
|
|
--account mock \
|
|
--ticker 005930 \
|
|
--dry-run
|
|
|
|
collect-kis-data-live:
|
|
if: github.event_name == 'schedule'
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- name: Checkout Code
|
|
run: |
|
|
if [ -d .git ]; then
|
|
git remote set-url origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git
|
|
else
|
|
git init
|
|
git remote add origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git
|
|
fi
|
|
TARGET_REF="${GITHUB_REF_NAME:-main}"
|
|
git fetch origin "$TARGET_REF" --depth=1
|
|
git reset --hard FETCH_HEAD
|
|
|
|
- name: Prepare Raw Seed Snapshot
|
|
run: |
|
|
if [ -f GatherTradingData.json ]; then
|
|
echo "GatherTradingData.json present"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -f .clasprc.json ]; then
|
|
echo "GatherTradingData.json missing; seed regeneration is not performed in this workflow."
|
|
echo "::error::Commit or pre-stage GatherTradingData.json before running this workflow."
|
|
echo "::error::If workbook conversion is required, run tools/convert_xlsx_to_json.py in a separate seed-prep step."
|
|
exit 1
|
|
fi
|
|
|
|
echo "::error::GatherTradingData.json is missing."
|
|
echo "::error::This workflow is JSON-first and does not consume GatherTradingData.xlsx directly."
|
|
echo "::error::Fix options:"
|
|
echo "::error:: 1) Commit GatherTradingData.json to the repository tree."
|
|
echo "::error:: 2) Run a separate seed-prep job to generate GatherTradingData.json from workbook sources."
|
|
exit 1
|
|
|
|
- name: Configure Runtime Paths
|
|
run: |
|
|
export PATH=/usr/local/bin:$PATH
|
|
echo "/usr/local/bin" >> $GITHUB_PATH
|
|
/usr/bin/python3 --version
|
|
|
|
- name: Setup Python Environment
|
|
run: |
|
|
VENV_BASE=/volume1/gitea/python_venv
|
|
REQ_HASH=$(md5sum tools/run_kis_data_collection_v1.py 2>/dev/null | cut -d' ' -f1 || echo "kis-default")
|
|
VENV="$VENV_BASE/$REQ_HASH"
|
|
|
|
if [ ! -f "$VENV/bin/python" ]; then
|
|
mkdir -p "$VENV_BASE"
|
|
/usr/bin/python3 -m venv "$VENV"
|
|
if [ ! -f "$VENV/bin/pip" ]; then
|
|
curl -sS https://bootstrap.pypa.io/pip/3.8/get-pip.py -o get-pip.py
|
|
"$VENV/bin/python" get-pip.py --quiet
|
|
rm get-pip.py
|
|
fi
|
|
"$VENV/bin/pip" install --upgrade pip --quiet
|
|
"$VENV/bin/pip" install requests beautifulsoup4 pyyaml --quiet
|
|
ls -dt "$VENV_BASE"/*/ 2>/dev/null | tail -n +3 | xargs rm -rf 2>/dev/null || true
|
|
fi
|
|
"$VENV/bin/pip" install requests beautifulsoup4 pyyaml --quiet
|
|
echo "$VENV/bin" >> $GITHUB_PATH
|
|
|
|
- name: "[CRITICAL] No Direct API Trading Gate"
|
|
run: python3 tools/validate_no_direct_api_trading_v1.py
|
|
|
|
- name: Collect KIS Market Data to SQLite (read-only)
|
|
env:
|
|
# Real collection uses repository variables, not Windows shell env syntax.
|
|
KIS_APP_Key: ${{ vars.KIS_APP_KEY }}
|
|
KIS_APP_Secret: ${{ vars.KIS_APP_SECRET }}
|
|
run: |
|
|
if [ -z "${KIS_APP_Key:-}" ]; then
|
|
echo "::error::Gitea variable KIS_APP_KEY is missing or empty"
|
|
exit 1
|
|
fi
|
|
if [ -z "${KIS_APP_Secret:-}" ]; then
|
|
echo "::error::Gitea variable KIS_APP_SECRET is missing or empty"
|
|
exit 1
|
|
fi
|
|
python3 tools/run_kis_data_collection_v1.py \
|
|
--input-json GatherTradingData.json \
|
|
--sqlite-db outputs/kis_data_collection/kis_data_collection.db \
|
|
--output-json Temp/kis_data_collection_v1.json \
|
|
--kis-account real
|
|
|
|
- name: Validate SQLite Artifact
|
|
run: |
|
|
python3 - <<'PY'
|
|
import json, sqlite3
|
|
from pathlib import Path
|
|
db = Path("outputs/kis_data_collection/kis_data_collection.db")
|
|
report = Path("Temp/kis_data_collection_v1.json")
|
|
assert db.exists(), f"missing db: {db}"
|
|
assert report.exists(), f"missing report: {report}"
|
|
conn = sqlite3.connect(db)
|
|
try:
|
|
run_count = conn.execute("SELECT COUNT(*) FROM collection_runs").fetchone()[0]
|
|
snap_count = conn.execute("SELECT COUNT(*) FROM collection_snapshots").fetchone()[0]
|
|
print(json.dumps({"run_count": run_count, "snapshot_count": snap_count}, ensure_ascii=False))
|
|
assert run_count >= 1
|
|
assert snap_count >= 1
|
|
finally:
|
|
conn.close()
|
|
PY
|
|
|
|
- name: Backup SQLite Database (WBS-9.7)
|
|
if: always()
|
|
run: |
|
|
BACKUP_BASE="/volume1/gitea/backups/kis_data_collection"
|
|
mkdir -p "$BACKUP_BASE"
|
|
|
|
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
|
SOURCE_DB="outputs/kis_data_collection/kis_data_collection.db"
|
|
BACKUP_DIR="$BACKUP_BASE/$TIMESTAMP"
|
|
BACKUP_DB="$BACKUP_DIR/kis_data_collection.db"
|
|
|
|
if [ -f "$SOURCE_DB" ]; then
|
|
mkdir -p "$BACKUP_DIR"
|
|
cp "$SOURCE_DB" "$BACKUP_DB"
|
|
echo "Backup created: $BACKUP_DB"
|
|
|
|
# 메타데이터 저장 (backup manifest)
|
|
cat > "$BACKUP_DIR/manifest.json" <<EOF
|
|
{
|
|
"timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"source_db": "$SOURCE_DB",
|
|
"backup_db": "$BACKUP_DB",
|
|
"job_id": "${{ github.run_id }}",
|
|
"branch": "${{ github.ref }}",
|
|
"status": "${{ job.status }}"
|
|
}
|
|
EOF
|
|
|
|
# 오래된 백업 정리 (7일 이상 된 것 삭제)
|
|
find "$BACKUP_BASE" -mindepth 1 -maxdepth 1 -type d -mtime +7 -exec rm -rf {} \; 2>/dev/null || true
|
|
else
|
|
echo "::warning::Source DB not found: $SOURCE_DB"
|
|
fi
|
|
|
|
- name: Notify Run Result
|
|
if: always()
|
|
run: |
|
|
STATUS="${{ job.status }}"
|
|
RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
SUMMARY_FILE="Temp/kis_data_collection_v1.json"
|
|
SUMMARY_TEXT="(요약 파일 없음)"
|
|
[ -f "$SUMMARY_FILE" ] && SUMMARY_TEXT=$(cat "$SUMMARY_FILE")
|
|
echo "=== KIS Data Collection Result ==="
|
|
echo "status: $STATUS"
|
|
echo "summary: $SUMMARY_TEXT"
|
|
echo "run log: $RUN_URL"
|