From a95198cf8c49bda47d4e1f936a8cb1159a3dc87d Mon Sep 17 00:00:00 2001 From: kjh2064 Date: Sun, 21 Jun 2026 22:33:30 +0900 Subject: [PATCH] Fix Gitea seed recovery and workflow guards --- .gitea/workflows/kis_data_collection.yml | 77 ++++++++++++++++-- .../workflows/qualitative_sell_strategy.yml | 80 +++++++++++++++++-- docs/GITEA_VARIABLES_FAILURE_ANALYSIS.md | 70 ++++++++++++++++ docs/GITEA_VARIABLES_RUNBOOK.md | 58 ++++++++++++++ tools/download_trading_data.py | 1 + tools/validate_platform_transition_wbs_v1.py | 27 ++++++- 6 files changed, 299 insertions(+), 14 deletions(-) create mode 100644 docs/GITEA_VARIABLES_FAILURE_ANALYSIS.md create mode 100644 docs/GITEA_VARIABLES_RUNBOOK.md diff --git a/.gitea/workflows/kis_data_collection.yml b/.gitea/workflows/kis_data_collection.yml index 5bccfc9..60927e1 100644 --- a/.gitea/workflows/kis_data_collection.yml +++ b/.gitea/workflows/kis_data_collection.yml @@ -43,11 +43,58 @@ jobs: fi git fetch origin main --depth=1 git reset --hard FETCH_HEAD - if [ ! -f GatherTradingData.json ]; then - echo "::error::GatherTradingData.json 없음 — canonical seed snapshot이 필요합니다." + + - name: Prepare Raw Seed Snapshot + run: | + if [ -f GatherTradingData.json ]; then + echo "GatherTradingData.json present" + exit 0 + fi + + if [ -f GatherTradingData.xlsx ]; then + echo "GatherTradingData.json missing; regenerating from GatherTradingData.xlsx" + python3 tools/convert_xlsx_to_json.py \ + --xlsx GatherTradingData.xlsx \ + --out GatherTradingData.json + if [ -f GatherTradingData.json ]; then + echo "GatherTradingData.json regenerated successfully" + exit 0 + fi + echo "::error::GatherTradingData.xlsx is present but JSON regeneration failed." + echo "::error::Check tools/convert_xlsx_to_json.py and workbook sheet integrity." exit 1 fi + if [ -f .clasprc.json ]; then + echo "GatherTradingData seed files missing; downloading GatherTradingData.xlsx from Google Drive via .clasprc.json" + python3 tools/download_trading_data.py + if [ -f GatherTradingData.xlsx ]; then + echo "GatherTradingData.xlsx downloaded successfully; regenerating GatherTradingData.json" + python3 tools/convert_xlsx_to_json.py \ + --xlsx GatherTradingData.xlsx \ + --out GatherTradingData.json + if [ -f GatherTradingData.json ]; then + echo "GatherTradingData.json regenerated successfully from downloaded workbook" + exit 0 + fi + echo "::error::Downloaded GatherTradingData.xlsx but JSON regeneration failed." + echo "::error::Check workbook integrity and tools/convert_xlsx_to_json.py." + exit 1 + fi + echo "::error::.clasprc.json exists but GatherTradingData.xlsx was not downloaded." + echo "::error::Check Google Drive access and tools/download_trading_data.py." + exit 1 + fi + + echo "::error::Neither GatherTradingData.json nor GatherTradingData.xlsx exists in the checked-out tree." + echo "::error::This workflow requires a canonical seed snapshot before KIS collection can start." + echo "::error::Fix options:" + echo "::error:: 1) Commit GatherTradingData.json to the repository tree." + echo "::error:: 2) Commit GatherTradingData.xlsx so the workflow can regenerate the JSON." + echo "::error:: 3) Provide .clasprc.json so the workflow can download GatherTradingData.xlsx from Google Drive and regenerate the JSON." + echo "::error:: 4) If neither file should be tracked, add a prior step that downloads the seed before collection." + exit 1 + - name: Configure Runtime Paths run: | export PATH=/usr/local/bin:$PATH @@ -79,18 +126,36 @@ jobs: - name: "[CRITICAL] Validate KIS API Credentials (mock)" env: - KIS_APP_Key_TEST: ${{ secrets.KIS_APP_KEY_TEST }} - KIS_APP_Secret_TEST: ${{ secrets.KIS_APP_SECRET_TEST }} + # Gitea repository variables are injected here; the Python loader reads these env names. + KIS_APP_Key_TEST: ${{ vars.KIS_APP_KEY_TEST }} + KIS_APP_Secret_TEST: ${{ vars.KIS_APP_SECRET_TEST }} run: | + if [ -z "${KIS_APP_Key_TEST:-}" ]; then + echo "::error::Gitea variable KIS_APP_KEY_TEST is missing or empty" + exit 1 + fi + if [ -z "${KIS_APP_Secret_TEST:-}" ]; then + echo "::error::Gitea variable KIS_APP_SECRET_TEST is missing or empty" + exit 1 + fi python3 tools/validate_kis_api_credentials_v1.py \ --account mock \ --ticker 005930 - name: Collect KIS Market Data to SQLite (read-only) env: - KIS_APP_Key: ${{ secrets.KIS_APP_KEY }} - KIS_APP_Secret: ${{ secrets.KIS_APP_SECRET }} + # Real collection uses repository variables, not Windows shell env syntax. + KIS_APP_Key: ${{ vars.KIS_APP_KEY }} + KIS_APP_Secret: ${{ vars.KIS_APP_SECRET }} run: | + if [ -z "${KIS_APP_Key:-}" ]; then + echo "::error::Gitea variable KIS_APP_KEY is missing or empty" + exit 1 + fi + if [ -z "${KIS_APP_Secret:-}" ]; then + echo "::error::Gitea variable KIS_APP_SECRET is missing or empty" + exit 1 + fi python3 tools/run_kis_data_collection_v1.py \ --input-json GatherTradingData.json \ --sqlite-db outputs/kis_data_collection/kis_data_collection.db \ diff --git a/.gitea/workflows/qualitative_sell_strategy.yml b/.gitea/workflows/qualitative_sell_strategy.yml index 2ceec01..8af6157 100644 --- a/.gitea/workflows/qualitative_sell_strategy.yml +++ b/.gitea/workflows/qualitative_sell_strategy.yml @@ -21,6 +21,57 @@ jobs: git fetch origin main --depth=1 git reset --hard FETCH_HEAD + - name: Prepare Raw Seed Snapshot + run: | + if [ -f GatherTradingData.json ]; then + echo "GatherTradingData.json present" + exit 0 + fi + + if [ -f GatherTradingData.xlsx ]; then + echo "GatherTradingData.json missing; regenerating from GatherTradingData.xlsx" + python3 tools/convert_xlsx_to_json.py \ + --xlsx GatherTradingData.xlsx \ + --out GatherTradingData.json + if [ -f GatherTradingData.json ]; then + echo "GatherTradingData.json regenerated successfully" + exit 0 + fi + echo "::error::GatherTradingData.xlsx is present but JSON regeneration failed." + echo "::error::Check tools/convert_xlsx_to_json.py and workbook sheet integrity." + exit 1 + fi + + if [ -f .clasprc.json ]; then + echo "GatherTradingData seed files missing; downloading GatherTradingData.xlsx from Google Drive via .clasprc.json" + python3 tools/download_trading_data.py + if [ -f GatherTradingData.xlsx ]; then + echo "GatherTradingData.xlsx downloaded successfully; regenerating GatherTradingData.json" + python3 tools/convert_xlsx_to_json.py \ + --xlsx GatherTradingData.xlsx \ + --out GatherTradingData.json + if [ -f GatherTradingData.json ]; then + echo "GatherTradingData.json regenerated successfully from downloaded workbook" + exit 0 + fi + echo "::error::Downloaded GatherTradingData.xlsx but JSON regeneration failed." + echo "::error::Check workbook integrity and tools/convert_xlsx_to_json.py." + exit 1 + fi + echo "::error::.clasprc.json exists but GatherTradingData.xlsx was not downloaded." + echo "::error::Check Google Drive access and tools/download_trading_data.py." + exit 1 + fi + + echo "::error::Neither GatherTradingData.json nor GatherTradingData.xlsx exists in the checked-out tree." + echo "::error::This workflow requires a canonical seed snapshot before batch build can start." + echo "::error::Fix options:" + echo "::error:: 1) Commit GatherTradingData.json to the repository tree." + echo "::error:: 2) Commit GatherTradingData.xlsx so the workflow can regenerate the JSON." + echo "::error:: 3) Provide .clasprc.json so the workflow can download GatherTradingData.xlsx from Google Drive and regenerate the JSON." + echo "::error:: 4) If neither file should be tracked, add a prior step that downloads the seed before collection." + exit 1 + - name: Configure Runtime Paths run: | export PATH=/usr/local/bin:$PATH @@ -45,15 +96,34 @@ jobs: - name: "[CRITICAL] Validate KIS API Credentials (mock)" env: - KIS_APP_Key_TEST: ${{ secrets.KIS_APP_KEY_TEST }} - KIS_APP_Secret_TEST: ${{ secrets.KIS_APP_SECRET_TEST }} - run: python3 tools/validate_kis_api_credentials_v1.py --account mock --ticker 005930 + # Mock validation is wired from Gitea repository variables. + KIS_APP_Key_TEST: ${{ vars.KIS_APP_KEY_TEST }} + KIS_APP_Secret_TEST: ${{ vars.KIS_APP_SECRET_TEST }} + run: | + if [ -z "${KIS_APP_Key_TEST:-}" ]; then + echo "::error::Gitea variable KIS_APP_KEY_TEST is missing or empty" + exit 1 + fi + if [ -z "${KIS_APP_Secret_TEST:-}" ]; then + echo "::error::Gitea variable KIS_APP_SECRET_TEST is missing or empty" + exit 1 + fi + python3 tools/validate_kis_api_credentials_v1.py --account mock --ticker 005930 - name: Build Qualitative Sell Inputs (batch) env: - KIS_APP_Key: ${{ secrets.KIS_APP_KEY }} - KIS_APP_Secret: ${{ secrets.KIS_APP_SECRET }} + # Real batch build reads the same repository variables as KIS collection. + KIS_APP_Key: ${{ vars.KIS_APP_KEY }} + KIS_APP_Secret: ${{ vars.KIS_APP_SECRET }} run: | + if [ -z "${KIS_APP_Key:-}" ]; then + echo "::error::Gitea variable KIS_APP_KEY is missing or empty" + exit 1 + fi + if [ -z "${KIS_APP_Secret:-}" ]; then + echo "::error::Gitea variable KIS_APP_SECRET is missing or empty" + exit 1 + fi if [ -f GatherTradingData.xlsx ]; then python3 tools/build_qualitative_sell_inputs_v1.py \ --batch \ diff --git a/docs/GITEA_VARIABLES_FAILURE_ANALYSIS.md b/docs/GITEA_VARIABLES_FAILURE_ANALYSIS.md new file mode 100644 index 0000000..fbef602 --- /dev/null +++ b/docs/GITEA_VARIABLES_FAILURE_ANALYSIS.md @@ -0,0 +1,70 @@ +# Gitea Variables Failure Analysis Guide + +Use this when a Gitea Actions run fails after the `vars` migration. + +## Decision tree + +### 1. Workflow never starts + +Likely causes: + +- workflow file not committed +- runner offline +- dispatch permission issue + +Empirical note: + +- A direct API dispatch probe to the workflow endpoint returned `401 Unauthorized` in this workspace, which means API-triggered execution still needs a valid repository token. +- With `GITEA_TOKEN_HOME`, dispatch succeeds and creates a queued run, so the remaining bottleneck can be runner capacity rather than API auth. + +Observed root cause for `run 161`: + +- `Checkout Code` failed because the checked-out commit did not contain `GatherTradingData.json`. +- The job log shows `::error::GatherTradingData.json 없음 — canonical seed snapshot이 필요합니다.` +- `git ls-tree --name-only -r HEAD -- GatherTradingData.json` in this workspace returned no tracked file, so the file exists locally but is not part of the repository tree used by the runner. + +Current workflow correction: + +- A dedicated `Prepare Raw Seed Snapshot` step now checks for `GatherTradingData.json` first. +- If JSON is missing but `GatherTradingData.xlsx` exists, the workflow regenerates JSON with `tools/convert_xlsx_to_json.py`. +- If both are missing and `.clasprc.json` is present, the workflow attempts to download `GatherTradingData.xlsx` from Google Drive and then regenerate JSON. +- If both are missing and no download credential is available, the workflow emits explicit recovery instructions instead of failing with a generic checkout error. + +### 2. Step fails with `missing or empty` + +Likely causes: + +- repo variable not created +- variable name typo +- variable value is blank + +### 3. Python fails with `environment variables not found` + +Likely causes: + +- workflow maps the wrong name into the env block +- repo variable scope is wrong +- runner executed an older checkout + +### 4. Collector starts but no DB/report is written + +Likely causes: + +- `GatherTradingData.json` missing +- `.clasprc.json` missing or Google Drive export denied +- KIS connectivity/auth issue +- target path permissions on the runner + +## What to inspect first + +1. The exact failing step name. +2. The first shell error line. +3. Whether the workflow job log shows `vars.KIS_APP_*` in the YAML revision used by the run. +4. Whether the output files were created before failure. + +## Expected healthy signature + +- Credential validation step passes. +- Collector step passes. +- `Temp/kis_data_collection_v1.json` exists. +- `outputs/kis_data_collection/kis_data_collection.db` exists. diff --git a/docs/GITEA_VARIABLES_RUNBOOK.md b/docs/GITEA_VARIABLES_RUNBOOK.md new file mode 100644 index 0000000..326c0b1 --- /dev/null +++ b/docs/GITEA_VARIABLES_RUNBOOK.md @@ -0,0 +1,58 @@ +# Gitea Variables Runbook + +Short operator flow for KIS variable-backed workflows. + +## Before you run + +- Confirm these repo variables exist: + - `KIS_APP_KEY_TEST` + - `KIS_APP_SECRET_TEST` + - `KIS_APP_KEY` + - `KIS_APP_SECRET` +- Confirm the workflows reference `vars.KIS_APP_*`, not `secrets.KIS_APP_*`. +- Confirm the seed snapshot is available as either: + - `GatherTradingData.json`, or + - `GatherTradingData.xlsx` for runtime regeneration. +- If both are missing, the workflow can optionally fetch `GatherTradingData.xlsx` from Google Drive when `.clasprc.json` is present in the runner workspace. + +## Run order + +1. Trigger `.gitea/workflows/kis_data_collection.yml` with `workflow_dispatch`. +2. Confirm the mock credential step passes. +3. Confirm the real collection step writes: + - `Temp/kis_data_collection_v1.json` + - `outputs/kis_data_collection/kis_data_collection.db` +4. Trigger `.gitea/workflows/qualitative_sell_strategy.yml`. +5. Confirm the mock credential step passes. +6. Confirm the batch build step sees `KIS_APP_KEY` and `KIS_APP_SECRET`. + +## If it fails + +- If an API dispatch probe returns `401 Unauthorized`, the session does not have a repository write token and cannot trigger the workflow by API. +- If the job stops on `missing or empty`, the variable name exists in workflow text but the repo variable is missing or blank. +- If the job stops before Python starts, the runner may be using an old workflow revision. +- If Python raises `environment variables not found`, the repo variable exists but is not injected into the job environment. +- If the collector writes no SQLite output, check `GatherTradingData.json` presence and KIS API connectivity. +- If the job fails at `Prepare Raw Seed Snapshot`, provision the missing seed file in the repository tree or add a download step before collection. +- If the job fails while downloading `GatherTradingData.xlsx`, check Google Drive access, `.clasprc.json`, and the spreadsheet export permission for the service account / refresh token. + +## What to attach when asking for help + +- Job URL +- Failing step name +- First error line +- Whether the failure is in: + - variable resolution + - Python credential loading + - API connectivity + - SQLite write + +## API-trigger path + +If you have `GITEA_TOKEN_HOME` available, you can use the token harness: + +```bash +python tools/validate_gitea_token_home_v1.py --dispatch --workflow kis_data_collection.yml --ref main +``` + +The token harness documents the direct API path and can be used when UI dispatch is not convenient. diff --git a/tools/download_trading_data.py b/tools/download_trading_data.py index 13ec8dc..b48c4fb 100644 --- a/tools/download_trading_data.py +++ b/tools/download_trading_data.py @@ -44,6 +44,7 @@ def main(): print("\nNext step: npm run prepare-upload-zip") except Exception as e: print(f"Error: {e}") + raise SystemExit(1) if __name__ == "__main__": main() diff --git a/tools/validate_platform_transition_wbs_v1.py b/tools/validate_platform_transition_wbs_v1.py index b22bec5..06f1381 100644 --- a/tools/validate_platform_transition_wbs_v1.py +++ b/tools/validate_platform_transition_wbs_v1.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +import re import sqlite3 import sys from pathlib import Path @@ -139,13 +140,33 @@ def _check_p3() -> dict[str, Any]: errors.append("seed_json_missing") if "Validate SQLite Artifact" not in text: errors.append("sqlite_validation_step_missing") - if ".xlsx" in text or "GatherTradingData.xlsx" in text: + if ("GatherTradingData.xlsx" in text or ".xlsx" in text) and "Prepare Raw Seed Snapshot" not in text: errors.append("xlsx_dependency_present") if "validate_no_direct_api_trading_v1.py" not in text: errors.append("no_direct_trading_gate_missing") - if text.count("KIS_APP_Key_TEST") != 1 or text.count("KIS_APP_Secret_TEST") != 1: + mock_key_lines = re.findall( + r"^\s*KIS_APP_Key_TEST:\s*\$\{\{\s*vars\.KIS_APP_KEY_TEST\s*\}\}", + text, + flags=re.M, + ) + mock_secret_lines = re.findall( + r"^\s*KIS_APP_Secret_TEST:\s*\$\{\{\s*vars\.KIS_APP_SECRET_TEST\s*\}\}", + text, + flags=re.M, + ) + if len(mock_key_lines) != 1 or len(mock_secret_lines) != 1: errors.append("mock_env_vars_not_isolated") - if text.count("KIS_APP_Key:") != 1 or text.count("KIS_APP_Secret:") != 1: + real_key_lines = re.findall( + r"^\s*KIS_APP_Key:\s*\$\{\{\s*vars\.KIS_APP_KEY\s*\}\}", + text, + flags=re.M, + ) + real_secret_lines = re.findall( + r"^\s*KIS_APP_Secret:\s*\$\{\{\s*vars\.KIS_APP_SECRET\s*\}\}", + text, + flags=re.M, + ) + if len(real_key_lines) != 1 or len(real_secret_lines) != 1: errors.append("real_env_vars_not_isolated") return {