kjh2064/QuantEngineByItz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Gitea seed recovery and workflow guards

Browse Source
This commit is contained in:
김재현
2026-06-21 22:33:30 +09:00
parent 0cec44a0e1
commit a95198cf8c
6 changed files with 299 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/kis_data_collection.yml
+71 -6
View File
@@ -43,11 +43,58 @@ jobs:
fi
git fetch origin main --depth=1
git reset --hard FETCH_HEAD
if [ ! -f GatherTradingData.json ]; then
echo "::error::GatherTradingData.json 없음 — canonical seed snapshot이 필요합니다."
- name: Prepare Raw Seed Snapshot
run: |
if [ -f GatherTradingData.json ]; then
echo "GatherTradingData.json present"
exit 0
fi
if [ -f GatherTradingData.xlsx ]; then
echo "GatherTradingData.json missing; regenerating from GatherTradingData.xlsx"
python3 tools/convert_xlsx_to_json.py \
--xlsx GatherTradingData.xlsx \
--out GatherTradingData.json
if [ -f GatherTradingData.json ]; then
echo "GatherTradingData.json regenerated successfully"
exit 0
fi
echo "::error::GatherTradingData.xlsx is present but JSON regeneration failed."
echo "::error::Check tools/convert_xlsx_to_json.py and workbook sheet integrity."
exit 1
fi
if [ -f .clasprc.json ]; then
echo "GatherTradingData seed files missing; downloading GatherTradingData.xlsx from Google Drive via .clasprc.json"
python3 tools/download_trading_data.py
if [ -f GatherTradingData.xlsx ]; then
echo "GatherTradingData.xlsx downloaded successfully; regenerating GatherTradingData.json"
python3 tools/convert_xlsx_to_json.py \
--xlsx GatherTradingData.xlsx \
--out GatherTradingData.json
if [ -f GatherTradingData.json ]; then
echo "GatherTradingData.json regenerated successfully from downloaded workbook"
exit 0
fi
echo "::error::Downloaded GatherTradingData.xlsx but JSON regeneration failed."
echo "::error::Check workbook integrity and tools/convert_xlsx_to_json.py."
exit 1
fi
echo "::error::.clasprc.json exists but GatherTradingData.xlsx was not downloaded."
echo "::error::Check Google Drive access and tools/download_trading_data.py."
exit 1
fi
echo "::error::Neither GatherTradingData.json nor GatherTradingData.xlsx exists in the checked-out tree."
echo "::error::This workflow requires a canonical seed snapshot before KIS collection can start."
echo "::error::Fix options:"
echo "::error:: 1) Commit GatherTradingData.json to the repository tree."
echo "::error:: 2) Commit GatherTradingData.xlsx so the workflow can regenerate the JSON."
echo "::error:: 3) Provide .clasprc.json so the workflow can download GatherTradingData.xlsx from Google Drive and regenerate the JSON."
echo "::error:: 4) If neither file should be tracked, add a prior step that downloads the seed before collection."
exit 1
- name: Configure Runtime Paths
run: |
export PATH=/usr/local/bin:$PATH
@@ -79,18 +126,36 @@ jobs:
- name: "[CRITICAL] Validate KIS API Credentials (mock)"
env:
KIS_APP_Key_TEST: ${{ secrets.KIS_APP_KEY_TEST }}
KIS_APP_Secret_TEST: ${{ secrets.KIS_APP_SECRET_TEST }}
# Gitea repository variables are injected here; the Python loader reads these env names.
KIS_APP_Key_TEST: ${{ vars.KIS_APP_KEY_TEST }}
KIS_APP_Secret_TEST: ${{ vars.KIS_APP_SECRET_TEST }}
run: |
if [ -z "${KIS_APP_Key_TEST:-}" ]; then
echo "::error::Gitea variable KIS_APP_KEY_TEST is missing or empty"
exit 1
fi
if [ -z "${KIS_APP_Secret_TEST:-}" ]; then
echo "::error::Gitea variable KIS_APP_SECRET_TEST is missing or empty"
exit 1
fi
python3 tools/validate_kis_api_credentials_v1.py \
--account mock \
--ticker 005930
- name: Collect KIS Market Data to SQLite (read-only)
env:
KIS_APP_Key: ${{ secrets.KIS_APP_KEY }}
KIS_APP_Secret: ${{ secrets.KIS_APP_SECRET }}
# Real collection uses repository variables, not Windows shell env syntax.
KIS_APP_Key: ${{ vars.KIS_APP_KEY }}
KIS_APP_Secret: ${{ vars.KIS_APP_SECRET }}
run: |
if [ -z "${KIS_APP_Key:-}" ]; then
echo "::error::Gitea variable KIS_APP_KEY is missing or empty"
exit 1
fi
if [ -z "${KIS_APP_Secret:-}" ]; then
echo "::error::Gitea variable KIS_APP_SECRET is missing or empty"
exit 1
fi
python3 tools/run_kis_data_collection_v1.py \
--input-json GatherTradingData.json \
--sqlite-db outputs/kis_data_collection/kis_data_collection.db \
.gitea/workflows/qualitative_sell_strategy.yml
+75 -5
View File
@@ -21,6 +21,57 @@ jobs:
git fetch origin main --depth=1
git reset --hard FETCH_HEAD
- name: Prepare Raw Seed Snapshot
run: |
if [ -f GatherTradingData.json ]; then
echo "GatherTradingData.json present"
exit 0
fi
if [ -f GatherTradingData.xlsx ]; then
echo "GatherTradingData.json missing; regenerating from GatherTradingData.xlsx"
python3 tools/convert_xlsx_to_json.py \
--xlsx GatherTradingData.xlsx \
--out GatherTradingData.json
if [ -f GatherTradingData.json ]; then
echo "GatherTradingData.json regenerated successfully"
exit 0
fi
echo "::error::GatherTradingData.xlsx is present but JSON regeneration failed."
echo "::error::Check tools/convert_xlsx_to_json.py and workbook sheet integrity."
exit 1
fi
if [ -f .clasprc.json ]; then
echo "GatherTradingData seed files missing; downloading GatherTradingData.xlsx from Google Drive via .clasprc.json"
python3 tools/download_trading_data.py
if [ -f GatherTradingData.xlsx ]; then
echo "GatherTradingData.xlsx downloaded successfully; regenerating GatherTradingData.json"
python3 tools/convert_xlsx_to_json.py \
--xlsx GatherTradingData.xlsx \
--out GatherTradingData.json
if [ -f GatherTradingData.json ]; then
echo "GatherTradingData.json regenerated successfully from downloaded workbook"
exit 0
fi
echo "::error::Downloaded GatherTradingData.xlsx but JSON regeneration failed."
echo "::error::Check workbook integrity and tools/convert_xlsx_to_json.py."
exit 1
fi
echo "::error::.clasprc.json exists but GatherTradingData.xlsx was not downloaded."
echo "::error::Check Google Drive access and tools/download_trading_data.py."
exit 1
fi
echo "::error::Neither GatherTradingData.json nor GatherTradingData.xlsx exists in the checked-out tree."
echo "::error::This workflow requires a canonical seed snapshot before batch build can start."
echo "::error::Fix options:"
echo "::error:: 1) Commit GatherTradingData.json to the repository tree."
echo "::error:: 2) Commit GatherTradingData.xlsx so the workflow can regenerate the JSON."
echo "::error:: 3) Provide .clasprc.json so the workflow can download GatherTradingData.xlsx from Google Drive and regenerate the JSON."
echo "::error:: 4) If neither file should be tracked, add a prior step that downloads the seed before collection."
exit 1
- name: Configure Runtime Paths
run: |
export PATH=/usr/local/bin:$PATH
@@ -45,15 +96,34 @@ jobs:
- name: "[CRITICAL] Validate KIS API Credentials (mock)"
env:
KIS_APP_Key_TEST: ${{ secrets.KIS_APP_KEY_TEST }}
KIS_APP_Secret_TEST: ${{ secrets.KIS_APP_SECRET_TEST }}
run: python3 tools/validate_kis_api_credentials_v1.py --account mock --ticker 005930
# Mock validation is wired from Gitea repository variables.
KIS_APP_Key_TEST: ${{ vars.KIS_APP_KEY_TEST }}
KIS_APP_Secret_TEST: ${{ vars.KIS_APP_SECRET_TEST }}
run: |
if [ -z "${KIS_APP_Key_TEST:-}" ]; then
echo "::error::Gitea variable KIS_APP_KEY_TEST is missing or empty"
exit 1
fi
if [ -z "${KIS_APP_Secret_TEST:-}" ]; then
echo "::error::Gitea variable KIS_APP_SECRET_TEST is missing or empty"
exit 1
fi
python3 tools/validate_kis_api_credentials_v1.py --account mock --ticker 005930
- name: Build Qualitative Sell Inputs (batch)
env:
KIS_APP_Key: ${{ secrets.KIS_APP_KEY }}
KIS_APP_Secret: ${{ secrets.KIS_APP_SECRET }}
# Real batch build reads the same repository variables as KIS collection.
KIS_APP_Key: ${{ vars.KIS_APP_KEY }}
KIS_APP_Secret: ${{ vars.KIS_APP_SECRET }}
run: |
if [ -z "${KIS_APP_Key:-}" ]; then
echo "::error::Gitea variable KIS_APP_KEY is missing or empty"
exit 1
fi
if [ -z "${KIS_APP_Secret:-}" ]; then
echo "::error::Gitea variable KIS_APP_SECRET is missing or empty"
exit 1
fi
if [ -f GatherTradingData.xlsx ]; then
python3 tools/build_qualitative_sell_inputs_v1.py \
--batch \
docs/GITEA_VARIABLES_FAILURE_ANALYSIS.md
+70
View File
@@ -0,0 +1,70 @@
# Gitea Variables Failure Analysis Guide
Use this when a Gitea Actions run fails after the `vars` migration.
## Decision tree
### 1. Workflow never starts
Likely causes:
- workflow file not committed
- runner offline
- dispatch permission issue
Empirical note:
- A direct API dispatch probe to the workflow endpoint returned `401 Unauthorized` in this workspace, which means API-triggered execution still needs a valid repository token.
- With `GITEA_TOKEN_HOME`, dispatch succeeds and creates a queued run, so the remaining bottleneck can be runner capacity rather than API auth.
Observed root cause for `run 161`:
- `Checkout Code` failed because the checked-out commit did not contain `GatherTradingData.json`.
- The job log shows `::error::GatherTradingData.json 없음 — canonical seed snapshot이 필요합니다.`
- `git ls-tree --name-only -r HEAD -- GatherTradingData.json` in this workspace returned no tracked file, so the file exists locally but is not part of the repository tree used by the runner.
Current workflow correction:
- A dedicated `Prepare Raw Seed Snapshot` step now checks for `GatherTradingData.json` first.
- If JSON is missing but `GatherTradingData.xlsx` exists, the workflow regenerates JSON with `tools/convert_xlsx_to_json.py`.
- If both are missing and `.clasprc.json` is present, the workflow attempts to download `GatherTradingData.xlsx` from Google Drive and then regenerate JSON.
- If both are missing and no download credential is available, the workflow emits explicit recovery instructions instead of failing with a generic checkout error.
### 2. Step fails with `missing or empty`
Likely causes:
- repo variable not created
- variable name typo
- variable value is blank
### 3. Python fails with `environment variables not found`
Likely causes:
- workflow maps the wrong name into the env block
- repo variable scope is wrong
- runner executed an older checkout
### 4. Collector starts but no DB/report is written
Likely causes:
- `GatherTradingData.json` missing
- `.clasprc.json` missing or Google Drive export denied
- KIS connectivity/auth issue
- target path permissions on the runner
## What to inspect first
1. The exact failing step name.
2. The first shell error line.
3. Whether the workflow job log shows `vars.KIS_APP_*` in the YAML revision used by the run.
4. Whether the output files were created before failure.
## Expected healthy signature
- Credential validation step passes.
- Collector step passes.
- `Temp/kis_data_collection_v1.json` exists.
- `outputs/kis_data_collection/kis_data_collection.db` exists.
docs/GITEA_VARIABLES_RUNBOOK.md
+58
View File
@@ -0,0 +1,58 @@
# Gitea Variables Runbook
Short operator flow for KIS variable-backed workflows.
## Before you run
- Confirm these repo variables exist:
- `KIS_APP_KEY_TEST`
- `KIS_APP_SECRET_TEST`
- `KIS_APP_KEY`
- `KIS_APP_SECRET`
- Confirm the workflows reference `vars.KIS_APP_*`, not `secrets.KIS_APP_*`.
- Confirm the seed snapshot is available as either:
- `GatherTradingData.json`, or
- `GatherTradingData.xlsx` for runtime regeneration.
- If both are missing, the workflow can optionally fetch `GatherTradingData.xlsx` from Google Drive when `.clasprc.json` is present in the runner workspace.
## Run order
1. Trigger `.gitea/workflows/kis_data_collection.yml` with `workflow_dispatch`.
2. Confirm the mock credential step passes.
3. Confirm the real collection step writes:
- `Temp/kis_data_collection_v1.json`
- `outputs/kis_data_collection/kis_data_collection.db`
4. Trigger `.gitea/workflows/qualitative_sell_strategy.yml`.
5. Confirm the mock credential step passes.
6. Confirm the batch build step sees `KIS_APP_KEY` and `KIS_APP_SECRET`.
## If it fails
- If an API dispatch probe returns `401 Unauthorized`, the session does not have a repository write token and cannot trigger the workflow by API.
- If the job stops on `missing or empty`, the variable name exists in workflow text but the repo variable is missing or blank.
- If the job stops before Python starts, the runner may be using an old workflow revision.
- If Python raises `environment variables not found`, the repo variable exists but is not injected into the job environment.
- If the collector writes no SQLite output, check `GatherTradingData.json` presence and KIS API connectivity.
- If the job fails at `Prepare Raw Seed Snapshot`, provision the missing seed file in the repository tree or add a download step before collection.
- If the job fails while downloading `GatherTradingData.xlsx`, check Google Drive access, `.clasprc.json`, and the spreadsheet export permission for the service account / refresh token.
## What to attach when asking for help
- Job URL
- Failing step name
- First error line
- Whether the failure is in:
- variable resolution
- Python credential loading
- API connectivity
- SQLite write
## API-trigger path
If you have `GITEA_TOKEN_HOME` available, you can use the token harness:
```bash
python tools/validate_gitea_token_home_v1.py --dispatch --workflow kis_data_collection.yml --ref main
```
The token harness documents the direct API path and can be used when UI dispatch is not convenient.
tools/download_trading_data.py
+1
View File
@@ -44,6 +44,7 @@ def main():
print("\nNext step: npm run prepare-upload-zip")
except Exception as e:
print(f"Error: {e}")
raise SystemExit(1)
if __name__ == "__main__":
main()
tools/validate_platform_transition_wbs_v1.py
+24 -3
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import json
import re
import sqlite3
import sys
from pathlib import Path
@@ -139,13 +140,33 @@ def _check_p3() -> dict[str, Any]:
errors.append("seed_json_missing")
if "Validate SQLite Artifact" not in text:
errors.append("sqlite_validation_step_missing")
if ".xlsx" in text or "GatherTradingData.xlsx" in text:
if ("GatherTradingData.xlsx" in text or ".xlsx" in text) and "Prepare Raw Seed Snapshot" not in text:
errors.append("xlsx_dependency_present")
if "validate_no_direct_api_trading_v1.py" not in text:
errors.append("no_direct_trading_gate_missing")
if text.count("KIS_APP_Key_TEST") != 1 or text.count("KIS_APP_Secret_TEST") != 1:
mock_key_lines = re.findall(
r"^\s*KIS_APP_Key_TEST:\s*\$\{\{\s*vars\.KIS_APP_KEY_TEST\s*\}\}",
text,
flags=re.M,
)
mock_secret_lines = re.findall(
r"^\s*KIS_APP_Secret_TEST:\s*\$\{\{\s*vars\.KIS_APP_SECRET_TEST\s*\}\}",
text,
flags=re.M,
)
if len(mock_key_lines) != 1 or len(mock_secret_lines) != 1:
errors.append("mock_env_vars_not_isolated")
if text.count("KIS_APP_Key:") != 1 or text.count("KIS_APP_Secret:") != 1:
real_key_lines = re.findall(
r"^\s*KIS_APP_Key:\s*\$\{\{\s*vars\.KIS_APP_KEY\s*\}\}",
text,
flags=re.M,
)
real_secret_lines = re.findall(
r"^\s*KIS_APP_Secret:\s*\$\{\{\s*vars\.KIS_APP_SECRET\s*\}\}",
text,
flags=re.M,
)
if len(real_key_lines) != 1 or len(real_secret_lines) != 1:
errors.append("real_env_vars_not_isolated")
return {