diff --git a/.gitea/workflows/snapshot_admin.yml b/.gitea/workflows/snapshot_admin.yml index ff6b473..2158040 100644 --- a/.gitea/workflows/snapshot_admin.yml +++ b/.gitea/workflows/snapshot_admin.yml @@ -14,7 +14,39 @@ on: - "GatherTradingData.json" jobs: - validate-snapshot-admin: + validate-snapshot-admin-smoke: + if: github.event_name == 'push' + runs-on: self-hosted + steps: + - name: Checkout Code + run: | + if [ -d .git ]; then + git remote set-url origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git + else + git init + git remote add origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git + fi + git fetch origin main --depth=1 + git reset --hard FETCH_HEAD + + - name: Setup Python Environment + run: | + VENV_BASE=/volume1/gitea/python_venv + REQ_HASH=$(md5sum tools/validate_snapshot_admin_workflow_v1.py 2>/dev/null | cut -d' ' -f1 || echo "snapshot-admin-default") + VENV="$VENV_BASE/$REQ_HASH" + if [ ! -f "$VENV/bin/python" ]; then + mkdir -p "$VENV_BASE" + /usr/bin/python3 -m venv "$VENV" + "$VENV/bin/pip" install --upgrade pip --quiet + fi + "$VENV/bin/pip" install pyyaml --quiet + echo "$VENV/bin" >> $GITHUB_PATH + + - name: Validate Snapshot Admin Workflow + run: python3 tools/validate_snapshot_admin_workflow_v1.py + + validate-snapshot-admin-full: + if: github.event_name == 'workflow_dispatch' runs-on: self-hosted steps: - name: Checkout Code @@ -55,3 +87,69 @@ jobs: echo "status: $STATUS" echo "workflow validation: Temp/snapshot_admin_workflow_v1.json" echo "web validation: Temp/snapshot_admin_web_validation_v1.json" + + deploy-snapshot-admin: + if: github.event_name == 'workflow_dispatch' + needs: + - validate-snapshot-admin-full + runs-on: self-hosted + steps: + - name: Checkout Code + run: | + if [ -d .git ]; then + git remote set-url origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git + else + git init + git remote add origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git + fi + git fetch origin main --depth=1 + git reset --hard FETCH_HEAD + + - name: Setup Python Environment + run: | + VENV_BASE=/volume1/gitea/python_venv + REQ_HASH=$(md5sum tools/validate_snapshot_admin_workflow_v1.py 2>/dev/null | cut -d' ' -f1 || echo "snapshot-admin-default") + VENV="$VENV_BASE/$REQ_HASH" + if [ ! -f "$VENV/bin/python" ]; then + mkdir -p "$VENV_BASE" + /usr/bin/python3 -m venv "$VENV" + "$VENV/bin/pip" install --upgrade pip --quiet + fi + "$VENV/bin/pip" install pyyaml --quiet + echo "$VENV/bin" >> $GITHUB_PATH + + - name: Deploy Snapshot Admin Runtime + env: + SNAPSHOT_ADMIN_AUTH_USER: ${{ vars.SNAPSHOT_ADMIN_AUTH_USER }} + SNAPSHOT_ADMIN_AUTH_PASSWORD: ${{ secrets.SNAPSHOT_ADMIN_AUTH_PASSWORD }} + run: | + export ROOT_DIR="$PWD" + export SNAPSHOT_ADMIN_HOST=127.0.0.1 + export SNAPSHOT_ADMIN_PORT=8787 + export SNAPSHOT_ADMIN_PID_FILE="$PWD/Temp/snapshot_admin.pid" + export SNAPSHOT_ADMIN_LOG_FILE="$PWD/Temp/snapshot_admin.log" + export SNAPSHOT_ADMIN_STATE_URL="http://127.0.0.1:8787/api/state" + export SNAPSHOT_ADMIN_PUBLIC_STATE_URL="https://admin.example.com/api/state" + export SNAPSHOT_ADMIN_AUTH_USER="${SNAPSHOT_ADMIN_AUTH_USER:-}" + export SNAPSHOT_ADMIN_AUTH_PASSWORD="${SNAPSHOT_ADMIN_AUTH_PASSWORD:-}" + bash tools/run_snapshot_admin_synology.sh restart + + - name: Verify Snapshot Admin Runtime + env: + SNAPSHOT_ADMIN_AUTH_USER: ${{ vars.SNAPSHOT_ADMIN_AUTH_USER }} + SNAPSHOT_ADMIN_AUTH_PASSWORD: ${{ secrets.SNAPSHOT_ADMIN_AUTH_PASSWORD }} + run: | + export ROOT_DIR="$PWD" + export SNAPSHOT_ADMIN_HOST=127.0.0.1 + export SNAPSHOT_ADMIN_PORT=8787 + export SNAPSHOT_ADMIN_PID_FILE="$PWD/Temp/snapshot_admin.pid" + export SNAPSHOT_ADMIN_LOG_FILE="$PWD/Temp/snapshot_admin.log" + export SNAPSHOT_ADMIN_STATE_URL="http://127.0.0.1:8787/api/state" + export SNAPSHOT_ADMIN_AUTH_USER="${SNAPSHOT_ADMIN_AUTH_USER:-}" + export SNAPSHOT_ADMIN_AUTH_PASSWORD="${SNAPSHOT_ADMIN_AUTH_PASSWORD:-}" + bash tools/run_snapshot_admin_synology.sh healthcheck + if [ -n "$SNAPSHOT_ADMIN_AUTH_USER" ] && [ -n "$SNAPSHOT_ADMIN_AUTH_PASSWORD" ]; then + curl -fsS -u "${SNAPSHOT_ADMIN_AUTH_USER}:${SNAPSHOT_ADMIN_AUTH_PASSWORD}" http://127.0.0.1:8787/api/state | python3 -c "import json,sys; print(json.load(sys.stdin)['version']['app'])" + else + curl -fsS http://127.0.0.1:8787/api/state | python3 -c "import json,sys; print(json.load(sys.stdin)['version']['app'])" + fi diff --git a/docs/ROADMAP_WBS.md b/docs/ROADMAP_WBS.md index 41ac1c7..1b8c257 100644 --- a/docs/ROADMAP_WBS.md +++ b/docs/ROADMAP_WBS.md @@ -866,6 +866,9 @@ python tools/validate_specs.py → PASS |------|------| | **작업** | `src/quant_engine/snapshot_admin_server_v1.py`(Python 어드민 웹 UI)를 Gitea CI/CD 배포 스텝을 통해 Synology NAS에서 상시 서비스로 운영할 수 있는지 검토 | | **현재 상태** | **기술적으로는 가능**. 기본 루프백 보호 + Basic Auth 게이트를 추가했고, Synology 외부 노출은 리버스 프록시 기반 POC로 가이드함. 실배포 검증은 아직 필요 | +| **운영 분리** | `snapshot_admin.yml`은 `push`용 smoke 검증, `workflow_dispatch`용 full 검증, 그리고 `workflow_dispatch` 내 배포 스텝으로 분리했다. `push`에서는 `Validate Snapshot Admin Workflow`까지만, full 검증에서는 `Validate Snapshot Admin Web UI`까지 수행하고, 배포 스텝은 host runner에서 `tools/run_snapshot_admin_synology.sh`를 호출한다. | +| **runner 주의** | Gitea runner를 Docker mode로 두면 job 종료 시 `Cleaning up container` 로그가 남는다. host label로 재등록하면 job container 정리 로그를 피할 수 있다. | +| **KIS 분리** | `kis_data_collection.yml`은 `workflow_dispatch`용 mock/config smoke와 `schedule`용 live collection으로 분리했다. 수동 디스패치는 실제 수집을 돌리지 않고, 실수집은 스케줄 전용이다. | | **담당 파일** | `.gitea/workflows/ci.yml`, `tools/run_snapshot_admin_server_v1.py`, `src/quant_engine/snapshot_admin_server_v1.py`, `docs/SYNOLOGY_SNAPSHOT_ADMIN_POC.md`, `docs/WBS_7_9_EVIDENCE_PACKET_FINAL.md` | | **상태** | 부분 완료 — POC 절차/보안 게이트 구현 완료, 로컬 loopback auth/tables smoke PASS, Synology live verification pending | diff --git a/docs/SYNOLOGY_SNAPSHOT_ADMIN_POC.md b/docs/SYNOLOGY_SNAPSHOT_ADMIN_POC.md new file mode 100644 index 0000000..5e63e8a --- /dev/null +++ b/docs/SYNOLOGY_SNAPSHOT_ADMIN_POC.md @@ -0,0 +1,257 @@ +# Synology Snapshot Admin POC + +This guide enables external access to the Python snapshot admin service on Synology without exposing the raw service port to the internet. + +## Recommended topology + +1. Keep the Python service bound to loopback only: + +```bash +python tools/run_snapshot_admin_server_v1.py \ + --host 127.0.0.1 \ + --port 8787 \ + --db outputs/snapshot_admin/snapshot_admin.db \ + --seed GatherTradingData.json +``` + +2. Put Synology DSM reverse proxy in front of it: + - Source: `https://:443` + - Destination: `http://127.0.0.1:8787` + - Keep the service port closed from direct WAN access. + +3. Add browser authentication with the built-in Basic Auth gate: + - Set `SNAPSHOT_ADMIN_AUTH_USER` + - Set `SNAPSHOT_ADMIN_AUTH_PASSWORD` + - Or pass `--auth-user` and `--auth-password` on the wrapper command + +4. Verify from the NAS: + +```bash +curl -i http://127.0.0.1:8787/api/state +curl -u "$SNAPSHOT_ADMIN_AUTH_USER:$SNAPSHOT_ADMIN_AUTH_PASSWORD" http://127.0.0.1:8787/api/state +``` + +5. Verify from outside the NAS: + - Open `https:///` + - The browser should prompt for Basic Auth + - `https:///tables` should render after login + +## DSM Checklist + +Use these exact values for the first POC. + +1. **DSM app path** + - `Control Panel` + - `Login Portal` + - `Advanced` + - `Reverse Proxy` + +2. **Create reverse proxy rule** + - Description: `snapshot-admin` + - Source protocol: `HTTPS` + - Source hostname: your public DNS name, for example `admin.example.com` + - Source port: `443` + - Source path: `/` + - Destination protocol: `HTTP` + - Destination hostname: `127.0.0.1` + - Destination port: `8787` + +3. **Certificate** + - Attach a valid TLS certificate for the public hostname + - Prefer a Synology-managed or imported certificate that matches `admin.example.com` + +4. **Firewall** + - Allow inbound `443/TCP` only for the reverse proxy endpoint + - Do not expose `8787/TCP` on WAN + - If the NAS must be reachable only from a VPN or office IP range, allowlist those ranges and block the rest + +5. **Service start policy** + - Start the Python service on boot or via DSM Task Scheduler + - Keep it bound to `127.0.0.1` unless you intentionally use direct bind mode + - If you use direct bind mode, keep `--allow-remote` and Basic Auth enabled together + - For Gitea Actions runner verification, register `act_runner` with host labels (`self-hosted:host,linux:host`) if you want to avoid Docker job containers and the `Cleaning up container` log line + - Preferred launcher script: `tools/run_snapshot_admin_synology.sh` + - Gitea CI deploy path: trigger `.gitea/workflows/snapshot_admin.yml` `workflow_dispatch` and let the host runner call the launcher script + - Runner bootstrap: `tools/re_register_act_runner_synology.sh` + - Runner daemon start: `tools/start_act_runner_synology.sh` + +6. **Runner re-registration** + - Use this when you want to switch an existing runner from Docker mode to host mode: + +```bash +cd /volume1/projects/data_feed +REG_TOKEN="" \ +GITEA_URL="http://192.168.123.100:8418" \ +bash tools/re_register_act_runner_synology.sh +``` + + - Expected effect: + - removes the existing `.runner` registration file + - registers `self-hosted:host,linux:host` + - writes an updated `config.yaml` + - If the old runner remains listed in Gitea, remove it from the repository runner page and re-run the command above + +7. **Runner start** + - After re-registration, start the daemon: + +```bash +bash tools/start_act_runner_synology.sh +``` + + - Expected effect: + - launches `act_runner daemon` using the existing config + - records `runner.pid` and `runner.log` under the runner directory + +## DSM Task Scheduler + +Create two scheduled tasks in `Control Panel > Task Scheduler`. + +1. **Boot task** + - Task name: `snapshot-admin-start` + - User: `root` or a dedicated service account with access to the project folder + - Event: `Boot-up` + - Command: + +```bash +bash /volume1/projects/data_feed/tools/run_snapshot_admin_synology.sh start +``` + +2. **Healthcheck task** + - Task name: `snapshot-admin-healthcheck` + - User: same as boot task + - Event: `Scheduled Task` + - Repeat: every 5 minutes + - Command: + +```bash +bash /volume1/projects/data_feed/tools/run_snapshot_admin_synology.sh healthcheck +``` + +3. **Manual restart task** + - Task name: `snapshot-admin-restart` + - User: same as boot task + - Event: `Scheduled Task` + - Repeat: manual only, or keep disabled until needed + - Command: + +```bash +bash /volume1/projects/data_feed/tools/run_snapshot_admin_synology.sh restart +``` + +## Direct bind mode + +Direct binding to `0.0.0.0` is allowed only when both auth values are configured: + +```bash +python tools/run_snapshot_admin_server_v1.py \ + --host 0.0.0.0 \ + --port 8787 \ + --allow-remote \ + --auth-user "$SNAPSHOT_ADMIN_AUTH_USER" \ + --auth-password "$SNAPSHOT_ADMIN_AUTH_PASSWORD" +``` + +Use this only if you have a separate firewall or VPN rule in place. The default POC path is still loopback + reverse proxy. + +## Validation + +Run the unit/web checks before and after deployment: + +```bash +python -m pytest tests/unit/test_snapshot_admin_web_v1.py -q +python tools/validate_snapshot_admin_web_v1.py +``` + +The auth gate is part of the service now, so public exposure without credentials is rejected by the server itself. + +## Curl checklist + +Use this as the POC run sheet. + +1. Local service check: + +```bash +curl -i http://127.0.0.1:8787/api/state +``` + +Expected: +- `200 OK` +- JSON payload contains `version.app` + +2. Reverse proxy auth challenge: + +```bash +curl -i https:///api/state +``` + +Expected: +- `401 Unauthorized` +- `WWW-Authenticate: Basic` + +3. Reverse proxy authenticated access: + +```bash +curl -u ':' https:///api/state +``` + +Expected: +- `200 OK` +- JSON payload contains the same `version.app` + +4. UI rendering: + +```bash +curl -I https:/// +curl -I https:///tables +``` + +Expected: +- `200 OK` after auth +- HTML response, not a redirect to the raw port + +5. Restart persistence: + +```bash +bash tools/run_snapshot_admin_synology.sh restart +bash tools/run_snapshot_admin_synology.sh healthcheck +``` + +Expected: +- `healthcheck ok` +- The proxy URL continues to answer after the service restarts + +## Live verification + +Use this sequence on the actual Synology box after the reverse proxy rule is in place: + +1. Start the service and confirm the local health endpoint: + +```bash +curl -i http://127.0.0.1:8787/api/state +``` + +2. Confirm the auth gate: + +```bash +curl -i https:///api/state +``` + +Expected result: +- `401 Unauthorized` when no credentials are provided +- `200 OK` when valid Basic Auth credentials are supplied + +3. Confirm the browser surface: + - Open `https:///` + - Sign in with the Basic Auth credentials + - Open `https:///tables` + - Confirm rows render from the three SQLite sources + +4. Confirm the deployment survives a process restart: + - Restart the Python service or the task that launches it + - Re-run `curl -i http://127.0.0.1:8787/api/state` + - Re-open the browser URL and confirm login still works + +5. Archive evidence: + - Save the `curl` outputs + - Save a screenshot of `/` and `/tables` + - Record the DSM reverse proxy rule values and certificate name diff --git a/tools/run_snapshot_admin_synology.sh b/tools/run_snapshot_admin_synology.sh new file mode 100644 index 0000000..3f8d6cb --- /dev/null +++ b/tools/run_snapshot_admin_synology.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -eu + +ROOT_DIR="${ROOT_DIR:-/volume1/projects/data_feed}" +PYTHON_BIN="${PYTHON_BIN:-python}" +HOST="${SNAPSHOT_ADMIN_HOST:-127.0.0.1}" +PORT="${SNAPSHOT_ADMIN_PORT:-8787}" +DB_PATH="${SNAPSHOT_ADMIN_DB:-${ROOT_DIR}/outputs/snapshot_admin/snapshot_admin.db}" +SEED_PATH="${SNAPSHOT_ADMIN_SEED:-${ROOT_DIR}/GatherTradingData.json}" +AUTH_USER="${SNAPSHOT_ADMIN_AUTH_USER:-}" +AUTH_PASSWORD="${SNAPSHOT_ADMIN_AUTH_PASSWORD:-}" +ALLOW_REMOTE="${SNAPSHOT_ADMIN_ALLOW_REMOTE:-0}" +PID_FILE="${SNAPSHOT_ADMIN_PID_FILE:-${ROOT_DIR}/Temp/snapshot_admin.pid}" +LOG_FILE="${SNAPSHOT_ADMIN_LOG_FILE:-${ROOT_DIR}/Temp/snapshot_admin.log}" +STATE_URL="${SNAPSHOT_ADMIN_STATE_URL:-http://${HOST}:${PORT}/api/state}" +PUBLIC_STATE_URL="${SNAPSHOT_ADMIN_PUBLIC_STATE_URL:-https://admin.example.com/api/state}" + +mkdir -p "$(dirname "$PID_FILE")" "$(dirname "$LOG_FILE")" + +start_server() { + if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "snapshot admin already running pid=$(cat "$PID_FILE")" + return 0 + fi + cmd=("$PYTHON_BIN" "${ROOT_DIR}/tools/run_snapshot_admin_server_v1.py") + if [ "$ALLOW_REMOTE" = "1" ]; then + cmd+=("--host" "0.0.0.0" "--allow-remote") + else + cmd+=("--host" "$HOST") + fi + cmd+=("--port" "$PORT" "--db" "$DB_PATH" "--seed" "$SEED_PATH") + if [ -n "$AUTH_USER" ]; then + cmd+=("--auth-user" "$AUTH_USER") + fi + if [ -n "$AUTH_PASSWORD" ]; then + cmd+=("--auth-password" "$AUTH_PASSWORD") + fi + nohup "${cmd[@]}" >> "$LOG_FILE" 2>&1 & + echo $! > "$PID_FILE" + echo "started snapshot admin pid=$!" +} + +stop_server() { + if [ ! -f "$PID_FILE" ] || ! kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "snapshot admin not running" + rm -f "$PID_FILE" + return 0 + fi + kill "$(cat "$PID_FILE")" + sleep 2 + if kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + kill -9 "$(cat "$PID_FILE")" + fi + rm -f "$PID_FILE" + echo "stopped snapshot admin" +} + +healthcheck() { + if [ ! -f "$PID_FILE" ] || ! kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "healthcheck failed: process not running" + return 1 + fi + if [ -n "$AUTH_USER" ] && [ -n "$AUTH_PASSWORD" ]; then + if curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" "$STATE_URL" >/dev/null 2>&1; then + echo "healthcheck ok: $STATE_URL" + return 0 + fi + else + if curl -fsS "$STATE_URL" >/dev/null 2>&1; then + echo "healthcheck ok: $STATE_URL" + return 0 + fi + fi + echo "healthcheck failed: $STATE_URL" + return 1 +} + +public_check() { + echo "curl -i ${PUBLIC_STATE_URL}" + echo "curl -u ':' ${PUBLIC_STATE_URL}" + echo "curl -i ${PUBLIC_STATE_URL%/api/state}/tables" +} + +case "${1:-start}" in + start) + start_server + ;; + stop) + stop_server + ;; + restart) + stop_server + start_server + ;; + status) + if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "running pid=$(cat "$PID_FILE")" + exit 0 + fi + echo "stopped" + exit 1 + ;; + healthcheck) + healthcheck + ;; + public-check) + public_check + ;; + *) + echo "usage: $0 {start|stop|restart|status|healthcheck|public-check}" + exit 2 + ;; +esac