WBS-7.9: wire snapshot admin CI deploy

2026-06-22 00:42:02 +09:00
parent f2e304a508
commit e32cbf49e5
4 changed files with 472 additions and 1 deletions
@@ -14,7 +14,39 @@ on:
      - "GatherTradingData.json"

 jobs:
-  validate-snapshot-admin:
+  validate-snapshot-admin-smoke:
+    if: github.event_name == 'push'
+    runs-on: self-hosted
+    steps:
+      - name: Checkout Code
+        run: |
+          if [ -d .git ]; then
+            git remote set-url origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git
+          else
+            git init
+            git remote add origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git
+          fi
+          git fetch origin main --depth=1
+          git reset --hard FETCH_HEAD
+
+      - name: Setup Python Environment
+        run: |
+          VENV_BASE=/volume1/gitea/python_venv
+          REQ_HASH=$(md5sum tools/validate_snapshot_admin_workflow_v1.py 2>/dev/null | cut -d' ' -f1 || echo "snapshot-admin-default")
+          VENV="$VENV_BASE/$REQ_HASH"
+          if [ ! -f "$VENV/bin/python" ]; then
+            mkdir -p "$VENV_BASE"
+            /usr/bin/python3 -m venv "$VENV"
+            "$VENV/bin/pip" install --upgrade pip --quiet
+          fi
+          "$VENV/bin/pip" install pyyaml --quiet
+          echo "$VENV/bin" >> $GITHUB_PATH
+
+      - name: Validate Snapshot Admin Workflow
+        run: python3 tools/validate_snapshot_admin_workflow_v1.py
+
+  validate-snapshot-admin-full:
+    if: github.event_name == 'workflow_dispatch'
    runs-on: self-hosted
    steps:
      - name: Checkout Code
@@ -55,3 +87,69 @@ jobs:
          echo "status: $STATUS"
          echo "workflow validation: Temp/snapshot_admin_workflow_v1.json"
          echo "web validation: Temp/snapshot_admin_web_validation_v1.json"
+
+  deploy-snapshot-admin:
+    if: github.event_name == 'workflow_dispatch'
+    needs:
+      - validate-snapshot-admin-full
+    runs-on: self-hosted
+    steps:
+      - name: Checkout Code
+        run: |
+          if [ -d .git ]; then
+            git remote set-url origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git
+          else
+            git init
+            git remote add origin http://x-access-token:${{ secrets.GITHUB_TOKEN }}@192.168.123.100:8418/KimJaeHyun/myfinance.git
+          fi
+          git fetch origin main --depth=1
+          git reset --hard FETCH_HEAD
+
+      - name: Setup Python Environment
+        run: |
+          VENV_BASE=/volume1/gitea/python_venv
+          REQ_HASH=$(md5sum tools/validate_snapshot_admin_workflow_v1.py 2>/dev/null | cut -d' ' -f1 || echo "snapshot-admin-default")
+          VENV="$VENV_BASE/$REQ_HASH"
+          if [ ! -f "$VENV/bin/python" ]; then
+            mkdir -p "$VENV_BASE"
+            /usr/bin/python3 -m venv "$VENV"
+            "$VENV/bin/pip" install --upgrade pip --quiet
+          fi
+          "$VENV/bin/pip" install pyyaml --quiet
+          echo "$VENV/bin" >> $GITHUB_PATH
+
+      - name: Deploy Snapshot Admin Runtime
+        env:
+          SNAPSHOT_ADMIN_AUTH_USER: ${{ vars.SNAPSHOT_ADMIN_AUTH_USER }}
+          SNAPSHOT_ADMIN_AUTH_PASSWORD: ${{ secrets.SNAPSHOT_ADMIN_AUTH_PASSWORD }}
+        run: |
+          export ROOT_DIR="$PWD"
+          export SNAPSHOT_ADMIN_HOST=127.0.0.1
+          export SNAPSHOT_ADMIN_PORT=8787
+          export SNAPSHOT_ADMIN_PID_FILE="$PWD/Temp/snapshot_admin.pid"
+          export SNAPSHOT_ADMIN_LOG_FILE="$PWD/Temp/snapshot_admin.log"
+          export SNAPSHOT_ADMIN_STATE_URL="http://127.0.0.1:8787/api/state"
+          export SNAPSHOT_ADMIN_PUBLIC_STATE_URL="https://admin.example.com/api/state"
+          export SNAPSHOT_ADMIN_AUTH_USER="${SNAPSHOT_ADMIN_AUTH_USER:-}"
+          export SNAPSHOT_ADMIN_AUTH_PASSWORD="${SNAPSHOT_ADMIN_AUTH_PASSWORD:-}"
+          bash tools/run_snapshot_admin_synology.sh restart
+
+      - name: Verify Snapshot Admin Runtime
+        env:
+          SNAPSHOT_ADMIN_AUTH_USER: ${{ vars.SNAPSHOT_ADMIN_AUTH_USER }}
+          SNAPSHOT_ADMIN_AUTH_PASSWORD: ${{ secrets.SNAPSHOT_ADMIN_AUTH_PASSWORD }}
+        run: |
+          export ROOT_DIR="$PWD"
+          export SNAPSHOT_ADMIN_HOST=127.0.0.1
+          export SNAPSHOT_ADMIN_PORT=8787
+          export SNAPSHOT_ADMIN_PID_FILE="$PWD/Temp/snapshot_admin.pid"
+          export SNAPSHOT_ADMIN_LOG_FILE="$PWD/Temp/snapshot_admin.log"
+          export SNAPSHOT_ADMIN_STATE_URL="http://127.0.0.1:8787/api/state"
+          export SNAPSHOT_ADMIN_AUTH_USER="${SNAPSHOT_ADMIN_AUTH_USER:-}"
+          export SNAPSHOT_ADMIN_AUTH_PASSWORD="${SNAPSHOT_ADMIN_AUTH_PASSWORD:-}"
+          bash tools/run_snapshot_admin_synology.sh healthcheck
+          if [ -n "$SNAPSHOT_ADMIN_AUTH_USER" ] && [ -n "$SNAPSHOT_ADMIN_AUTH_PASSWORD" ]; then
+            curl -fsS -u "${SNAPSHOT_ADMIN_AUTH_USER}:${SNAPSHOT_ADMIN_AUTH_PASSWORD}" http://127.0.0.1:8787/api/state | python3 -c "import json,sys; print(json.load(sys.stdin)['version']['app'])"
+          else
+            curl -fsS http://127.0.0.1:8787/api/state | python3 -c "import json,sys; print(json.load(sys.stdin)['version']['app'])"
+          fi
@@ -866,6 +866,9 @@ python tools/validate_specs.py → PASS
 |------|------|
 | **작업** | `src/quant_engine/snapshot_admin_server_v1.py`(Python 어드민 웹 UI)를 Gitea CI/CD 배포 스텝을 통해 Synology NAS에서 상시 서비스로 운영할 수 있는지 검토 |
 | **현재 상태** | **기술적으로는 가능**. 기본 루프백 보호 + Basic Auth 게이트를 추가했고, Synology 외부 노출은 리버스 프록시 기반 POC로 가이드함. 실배포 검증은 아직 필요 |
+| **운영 분리** | `snapshot_admin.yml`은 `push`용 smoke 검증, `workflow_dispatch`용 full 검증, 그리고 `workflow_dispatch` 내 배포 스텝으로 분리했다. `push`에서는 `Validate Snapshot Admin Workflow`까지만, full 검증에서는 `Validate Snapshot Admin Web UI`까지 수행하고, 배포 스텝은 host runner에서 `tools/run_snapshot_admin_synology.sh`를 호출한다. |
+| **runner 주의** | Gitea runner를 Docker mode로 두면 job 종료 시 `Cleaning up container` 로그가 남는다. host label로 재등록하면 job container 정리 로그를 피할 수 있다. |
+| **KIS 분리** | `kis_data_collection.yml`은 `workflow_dispatch`용 mock/config smoke와 `schedule`용 live collection으로 분리했다. 수동 디스패치는 실제 수집을 돌리지 않고, 실수집은 스케줄 전용이다. |
 | **담당 파일** | `.gitea/workflows/ci.yml`, `tools/run_snapshot_admin_server_v1.py`, `src/quant_engine/snapshot_admin_server_v1.py`, `docs/SYNOLOGY_SNAPSHOT_ADMIN_POC.md`, `docs/WBS_7_9_EVIDENCE_PACKET_FINAL.md` |
 | **상태** | 부분 완료 — POC 절차/보안 게이트 구현 완료, 로컬 loopback auth/tables smoke PASS, Synology live verification pending |

@@ -0,0 +1,257 @@
+# Synology Snapshot Admin POC
+
+This guide enables external access to the Python snapshot admin service on Synology without exposing the raw service port to the internet.
+
+## Recommended topology
+
+1. Keep the Python service bound to loopback only:
+
+```bash
+python tools/run_snapshot_admin_server_v1.py \
+  --host 127.0.0.1 \
+  --port 8787 \
+  --db outputs/snapshot_admin/snapshot_admin.db \
+  --seed GatherTradingData.json
+```
+
+2. Put Synology DSM reverse proxy in front of it:
+   - Source: `https://<public-host>:443`
+   - Destination: `http://127.0.0.1:8787`
+   - Keep the service port closed from direct WAN access.
+
+3. Add browser authentication with the built-in Basic Auth gate:
+   - Set `SNAPSHOT_ADMIN_AUTH_USER`
+   - Set `SNAPSHOT_ADMIN_AUTH_PASSWORD`
+   - Or pass `--auth-user` and `--auth-password` on the wrapper command
+
+4. Verify from the NAS:
+
+```bash
+curl -i http://127.0.0.1:8787/api/state
+curl -u "$SNAPSHOT_ADMIN_AUTH_USER:$SNAPSHOT_ADMIN_AUTH_PASSWORD" http://127.0.0.1:8787/api/state
+```
+
+5. Verify from outside the NAS:
+   - Open `https://<public-host>/`
+   - The browser should prompt for Basic Auth
+   - `https://<public-host>/tables` should render after login
+
+## DSM Checklist
+
+Use these exact values for the first POC.
+
+1. **DSM app path**
+   - `Control Panel`
+   - `Login Portal`
+   - `Advanced`
+   - `Reverse Proxy`
+
+2. **Create reverse proxy rule**
+   - Description: `snapshot-admin`
+   - Source protocol: `HTTPS`
+   - Source hostname: your public DNS name, for example `admin.example.com`
+   - Source port: `443`
+   - Source path: `/`
+   - Destination protocol: `HTTP`
+   - Destination hostname: `127.0.0.1`
+   - Destination port: `8787`
+
+3. **Certificate**
+   - Attach a valid TLS certificate for the public hostname
+   - Prefer a Synology-managed or imported certificate that matches `admin.example.com`
+
+4. **Firewall**
+   - Allow inbound `443/TCP` only for the reverse proxy endpoint
+   - Do not expose `8787/TCP` on WAN
+   - If the NAS must be reachable only from a VPN or office IP range, allowlist those ranges and block the rest
+
+5. **Service start policy**
+   - Start the Python service on boot or via DSM Task Scheduler
+   - Keep it bound to `127.0.0.1` unless you intentionally use direct bind mode
+   - If you use direct bind mode, keep `--allow-remote` and Basic Auth enabled together
+   - For Gitea Actions runner verification, register `act_runner` with host labels (`self-hosted:host,linux:host`) if you want to avoid Docker job containers and the `Cleaning up container` log line
+   - Preferred launcher script: `tools/run_snapshot_admin_synology.sh`
+   - Gitea CI deploy path: trigger `.gitea/workflows/snapshot_admin.yml` `workflow_dispatch` and let the host runner call the launcher script
+   - Runner bootstrap: `tools/re_register_act_runner_synology.sh`
+   - Runner daemon start: `tools/start_act_runner_synology.sh`
+
+6. **Runner re-registration**
+   - Use this when you want to switch an existing runner from Docker mode to host mode:
+
+```bash
+cd /volume1/projects/data_feed
+REG_TOKEN="<runner-registration-token>" \
+GITEA_URL="http://192.168.123.100:8418" \
+bash tools/re_register_act_runner_synology.sh
+```
+
+   - Expected effect:
+     - removes the existing `.runner` registration file
+     - registers `self-hosted:host,linux:host`
+     - writes an updated `config.yaml`
+   - If the old runner remains listed in Gitea, remove it from the repository runner page and re-run the command above
+
+7. **Runner start**
+   - After re-registration, start the daemon:
+
+```bash
+bash tools/start_act_runner_synology.sh
+```
+
+   - Expected effect:
+     - launches `act_runner daemon` using the existing config
+     - records `runner.pid` and `runner.log` under the runner directory
+
+## DSM Task Scheduler
+
+Create two scheduled tasks in `Control Panel > Task Scheduler`.
+
+1. **Boot task**
+   - Task name: `snapshot-admin-start`
+   - User: `root` or a dedicated service account with access to the project folder
+   - Event: `Boot-up`
+   - Command:
+
+```bash
+bash /volume1/projects/data_feed/tools/run_snapshot_admin_synology.sh start
+```
+
+2. **Healthcheck task**
+   - Task name: `snapshot-admin-healthcheck`
+   - User: same as boot task
+   - Event: `Scheduled Task`
+   - Repeat: every 5 minutes
+   - Command:
+
+```bash
+bash /volume1/projects/data_feed/tools/run_snapshot_admin_synology.sh healthcheck
+```
+
+3. **Manual restart task**
+   - Task name: `snapshot-admin-restart`
+   - User: same as boot task
+   - Event: `Scheduled Task`
+   - Repeat: manual only, or keep disabled until needed
+   - Command:
+
+```bash
+bash /volume1/projects/data_feed/tools/run_snapshot_admin_synology.sh restart
+```
+
+## Direct bind mode
+
+Direct binding to `0.0.0.0` is allowed only when both auth values are configured:
+
+```bash
+python tools/run_snapshot_admin_server_v1.py \
+  --host 0.0.0.0 \
+  --port 8787 \
+  --allow-remote \
+  --auth-user "$SNAPSHOT_ADMIN_AUTH_USER" \
+  --auth-password "$SNAPSHOT_ADMIN_AUTH_PASSWORD"
+```
+
+Use this only if you have a separate firewall or VPN rule in place. The default POC path is still loopback + reverse proxy.
+
+## Validation
+
+Run the unit/web checks before and after deployment:
+
+```bash
+python -m pytest tests/unit/test_snapshot_admin_web_v1.py -q
+python tools/validate_snapshot_admin_web_v1.py
+```
+
+The auth gate is part of the service now, so public exposure without credentials is rejected by the server itself.
+
+## Curl checklist
+
+Use this as the POC run sheet.
+
+1. Local service check:
+
+```bash
+curl -i http://127.0.0.1:8787/api/state
+```
+
+Expected:
+- `200 OK`
+- JSON payload contains `version.app`
+
+2. Reverse proxy auth challenge:
+
+```bash
+curl -i https://<public-host>/api/state
+```
+
+Expected:
+- `401 Unauthorized`
+- `WWW-Authenticate: Basic`
+
+3. Reverse proxy authenticated access:
+
+```bash
+curl -u '<user>:<password>' https://<public-host>/api/state
+```
+
+Expected:
+- `200 OK`
+- JSON payload contains the same `version.app`
+
+4. UI rendering:
+
+```bash
+curl -I https://<public-host>/
+curl -I https://<public-host>/tables
+```
+
+Expected:
+- `200 OK` after auth
+- HTML response, not a redirect to the raw port
+
+5. Restart persistence:
+
+```bash
+bash tools/run_snapshot_admin_synology.sh restart
+bash tools/run_snapshot_admin_synology.sh healthcheck
+```
+
+Expected:
+- `healthcheck ok`
+- The proxy URL continues to answer after the service restarts
+
+## Live verification
+
+Use this sequence on the actual Synology box after the reverse proxy rule is in place:
+
+1. Start the service and confirm the local health endpoint:
+
+```bash
+curl -i http://127.0.0.1:8787/api/state
+```
+
+2. Confirm the auth gate:
+
+```bash
+curl -i https://<public-host>/api/state
+```
+
+Expected result:
+- `401 Unauthorized` when no credentials are provided
+- `200 OK` when valid Basic Auth credentials are supplied
+
+3. Confirm the browser surface:
+   - Open `https://<public-host>/`
+   - Sign in with the Basic Auth credentials
+   - Open `https://<public-host>/tables`
+   - Confirm rows render from the three SQLite sources
+
+4. Confirm the deployment survives a process restart:
+   - Restart the Python service or the task that launches it
+   - Re-run `curl -i http://127.0.0.1:8787/api/state`
+   - Re-open the browser URL and confirm login still works
+
+5. Archive evidence:
+   - Save the `curl` outputs
+   - Save a screenshot of `/` and `/tables`
+   - Record the DSM reverse proxy rule values and certificate name
@@ -0,0 +1,113 @@
+#!/bin/bash
+set -eu
+
+ROOT_DIR="${ROOT_DIR:-/volume1/projects/data_feed}"
+PYTHON_BIN="${PYTHON_BIN:-python}"
+HOST="${SNAPSHOT_ADMIN_HOST:-127.0.0.1}"
+PORT="${SNAPSHOT_ADMIN_PORT:-8787}"
+DB_PATH="${SNAPSHOT_ADMIN_DB:-${ROOT_DIR}/outputs/snapshot_admin/snapshot_admin.db}"
+SEED_PATH="${SNAPSHOT_ADMIN_SEED:-${ROOT_DIR}/GatherTradingData.json}"
+AUTH_USER="${SNAPSHOT_ADMIN_AUTH_USER:-}"
+AUTH_PASSWORD="${SNAPSHOT_ADMIN_AUTH_PASSWORD:-}"
+ALLOW_REMOTE="${SNAPSHOT_ADMIN_ALLOW_REMOTE:-0}"
+PID_FILE="${SNAPSHOT_ADMIN_PID_FILE:-${ROOT_DIR}/Temp/snapshot_admin.pid}"
+LOG_FILE="${SNAPSHOT_ADMIN_LOG_FILE:-${ROOT_DIR}/Temp/snapshot_admin.log}"
+STATE_URL="${SNAPSHOT_ADMIN_STATE_URL:-http://${HOST}:${PORT}/api/state}"
+PUBLIC_STATE_URL="${SNAPSHOT_ADMIN_PUBLIC_STATE_URL:-https://admin.example.com/api/state}"
+
+mkdir -p "$(dirname "$PID_FILE")" "$(dirname "$LOG_FILE")"
+
+start_server() {
+  if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then
+    echo "snapshot admin already running pid=$(cat "$PID_FILE")"
+    return 0
+  fi
+  cmd=("$PYTHON_BIN" "${ROOT_DIR}/tools/run_snapshot_admin_server_v1.py")
+  if [ "$ALLOW_REMOTE" = "1" ]; then
+    cmd+=("--host" "0.0.0.0" "--allow-remote")
+  else
+    cmd+=("--host" "$HOST")
+  fi
+  cmd+=("--port" "$PORT" "--db" "$DB_PATH" "--seed" "$SEED_PATH")
+  if [ -n "$AUTH_USER" ]; then
+    cmd+=("--auth-user" "$AUTH_USER")
+  fi
+  if [ -n "$AUTH_PASSWORD" ]; then
+    cmd+=("--auth-password" "$AUTH_PASSWORD")
+  fi
+  nohup "${cmd[@]}" >> "$LOG_FILE" 2>&1 &
+  echo $! > "$PID_FILE"
+  echo "started snapshot admin pid=$!"
+}
+
+stop_server() {
+  if [ ! -f "$PID_FILE" ] || ! kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then
+    echo "snapshot admin not running"
+    rm -f "$PID_FILE"
+    return 0
+  fi
+  kill "$(cat "$PID_FILE")"
+  sleep 2
+  if kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then
+    kill -9 "$(cat "$PID_FILE")"
+  fi
+  rm -f "$PID_FILE"
+  echo "stopped snapshot admin"
+}
+
+healthcheck() {
+  if [ ! -f "$PID_FILE" ] || ! kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then
+    echo "healthcheck failed: process not running"
+    return 1
+  fi
+  if [ -n "$AUTH_USER" ] && [ -n "$AUTH_PASSWORD" ]; then
+    if curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" "$STATE_URL" >/dev/null 2>&1; then
+      echo "healthcheck ok: $STATE_URL"
+      return 0
+    fi
+  else
+    if curl -fsS "$STATE_URL" >/dev/null 2>&1; then
+      echo "healthcheck ok: $STATE_URL"
+      return 0
+    fi
+  fi
+  echo "healthcheck failed: $STATE_URL"
+  return 1
+}
+
+public_check() {
+  echo "curl -i ${PUBLIC_STATE_URL}"
+  echo "curl -u '<user>:<password>' ${PUBLIC_STATE_URL}"
+  echo "curl -i ${PUBLIC_STATE_URL%/api/state}/tables"
+}
+
+case "${1:-start}" in
+  start)
+    start_server
+    ;;
+  stop)
+    stop_server
+    ;;
+  restart)
+    stop_server
+    start_server
+    ;;
+  status)
+    if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then
+      echo "running pid=$(cat "$PID_FILE")"
+      exit 0
+    fi
+    echo "stopped"
+    exit 1
+    ;;
+  healthcheck)
+    healthcheck
+    ;;
+  public-check)
+    public_check
+    ;;
+  *)
+    echo "usage: $0 {start|stop|restart|status|healthcheck|public-check}"
+    exit 2
+    ;;
+esac