kjh2064
1db1c46b32
Run #2005's Health Check job hung for 18+ minutes (well past its own timeout-minutes: 10) instead of failing within seconds. Killed the zombie container manually via 'docker kill' on the runner host. Root cause: the pre-fix curl calls to the unreachable $DEPLOY_HOST:5000 had no --connect-timeout/--max-time, so each of the 20 retry attempts could hang on the OS's default TCP timeout instead of failing fast; the job-level timeout-minutes didn't reliably cut it off either (act_runner enforcement gap, not something we control from the workflow file). This is now largely moot after the previous commit (health checks run against 127.0.0.1 on the server itself, where curl returns near-instantly), but added explicit timeouts everywhere as a second line of defense against the same failure mode recurring: - Gitea API curl calls (release fetch, artifact download): --connect-timeout 10 --max-time 30/120 - Local 127.0.0.1 health-check curls: --connect-timeout 5 --max-time 10 - All ssh/scp invocations: -o ConnectTimeout=10 No single curl or ssh call in this workflow should now be able to hang indefinitely.
381 lines
16 KiB
YAML
381 lines
16 KiB
YAML
name: Deploy to Production
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
release:
|
|
description: 'Release version to deploy (e.g., v0.1.20260711, or leave empty for latest)'
|
|
required: false
|
|
type: string
|
|
|
|
concurrency:
|
|
group: deploy-prod-main
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
DEPLOY_HOST: 178.104.200.7
|
|
DEPLOY_USER: kjh2064
|
|
DEPLOY_PORT: 22
|
|
SERVICE_NAME: quantengine
|
|
REPO: kjh2064/QuantEngineByItz
|
|
|
|
jobs:
|
|
deploy:
|
|
name: Deploy to Production
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
outputs:
|
|
release-tag: ${{ steps.fetch.outputs.tag }}
|
|
artifact-name: ${{ steps.fetch.outputs.artifact }}
|
|
commit-hash: ${{ steps.fetch.outputs.commit }}
|
|
|
|
steps:
|
|
- name: Verify SSH Key and Secrets
|
|
run: |
|
|
# SSH_PRIVATE_KEY is the actual secret name registered in this repo
|
|
# (verified via GET /repos/{r}/actions/secrets -- DEPLOY_SSH_KEY_B64 /
|
|
# DEPLOY_SSH_KEY were never actually created despite CLAUDE.md
|
|
# claiming so; kept as fallback names in case they're added later).
|
|
SSH_KEY="${{ secrets.SSH_PRIVATE_KEY }}"
|
|
SSH_KEY_B64="${{ secrets.DEPLOY_SSH_KEY_B64 }}"
|
|
SSH_KEY_RAW="${{ secrets.DEPLOY_SSH_KEY }}"
|
|
if [ -z "$SSH_KEY" ] && [ -z "$SSH_KEY_B64" ] && [ -z "$SSH_KEY_RAW" ]; then
|
|
echo "ERROR: No SSH key secret configured (checked SSH_PRIVATE_KEY, DEPLOY_SSH_KEY_B64, DEPLOY_SSH_KEY)"
|
|
exit 1
|
|
fi
|
|
[ -z "${{ secrets.GITEA_TOKEN }}" ] && { echo "ERROR: GITEA_TOKEN not configured"; exit 1; }
|
|
echo "✓ SSH key and GITEA_TOKEN configured"
|
|
|
|
- name: Fetch Release Info
|
|
id: fetch
|
|
run: |
|
|
RELEASE_INPUT="${{ github.event.inputs.release }}"
|
|
TOKEN="${{ secrets.GITEA_TOKEN }}"
|
|
REPO="${{ env.REPO }}"
|
|
|
|
if [ -z "$RELEASE_INPUT" ]; then
|
|
RELEASE_URL="https://gitea.taxbaik.com/api/v1/repos/$REPO/releases/latest"
|
|
else
|
|
RELEASE_URL="https://gitea.taxbaik.com/api/v1/repos/$REPO/releases/tags/$RELEASE_INPUT"
|
|
fi
|
|
|
|
RELEASE=$(curl -sf --connect-timeout 10 --max-time 30 -H "Authorization: token $TOKEN" "$RELEASE_URL")
|
|
TAG=$(echo "$RELEASE" | jq -r '.tag_name')
|
|
# NOTE: '.target_commitish' is the branch name the tag was cut from
|
|
# (e.g. "main"), NOT a commit SHA -- do not use it as a commit hash.
|
|
# Our tags are always "quant_YYYYMMDD.count.hash" (see
|
|
# prepare-release.yml), so pull the hash back out of the tag name.
|
|
COMMIT="${TAG##*.}"
|
|
ARTIFACT=$(echo "$RELEASE" | jq -r '.assets[0].name')
|
|
DOWNLOAD_URL=$(echo "$RELEASE" | jq -r '.assets[0].browser_download_url')
|
|
|
|
if [ "$TAG" = "null" ] || [ -z "$TAG" ]; then
|
|
echo "ERROR: Release not found"; exit 1
|
|
fi
|
|
if [ "$ARTIFACT" = "null" ] || [ -z "$ARTIFACT" ]; then
|
|
echo "ERROR: No artifacts found in release $TAG"; exit 1
|
|
fi
|
|
if [ "$DOWNLOAD_URL" = "null" ] || [ -z "$DOWNLOAD_URL" ]; then
|
|
echo "ERROR: No browser_download_url found for asset"; exit 1
|
|
fi
|
|
|
|
echo "tag=${TAG}" >> $GITHUB_OUTPUT
|
|
echo "artifact=${ARTIFACT}" >> $GITHUB_OUTPUT
|
|
echo "download_url=${DOWNLOAD_URL}" >> $GITHUB_OUTPUT
|
|
echo "commit=${COMMIT}" >> $GITHUB_OUTPUT
|
|
|
|
echo "✓ Release: $TAG"
|
|
echo "✓ Artifact: $ARTIFACT"
|
|
echo "✓ Download URL: $DOWNLOAD_URL"
|
|
|
|
- name: Download Release Artifact
|
|
run: |
|
|
ARTIFACT="${{ steps.fetch.outputs.artifact }}"
|
|
TOKEN="${{ secrets.GITEA_TOKEN }}"
|
|
DOWNLOAD_URL="${{ steps.fetch.outputs.download_url }}"
|
|
|
|
echo "Downloading: $DOWNLOAD_URL"
|
|
curl -sfL --connect-timeout 10 --max-time 120 -H "Authorization: token $TOKEN" -o "$ARTIFACT" "$DOWNLOAD_URL"
|
|
|
|
# A 404/error page would still create a small file -- verify it's a
|
|
# real gzip archive, not an HTML/JSON error body (this is exactly
|
|
# how the old /releases/download/{tag}/{file} guessed URL failed
|
|
# silently: curl exited 0 but wrote a 19-byte "404 page not found").
|
|
file "$ARTIFACT" | grep -q "gzip compressed" || {
|
|
echo "ERROR: Downloaded file is not a valid gzip archive:"
|
|
file "$ARTIFACT"
|
|
cat "$ARTIFACT"
|
|
exit 1
|
|
}
|
|
|
|
echo "✓ Downloaded: $(du -sh $ARTIFACT)"
|
|
|
|
- name: Setup SSH
|
|
run: |
|
|
mkdir -p ~/.ssh
|
|
SSH_KEY="${{ secrets.SSH_PRIVATE_KEY }}"
|
|
SSH_KEY_B64="${{ secrets.DEPLOY_SSH_KEY_B64 }}"
|
|
SSH_KEY_RAW="${{ secrets.DEPLOY_SSH_KEY }}"
|
|
|
|
write_key() {
|
|
# $1 = raw secret value; auto-detects PEM vs base64
|
|
if printf '%s' "$1" | grep -q 'BEGIN.*PRIVATE KEY'; then
|
|
printf '%b\n' "$1" > ~/.ssh/deploy_key
|
|
else
|
|
printf '%s' "$1" | base64 -d > ~/.ssh/deploy_key
|
|
fi
|
|
}
|
|
|
|
if [ -n "$SSH_KEY" ]; then
|
|
write_key "$SSH_KEY"
|
|
elif [ -n "$SSH_KEY_B64" ]; then
|
|
printf '%s' "$SSH_KEY_B64" | base64 -d > ~/.ssh/deploy_key
|
|
elif [ -n "$SSH_KEY_RAW" ]; then
|
|
write_key "$SSH_KEY_RAW"
|
|
else
|
|
echo "ERROR: No SSH key configured"
|
|
exit 1
|
|
fi
|
|
|
|
sed -i 's/\r$//' ~/.ssh/deploy_key
|
|
chmod 600 ~/.ssh/deploy_key
|
|
ssh-keyscan -p ${{ env.DEPLOY_PORT }} ${{ env.DEPLOY_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
echo "✓ SSH configured"
|
|
|
|
- name: Upload Release Artifact
|
|
run: |
|
|
ARTIFACT="${{ steps.fetch.outputs.artifact }}"
|
|
echo "Uploading: $ARTIFACT"
|
|
ls -lh "$ARTIFACT"
|
|
|
|
scp -i ~/.ssh/deploy_key \
|
|
-P ${{ env.DEPLOY_PORT }} \
|
|
-o StrictHostKeyChecking=accept-new \
|
|
-o ConnectTimeout=10 \
|
|
"$ARTIFACT" ${{ env.DEPLOY_USER }}@${{ env.DEPLOY_HOST }}:/tmp/
|
|
echo "✓ Release artifact uploaded"
|
|
|
|
- name: Deploy & Verify
|
|
run: |
|
|
ARTIFACT="${{ steps.fetch.outputs.artifact }}"
|
|
RELEASE_TAG="${{ steps.fetch.outputs.tag }}"
|
|
COMMIT="${{ steps.fetch.outputs.commit }}"
|
|
SERVICE_NAME="${{ env.SERVICE_NAME }}"
|
|
|
|
# IMPORTANT: the heredoc below uses a QUOTED delimiter ('REMOTE'),
|
|
# so none of $ARTIFACT/$RELEASE_TAG/etc inside it are expanded by
|
|
# this (local runner) shell -- they must arrive as real
|
|
# environment variables on the remote bash process instead. The
|
|
# previous version of this script had the same quoted heredoc but
|
|
# relied on local expansion anyway, so every deploy printed the
|
|
# literal text "$ARTIFACT" and then failed on
|
|
# "tar: /tmp/$ARTIFACT: No such file or directory". Passing them
|
|
# as a prefix to `bash -s` is what actually gets them into the
|
|
# remote script's environment.
|
|
ssh -i ~/.ssh/deploy_key \
|
|
-p ${{ env.DEPLOY_PORT }} \
|
|
-o StrictHostKeyChecking=accept-new \
|
|
-o ConnectTimeout=10 \
|
|
${{ env.DEPLOY_USER }}@${{ env.DEPLOY_HOST }} \
|
|
"ARTIFACT='$ARTIFACT' RELEASE_TAG='$RELEASE_TAG' COMMIT='$COMMIT' SERVICE_NAME='$SERVICE_NAME' bash -s" << 'REMOTE'
|
|
set -e
|
|
|
|
DEPLOY_HOME=$HOME
|
|
DEPLOY_DIR="$DEPLOY_HOME/deployments/quantengine_${RELEASE_TAG}_${COMMIT}"
|
|
|
|
echo "=== Deployment Start ==="
|
|
echo "Release: $RELEASE_TAG"
|
|
echo "Artifact: $ARTIFACT"
|
|
echo "Commit: $COMMIT"
|
|
echo "Deploy Dir: $DEPLOY_DIR"
|
|
echo ""
|
|
|
|
# 1. Extract
|
|
echo "【 1/4 Extract Artifact 】"
|
|
mkdir -p "$DEPLOY_DIR"
|
|
tar -xzf "/tmp/$ARTIFACT" -C "$DEPLOY_DIR"
|
|
rm -f "/tmp/$ARTIFACT"
|
|
echo "✓ Extraction complete"
|
|
|
|
# 2. Verify
|
|
echo ""
|
|
echo "【 2/4 Verify Deployment 】"
|
|
if [ ! -f "$DEPLOY_DIR/QuantEngine.Web.dll" ]; then
|
|
echo "ERROR: QuantEngine.Web.dll not found"
|
|
exit 1
|
|
fi
|
|
if [ ! -f "$DEPLOY_DIR/appsettings.Production.json" ]; then
|
|
echo "ERROR: appsettings.Production.json not found"
|
|
exit 1
|
|
fi
|
|
echo "✓ DLL verified"
|
|
echo "✓ Config verified"
|
|
|
|
# 3. Update Symlink
|
|
echo ""
|
|
echo "【 3/4 Update Symlink 】"
|
|
ln -sfn "$DEPLOY_DIR" "$DEPLOY_HOME/quantengine_active"
|
|
echo "✓ Active: $(readlink $DEPLOY_HOME/quantengine_active)"
|
|
|
|
# 4. Restart Service
|
|
echo ""
|
|
echo "【 4/4 Restart Service 】"
|
|
sudo systemctl restart "$SERVICE_NAME"
|
|
echo "✓ Service restarted"
|
|
|
|
REMOTE
|
|
|
|
post-deploy-check:
|
|
name: Health Check & Verification
|
|
runs-on: ubuntu-latest
|
|
needs: deploy
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- name: Setup SSH (for service check)
|
|
run: |
|
|
mkdir -p ~/.ssh
|
|
SSH_KEY="${{ secrets.SSH_PRIVATE_KEY }}"
|
|
SSH_KEY_B64="${{ secrets.DEPLOY_SSH_KEY_B64 }}"
|
|
SSH_KEY_RAW="${{ secrets.DEPLOY_SSH_KEY }}"
|
|
|
|
if [ -n "$SSH_KEY" ]; then
|
|
if printf '%s' "$SSH_KEY" | grep -q 'BEGIN.*PRIVATE KEY'; then
|
|
printf '%b\n' "$SSH_KEY" > ~/.ssh/deploy_key
|
|
else
|
|
printf '%s' "$SSH_KEY" | base64 -d > ~/.ssh/deploy_key
|
|
fi
|
|
elif [ -n "$SSH_KEY_B64" ]; then
|
|
printf '%s' "$SSH_KEY_B64" | base64 -d > ~/.ssh/deploy_key
|
|
elif [ -n "$SSH_KEY_RAW" ]; then
|
|
printf '%s' "$SSH_KEY_RAW" | base64 -d > ~/.ssh/deploy_key
|
|
fi
|
|
|
|
chmod 600 ~/.ssh/deploy_key 2>/dev/null || true
|
|
ssh-keyscan -p 22 ${{ env.DEPLOY_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
|
|
|
|
- name: Health Check
|
|
run: |
|
|
# IMPORTANT: quantengine.service binds ASPNETCORE_URLS to
|
|
# http://127.0.0.1:5000 (loopback only) -- Nginx is the only
|
|
# thing that reaches it from outside, via quant.taxbaik.com.
|
|
# The Gitea Actions runner is a separate host/container, so
|
|
# `curl http://$DEPLOY_HOST:5000/...` from here always hits a
|
|
# closed port and times out ("000") -- confirmed directly:
|
|
# curl --connect-timeout 5 http://178.104.200.7:5000/... -> 000
|
|
# Every previous run's Health Check silently burned through all
|
|
# 20 retries on this before failing, even on deployments that
|
|
# actually worked (see Run #2005: Deploy job succeeded, site was
|
|
# reachable over HTTPS and journalctl was clean the whole time).
|
|
# Fix: run the HTTP/CSS checks *on* the server against
|
|
# 127.0.0.1:5000, the same way the service-status and DB-error
|
|
# checks already correctly do via SSH.
|
|
ssh -i ~/.ssh/deploy_key \
|
|
-p ${{ env.DEPLOY_PORT }} \
|
|
-o StrictHostKeyChecking=accept-new \
|
|
-o ConnectTimeout=10 \
|
|
${{ env.DEPLOY_USER }}@${{ env.DEPLOY_HOST }} bash -s << 'REMOTE'
|
|
set -e
|
|
ATTEMPTS=20
|
|
|
|
echo "【 Health Checks (max ${ATTEMPTS} attempts) 】"
|
|
|
|
for i in $(seq 1 $ATTEMPTS); do
|
|
HTTP_CODE=$(curl -s --connect-timeout 5 --max-time 10 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/Account/Login 2>/dev/null || echo "000")
|
|
if [ "$HTTP_CODE" = "200" ]; then
|
|
echo "✓ [1/6] HTTP 200 OK (attempt $i)"
|
|
|
|
LOGIN_BODY=$(curl -s --connect-timeout 5 --max-time 10 http://127.0.0.1:5000/Account/Login 2>/dev/null || echo "")
|
|
if echo "$LOGIN_BODY" | grep -q "login\|Login\|로그인"; then
|
|
echo "✓ [2/6] Login page content verified"
|
|
else
|
|
echo "⚠ [2/6] Login page content verification skipped"
|
|
fi
|
|
|
|
CSS_CODE=$(curl -s --connect-timeout 5 --max-time 10 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/css/admin.css 2>/dev/null || echo "000")
|
|
if [ "$CSS_CODE" = "200" ]; then
|
|
echo "✓ [3/6] CSS file loaded"
|
|
else
|
|
echo "⚠ [3/6] CSS file check skipped (status: $CSS_CODE)"
|
|
fi
|
|
|
|
SERVICE_STATUS=$(systemctl is-active quantengine 2>/dev/null || echo "unknown")
|
|
if [ "$SERVICE_STATUS" = "active" ]; then
|
|
echo "✓ [4/6] Service active (running)"
|
|
else
|
|
echo "⚠ [4/6] Service status: $SERVICE_STATUS"
|
|
fi
|
|
|
|
echo "✓ [5/6] Deployment release: ${{ needs.deploy.outputs.release-tag }} (commit: ${{ needs.deploy.outputs.commit-hash }})"
|
|
|
|
# Check 6: DB connectivity (GET /Account/Login returns 200 even when
|
|
# the DB password is stale -- the page itself has no DB dependency.
|
|
# Only an actual login POST, or the app logs, reveal a broken
|
|
# connection string. See CLAUDE.md "DB Secret Management" incident
|
|
# 2026-07-12: this check would have caught it, the HTTP check alone
|
|
# did not.)
|
|
sleep 2
|
|
DB_ERRORS=$(journalctl -u quantengine --since '1 minute ago' --no-pager 2>/dev/null | grep -c '28P01\|password authentication failed' || echo "0")
|
|
if [ "$DB_ERRORS" = "0" ]; then
|
|
echo "✓ [6/6] No DB authentication errors in recent logs"
|
|
else
|
|
echo "❌ [6/6] DB authentication errors found in logs ($DB_ERRORS occurrences)"
|
|
echo ""
|
|
echo "❌ FAILED: Deployment reachable over HTTP but DB connection is broken"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "✅ All health checks passed!"
|
|
exit 0
|
|
fi
|
|
|
|
if [ $i -lt $ATTEMPTS ]; then
|
|
echo " Attempt $i/$ATTEMPTS... (HTTP $HTTP_CODE, retrying in 3s)"
|
|
sleep 3
|
|
else
|
|
echo ""
|
|
echo "❌ FAILED: Service did not respond after $ATTEMPTS attempts"
|
|
exit 1
|
|
fi
|
|
done
|
|
REMOTE
|
|
|
|
post-deploy-report:
|
|
name: Deployment Report
|
|
runs-on: ubuntu-latest
|
|
if: always()
|
|
needs: [ deploy, post-deploy-check ]
|
|
|
|
steps:
|
|
- name: Report Status
|
|
run: |
|
|
RELEASE="${{ needs.deploy.outputs.release-tag }}"
|
|
COMMIT="${{ needs.deploy.outputs.commit-hash }}"
|
|
ARTIFACT="${{ needs.deploy.outputs.artifact-name }}"
|
|
DEPLOY_STATUS="${{ needs.deploy.result }}"
|
|
CHECK_STATUS="${{ needs.post-deploy-check.result }}"
|
|
|
|
echo "╔════════════════════════════════════════════╗"
|
|
echo "║ Deployment Report ║"
|
|
echo "╚════════════════════════════════════════════╝"
|
|
echo ""
|
|
echo "Release: $RELEASE"
|
|
echo "Commit: $COMMIT"
|
|
echo "Artifact: $ARTIFACT"
|
|
echo ""
|
|
echo "【 Status 】"
|
|
echo "Deploy: $([ "$DEPLOY_STATUS" = "success" ] && echo "✓" || echo "✗") $DEPLOY_STATUS"
|
|
echo "Health: $([ "$CHECK_STATUS" = "success" ] && echo "✓" || echo "✗") $CHECK_STATUS"
|
|
echo ""
|
|
|
|
if [ "$DEPLOY_STATUS" = "success" ] && [ "$CHECK_STATUS" = "success" ]; then
|
|
echo "✅ Deployment Successful"
|
|
echo "Server: 178.104.200.7"
|
|
echo "Release: $RELEASE"
|
|
exit 0
|
|
else
|
|
echo "❌ Deployment Failed"
|
|
exit 1
|
|
fi
|