QuantEngineByItz/.gitea/workflows/deploy-prod.yml

name: Deploy to Production

on:
  workflow_dispatch:
    inputs:
      release:
        description: 'Release version to deploy (e.g., v0.1.20260711, or leave empty for latest)'
        required: false
        type: string

concurrency:
  group: deploy-prod-${{ github.sha }}
  cancel-in-progress: false

env:
  DEPLOY_HOST: 178.104.200.7
  DEPLOY_USER: kjh2064
  DEPLOY_PORT: 22
  SERVICE_NAME: quantengine
  REPO: kjh2064/QuantEngineByItz

jobs:
  deploy:
    name: Deploy to Production
    if: ${{ github.event_name == 'workflow_dispatch' }}
    runs-on: ubuntu-latest
    timeout-minutes: 30
    outputs:
      release-tag: ${{ steps.fetch.outputs.tag }}
      artifact-name: ${{ steps.fetch.outputs.artifact }}
      commit-hash: ${{ steps.fetch.outputs.commit }}

    steps:
      - name: Verify SSH Key and Secrets
        run: |
          # SSH_PRIVATE_KEY is the actual secret name registered in this repo
          # (verified via GET /repos/{r}/actions/secrets -- DEPLOY_SSH_KEY_B64 /
          # DEPLOY_SSH_KEY were never actually created despite CLAUDE.md
          # claiming so; kept as fallback names in case they're added later).
          SSH_KEY="${{ secrets.SSH_PRIVATE_KEY }}"
          SSH_KEY_B64="${{ secrets.DEPLOY_SSH_KEY_B64 }}"
          SSH_KEY_RAW="${{ secrets.DEPLOY_SSH_KEY }}"
          if [ -z "$SSH_KEY" ] && [ -z "$SSH_KEY_B64" ] && [ -z "$SSH_KEY_RAW" ]; then
            echo "ERROR: No SSH key secret configured (checked SSH_PRIVATE_KEY, DEPLOY_SSH_KEY_B64, DEPLOY_SSH_KEY)"
            exit 1
          fi
          [ -z "${{ secrets.GITEA_TOKEN }}" ] && { echo "ERROR: GITEA_TOKEN not configured"; exit 1; }
          echo "✓ SSH key and GITEA_TOKEN configured"

      - name: Fetch Release Info
        id: fetch
        run: |
          RELEASE_INPUT="${{ github.event.inputs.release }}"
          TOKEN="${{ secrets.GITEA_TOKEN }}"
          REPO="${{ env.REPO }}"

          if [ -z "$RELEASE_INPUT" ]; then
            RELEASE_URL="https://gitea.taxbaik.com/api/v1/repos/$REPO/releases/latest"
          else
            RELEASE_URL="https://gitea.taxbaik.com/api/v1/repos/$REPO/releases/tags/$RELEASE_INPUT"
          fi

          RELEASE=$(curl -sf --connect-timeout 10 --max-time 30 -H "Authorization: token $TOKEN" "$RELEASE_URL")
          TAG=$(echo "$RELEASE" | jq -r '.tag_name')
          # NOTE: '.target_commitish' is the branch name the tag was cut from
          # (e.g. "main"), NOT a commit SHA -- do not use it as a commit hash.
          # Our tags are always "quant_YYYYMMDD.count.hash" (see
          # prepare-release.yml), so pull the hash back out of the tag name.
          COMMIT="${TAG##*.}"
          ARTIFACT=$(echo "$RELEASE" | jq -r '.assets[0].name')
          DOWNLOAD_URL=$(echo "$RELEASE" | jq -r '.assets[0].browser_download_url')

          if [ "$TAG" = "null" ] || [ -z "$TAG" ]; then
            echo "ERROR: Release not found"; exit 1
          fi
          if [ "$ARTIFACT" = "null" ] || [ -z "$ARTIFACT" ]; then
            echo "ERROR: No artifacts found in release $TAG"; exit 1
          fi
          if [ "$DOWNLOAD_URL" = "null" ] || [ -z "$DOWNLOAD_URL" ]; then
            echo "ERROR: No browser_download_url found for asset"; exit 1
          fi

          echo "tag=${TAG}" >> $GITHUB_OUTPUT
          echo "artifact=${ARTIFACT}" >> $GITHUB_OUTPUT
          echo "download_url=${DOWNLOAD_URL}" >> $GITHUB_OUTPUT
          echo "commit=${COMMIT}" >> $GITHUB_OUTPUT

          echo "✓ Release: $TAG"
          echo "✓ Artifact: $ARTIFACT"
          echo "✓ Download URL: $DOWNLOAD_URL"

      - name: Validate Release Chain
        run: |
          RELEASE_TAG="${{ steps.fetch.outputs.tag }}"
          RELEASE_SHA="${RELEASE_TAG##*.}"
          echo "✓ Workflow dispatch mode — release chain verification is manual"
          echo "  Selected release: $RELEASE_TAG"
          echo "  Extracted commit suffix: $RELEASE_SHA"

      - name: Validate Upstream CI Success
        env:
          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
          REPO: ${{ env.REPO }}
          EXPECTED_SHA: ${{ steps.fetch.outputs.commit }}
        run: |
          python3 - <<'PY'
          import json
          import os
          import sys
          import urllib.request

          token = os.environ["GITEA_TOKEN"]
          repo = os.environ["REPO"]
          expected_sha = os.environ.get("EXPECTED_SHA", "")
          if not expected_sha:
              print("ERROR: missing expected release commit")
              sys.exit(1)

          matched_ci = None
          for page in range(1, 6):
              url = f"https://gitea.taxbaik.com/api/v1/repos/{repo}/actions/runs?limit=50&page={page}"
              req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
              with urllib.request.urlopen(req, timeout=30) as resp:
                  payload = json.load(resp)

              for run in payload.get("workflow_runs", []):
                  path = str(run.get("path") or "")
                  if "ci.yml@" not in path:
                      continue
                  if run.get("status") != "completed" or run.get("conclusion") != "success":
                      continue
                  actual_sha = str(run.get("head_sha") or "")
                  if actual_sha != expected_sha:
                      continue
                  matched_ci = run
                  break
              if matched_ci:
                  break

          if not matched_ci:
              print("ERROR: No successful ci.yml run found for the release SHA")
              sys.exit(1)

          print(f"✓ Upstream CI verified: {expected_sha} (run {matched_ci.get('id')})")
          PY

      - name: Download Release Artifact
        run: |
          ARTIFACT="${{ steps.fetch.outputs.artifact }}"
          TOKEN="${{ secrets.GITEA_TOKEN }}"
          DOWNLOAD_URL="${{ steps.fetch.outputs.download_url }}"

          echo "Downloading: $DOWNLOAD_URL"
          curl -sfL --connect-timeout 10 --max-time 120 -H "Authorization: token $TOKEN" -o "$ARTIFACT" "$DOWNLOAD_URL"

          # A 404/error page would still create a small file -- verify it's a
          # real gzip archive, not an HTML/JSON error body (this is exactly
          # how the old /releases/download/{tag}/{file} guessed URL failed
          # silently: curl exited 0 but wrote a 19-byte "404 page not found").
          file "$ARTIFACT" | grep -q "gzip compressed" || {
            echo "ERROR: Downloaded file is not a valid gzip archive:"
            file "$ARTIFACT"
            cat "$ARTIFACT"
            exit 1
          }

          echo "✓ Downloaded: $(du -sh $ARTIFACT)"

      - name: Download Release Checksum
        run: |
          ARTIFACT="${{ steps.fetch.outputs.artifact }}"
          TOKEN="${{ secrets.GITEA_TOKEN }}"
          RELEASE_TAG="${{ steps.fetch.outputs.tag }}"
          CHECKSUM_URL="https://gitea.taxbaik.com/api/v1/repos/${{ env.REPO }}/releases/tags/${RELEASE_TAG}"

          RELEASE=$(curl -sf --connect-timeout 10 --max-time 30 -H "Authorization: token $TOKEN" "$CHECKSUM_URL")
          CHECKSUM_DOWNLOAD_URL=$(echo "$RELEASE" | jq -r '.assets[] | select(.name == "'"${ARTIFACT}"'.sha256") | .browser_download_url')

          if [ -z "$CHECKSUM_DOWNLOAD_URL" ] || [ "$CHECKSUM_DOWNLOAD_URL" = "null" ]; then
            echo "ERROR: No checksum asset found for release $RELEASE_TAG"
            exit 1
          fi

          curl -sfL --connect-timeout 10 --max-time 120 -H "Authorization: token $TOKEN" -o "${ARTIFACT}.sha256" "$CHECKSUM_DOWNLOAD_URL"
          test -s "${ARTIFACT}.sha256" || { echo "ERROR: checksum file missing"; exit 1; }
          echo "✓ Checksum downloaded"

      - name: Download Release Manifest
        run: |
          ARTIFACT="${{ steps.fetch.outputs.artifact }}"
          TOKEN="${{ secrets.GITEA_TOKEN }}"
          RELEASE_TAG="${{ steps.fetch.outputs.tag }}"
          MANIFEST_URL="https://gitea.taxbaik.com/api/v1/repos/${{ env.REPO }}/releases/tags/${RELEASE_TAG}"

          RELEASE=$(curl -sf --connect-timeout 10 --max-time 30 -H "Authorization: token $TOKEN" "$MANIFEST_URL")
          MANIFEST_DOWNLOAD_URL=$(echo "$RELEASE" | jq -r '.assets[] | select(.name == "'"${ARTIFACT}"'.manifest.json") | .browser_download_url')

          if [ -z "$MANIFEST_DOWNLOAD_URL" ] || [ "$MANIFEST_DOWNLOAD_URL" = "null" ]; then
            echo "ERROR: No manifest asset found for release $RELEASE_TAG"
            exit 1
          fi

          curl -sfL --connect-timeout 10 --max-time 120 -H "Authorization: token $TOKEN" -o "${ARTIFACT}.manifest.json" "$MANIFEST_DOWNLOAD_URL"
          test -s "${ARTIFACT}.manifest.json" || { echo "ERROR: manifest file missing"; exit 1; }
          echo "✓ Manifest downloaded"

      - name: Validate Release Checksum
        run: |
          ARTIFACT="${{ steps.fetch.outputs.artifact }}"
          EXPECTED=$(cat "${ARTIFACT}.sha256" | tr -d '\r\n[:space:]')
          ACTUAL=$(sha256sum "$ARTIFACT" | awk '{print $1}')
          if [ "$EXPECTED" != "$ACTUAL" ]; then
            echo "ERROR: Artifact checksum mismatch"
            echo "Expected: $EXPECTED"
            echo "Actual:   $ACTUAL"
            exit 1
          fi
          echo "✓ Artifact checksum verified"

      - name: Validate Release Manifest
        env:
          ARTIFACT_NAME: ${{ steps.fetch.outputs.artifact }}
          RELEASE_TAG: ${{ steps.fetch.outputs.tag }}
          COMMIT_SHA: ${{ steps.fetch.outputs.commit }}
        run: |
          python3 - <<'PY'
          import json
          import hashlib
          import os
          import pathlib
          import sys

          artifact_name = os.environ["ARTIFACT_NAME"]
          release_tag = os.environ["RELEASE_TAG"]
          commit_sha = os.environ["COMMIT_SHA"]

          artifact = pathlib.Path(artifact_name)
          manifest_path = pathlib.Path(f"{artifact_name}.manifest.json")
          
          if not manifest_path.exists():
              print(f"ERROR: Manifest file not found: {manifest_path}")
              sys.exit(1)

          manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
          expected = {
              "artifact": artifact.name,
              "version": release_tag,
              "commit": commit_sha,
          }

          for key, value in expected.items():
              if manifest.get(key) != value:
                  print(f"ERROR: manifest {key} mismatch: {manifest.get(key)!r} != {value!r}")
                  sys.exit(1)

          actual_sha = hashlib.sha256(artifact.read_bytes()).hexdigest()
          if manifest.get("sha256") != actual_sha:
              print("ERROR: manifest sha256 mismatch")
              print(f"Expected: {manifest.get('sha256')}")
              print(f"Actual:   {actual_sha}")
              sys.exit(1)

          print("✓ Manifest verified")
          PY

      - name: Setup SSH
        run: |
          mkdir -p ~/.ssh
          SSH_KEY="${{ secrets.SSH_PRIVATE_KEY }}"
          SSH_KEY_B64="${{ secrets.DEPLOY_SSH_KEY_B64 }}"
          SSH_KEY_RAW="${{ secrets.DEPLOY_SSH_KEY }}"

          write_key() {
            # $1 = raw secret value; auto-detects PEM vs base64
            if printf '%s' "$1" | grep -q 'BEGIN.*PRIVATE KEY'; then
              printf '%b\n' "$1" > ~/.ssh/deploy_key
            else
              printf '%s' "$1" | base64 -d > ~/.ssh/deploy_key
            fi
          }

          if [ -n "$SSH_KEY" ]; then
            write_key "$SSH_KEY"
          elif [ -n "$SSH_KEY_B64" ]; then
            printf '%s' "$SSH_KEY_B64" | base64 -d > ~/.ssh/deploy_key
          elif [ -n "$SSH_KEY_RAW" ]; then
            write_key "$SSH_KEY_RAW"
          else
            echo "ERROR: No SSH key configured"
            exit 1
          fi

          sed -i 's/\r$//' ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
          ssh-keyscan -p ${{ env.DEPLOY_PORT }} ${{ env.DEPLOY_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
          echo "✓ SSH configured"

      - name: Upload Release Artifact
        run: |
          ARTIFACT="${{ steps.fetch.outputs.artifact }}"
          echo "Uploading: $ARTIFACT"
          ls -lh "$ARTIFACT"

          scp -i ~/.ssh/deploy_key \
            -P ${{ env.DEPLOY_PORT }} \
            -o StrictHostKeyChecking=accept-new \
            -o ConnectTimeout=10 \
            "$ARTIFACT" ${{ env.DEPLOY_USER }}@${{ env.DEPLOY_HOST }}:/tmp/
          echo "✓ Release artifact uploaded"

      - name: Deploy & Verify
        run: |
          ARTIFACT="${{ steps.fetch.outputs.artifact }}"
          RELEASE_TAG="${{ steps.fetch.outputs.tag }}"
          COMMIT="${{ steps.fetch.outputs.commit }}"
          SERVICE_NAME="${{ env.SERVICE_NAME }}"

          # IMPORTANT: the heredoc below uses a QUOTED delimiter ('REMOTE'),
          # so none of $ARTIFACT/$RELEASE_TAG/etc inside it are expanded by
          # this (local runner) shell -- they must arrive as real
          # environment variables on the remote bash process instead. The
          # previous version of this script had the same quoted heredoc but
          # relied on local expansion anyway, so every deploy printed the
          # literal text "$ARTIFACT" and then failed on
          # "tar: /tmp/$ARTIFACT: No such file or directory". Passing them
          # as a prefix to `bash -s` is what actually gets them into the
          # remote script's environment.
          ssh -i ~/.ssh/deploy_key \
            -p ${{ env.DEPLOY_PORT }} \
            -o StrictHostKeyChecking=accept-new \
            -o ConnectTimeout=10 \
            ${{ env.DEPLOY_USER }}@${{ env.DEPLOY_HOST }} \
            "ARTIFACT='$ARTIFACT' RELEASE_TAG='$RELEASE_TAG' COMMIT='$COMMIT' SERVICE_NAME='$SERVICE_NAME' bash -s" << 'REMOTE'
          set -e

          DEPLOY_HOME=$HOME
          DEPLOY_DIR="$DEPLOY_HOME/deployments/quantengine_${RELEASE_TAG}_${COMMIT}"

          echo "=== Deployment Start ==="
          echo "Release: $RELEASE_TAG"
          echo "Artifact: $ARTIFACT"
          echo "Commit: $COMMIT"
          echo "Deploy Dir: $DEPLOY_DIR"
          echo ""

          # 1. Extract
          echo "【 1/4 Extract Artifact 】"
          mkdir -p "$DEPLOY_DIR"
          tar -xzf "/tmp/$ARTIFACT" -C "$DEPLOY_DIR"
          rm -f "/tmp/$ARTIFACT"
          echo "✓ Extraction complete"

          # 2. Verify
          echo ""
          echo "【 2/4 Verify Deployment 】"
          if [ ! -f "$DEPLOY_DIR/QuantEngine.Web.dll" ]; then
            echo "ERROR: QuantEngine.Web.dll not found"
            exit 1
          fi
          echo "✓ DLL verified"
          echo "✓ Runtime configuration is managed outside the release artifact"

          # 3. Update Symlink
          echo ""
          echo "【 3/4 Update Symlink 】"
          ln -sfn "$DEPLOY_DIR" "$DEPLOY_HOME/quantengine_active"
          echo "✓ Active: $(readlink $DEPLOY_HOME/quantengine_active)"

          # 4. Restart Service
          echo ""
          echo "【 4/4 Restart Service 】"
          sudo systemctl restart "$SERVICE_NAME"
          echo "✓ Service restarted"

          REMOTE

  post-deploy-check:
    name: Health Check & Verification
    runs-on: ubuntu-latest
    needs: deploy
    timeout-minutes: 10

    steps:
      - name: Setup SSH (reuse deploy credentials)
        run: |
          mkdir -p ~/.ssh
          SSH_KEY="${{ secrets.SSH_PRIVATE_KEY }}"
          SSH_KEY_B64="${{ secrets.DEPLOY_SSH_KEY_B64 }}"
          SSH_KEY_RAW="${{ secrets.DEPLOY_SSH_KEY }}"

          write_ssh_key() {
            if printf '%s' "$1" | grep -q 'BEGIN.*PRIVATE KEY'; then
              printf '%b\n' "$1" > ~/.ssh/deploy_key
            else
              printf '%s' "$1" | base64 -d > ~/.ssh/deploy_key
            fi
          }

          if [ -n "$SSH_KEY" ]; then
            write_ssh_key "$SSH_KEY"
          elif [ -n "$SSH_KEY_B64" ]; then
            printf '%s' "$SSH_KEY_B64" | base64 -d > ~/.ssh/deploy_key
          elif [ -n "$SSH_KEY_RAW" ]; then
            write_ssh_key "$SSH_KEY_RAW"
          else
            echo "ERROR: No SSH key configured"; exit 1
          fi

          chmod 600 ~/.ssh/deploy_key 2>/dev/null || true
          ssh-keyscan -p 22 ${{ env.DEPLOY_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true
          echo "✓ SSH configured"

      - name: Health Check
        run: |
          # IMPORTANT: quantengine.service binds ASPNETCORE_URLS to
          # http://127.0.0.1:5000 (loopback only) -- Nginx is the only
          # thing that reaches it from outside, via quant.taxbaik.com.
          # The Gitea Actions runner is a separate host/container, so
          # `curl http://$DEPLOY_HOST:5000/...` from here always hits a
          # closed port and times out ("000") -- confirmed directly:
          #   curl --connect-timeout 5 http://178.104.200.7:5000/... -> 000
          # Every previous run's Health Check silently burned through all
          # 20 retries on this before failing, even on deployments that
          # actually worked (see Run #2005: Deploy job succeeded, site was
          # reachable over HTTPS and journalctl was clean the whole time).
          # Fix: run the HTTP/CSS checks *on* the server against
          # 127.0.0.1:5000, the same way the service-status and DB-error
          # checks already correctly do via SSH.
          ssh -i ~/.ssh/deploy_key \
            -p ${{ env.DEPLOY_PORT }} \
            -o StrictHostKeyChecking=accept-new \
            -o ConnectTimeout=10 \
            ${{ env.DEPLOY_USER }}@${{ env.DEPLOY_HOST }} bash -s << 'REMOTE'
          set -e
          ATTEMPTS=20

          echo "【 Health Checks (max ${ATTEMPTS} attempts) 】"

          for i in $(seq 1 $ATTEMPTS); do
            HTTP_CODE=$(curl -s --connect-timeout 5 --max-time 10 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/Account/Login 2>/dev/null || echo "000")
            if [ "$HTTP_CODE" = "200" ]; then
              echo "✓ [1/6] HTTP 200 OK (attempt $i)"

              LOGIN_BODY=$(curl -s --connect-timeout 5 --max-time 10 http://127.0.0.1:5000/Account/Login 2>/dev/null || echo "")
              if echo "$LOGIN_BODY" | grep -q "login\|Login\|로그인"; then
                echo "✓ [2/6] Login page content verified"
              else
                echo "⚠ [2/6] Login page content verification skipped"
              fi

              CSS_CODE=$(curl -s --connect-timeout 5 --max-time 10 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/css/admin.css 2>/dev/null || echo "000")
              if [ "$CSS_CODE" = "200" ]; then
                echo "✓ [3/6] CSS file loaded"
              else
                echo "⚠ [3/6] CSS file check skipped (status: $CSS_CODE)"
              fi

              SERVICE_STATUS=$(systemctl is-active quantengine 2>/dev/null || echo "unknown")
              if [ "$SERVICE_STATUS" = "active" ]; then
                echo "✓ [4/6] Service active (running)"
              else
                echo "⚠ [4/6] Service status: $SERVICE_STATUS"
              fi

              echo "✓ [5/6] Deployment release: ${{ needs.deploy.outputs.release-tag }} (commit: ${{ needs.deploy.outputs.commit-hash }})"

              # Check 6: DB connectivity (GET /Account/Login returns 200 even when
              # the DB password is stale -- the page itself has no DB dependency.
              # Only an actual login POST, or the app logs, reveal a broken
              # connection string. See CLAUDE.md "DB Secret Management" incident
              # 2026-07-12: this check would have caught it, the HTTP check alone
              # did not.)
              sleep 2
              # NOTE: `grep -c` exits 1 when the count is 0 (no matches),
              # even though it correctly prints "0". Combined with
              # `|| echo "0"`, a healthy zero-error result triggered BOTH
              # grep's own "0" output AND the fallback's "0", producing a
              # two-line "0\n0" that never equals the string "0" below.
              # Use `|| true` instead, which only neutralizes the exit
              # code without adding a second line.
              DB_ERRORS=$(journalctl -u quantengine --since '1 minute ago' --no-pager 2>/dev/null | grep -c '28P01\|password authentication failed' || true)
              if [ "$DB_ERRORS" = "0" ]; then
                echo "✓ [6/6] No DB authentication errors in recent logs"
              else
                echo "❌ [6/6] DB authentication errors found in logs ($DB_ERRORS occurrences)"
                echo ""
                echo "❌ FAILED: Deployment reachable over HTTP but DB connection is broken"
                exit 1
              fi

              echo ""
              echo "✅ All health checks passed!"
              exit 0
            fi

            if [ $i -lt $ATTEMPTS ]; then
              echo "  Attempt $i/$ATTEMPTS... (HTTP $HTTP_CODE, retrying in 3s)"
              sleep 3
            else
              echo ""
              echo "❌ FAILED: Service did not respond after $ATTEMPTS attempts"
              exit 1
            fi
          done
          REMOTE

  post-deploy-report:
    name: Deployment Report
    runs-on: ubuntu-latest
    if: always()
    needs: [ deploy, post-deploy-check ]

    steps:
      - name: Report Status
        run: |
          RELEASE="${{ needs.deploy.outputs.release-tag }}"
          COMMIT="${{ needs.deploy.outputs.commit-hash }}"
          ARTIFACT="${{ needs.deploy.outputs.artifact-name }}"
          DEPLOY_STATUS="${{ needs.deploy.result }}"
          CHECK_STATUS="${{ needs.post-deploy-check.result }}"

          echo "╔════════════════════════════════════════════╗"
          echo "║      Deployment Report                     ║"
          echo "╚════════════════════════════════════════════╝"
          echo ""
          echo "Release: $RELEASE"
          echo "Commit: $COMMIT"
          echo "Artifact: $ARTIFACT"
          echo ""
          echo "【 Status 】"
          echo "Deploy: $([ "$DEPLOY_STATUS" = "success" ] && echo "✓" || echo "✗") $DEPLOY_STATUS"
          echo "Health: $([ "$CHECK_STATUS" = "success" ] && echo "✓" || echo "✗") $CHECK_STATUS"
          echo ""

          if [ "$DEPLOY_STATUS" = "success" ] && [ "$CHECK_STATUS" = "success" ]; then
            echo "✅ Deployment Successful"
            echo "Server: 178.104.200.7"
            echo "Release: $RELEASE"
            exit 0
          else
            echo "❌ Deployment Failed"
            exit 1
          fi