KArtSell.Aegis/scripts/enhanced-monitoring.ps1

# Enhanced Host Monitoring (Phase 1: Job 893 Execution)
# Purpose: Comprehensive health checks during 50-90+ day execution
# Governance: AGENTS.md v16.0 (Evidence-based, Traceability)
# Update Interval: 30 minutes (detailed) + 5 minutes (quick health check)

param(
    [int]$DetailedIntervalMinutes = 30,
    [int]$QuickIntervalMinutes = 5,
    [string]$LogFile = "logs/host-monitoring.log",
    [string]$MetricsFile = "logs/monitoring-metrics.csv"
)

$ErrorActionPreference = "SilentlyContinue"

function Write-Log {
    param([string]$Message, [string]$Level = "INFO")

    $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
    $logEntry = "[$timestamp] [$Level] $Message"

    Write-Host $logEntry -ForegroundColor $(
        if ($Level -eq "ERROR") { "Red" }
        elseif ($Level -eq "WARN") { "Yellow" }
        else { "Green" }
    )

    Add-Content -Path $LogFile -Value $logEntry
}

function Test-HostHealth {
    # Quick health check (5-min interval)
    $status = @{
        Timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
        HostRunning = $false
        PortOpen = $false
        ResponseTime = $null
        LastCheck = Get-Date
    }

    try {
        $tcp = New-Object System.Net.Sockets.TcpClient
        $stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
        $tcp.Connect("127.0.0.1", 5002)
        $stopwatch.Stop()

        if ($tcp.Connected) {
            $status.HostRunning = $true
            $status.PortOpen = $true
            $status.ResponseTime = $stopwatch.ElapsedMilliseconds
            $tcp.Close()
        }
    } catch {
        $status.HostRunning = $false
        $status.PortOpen = $false
    }

    return $status
}

function Get-DetailedMetrics {
    # Detailed checks (30-min interval)
    $metrics = @{
        Timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
        ProcessHealth = @{}
        DatabaseHealth = @{}
        JobStatus = @{}
        Errors = @()
    }

    # Process Health
    $proc = Get-Process -Name "KArtSell.Host" -ErrorAction SilentlyContinue
    if ($proc) {
        $metrics.ProcessHealth = @{
            ProcessId = $proc.Id
            MemoryMB = [Math]::Round($proc.WorkingSet / 1MB)
            CpuPercent = $proc.CPU  # Requires performance counter setup
            ThreadCount = $proc.Threads.Count
            HandleCount = $proc.HandleCount
            Uptime = $(if ($proc.StartTime) { ((Get-Date) - $proc.StartTime).TotalHours } else { 0 })
        }
        Write-Log "Process health: PID=$($proc.Id), Memory=$($metrics.ProcessHealth.MemoryMB)MB, Threads=$($metrics.ProcessHealth.ThreadCount)" "INFO"
    } else {
        $metrics.Errors += "Host process not found"
        Write-Log "ERROR: Host process not running" "ERROR"
    }

    # Database Health (via SSH query)
    try {
        $dbCheck = ssh kjh2064@178.104.200.7 "PGPASSWORD='kartsell4321@!' psql -h localhost -p 5432 -U kartsell -d kartselldb -c 'SELECT COUNT(*) as job_count FROM hangfire.job;'" 2>$null

        if ($dbCheck -match "(\d+)") {
            $jobCount = [int]$matches[1]
            $metrics.DatabaseHealth = @{
                ConnectionStatus = "OK"
                JobCount = $jobCount
                LastQueryTime = Get-Date -Format "HH:mm:ss"
            }
            Write-Log "Database health: $jobCount Hangfire jobs" "INFO"
        } else {
            $metrics.DatabaseHealth = @{
                ConnectionStatus = "FAILED"
                JobCount = 0
                Error = "Query returned no results"
            }
            Write-Log "Database query inconclusive" "WARN"
        }
    } catch {
        $metrics.Errors += "Database health check failed: $_"
        Write-Log "Database connection failed" "ERROR"
    }

    # Job 893 Status
    try {
        $jobStatus = ssh kjh2064@178.104.200.7 "PGPASSWORD='kartsell4321@!' psql -h localhost -p 5432 -U kartsell -d kartselldb -c \"SELECT State, CreatedAt FROM hangfire.job WHERE Id = 893;\" 2>/dev/null"

        if ($jobStatus) {
            $metrics.JobStatus = @{
                JobId = 893
                Status = "Queried"
                Details = $jobStatus -split "`n" | Where-Object { $_ -match "^\|" } | Select-Object -First 1
            }
            Write-Log "Job 893 status: $($metrics.JobStatus.Details)" "INFO"
        }
    } catch {
        Write-Log "Job 893 status query failed" "WARN"
    }

    return $metrics
}

function Export-Metrics {
    param($metrics, $filePath)

    $csv = "$($metrics.Timestamp),$($metrics.ProcessHealth.ProcessId),$($metrics.ProcessHealth.MemoryMB),$($metrics.ProcessHealth.ThreadCount),$($metrics.DatabaseHealth.JobCount)"

    # Create header if file doesn't exist
    if (-not (Test-Path $filePath)) {
        $header = "Timestamp,ProcessId,MemoryMB,ThreadCount,JobCount"
        Add-Content -Path $filePath -Value $header
    }

    Add-Content -Path $filePath -Value $csv
}

function Invoke-AlertCheck {
    param($metrics, $status)

    # Alert thresholds
    $alerts = @()

    # Memory threshold: 500MB
    if ($metrics.ProcessHealth.MemoryMB -gt 500) {
        $alerts += "WARN: High memory usage ($($metrics.ProcessHealth.MemoryMB)MB > 500MB)"
    }

    # Connection failure
    if (-not $status.PortOpen) {
        $alerts += "ERROR: Host not responding on port 5002"
    }

    # Database connection failure
    if ($metrics.DatabaseHealth.ConnectionStatus -eq "FAILED") {
        $alerts += "ERROR: Database connection failed"
    }

    # No job progress (same job count for 6 consecutive checks)
    # TODO: Implement with state tracking

    foreach ($alert in $alerts) {
        Write-Log $alert $(if ($alert -like "ERROR*") { "ERROR" } else { "WARN" })
    }

    return $alerts.Count -eq 0  # Return $true if no alerts
}

# ============================================================================
# MAIN LOOP
# ============================================================================

Write-Log "=== HOST MONITORING STARTED ===" "INFO"
Write-Log "Detail interval: $DetailedIntervalMinutes min, Quick interval: $QuickIntervalMinutes min" "INFO"

$detailedCounter = 0
$lastDetailedCheck = (Get-Date).AddMinutes(-$DetailedIntervalMinutes)

while ($true) {
    # Quick health check (every 5 minutes)
    $status = Test-HostHealth

    if ($status.HostRunning) {
        Write-Log "✓ Host healthy (Response: $($status.ResponseTime)ms)" "INFO"
    } else {
        Write-Log "✗ Host UNHEALTHY - Not responding" "ERROR"
    }

    # Detailed check (every 30 minutes)
    $now = Get-Date
    if (($now - $lastDetailedCheck).TotalMinutes -ge $DetailedIntervalMinutes) {
        Write-Log "--- DETAILED METRICS COLLECTION ---" "INFO"

        $metrics = Get-DetailedMetrics
        Export-Metrics $metrics $MetricsFile

        $alertStatus = Invoke-AlertCheck $metrics $status
        if ($alertStatus) {
            Write-Log "All health checks passed" "INFO"
        } else {
            Write-Log "Health alerts detected - review logs" "WARN"
        }

        $lastDetailedCheck = $now
    }

    # Wait for next check
    Start-Sleep -Seconds ($QuickIntervalMinutes * 60)
}