diff --git a/src/Cli/CronJobs/ServersCronJob.php b/src/Cli/CronJobs/ServersCronJob.php index 12e1a436..c609eb3d 100644 --- a/src/Cli/CronJobs/ServersCronJob.php +++ b/src/Cli/CronJobs/ServersCronJob.php @@ -101,7 +101,22 @@ class ServersCronJob implements CommandInterface { shell_exec(BIN_PATH . 'network > /dev/null 2>/dev/null &'); } - if (!ProcessManager::isAnyProcessRunning(array('XC_VM[Watchdog]', 'console.php watchdog'))) { + // A watchdog generation lives only a few seconds. If one is still + // present but far older it is wedged — typically blocked in poll() + // on a half-open MariaDB socket (CLOSE_WAIT) inside its DB ping — and + // will never refresh last_check_ago, so the panel marks the node + // offline while nginx/php/redis/streams are all fine. Kill the stale + // process and start a fresh generation; a plain presence check alone + // would keep trusting the wedged one forever. + $rWatchdogAlive = false; + foreach (ProcessManager::findProcessPIDs(array('XC_VM[Watchdog]', 'console.php watchdog')) as $rWatchdogPID) { + if (ProcessManager::getProcessAge($rWatchdogPID) > 90) { + ProcessManager::kill($rWatchdogPID); + continue; + } + $rWatchdogAlive = true; + } + if (!$rWatchdogAlive) { shell_exec(PHP_BIN . ' ' . MAIN_HOME . 'console.php watchdog > /dev/null 2>/dev/null &'); } diff --git a/src/Core/Process/ProcessManager.php b/src/Core/Process/ProcessManager.php index f7229480..aef3415b 100644 --- a/src/Core/Process/ProcessManager.php +++ b/src/Core/Process/ProcessManager.php @@ -254,6 +254,40 @@ class ProcessManager { return getmypid(); } + /** + * Get the age of a process in seconds (how long it has been running). + * + * On Linux the mtime of the /proc/PID directory is fixed to the + * process start time, so `time() - filemtime()` yields the same figure + * as `ps -o etimes` without shelling out to ps — consistent with the + * rest of this class reading /proc directly. + * + * Used to detect daemons that are still present but wedged (e.g. a + * watchdog blocked in poll() on a half-open MariaDB socket): a normally + * short-lived generation that has been alive far too long is stale. + * + * @param int $pid Process ID + * @return int Age in seconds, or -1 if it cannot be determined + */ + public static function getProcessAge($pid) { + $pid = (int)$pid; + + if ($pid <= 0 || !self::procExists($pid)) { + return -1; + } + + clearstatcache(true, '/proc/' . $pid); + $rStart = @filemtime('/proc/' . $pid); + + if ($rStart === false) { + return -1; + } + + $rAge = time() - $rStart; + + return $rAge > 0 ? $rAge : 0; + } + // ─────────────────────────────────────────────────────────── // Cron Lock Management // ───────────────────────────────────────────────────────────