server->uuid))->expireAfter(25)->dontRelease()]; } private function disableSshMux(): void { $configRepository = app(ConfigurationRepository::class); $configRepository->disableSshMux(); } public function handle(): void { if ($this->server->hasPlaceholderIp()) { return; } $wasReachable = (bool) $this->server->settings->is_reachable; $wasNotified = (bool) $this->server->unreachable_notification_sent; try { // Check if server is disabled if ($this->server->settings->force_disabled) { $this->server->settings->update([ 'is_reachable' => false, 'is_usable' => false, ]); Log::debug('ServerConnectionCheck: Server is disabled', [ 'server_id' => $this->server->id, 'server_name' => $this->server->name, ]); return; } // Temporarily disable mux if requested if ($this->disableMux) { $this->disableSshMux(); } // Check basic connectivity first $isReachable = $this->checkConnection(); if (! $isReachable) { $this->server->settings->update([ 'is_reachable' => false, 'is_usable' => false, ]); $this->server->increment('unreachable_count'); Log::warning('ServerConnectionCheck: Server not reachable', [ 'server_id' => $this->server->id, 'server_name' => $this->server->name, 'server_ip' => $this->server->ip, ]); $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false); return; } // Server is reachable, check if Docker is available $isUsable = $this->checkDockerAvailability(); $this->server->settings->update([ 'is_reachable' => true, 'is_usable' => $isUsable, ]); if ($this->server->unreachable_count > 0) { $this->server->update(['unreachable_count' => 0]); } $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, true); } catch (\Throwable $e) { Log::error('ServerConnectionCheckJob failed', [ 'error' => $e->getMessage(), 'server_id' => $this->server->id, ]); $this->server->settings->update([ 'is_reachable' => false, 'is_usable' => false, ]); $this->server->increment('unreachable_count'); $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false); return; } } public function failed(?\Throwable $exception): void { if ($exception instanceof TimeoutExceededException) { // Delete the queue job so it doesn't appear in Horizon's failed list. $this->job?->delete(); } } /** * Fire ServerReachabilityChanged when state crosses the unreachable threshold (count >= 2) * or when a previously-notified server recovers. Skips noise from single transient flaps. */ private function dispatchReachabilityChangedIfNeeded(bool $wasReachable, bool $wasNotified, bool $isReachable): void { if ($isReachable) { if (! $wasReachable || $wasNotified) { ServerReachabilityChanged::dispatch($this->server); } return; } if ($this->server->unreachable_count >= 2 && ! $wasNotified) { ServerReachabilityChanged::dispatch($this->server); } } private function checkConnection(): bool { try { // Single SSH attempt without SshRetryHandler — retries waste time for connectivity checks. // Backoff is managed at the dispatch level via unreachable_count. $commands = ['ls -la /']; if ($this->server->isNonRoot()) { $commands = parseCommandsByLineForSudo(collect($commands), $this->server); } $commandString = implode("\n", $commands); $sshCommand = SshMultiplexingHelper::generateSshCommand($this->server, $commandString, true); $process = Process::timeout(10)->run($sshCommand); return $process->exitCode() === 0; } catch (\Throwable $e) { Log::debug('ServerConnectionCheck: Connection check failed', [ 'server_id' => $this->server->id, 'error' => $e->getMessage(), ]); return false; } } private function checkDockerAvailability(): bool { try { // Use instant_remote_process to check Docker // The function will automatically handle sudo for non-root users $output = instant_remote_process_with_timeout( ['docker version --format json'], $this->server, false // don't throw error ); if ($output === null) { return false; } // Try to parse the JSON output to ensure Docker is really working $output = trim($output); if (! empty($output)) { $dockerInfo = json_decode($output, true); return isset($dockerInfo['Server']['Version']); } return false; } catch (\Throwable $e) { Log::debug('ServerConnectionCheck: Docker check failed', [ 'server_id' => $this->server->id, 'error' => $e->getMessage(), ]); return false; } } }