diff --git a/app/Jobs/CheckTraefikVersionForServerJob.php b/app/Jobs/CheckTraefikVersionForServerJob.php index 3e2c85df5..27780553b 100644 --- a/app/Jobs/CheckTraefikVersionForServerJob.php +++ b/app/Jobs/CheckTraefikVersionForServerJob.php @@ -8,7 +8,6 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Illuminate\Support\Facades\Log; class CheckTraefikVersionForServerJob implements ShouldQueue { @@ -33,80 +32,78 @@ public function __construct( */ public function handle(): void { - try { - Log::debug("CheckTraefikVersionForServerJob: Processing server '{$this->server->name}' (ID: {$this->server->id})"); + // Detect current version (makes SSH call) + $currentVersion = getTraefikVersionFromDockerCompose($this->server); - // Detect current version (makes SSH call) - $currentVersion = getTraefikVersionFromDockerCompose($this->server); + // Update detected version in database + $this->server->update(['detected_traefik_version' => $currentVersion]); - Log::info("CheckTraefikVersionForServerJob: Server '{$this->server->name}' - Detected version: ".($currentVersion ?? 'unable to detect')); + if (! $currentVersion) { + return; + } - // Update detected version in database - $this->server->update(['detected_traefik_version' => $currentVersion]); + // Check if image tag is 'latest' by inspecting the image (makes SSH call) + $imageTag = instant_remote_process([ + "docker inspect coolify-proxy --format '{{.Config.Image}}' 2>/dev/null", + ], $this->server, false); - if (! $currentVersion) { - Log::warning("CheckTraefikVersionForServerJob: Server '{$this->server->name}' - Unable to detect version, skipping"); + // Handle empty/null response from SSH command + if (empty(trim($imageTag))) { + return; + } - return; - } + if (str_contains(strtolower(trim($imageTag)), ':latest')) { + return; + } - // Check if image tag is 'latest' by inspecting the image (makes SSH call) - $imageTag = instant_remote_process([ - "docker inspect coolify-proxy --format '{{.Config.Image}}' 2>/dev/null", - ], $this->server, false); + // Parse current version to extract major.minor.patch + $current = ltrim($currentVersion, 'v'); + if (! preg_match('/^(\d+\.\d+)\.(\d+)$/', $current, $matches)) { + return; + } - if (str_contains(strtolower(trim($imageTag)), ':latest')) { - Log::info("CheckTraefikVersionForServerJob: Server '{$this->server->name}' uses 'latest' tag, skipping notification (UI warning only)"); + $currentBranch = $matches[1]; // e.g., "3.6" + $currentPatch = $matches[2]; // e.g., "0" - return; - } + // Find the latest version for this branch + $latestForBranch = $this->traefikVersions["v{$currentBranch}"] ?? null; - // Parse current version to extract major.minor.patch - $current = ltrim($currentVersion, 'v'); - if (! preg_match('/^(\d+\.\d+)\.(\d+)$/', $current, $matches)) { - Log::warning("CheckTraefikVersionForServerJob: Server '{$this->server->name}' - Invalid version format '{$current}', skipping"); + if (! $latestForBranch) { + // User is on a branch we don't track - check if newer branches exist + $newerBranchInfo = $this->getNewerBranchInfo($current, $currentBranch); - return; - } - - $currentBranch = $matches[1]; // e.g., "3.6" - $currentPatch = $matches[2]; // e.g., "0" - - Log::debug("CheckTraefikVersionForServerJob: Server '{$this->server->name}' - Parsed branch: {$currentBranch}, patch: {$currentPatch}"); - - // Find the latest version for this branch - $latestForBranch = $this->traefikVersions["v{$currentBranch}"] ?? null; - - if (! $latestForBranch) { - // User is on a branch we don't track - check if newer branches exist - $this->checkForNewerBranch($current, $currentBranch); - - return; - } - - // Compare patch version within the same branch - $latest = ltrim($latestForBranch, 'v'); - - if (version_compare($current, $latest, '<')) { - Log::info("CheckTraefikVersionForServerJob: Server '{$this->server->name}' is outdated - current: {$current}, latest for branch: {$latest}"); - $this->storeOutdatedInfo($current, $latest, 'patch_update'); + if ($newerBranchInfo) { + $this->storeOutdatedInfo($current, $newerBranchInfo['latest'], 'minor_upgrade', $newerBranchInfo['target']); } else { - // Check if newer branches exist - $this->checkForNewerBranch($current, $currentBranch); + // No newer branch found, clear outdated info + $this->server->update(['traefik_outdated_info' => null]); } - } catch (\Throwable $e) { - Log::error("CheckTraefikVersionForServerJob: Error checking server '{$this->server->name}': ".$e->getMessage(), [ - 'server_id' => $this->server->id, - 'exception' => $e, - ]); - throw $e; + + return; + } + + // Compare patch version within the same branch + $latest = ltrim($latestForBranch, 'v'); + + // Always check for newer branches first + $newerBranchInfo = $this->getNewerBranchInfo($current, $currentBranch); + + if (version_compare($current, $latest, '<')) { + // Patch update available + $this->storeOutdatedInfo($current, $latest, 'patch_update', null, $newerBranchInfo); + } elseif ($newerBranchInfo) { + // Only newer branch available (no patch update) + $this->storeOutdatedInfo($current, $newerBranchInfo['latest'], 'minor_upgrade', $newerBranchInfo['target']); + } else { + // Fully up to date + $this->server->update(['traefik_outdated_info' => null]); } } /** - * Check if there are newer branches available. + * Get information about newer branches if available. */ - private function checkForNewerBranch(string $current, string $currentBranch): void + private function getNewerBranchInfo(string $current, string $currentBranch): ?array { $newestBranch = null; $newestVersion = null; @@ -122,28 +119,38 @@ private function checkForNewerBranch(string $current, string $currentBranch): vo } if ($newestVersion) { - Log::info("CheckTraefikVersionForServerJob: Server '{$this->server->name}' - newer branch {$newestBranch} available ({$newestVersion})"); - $this->storeOutdatedInfo($current, $newestVersion, 'minor_upgrade'); - } else { - Log::info("CheckTraefikVersionForServerJob: Server '{$this->server->name}' is fully up to date - version: {$current}"); - // Clear any outdated info using schemaless attributes - $this->server->extra_attributes->forget('traefik_outdated_info'); - $this->server->save(); + return [ + 'target' => "v{$newestBranch}", + 'latest' => ltrim($newestVersion, 'v'), + ]; } + + return null; } /** - * Store outdated information using schemaless attributes. + * Store outdated information in database. */ - private function storeOutdatedInfo(string $current, string $latest, string $type): void + private function storeOutdatedInfo(string $current, string $latest, string $type, ?string $upgradeTarget = null, ?array $newerBranchInfo = null): void { - // Store in schemaless attributes for persistence - $this->server->extra_attributes->set('traefik_outdated_info', [ + $outdatedInfo = [ 'current' => $current, 'latest' => $latest, 'type' => $type, 'checked_at' => now()->toIso8601String(), - ]); - $this->server->save(); + ]; + + // For minor upgrades, add the upgrade_target field (e.g., "v3.6") + if ($type === 'minor_upgrade' && $upgradeTarget) { + $outdatedInfo['upgrade_target'] = $upgradeTarget; + } + + // If there's a newer branch available (even for patch updates), include that info + if ($newerBranchInfo) { + $outdatedInfo['newer_branch_target'] = $newerBranchInfo['target']; + $outdatedInfo['newer_branch_latest'] = $newerBranchInfo['latest']; + } + + $this->server->update(['traefik_outdated_info' => $outdatedInfo]); } } diff --git a/app/Jobs/CheckTraefikVersionJob.php b/app/Jobs/CheckTraefikVersionJob.php index 653849fef..3fb1d6601 100644 --- a/app/Jobs/CheckTraefikVersionJob.php +++ b/app/Jobs/CheckTraefikVersionJob.php @@ -10,7 +10,6 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\File; -use Illuminate\Support\Facades\Log; class CheckTraefikVersionJob implements ShouldQueue { @@ -20,69 +19,85 @@ class CheckTraefikVersionJob implements ShouldQueue public function handle(): void { - try { - Log::info('CheckTraefikVersionJob: Starting Traefik version check with parallel processing'); - - // Load versions from versions.json - $versionsPath = base_path('versions.json'); - if (! File::exists($versionsPath)) { - Log::warning('CheckTraefikVersionJob: versions.json not found, skipping check'); - - return; - } - - $allVersions = json_decode(File::get($versionsPath), true); - $traefikVersions = data_get($allVersions, 'traefik'); - - if (empty($traefikVersions) || ! is_array($traefikVersions)) { - Log::warning('CheckTraefikVersionJob: Traefik versions not found or invalid in versions.json'); - - return; - } - - $branches = array_keys($traefikVersions); - Log::info('CheckTraefikVersionJob: Loaded Traefik version branches', ['branches' => $branches]); - - // Query all servers with Traefik proxy that are reachable - $servers = Server::whereNotNull('proxy') - ->whereProxyType(ProxyTypes::TRAEFIK->value) - ->whereRelation('settings', 'is_reachable', true) - ->whereRelation('settings', 'is_usable', true) - ->get(); - - $serverCount = $servers->count(); - Log::info("CheckTraefikVersionJob: Found {$serverCount} server(s) with Traefik proxy"); - - if ($serverCount === 0) { - Log::info('CheckTraefikVersionJob: No Traefik servers found, job completed'); - - return; - } - - // Dispatch individual server check jobs in parallel - Log::info('CheckTraefikVersionJob: Dispatching parallel server check jobs'); - - foreach ($servers as $server) { - CheckTraefikVersionForServerJob::dispatch($server, $traefikVersions); - } - - Log::info("CheckTraefikVersionJob: Dispatched {$serverCount} parallel server check jobs"); - - // Dispatch notification job with delay to allow server checks to complete - // For 1000 servers with 60s timeout each, we need at least 60s delay - // But jobs run in parallel via queue workers, so we only need enough time - // for the slowest server to complete - $delaySeconds = min(300, max(60, (int) ($serverCount / 10))); // 60s minimum, 300s maximum, 0.1s per server - NotifyOutdatedTraefikServersJob::dispatch()->delay(now()->addSeconds($delaySeconds)); - - Log::info("CheckTraefikVersionJob: Scheduled notification job with {$delaySeconds}s delay"); - Log::info('CheckTraefikVersionJob: Job completed successfully - parallel processing initiated'); - } catch (\Throwable $e) { - Log::error('CheckTraefikVersionJob: Error checking Traefik versions: '.$e->getMessage(), [ - 'exception' => $e, - 'trace' => $e->getTraceAsString(), - ]); - throw $e; + // Load versions from versions.json + $versionsPath = base_path('versions.json'); + if (! File::exists($versionsPath)) { + return; } + + $allVersions = json_decode(File::get($versionsPath), true); + $traefikVersions = data_get($allVersions, 'traefik'); + + if (empty($traefikVersions) || ! is_array($traefikVersions)) { + return; + } + + // Query all servers with Traefik proxy that are reachable + $servers = Server::whereNotNull('proxy') + ->whereProxyType(ProxyTypes::TRAEFIK->value) + ->whereRelation('settings', 'is_reachable', true) + ->whereRelation('settings', 'is_usable', true) + ->get(); + + $serverCount = $servers->count(); + + if ($serverCount === 0) { + return; + } + + // Dispatch individual server check jobs in parallel + foreach ($servers as $server) { + CheckTraefikVersionForServerJob::dispatch($server, $traefikVersions); + } + + // Dispatch notification job with delay to allow server checks to complete + // Jobs run in parallel via queue workers, but we need to account for: + // - Queue worker capacity (workers process jobs concurrently) + // - Job timeout (60s per server check) + // - Retry attempts (3 retries with exponential backoff) + // - Network latency and SSH connection overhead + // + // Calculation strategy: + // - Assume ~10-20 workers processing the high queue + // - Each server check takes up to 60s (timeout) + // - With retries, worst case is ~180s per job + // - More conservative: 0.2s per server (instead of 0.1s) + // - Higher minimum: 120s (instead of 60s) to account for retries + // - Keep 300s maximum to avoid excessive delays + $delaySeconds = $this->calculateNotificationDelay($serverCount); + if (isDev()) { + $delaySeconds = 1; + } + NotifyOutdatedTraefikServersJob::dispatch()->delay(now()->addSeconds($delaySeconds)); + } + + /** + * Calculate the delay in seconds before sending notifications. + * + * This method calculates an appropriate delay to allow all parallel + * CheckTraefikVersionForServerJob instances to complete before sending + * notifications to teams. + * + * The calculation considers: + * - Server count (more servers = longer delay) + * - Queue worker capacity + * - Job timeout (60s) and retry attempts (3x) + * - Network latency and SSH connection overhead + * + * @param int $serverCount Number of servers being checked + * @return int Delay in seconds + */ + protected function calculateNotificationDelay(int $serverCount): int + { + $minDelay = config('constants.server_checks.notification_delay_min'); + $maxDelay = config('constants.server_checks.notification_delay_max'); + $scalingFactor = config('constants.server_checks.notification_delay_scaling'); + + // Calculate delay based on server count + // More conservative approach: 0.2s per server + $calculatedDelay = (int) ($serverCount * $scalingFactor); + + // Apply min/max boundaries + return min($maxDelay, max($minDelay, $calculatedDelay)); } } diff --git a/app/Jobs/NotifyOutdatedTraefikServersJob.php b/app/Jobs/NotifyOutdatedTraefikServersJob.php index 041e04709..59c79cbdb 100644 --- a/app/Jobs/NotifyOutdatedTraefikServersJob.php +++ b/app/Jobs/NotifyOutdatedTraefikServersJob.php @@ -11,7 +11,6 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Illuminate\Support\Facades\Log; class NotifyOutdatedTraefikServersJob implements ShouldQueue { @@ -32,67 +31,38 @@ public function __construct() */ public function handle(): void { - try { - Log::info('NotifyOutdatedTraefikServersJob: Starting notification aggregation'); + // Query servers that have outdated info stored + $servers = Server::whereNotNull('proxy') + ->whereProxyType(ProxyTypes::TRAEFIK->value) + ->whereRelation('settings', 'is_reachable', true) + ->whereRelation('settings', 'is_usable', true) + ->get(); - // Query servers that have outdated info stored - $servers = Server::whereNotNull('proxy') - ->whereProxyType(ProxyTypes::TRAEFIK->value) - ->whereRelation('settings', 'is_reachable', true) - ->whereRelation('settings', 'is_usable', true) - ->get(); + $outdatedServers = collect(); - $outdatedServers = collect(); + foreach ($servers as $server) { + if ($server->traefik_outdated_info) { + // Attach the outdated info as a dynamic property for the notification + $server->outdatedInfo = $server->traefik_outdated_info; + $outdatedServers->push($server); + } + } - foreach ($servers as $server) { - $outdatedInfo = $server->extra_attributes->get('traefik_outdated_info'); + if ($outdatedServers->isEmpty()) { + return; + } - if ($outdatedInfo) { - // Attach the outdated info as a dynamic property for the notification - $server->outdatedInfo = $outdatedInfo; - $outdatedServers->push($server); - } + // Group by team and send notifications + $serversByTeam = $outdatedServers->groupBy('team_id'); + + foreach ($serversByTeam as $teamId => $teamServers) { + $team = Team::find($teamId); + if (! $team) { + continue; } - $outdatedCount = $outdatedServers->count(); - Log::info("NotifyOutdatedTraefikServersJob: Found {$outdatedCount} outdated server(s)"); - - if ($outdatedCount === 0) { - Log::info('NotifyOutdatedTraefikServersJob: No outdated servers found, no notifications to send'); - - return; - } - - // Group by team and send notifications - $serversByTeam = $outdatedServers->groupBy('team_id'); - $teamCount = $serversByTeam->count(); - - Log::info("NotifyOutdatedTraefikServersJob: Grouped outdated servers into {$teamCount} team(s)"); - - foreach ($serversByTeam as $teamId => $teamServers) { - $team = Team::find($teamId); - if (! $team) { - Log::warning("NotifyOutdatedTraefikServersJob: Team ID {$teamId} not found, skipping"); - - continue; - } - - $serverNames = $teamServers->pluck('name')->join(', '); - Log::info("NotifyOutdatedTraefikServersJob: Sending notification to team '{$team->name}' for {$teamServers->count()} server(s): {$serverNames}"); - - // Send one notification per team with all outdated servers - $team->notify(new TraefikVersionOutdated($teamServers)); - - Log::info("NotifyOutdatedTraefikServersJob: Notification sent to team '{$team->name}'"); - } - - Log::info('NotifyOutdatedTraefikServersJob: Job completed successfully'); - } catch (\Throwable $e) { - Log::error('NotifyOutdatedTraefikServersJob: Error sending notifications: '.$e->getMessage(), [ - 'exception' => $e, - 'trace' => $e->getTraceAsString(), - ]); - throw $e; + // Send one notification per team with all outdated servers + $team->notify(new TraefikVersionOutdated($teamServers)); } } } diff --git a/app/Livewire/Server/Proxy.php b/app/Livewire/Server/Proxy.php index e95eb4d3b..fb4da0c1b 100644 --- a/app/Livewire/Server/Proxy.php +++ b/app/Livewire/Server/Proxy.php @@ -230,6 +230,17 @@ public function getNewerTraefikBranchAvailableProperty(): ?string return null; } + // Check if we have outdated info stored + $outdatedInfo = $this->server->traefik_outdated_info; + if ($outdatedInfo && isset($outdatedInfo['type']) && $outdatedInfo['type'] === 'minor_upgrade') { + // Use the upgrade_target field if available (e.g., "v3.6") + if (isset($outdatedInfo['upgrade_target'])) { + return str_starts_with($outdatedInfo['upgrade_target'], 'v') + ? $outdatedInfo['upgrade_target'] + : "v{$outdatedInfo['upgrade_target']}"; + } + } + $versionsPath = base_path('versions.json'); if (! File::exists($versionsPath)) { return null; @@ -251,18 +262,17 @@ public function getNewerTraefikBranchAvailableProperty(): ?string $currentBranch = $matches[1]; // Find the newest branch that's greater than current - $newestVersion = null; + $newestBranch = null; foreach ($traefikVersions as $branch => $version) { $branchNum = ltrim($branch, 'v'); if (version_compare($branchNum, $currentBranch, '>')) { - $cleanVersion = ltrim($version, 'v'); - if (! $newestVersion || version_compare($cleanVersion, $newestVersion, '>')) { - $newestVersion = $cleanVersion; + if (! $newestBranch || version_compare($branchNum, $newestBranch, '>')) { + $newestBranch = $branchNum; } } } - return $newestVersion ? "v{$newestVersion}" : null; + return $newestBranch ? "v{$newestBranch}" : null; } catch (\Throwable $e) { return null; } diff --git a/app/Models/Server.php b/app/Models/Server.php index 157666d66..0f7db5ae4 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -142,6 +142,7 @@ protected static function booted() protected $casts = [ 'proxy' => SchemalessAttributes::class, + 'traefik_outdated_info' => 'array', 'logdrain_axiom_api_key' => 'encrypted', 'logdrain_newrelic_license_key' => 'encrypted', 'delete_unused_volumes' => 'boolean', @@ -168,6 +169,7 @@ protected static function booted() 'hetzner_server_status', 'is_validating', 'detected_traefik_version', + 'traefik_outdated_info', ]; protected $guarded = []; diff --git a/app/Notifications/Server/TraefikVersionOutdated.php b/app/Notifications/Server/TraefikVersionOutdated.php index 61c2d2497..09ef4257d 100644 --- a/app/Notifications/Server/TraefikVersionOutdated.php +++ b/app/Notifications/Server/TraefikVersionOutdated.php @@ -27,6 +27,17 @@ private function formatVersion(string $version): string return str_starts_with($version, 'v') ? $version : "v{$version}"; } + private function getUpgradeTarget(array $info): string + { + // For minor upgrades, use the upgrade_target field (e.g., "v3.6") + if (($info['type'] ?? 'patch_update') === 'minor_upgrade' && isset($info['upgrade_target'])) { + return $this->formatVersion($info['upgrade_target']); + } + + // For patch updates, show the full version + return $this->formatVersion($info['latest'] ?? 'unknown'); + } + public function toMail($notifiable = null): MailMessage { $mail = new MailMessage; @@ -44,24 +55,37 @@ public function toMail($notifiable = null): MailMessage public function toDiscord(): DiscordMessage { $count = $this->servers->count(); - $hasUpgrades = $this->servers->contains(fn ($s) => ($s->outdatedInfo['type'] ?? 'patch_update') === 'minor_upgrade'); + $hasUpgrades = $this->servers->contains(fn ($s) => ($s->outdatedInfo['type'] ?? 'patch_update') === 'minor_upgrade' || + isset($s->outdatedInfo['newer_branch_target']) + ); $description = "**{$count} server(s)** running outdated Traefik proxy. Update recommended for security and features.\n\n"; - $description .= "*Based on actual running container version*\n\n"; $description .= "**Affected servers:**\n"; foreach ($this->servers as $server) { $info = $server->outdatedInfo ?? []; $current = $this->formatVersion($info['current'] ?? 'unknown'); $latest = $this->formatVersion($info['latest'] ?? 'unknown'); - $type = ($info['type'] ?? 'patch_update') === 'patch_update' ? '(patch)' : '(upgrade)'; - $description .= "• {$server->name}: {$current} → {$latest} {$type}\n"; + $upgradeTarget = $this->getUpgradeTarget($info); + $isPatch = ($info['type'] ?? 'patch_update') === 'patch_update'; + $hasNewerBranch = isset($info['newer_branch_target']); + + if ($isPatch && $hasNewerBranch) { + $newerBranchTarget = $info['newer_branch_target']; + $newerBranchLatest = $this->formatVersion($info['newer_branch_latest']); + $description .= "• {$server->name}: {$current} → {$upgradeTarget} (patch update available)\n"; + $description .= " ↳ Also available: {$newerBranchTarget} (latest patch: {$newerBranchLatest}) - new minor version\n"; + } elseif ($isPatch) { + $description .= "• {$server->name}: {$current} → {$upgradeTarget} (patch update available)\n"; + } else { + $description .= "• {$server->name}: {$current} (latest patch: {$latest}) → {$upgradeTarget} (new minor version available)\n"; + } } $description .= "\n⚠️ It is recommended to test before switching the production version."; if ($hasUpgrades) { - $description .= "\n\n📖 **For major/minor upgrades**: Read the Traefik changelog before upgrading to understand breaking changes."; + $description .= "\n\n📖 **For minor version upgrades**: Read the Traefik changelog before upgrading to understand breaking changes and new features."; } return new DiscordMessage( @@ -74,25 +98,38 @@ public function toDiscord(): DiscordMessage public function toTelegram(): array { $count = $this->servers->count(); - $hasUpgrades = $this->servers->contains(fn ($s) => ($s->outdatedInfo['type'] ?? 'patch_update') === 'minor_upgrade'); + $hasUpgrades = $this->servers->contains(fn ($s) => ($s->outdatedInfo['type'] ?? 'patch_update') === 'minor_upgrade' || + isset($s->outdatedInfo['newer_branch_target']) + ); $message = "⚠️ Coolify: Traefik proxy outdated on {$count} server(s)!\n\n"; $message .= "Update recommended for security and features.\n"; - $message .= "ℹ️ Based on actual running container version\n\n"; $message .= "📊 Affected servers:\n"; foreach ($this->servers as $server) { $info = $server->outdatedInfo ?? []; $current = $this->formatVersion($info['current'] ?? 'unknown'); $latest = $this->formatVersion($info['latest'] ?? 'unknown'); - $type = ($info['type'] ?? 'patch_update') === 'patch_update' ? '(patch)' : '(upgrade)'; - $message .= "• {$server->name}: {$current} → {$latest} {$type}\n"; + $upgradeTarget = $this->getUpgradeTarget($info); + $isPatch = ($info['type'] ?? 'patch_update') === 'patch_update'; + $hasNewerBranch = isset($info['newer_branch_target']); + + if ($isPatch && $hasNewerBranch) { + $newerBranchTarget = $info['newer_branch_target']; + $newerBranchLatest = $this->formatVersion($info['newer_branch_latest']); + $message .= "• {$server->name}: {$current} → {$upgradeTarget} (patch update available)\n"; + $message .= " ↳ Also available: {$newerBranchTarget} (latest patch: {$newerBranchLatest}) - new minor version\n"; + } elseif ($isPatch) { + $message .= "• {$server->name}: {$current} → {$upgradeTarget} (patch update available)\n"; + } else { + $message .= "• {$server->name}: {$current} (latest patch: {$latest}) → {$upgradeTarget} (new minor version available)\n"; + } } $message .= "\n⚠️ It is recommended to test before switching the production version."; if ($hasUpgrades) { - $message .= "\n\n📖 For major/minor upgrades: Read the Traefik changelog before upgrading to understand breaking changes."; + $message .= "\n\n📖 For minor version upgrades: Read the Traefik changelog before upgrading to understand breaking changes and new features."; } return [ @@ -104,24 +141,37 @@ public function toTelegram(): array public function toPushover(): PushoverMessage { $count = $this->servers->count(); - $hasUpgrades = $this->servers->contains(fn ($s) => ($s->outdatedInfo['type'] ?? 'patch_update') === 'minor_upgrade'); + $hasUpgrades = $this->servers->contains(fn ($s) => ($s->outdatedInfo['type'] ?? 'patch_update') === 'minor_upgrade' || + isset($s->outdatedInfo['newer_branch_target']) + ); $message = "Traefik proxy outdated on {$count} server(s)!\n"; - $message .= "Based on actual running container version\n\n"; $message .= "Affected servers:\n"; foreach ($this->servers as $server) { $info = $server->outdatedInfo ?? []; $current = $this->formatVersion($info['current'] ?? 'unknown'); $latest = $this->formatVersion($info['latest'] ?? 'unknown'); - $type = ($info['type'] ?? 'patch_update') === 'patch_update' ? '(patch)' : '(upgrade)'; - $message .= "• {$server->name}: {$current} → {$latest} {$type}\n"; + $upgradeTarget = $this->getUpgradeTarget($info); + $isPatch = ($info['type'] ?? 'patch_update') === 'patch_update'; + $hasNewerBranch = isset($info['newer_branch_target']); + + if ($isPatch && $hasNewerBranch) { + $newerBranchTarget = $info['newer_branch_target']; + $newerBranchLatest = $this->formatVersion($info['newer_branch_latest']); + $message .= "• {$server->name}: {$current} → {$upgradeTarget} (patch update available)\n"; + $message .= " Also: {$newerBranchTarget} (latest: {$newerBranchLatest}) - new minor version\n"; + } elseif ($isPatch) { + $message .= "• {$server->name}: {$current} → {$upgradeTarget} (patch update available)\n"; + } else { + $message .= "• {$server->name}: {$current} (latest patch: {$latest}) → {$upgradeTarget} (new minor version available)\n"; + } } $message .= "\nIt is recommended to test before switching the production version."; if ($hasUpgrades) { - $message .= "\n\nFor major/minor upgrades: Read the Traefik changelog before upgrading."; + $message .= "\n\nFor minor version upgrades: Read the Traefik changelog before upgrading."; } return new PushoverMessage( @@ -134,24 +184,37 @@ public function toPushover(): PushoverMessage public function toSlack(): SlackMessage { $count = $this->servers->count(); - $hasUpgrades = $this->servers->contains(fn ($s) => ($s->outdatedInfo['type'] ?? 'patch_update') === 'minor_upgrade'); + $hasUpgrades = $this->servers->contains(fn ($s) => ($s->outdatedInfo['type'] ?? 'patch_update') === 'minor_upgrade' || + isset($s->outdatedInfo['newer_branch_target']) + ); $description = "Traefik proxy outdated on {$count} server(s)!\n"; - $description .= "_Based on actual running container version_\n\n"; $description .= "*Affected servers:*\n"; foreach ($this->servers as $server) { $info = $server->outdatedInfo ?? []; $current = $this->formatVersion($info['current'] ?? 'unknown'); $latest = $this->formatVersion($info['latest'] ?? 'unknown'); - $type = ($info['type'] ?? 'patch_update') === 'patch_update' ? '(patch)' : '(upgrade)'; - $description .= "• `{$server->name}`: {$current} → {$latest} {$type}\n"; + $upgradeTarget = $this->getUpgradeTarget($info); + $isPatch = ($info['type'] ?? 'patch_update') === 'patch_update'; + $hasNewerBranch = isset($info['newer_branch_target']); + + if ($isPatch && $hasNewerBranch) { + $newerBranchTarget = $info['newer_branch_target']; + $newerBranchLatest = $this->formatVersion($info['newer_branch_latest']); + $description .= "• `{$server->name}`: {$current} → {$upgradeTarget} (patch update available)\n"; + $description .= " ↳ Also available: {$newerBranchTarget} (latest patch: {$newerBranchLatest}) - new minor version\n"; + } elseif ($isPatch) { + $description .= "• `{$server->name}`: {$current} → {$upgradeTarget} (patch update available)\n"; + } else { + $description .= "• `{$server->name}`: {$current} (latest patch: {$latest}) → {$upgradeTarget} (new minor version available)\n"; + } } $description .= "\n:warning: It is recommended to test before switching the production version."; if ($hasUpgrades) { - $description .= "\n\n:book: For major/minor upgrades: Read the Traefik changelog before upgrading to understand breaking changes."; + $description .= "\n\n:book: For minor version upgrades: Read the Traefik changelog before upgrading to understand breaking changes and new features."; } return new SlackMessage( @@ -166,13 +229,26 @@ public function toWebhook(): array $servers = $this->servers->map(function ($server) { $info = $server->outdatedInfo ?? []; - return [ + $webhookData = [ 'name' => $server->name, 'uuid' => $server->uuid, 'current_version' => $info['current'] ?? 'unknown', 'latest_version' => $info['latest'] ?? 'unknown', 'update_type' => $info['type'] ?? 'patch_update', ]; + + // For minor upgrades, include the upgrade target (e.g., "v3.6") + if (($info['type'] ?? 'patch_update') === 'minor_upgrade' && isset($info['upgrade_target'])) { + $webhookData['upgrade_target'] = $info['upgrade_target']; + } + + // Include newer branch info if available + if (isset($info['newer_branch_target'])) { + $webhookData['newer_branch_target'] = $info['newer_branch_target']; + $webhookData['newer_branch_latest'] = $info['newer_branch_latest']; + } + + return $webhookData; })->toArray(); return [ diff --git a/config/constants.php b/config/constants.php index 6ad70b31a..58191e0b2 100644 --- a/config/constants.php +++ b/config/constants.php @@ -95,4 +95,27 @@ 'storage_api_key' => env('BUNNY_STORAGE_API_KEY'), 'api_key' => env('BUNNY_API_KEY'), ], + + 'server_checks' => [ + // Notification delay configuration for parallel server checks + // Used for Traefik version checks and other future server check jobs + // These settings control how long to wait before sending notifications + // after dispatching parallel check jobs for all servers + + // Minimum delay in seconds (120s = 2 minutes) + // Accounts for job processing time, retries, and network latency + 'notification_delay_min' => 120, + + // Maximum delay in seconds (300s = 5 minutes) + // Prevents excessive waiting for very large server counts + 'notification_delay_max' => 300, + + // Scaling factor: seconds to add per server (0.2) + // Formula: delay = min(max, max(min, serverCount * scaling)) + // Examples: + // - 100 servers: 120s (uses minimum) + // - 1000 servers: 200s + // - 2000 servers: 300s (hits maximum) + 'notification_delay_scaling' => 0.2, + ], ]; diff --git a/database/migrations/2025_11_14_114632_add_traefik_outdated_info_to_servers_table.php b/database/migrations/2025_11_14_114632_add_traefik_outdated_info_to_servers_table.php new file mode 100644 index 000000000..99e10707d --- /dev/null +++ b/database/migrations/2025_11_14_114632_add_traefik_outdated_info_to_servers_table.php @@ -0,0 +1,28 @@ +json('traefik_outdated_info')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('servers', function (Blueprint $table) { + $table->dropColumn('traefik_outdated_info'); + }); + } +}; diff --git a/resources/views/emails/traefik-version-outdated.blade.php b/resources/views/emails/traefik-version-outdated.blade.php index 3efb91231..28effabf3 100644 --- a/resources/views/emails/traefik-version-outdated.blade.php +++ b/resources/views/emails/traefik-version-outdated.blade.php @@ -1,8 +1,6 @@ {{ $count }} server(s) are running outdated Traefik proxy. Update recommended for security and features. -**Note:** This check is based on the actual running container version, not the configuration file. - ## Affected Servers @foreach ($servers as $server) @@ -10,16 +8,37 @@ $info = $server->outdatedInfo ?? []; $current = $info['current'] ?? 'unknown'; $latest = $info['latest'] ?? 'unknown'; - $type = ($info['type'] ?? 'patch_update') === 'patch_update' ? '(patch)' : '(upgrade)'; + $isPatch = ($info['type'] ?? 'patch_update') === 'patch_update'; + $hasNewerBranch = isset($info['newer_branch_target']); $hasUpgrades = $hasUpgrades ?? false; - if ($type === 'upgrade') { + if (!$isPatch || $hasNewerBranch) { $hasUpgrades = true; } // Add 'v' prefix for display $current = str_starts_with($current, 'v') ? $current : "v{$current}"; $latest = str_starts_with($latest, 'v') ? $latest : "v{$latest}"; + + // For minor upgrades, use the upgrade_target (e.g., "v3.6") + if (!$isPatch && isset($info['upgrade_target'])) { + $upgradeTarget = str_starts_with($info['upgrade_target'], 'v') ? $info['upgrade_target'] : "v{$info['upgrade_target']}"; + } else { + // For patch updates, show the full version + $upgradeTarget = $latest; + } + + // Get newer branch info if available + if ($hasNewerBranch) { + $newerBranchTarget = $info['newer_branch_target']; + $newerBranchLatest = str_starts_with($info['newer_branch_latest'], 'v') ? $info['newer_branch_latest'] : "v{$info['newer_branch_latest']}"; + } @endphp -- **{{ $server->name }}**: {{ $current }} → {{ $latest }} {{ $type }} +@if ($isPatch && $hasNewerBranch) +- **{{ $server->name }}**: {{ $current }} → {{ $upgradeTarget }} (patch update available) | Also available: {{ $newerBranchTarget }} (latest patch: {{ $newerBranchLatest }}) - new minor version +@elseif ($isPatch) +- **{{ $server->name }}**: {{ $current }} → {{ $upgradeTarget }} (patch update available) +@else +- **{{ $server->name }}**: {{ $current }} (latest patch: {{ $latest }}) → {{ $upgradeTarget }} (new minor version available) +@endif @endforeach ## Recommendation @@ -27,7 +46,7 @@ It is recommended to test the new Traefik version before switching it in production environments. You can update your proxy configuration through your [Coolify Dashboard]({{ config('app.url') }}). @if ($hasUpgrades ?? false) -**Important for major/minor upgrades:** Before upgrading to a new major or minor version, please read the [Traefik changelog](https://github.com/traefik/traefik/releases) to understand breaking changes and new features. +**Important for minor version upgrades:** Before upgrading to a new minor version, please read the [Traefik changelog](https://github.com/traefik/traefik/releases) to understand breaking changes and new features. @endif ## Next Steps diff --git a/resources/views/livewire/server/proxy.blade.php b/resources/views/livewire/server/proxy.blade.php index 5f68fd939..77e856864 100644 --- a/resources/views/livewire/server/proxy.blade.php +++ b/resources/views/livewire/server/proxy.blade.php @@ -115,12 +115,14 @@ class="font-mono">{{ $this->latestTraefikVersion }} is available. @endif @if ($this->newerTraefikBranchAvailable) - - A newer version of Traefik is available: + A new minor version of Traefik is available: {{ $this->newerTraefikBranchAvailable }}

- Important: Before upgrading to a new major or minor version, please - read + You are currently running v{{ $server->detected_traefik_version }}. + Upgrading to {{ $this->newerTraefikBranchAvailable }} will give you access to new features and improvements. +

+ Important: Before upgrading to a new minor version, please read the Traefik changelog to understand breaking changes and new features. diff --git a/tests/Feature/CheckTraefikVersionJobTest.php b/tests/Feature/CheckTraefikVersionJobTest.php index 9ae4a5b3d..67c04d2c4 100644 --- a/tests/Feature/CheckTraefikVersionJobTest.php +++ b/tests/Feature/CheckTraefikVersionJobTest.php @@ -195,21 +195,32 @@ }); it('calculates delay seconds correctly for notification job', function () { - // Test delay calculation logic - $serverCounts = [10, 100, 500, 1000, 5000]; + // Test the delay calculation logic + // Values: min=120s, max=300s, scaling=0.2 + $testCases = [ + ['servers' => 10, 'expected' => 120], // 10 * 0.2 = 2s -> uses min of 120s + ['servers' => 100, 'expected' => 120], // 100 * 0.2 = 20s -> uses min of 120s + ['servers' => 600, 'expected' => 120], // 600 * 0.2 = 120s (exactly at min) + ['servers' => 1000, 'expected' => 200], // 1000 * 0.2 = 200s + ['servers' => 1500, 'expected' => 300], // 1500 * 0.2 = 300s (at max) + ['servers' => 5000, 'expected' => 300], // 5000 * 0.2 = 1000s -> uses max of 300s + ]; - foreach ($serverCounts as $count) { - $delaySeconds = min(300, max(60, (int) ($count / 10))); + foreach ($testCases as $case) { + $count = $case['servers']; + $expected = $case['expected']; - // Should be at least 60 seconds - expect($delaySeconds)->toBeGreaterThanOrEqual(60); + // Use the same logic as the job's calculateNotificationDelay method + $minDelay = 120; + $maxDelay = 300; + $scalingFactor = 0.2; + $calculatedDelay = (int) ($count * $scalingFactor); + $delaySeconds = min($maxDelay, max($minDelay, $calculatedDelay)); - // Should not exceed 300 seconds - expect($delaySeconds)->toBeLessThanOrEqual(300); + expect($delaySeconds)->toBe($expected, "Failed for {$count} servers"); + + // Should always be within bounds + expect($delaySeconds)->toBeGreaterThanOrEqual($minDelay); + expect($delaySeconds)->toBeLessThanOrEqual($maxDelay); } - - // Specific test cases - expect(min(300, max(60, (int) (10 / 10))))->toBe(60); // 10 servers = 60s (minimum) - expect(min(300, max(60, (int) (1000 / 10))))->toBe(100); // 1000 servers = 100s - expect(min(300, max(60, (int) (5000 / 10))))->toBe(300); // 5000 servers = 300s (maximum) }); diff --git a/tests/Unit/CheckTraefikVersionForServerJobTest.php b/tests/Unit/CheckTraefikVersionForServerJobTest.php index cb5190271..5da6f97d8 100644 --- a/tests/Unit/CheckTraefikVersionForServerJobTest.php +++ b/tests/Unit/CheckTraefikVersionForServerJobTest.php @@ -103,3 +103,39 @@ expect($result)->toBe(0); expect($matches)->toBeEmpty(); }); + +it('handles empty image tag correctly', function () { + // Test that empty string after trim doesn't cause issues with str_contains + $emptyImageTag = ''; + $trimmed = trim($emptyImageTag); + + // This should be false, not an error + expect(empty($trimmed))->toBeTrue(); + + // Test with whitespace only + $whitespaceTag = " \n "; + $trimmed = trim($whitespaceTag); + expect(empty($trimmed))->toBeTrue(); +}); + +it('detects latest tag in image name', function () { + // Test various formats where :latest appears + $testCases = [ + 'traefik:latest' => true, + 'traefik:Latest' => true, + 'traefik:LATEST' => true, + 'traefik:v3.6.0' => false, + 'traefik:3.6.0' => false, + '' => false, + ]; + + foreach ($testCases as $imageTag => $expected) { + if (empty(trim($imageTag))) { + $result = false; // Should return false for empty tags + } else { + $result = str_contains(strtolower(trim($imageTag)), ':latest'); + } + + expect($result)->toBe($expected, "Failed for imageTag: '{$imageTag}'"); + } +}); diff --git a/tests/Unit/CheckTraefikVersionJobTest.php b/tests/Unit/CheckTraefikVersionJobTest.php new file mode 100644 index 000000000..78e7ee695 --- /dev/null +++ b/tests/Unit/CheckTraefikVersionJobTest.php @@ -0,0 +1,122 @@ + server_checks +const MIN_DELAY = 120; +const MAX_DELAY = 300; +const SCALING_FACTOR = 0.2; + +it('calculates notification delay correctly using formula', function () { + // Test the delay calculation formula directly + // Formula: min(max, max(min, serverCount * scaling)) + + $testCases = [ + ['servers' => 10, 'expected' => 120], // 10 * 0.2 = 2 -> uses min 120 + ['servers' => 600, 'expected' => 120], // 600 * 0.2 = 120 (at min) + ['servers' => 1000, 'expected' => 200], // 1000 * 0.2 = 200 + ['servers' => 1500, 'expected' => 300], // 1500 * 0.2 = 300 (at max) + ['servers' => 5000, 'expected' => 300], // 5000 * 0.2 = 1000 -> uses max 300 + ]; + + foreach ($testCases as $case) { + $count = $case['servers']; + $calculatedDelay = (int) ($count * SCALING_FACTOR); + $result = min(MAX_DELAY, max(MIN_DELAY, $calculatedDelay)); + + expect($result)->toBe($case['expected'], "Failed for {$count} servers"); + } +}); + +it('respects minimum delay boundary', function () { + // Test that delays never go below minimum + $serverCounts = [1, 10, 50, 100, 500, 599]; + + foreach ($serverCounts as $count) { + $calculatedDelay = (int) ($count * SCALING_FACTOR); + $result = min(MAX_DELAY, max(MIN_DELAY, $calculatedDelay)); + + expect($result)->toBeGreaterThanOrEqual(MIN_DELAY, + "Delay for {$count} servers should be >= ".MIN_DELAY); + } +}); + +it('respects maximum delay boundary', function () { + // Test that delays never exceed maximum + $serverCounts = [1500, 2000, 5000, 10000]; + + foreach ($serverCounts as $count) { + $calculatedDelay = (int) ($count * SCALING_FACTOR); + $result = min(MAX_DELAY, max(MIN_DELAY, $calculatedDelay)); + + expect($result)->toBeLessThanOrEqual(MAX_DELAY, + "Delay for {$count} servers should be <= ".MAX_DELAY); + } +}); + +it('provides more conservative delays than old calculation', function () { + // Compare new formula with old one + // Old: min(300, max(60, count/10)) + // New: min(300, max(120, count*0.2)) + + $testServers = [100, 500, 1000, 2000, 3000]; + + foreach ($testServers as $count) { + // Old calculation + $oldDelay = min(300, max(60, (int) ($count / 10))); + + // New calculation + $newDelay = min(300, max(120, (int) ($count * 0.2))); + + // For counts >= 600, new delay should be >= old delay + if ($count >= 600) { + expect($newDelay)->toBeGreaterThanOrEqual($oldDelay, + "New delay should be >= old delay for {$count} servers (old: {$oldDelay}s, new: {$newDelay}s)"); + } + + // Both should respect the 300s maximum + expect($newDelay)->toBeLessThanOrEqual(300); + expect($oldDelay)->toBeLessThanOrEqual(300); + } +}); + +it('scales linearly within bounds', function () { + // Test that scaling is linear between min and max thresholds + + // Find threshold where calculated delay equals min: 120 / 0.2 = 600 servers + $minThreshold = (int) (MIN_DELAY / SCALING_FACTOR); + expect($minThreshold)->toBe(600); + + // Find threshold where calculated delay equals max: 300 / 0.2 = 1500 servers + $maxThreshold = (int) (MAX_DELAY / SCALING_FACTOR); + expect($maxThreshold)->toBe(1500); + + // Test linear scaling between thresholds + $delay700 = min(MAX_DELAY, max(MIN_DELAY, (int) (700 * SCALING_FACTOR))); + $delay900 = min(MAX_DELAY, max(MIN_DELAY, (int) (900 * SCALING_FACTOR))); + $delay1100 = min(MAX_DELAY, max(MIN_DELAY, (int) (1100 * SCALING_FACTOR))); + + expect($delay700)->toBe(140); // 700 * 0.2 = 140 + expect($delay900)->toBe(180); // 900 * 0.2 = 180 + expect($delay1100)->toBe(220); // 1100 * 0.2 = 220 + + // Verify linear progression + expect($delay900 - $delay700)->toBe(40); // 200 servers * 0.2 = 40s difference + expect($delay1100 - $delay900)->toBe(40); // 200 servers * 0.2 = 40s difference +}); + +it('handles edge cases in formula', function () { + // Zero servers + $result = min(MAX_DELAY, max(MIN_DELAY, (int) (0 * SCALING_FACTOR))); + expect($result)->toBe(120); + + // One server + $result = min(MAX_DELAY, max(MIN_DELAY, (int) (1 * SCALING_FACTOR))); + expect($result)->toBe(120); + + // Exactly at boundaries + $result = min(MAX_DELAY, max(MIN_DELAY, (int) (600 * SCALING_FACTOR))); // 600 * 0.2 = 120 + expect($result)->toBe(120); + + $result = min(MAX_DELAY, max(MIN_DELAY, (int) (1500 * SCALING_FACTOR))); // 1500 * 0.2 = 300 + expect($result)->toBe(300); +}); diff --git a/tests/Unit/NotifyOutdatedTraefikServersJobTest.php b/tests/Unit/NotifyOutdatedTraefikServersJobTest.php new file mode 100644 index 000000000..82edfb0d9 --- /dev/null +++ b/tests/Unit/NotifyOutdatedTraefikServersJobTest.php @@ -0,0 +1,56 @@ +tries)->toBe(3); +}); + +it('handles servers with null traefik_outdated_info gracefully', function () { + // Create a mock server with null traefik_outdated_info + $server = \Mockery::mock('App\Models\Server')->makePartial(); + $server->traefik_outdated_info = null; + + // Accessing the property should not throw an error + $result = $server->traefik_outdated_info; + + expect($result)->toBeNull(); +}); + +it('handles servers with traefik_outdated_info data', function () { + $expectedInfo = [ + 'current' => '3.5.0', + 'latest' => '3.6.2', + 'type' => 'minor_upgrade', + 'upgrade_target' => 'v3.6', + 'checked_at' => '2025-11-14T10:00:00Z', + ]; + + $server = \Mockery::mock('App\Models\Server')->makePartial(); + $server->traefik_outdated_info = $expectedInfo; + + // Should return the outdated info + $result = $server->traefik_outdated_info; + + expect($result)->toBe($expectedInfo); +}); + +it('handles servers with patch update info without upgrade_target', function () { + $expectedInfo = [ + 'current' => '3.5.0', + 'latest' => '3.5.2', + 'type' => 'patch_update', + 'checked_at' => '2025-11-14T10:00:00Z', + ]; + + $server = \Mockery::mock('App\Models\Server')->makePartial(); + $server->traefik_outdated_info = $expectedInfo; + + // Should return the outdated info without upgrade_target + $result = $server->traefik_outdated_info; + + expect($result)->toBe($expectedInfo); + expect($result)->not->toHaveKey('upgrade_target'); +}); diff --git a/versions.json b/versions.json index 46b1a9c78..18fe45b1a 100644 --- a/versions.json +++ b/versions.json @@ -17,7 +17,7 @@ } }, "traefik": { - "v3.6": "3.6.0", + "v3.6": "3.6.1", "v3.5": "3.5.6", "v3.4": "3.4.5", "v3.3": "3.3.7",