diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index ef5cc37aa..72ece0562 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -224,14 +224,40 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti if ($serviceLabelId) { $subType = data_get($labels, 'coolify.service.subType'); $subId = data_get($labels, 'coolify.service.subId'); - $service = $services->where('id', $serviceLabelId)->first(); - if (! $service) { + $parentService = $services->where('id', $serviceLabelId)->first(); + if (! $parentService) { continue; } + + // Check if this container is excluded from health checks + $containerName = data_get($labels, 'com.docker.compose.service'); + $isExcluded = false; + if ($containerName) { + $dockerComposeRaw = data_get($parentService, 'docker_compose_raw'); + if ($dockerComposeRaw) { + try { + $dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw); + $serviceConfig = data_get($dockerCompose, "services.{$containerName}", []); + $excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false); + $restartPolicy = data_get($serviceConfig, 'restart', 'always'); + if ($excludeFromHc || $restartPolicy === 'no') { + $isExcluded = true; + } + } catch (\Exception $e) { + // If we can't parse, treat as not excluded + } + } + } + + // Append :excluded suffix if container is excluded + if ($isExcluded) { + $containerStatus = str_replace(')', ':excluded)', $containerStatus); + } + if ($subType === 'application') { - $service = $service->applications()->where('id', $subId)->first(); + $service = $parentService->applications()->where('id', $subId)->first(); } else { - $service = $service->databases()->where('id', $subId)->first(); + $service = $parentService->databases()->where('id', $subId)->first(); } if ($service) { $foundServices[] = "$service->id-$service->name"; @@ -461,7 +487,11 @@ private function aggregateApplicationStatus($application, Collection $containerS $hasRunning = false; $hasRestarting = false; $hasUnhealthy = false; + $hasUnknown = false; $hasExited = false; + $hasStarting = false; + $hasPaused = false; + $hasDead = false; foreach ($relevantStatuses as $status) { if (str($status)->contains('restarting')) { @@ -471,9 +501,18 @@ private function aggregateApplicationStatus($application, Collection $containerS if (str($status)->contains('unhealthy')) { $hasUnhealthy = true; } + if (str($status)->contains('unknown')) { + $hasUnknown = true; + } } elseif (str($status)->contains('exited')) { $hasExited = true; $hasUnhealthy = true; + } elseif (str($status)->contains('created') || str($status)->contains('starting')) { + $hasStarting = true; + } elseif (str($status)->contains('paused')) { + $hasPaused = true; + } elseif (str($status)->contains('dead') || str($status)->contains('removing')) { + $hasDead = true; } } @@ -491,7 +530,25 @@ private function aggregateApplicationStatus($application, Collection $containerS } if ($hasRunning) { - return $hasUnhealthy ? 'running (unhealthy)' : 'running (healthy)'; + if ($hasUnhealthy) { + return 'running (unhealthy)'; + } elseif ($hasUnknown) { + return 'running (unknown)'; + } else { + return 'running (healthy)'; + } + } + + if ($hasDead) { + return 'degraded (unhealthy)'; + } + + if ($hasPaused) { + return 'paused (unknown)'; + } + + if ($hasStarting) { + return 'starting (unknown)'; } // All containers are exited with no restart count - truly stopped diff --git a/app/Actions/Shared/ComplexStatusCheck.php b/app/Actions/Shared/ComplexStatusCheck.php index 1013c73e0..eaf34e227 100644 --- a/app/Actions/Shared/ComplexStatusCheck.php +++ b/app/Actions/Shared/ComplexStatusCheck.php @@ -84,7 +84,11 @@ private function aggregateContainerStatuses($application, $containers) $hasRunning = false; $hasRestarting = false; $hasUnhealthy = false; + $hasUnknown = false; $hasExited = false; + $hasStarting = false; + $hasPaused = false; + $hasDead = false; $relevantContainerCount = 0; foreach ($containers as $container) { @@ -104,12 +108,20 @@ private function aggregateContainerStatuses($application, $containers) $hasUnhealthy = true; } elseif ($containerStatus === 'running') { $hasRunning = true; - if ($containerHealth && $containerHealth === 'unhealthy') { + if ($containerHealth === 'unhealthy') { $hasUnhealthy = true; + } elseif ($containerHealth === null) { + $hasUnknown = true; } } elseif ($containerStatus === 'exited') { $hasExited = true; $hasUnhealthy = true; + } elseif ($containerStatus === 'created' || $containerStatus === 'starting') { + $hasStarting = true; + } elseif ($containerStatus === 'paused') { + $hasPaused = true; + } elseif ($containerStatus === 'dead' || $containerStatus === 'removing') { + $hasDead = true; } } @@ -119,7 +131,11 @@ private function aggregateContainerStatuses($application, $containers) $excludedHasRunning = false; $excludedHasRestarting = false; $excludedHasUnhealthy = false; + $excludedHasUnknown = false; $excludedHasExited = false; + $excludedHasStarting = false; + $excludedHasPaused = false; + $excludedHasDead = false; foreach ($containers as $container) { $labels = data_get($container, 'Config.Labels', []); @@ -138,12 +154,20 @@ private function aggregateContainerStatuses($application, $containers) $excludedHasUnhealthy = true; } elseif ($containerStatus === 'running') { $excludedHasRunning = true; - if ($containerHealth && $containerHealth === 'unhealthy') { + if ($containerHealth === 'unhealthy') { $excludedHasUnhealthy = true; + } elseif ($containerHealth === null) { + $excludedHasUnknown = true; } } elseif ($containerStatus === 'exited') { $excludedHasExited = true; $excludedHasUnhealthy = true; + } elseif ($containerStatus === 'created' || $containerStatus === 'starting') { + $excludedHasStarting = true; + } elseif ($containerStatus === 'paused') { + $excludedHasPaused = true; + } elseif ($containerStatus === 'dead' || $containerStatus === 'removing') { + $excludedHasDead = true; } } @@ -156,7 +180,25 @@ private function aggregateContainerStatuses($application, $containers) } if ($excludedHasRunning) { - return 'running:excluded'; + if ($excludedHasUnhealthy) { + return 'running:unhealthy:excluded'; + } elseif ($excludedHasUnknown) { + return 'running:unknown:excluded'; + } else { + return 'running:healthy:excluded'; + } + } + + if ($excludedHasDead) { + return 'degraded:excluded'; + } + + if ($excludedHasPaused) { + return 'paused:excluded'; + } + + if ($excludedHasStarting) { + return 'starting:excluded'; } return 'exited:excluded'; @@ -171,7 +213,25 @@ private function aggregateContainerStatuses($application, $containers) } if ($hasRunning) { - return $hasUnhealthy ? 'running:unhealthy' : 'running:healthy'; + if ($hasUnhealthy) { + return 'running:unhealthy'; + } elseif ($hasUnknown) { + return 'running:unknown'; + } else { + return 'running:healthy'; + } + } + + if ($hasDead) { + return 'degraded:unhealthy'; + } + + if ($hasPaused) { + return 'paused:unknown'; + } + + if ($hasStarting) { + return 'starting:unknown'; } return 'exited:unhealthy'; diff --git a/app/Models/Service.php b/app/Models/Service.php index c98c20121..d4366d6d8 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -190,9 +190,15 @@ public function getStatusAttribute() if ($application->exclude_from_status) { continue; } - $hasNonExcluded = true; $status = str($application->status)->before('(')->trim(); $health = str($application->status)->between('(', ')')->trim(); + + // Skip containers with :excluded suffix (they are excluded from health checks) + if ($health->contains(':excluded')) { + continue; + } + + $hasNonExcluded = true; if ($complexStatus === 'degraded') { continue; } @@ -206,12 +212,26 @@ public function getStatusAttribute() $complexStatus = 'degraded'; } elseif ($status->startsWith('exited')) { $complexStatus = 'exited'; + } elseif ($status->startsWith('created') || $status->startsWith('starting')) { + if ($complexStatus === null) { + $complexStatus = 'starting'; + } + } elseif ($status->startsWith('paused')) { + if ($complexStatus === null) { + $complexStatus = 'paused'; + } + } elseif ($status->startsWith('dead') || $status->startsWith('removing')) { + $complexStatus = 'degraded'; } if ($health->value() === 'healthy') { if ($complexHealth === 'unhealthy') { continue; } $complexHealth = 'healthy'; + } elseif ($health->value() === 'unknown') { + if ($complexHealth !== 'unhealthy') { + $complexHealth = 'unknown'; + } } else { $complexHealth = 'unhealthy'; } @@ -220,9 +240,15 @@ public function getStatusAttribute() if ($database->exclude_from_status) { continue; } - $hasNonExcluded = true; $status = str($database->status)->before('(')->trim(); $health = str($database->status)->between('(', ')')->trim(); + + // Skip containers with :excluded suffix (they are excluded from health checks) + if ($health->contains(':excluded')) { + continue; + } + + $hasNonExcluded = true; if ($complexStatus === 'degraded') { continue; } @@ -236,12 +262,26 @@ public function getStatusAttribute() $complexStatus = 'degraded'; } elseif ($status->startsWith('exited')) { $complexStatus = 'exited'; + } elseif ($status->startsWith('created') || $status->startsWith('starting')) { + if ($complexStatus === null) { + $complexStatus = 'starting'; + } + } elseif ($status->startsWith('paused')) { + if ($complexStatus === null) { + $complexStatus = 'paused'; + } + } elseif ($status->startsWith('dead') || $status->startsWith('removing')) { + $complexStatus = 'degraded'; } if ($health->value() === 'healthy') { if ($complexHealth === 'unhealthy') { continue; } $complexHealth = 'healthy'; + } elseif ($health->value() === 'unknown') { + if ($complexHealth !== 'unhealthy') { + $complexHealth = 'unknown'; + } } else { $complexHealth = 'unhealthy'; } @@ -257,6 +297,15 @@ public function getStatusAttribute() foreach ($applications as $application) { $status = str($application->status)->before('(')->trim(); $health = str($application->status)->between('(', ')')->trim(); + + // Only process containers with :excluded suffix (or truly excluded ones) + if (! $health->contains(':excluded') && ! $application->exclude_from_status) { + continue; + } + + // Strip :excluded suffix for health comparison + $health = str($health)->replace(':excluded', ''); + if ($excludedStatus === 'degraded') { continue; } @@ -270,12 +319,26 @@ public function getStatusAttribute() $excludedStatus = 'degraded'; } elseif ($status->startsWith('exited')) { $excludedStatus = 'exited'; + } elseif ($status->startsWith('created') || $status->startsWith('starting')) { + if ($excludedStatus === null) { + $excludedStatus = 'starting'; + } + } elseif ($status->startsWith('paused')) { + if ($excludedStatus === null) { + $excludedStatus = 'paused'; + } + } elseif ($status->startsWith('dead') || $status->startsWith('removing')) { + $excludedStatus = 'degraded'; } if ($health->value() === 'healthy') { if ($excludedHealth === 'unhealthy') { continue; } $excludedHealth = 'healthy'; + } elseif ($health->value() === 'unknown') { + if ($excludedHealth !== 'unhealthy') { + $excludedHealth = 'unknown'; + } } else { $excludedHealth = 'unhealthy'; } @@ -284,6 +347,15 @@ public function getStatusAttribute() foreach ($databases as $database) { $status = str($database->status)->before('(')->trim(); $health = str($database->status)->between('(', ')')->trim(); + + // Only process containers with :excluded suffix (or truly excluded ones) + if (! $health->contains(':excluded') && ! $database->exclude_from_status) { + continue; + } + + // Strip :excluded suffix for health comparison + $health = str($health)->replace(':excluded', ''); + if ($excludedStatus === 'degraded') { continue; } @@ -297,12 +369,26 @@ public function getStatusAttribute() $excludedStatus = 'degraded'; } elseif ($status->startsWith('exited')) { $excludedStatus = 'exited'; + } elseif ($status->startsWith('created') || $status->startsWith('starting')) { + if ($excludedStatus === null) { + $excludedStatus = 'starting'; + } + } elseif ($status->startsWith('paused')) { + if ($excludedStatus === null) { + $excludedStatus = 'paused'; + } + } elseif ($status->startsWith('dead') || $status->startsWith('removing')) { + $excludedStatus = 'degraded'; } if ($health->value() === 'healthy') { if ($excludedHealth === 'unhealthy') { continue; } $excludedHealth = 'healthy'; + } elseif ($health->value() === 'unknown') { + if ($excludedHealth !== 'unhealthy') { + $excludedHealth = 'unknown'; + } } else { $excludedHealth = 'unhealthy'; } diff --git a/tests/Unit/ContainerHealthStatusTest.php b/tests/Unit/ContainerHealthStatusTest.php index 98328630d..dbda8b8c7 100644 --- a/tests/Unit/ContainerHealthStatusTest.php +++ b/tests/Unit/ContainerHealthStatusTest.php @@ -65,22 +65,22 @@ ->not->toContain("data_get(\$container, 'State.Health.Status', 'unhealthy')") ->toContain("data_get(\$container, 'State.Health.Status')"); - // Verify the null check exists for non-excluded containers + // Verify the health check logic for non-excluded containers expect($complexStatusCheckFile) - ->toContain('if ($containerHealth && $containerHealth === \'unhealthy\') {'); + ->toContain('if ($containerHealth === \'unhealthy\') {'); }); it('only marks containers as unhealthy when health status explicitly equals unhealthy', function () { $complexStatusCheckFile = file_get_contents(__DIR__.'/../../app/Actions/Shared/ComplexStatusCheck.php'); - // For non-excluded containers (line ~107) + // For non-excluded containers (line ~108) expect($complexStatusCheckFile) - ->toContain('if ($containerHealth && $containerHealth === \'unhealthy\') {') + ->toContain('if ($containerHealth === \'unhealthy\') {') ->toContain('$hasUnhealthy = true;'); - // For excluded containers (line ~141) + // For excluded containers (line ~145) expect($complexStatusCheckFile) - ->toContain('if ($containerHealth && $containerHealth === \'unhealthy\') {') + ->toContain('if ($containerHealth === \'unhealthy\') {') ->toContain('$excludedHasUnhealthy = true;'); }); @@ -105,12 +105,236 @@ // 2. Only mark as unhealthy if health status EXISTS and equals 'unhealthy' // 3. Don't mark as unhealthy if health status is null/missing - // Verify the condition requires both health to exist AND be unhealthy + // Verify the condition explicitly checks for unhealthy expect($complexStatusCheckFile) - ->toContain('if ($containerHealth && $containerHealth === \'unhealthy\')'); + ->toContain('if ($containerHealth === \'unhealthy\')'); // Verify this check is done for running containers expect($complexStatusCheckFile) ->toContain('} elseif ($containerStatus === \'running\') {') ->toContain('$hasRunning = true;'); }); + +it('tracks unknown health state in aggregation', function () { + $getContainersStatusFile = file_get_contents(__DIR__.'/../../app/Actions/Docker/GetContainersStatus.php'); + + // Verify that $hasUnknown tracking variable exists + expect($getContainersStatusFile) + ->toContain('$hasUnknown = false;'); + + // Verify that unknown state is detected in status parsing + expect($getContainersStatusFile) + ->toContain("if (str(\$status)->contains('unknown')) {") + ->toContain('$hasUnknown = true;'); +}); + +it('preserves unknown health state in aggregated status with correct priority', function () { + $getContainersStatusFile = file_get_contents(__DIR__.'/../../app/Actions/Docker/GetContainersStatus.php'); + + // Verify three-way priority in aggregation: + // 1. Unhealthy (highest priority) + // 2. Unknown (medium priority) + // 3. Healthy (only when all explicitly healthy) + + expect($getContainersStatusFile) + ->toContain('if ($hasUnhealthy) {') + ->toContain("return 'running (unhealthy)';") + ->toContain('} elseif ($hasUnknown) {') + ->toContain("return 'running (unknown)';") + ->toContain('} else {') + ->toContain("return 'running (healthy)';"); +}); + +it('tracks unknown health state in ComplexStatusCheck for multi-server applications', function () { + $complexStatusCheckFile = file_get_contents(__DIR__.'/../../app/Actions/Shared/ComplexStatusCheck.php'); + + // Verify that $hasUnknown tracking variable exists + expect($complexStatusCheckFile) + ->toContain('$hasUnknown = false;'); + + // Verify that unknown state is detected when containerHealth is null + expect($complexStatusCheckFile) + ->toContain('} elseif ($containerHealth === null) {') + ->toContain('$hasUnknown = true;'); + + // Verify excluded containers also track unknown + expect($complexStatusCheckFile) + ->toContain('$excludedHasUnknown = false;'); +}); + +it('preserves unknown health state in ComplexStatusCheck aggregated status', function () { + $complexStatusCheckFile = file_get_contents(__DIR__.'/../../app/Actions/Shared/ComplexStatusCheck.php'); + + // Verify three-way priority for non-excluded containers + expect($complexStatusCheckFile) + ->toContain('if ($hasUnhealthy) {') + ->toContain("return 'running:unhealthy';") + ->toContain('} elseif ($hasUnknown) {') + ->toContain("return 'running:unknown';") + ->toContain('} else {') + ->toContain("return 'running:healthy';"); + + // Verify three-way priority for excluded containers + expect($complexStatusCheckFile) + ->toContain('if ($excludedHasUnhealthy) {') + ->toContain("return 'running:unhealthy:excluded';") + ->toContain('} elseif ($excludedHasUnknown) {') + ->toContain("return 'running:unknown:excluded';") + ->toContain("return 'running:healthy:excluded';"); +}); + +it('preserves unknown health state in Service model aggregation', function () { + $serviceFile = file_get_contents(__DIR__.'/../../app/Models/Service.php'); + + // Verify unknown is handled in non-excluded applications + expect($serviceFile) + ->toContain("} elseif (\$health->value() === 'unknown') {") + ->toContain("if (\$complexHealth !== 'unhealthy') {") + ->toContain("\$complexHealth = 'unknown';"); + + // The pattern should appear 4 times (non-excluded apps, non-excluded databases, + // excluded apps, excluded databases) + $unknownCount = substr_count($serviceFile, "} elseif (\$health->value() === 'unknown') {"); + expect($unknownCount)->toBe(4); +}); + +it('handles starting state (created/starting) in GetContainersStatus', function () { + $getContainersStatusFile = file_get_contents(__DIR__.'/../../app/Actions/Docker/GetContainersStatus.php'); + + // Verify tracking variable exists + expect($getContainersStatusFile) + ->toContain('$hasStarting = false;'); + + // Verify detection for created/starting states + expect($getContainersStatusFile) + ->toContain("} elseif (str(\$status)->contains('created') || str(\$status)->contains('starting')) {") + ->toContain('$hasStarting = true;'); + + // Verify aggregation returns starting status + expect($getContainersStatusFile) + ->toContain('if ($hasStarting) {') + ->toContain("return 'starting (unknown)';"); +}); + +it('handles paused state in GetContainersStatus', function () { + $getContainersStatusFile = file_get_contents(__DIR__.'/../../app/Actions/Docker/GetContainersStatus.php'); + + // Verify tracking variable exists + expect($getContainersStatusFile) + ->toContain('$hasPaused = false;'); + + // Verify detection for paused state + expect($getContainersStatusFile) + ->toContain("} elseif (str(\$status)->contains('paused')) {") + ->toContain('$hasPaused = true;'); + + // Verify aggregation returns paused status + expect($getContainersStatusFile) + ->toContain('if ($hasPaused) {') + ->toContain("return 'paused (unknown)';"); +}); + +it('handles dead/removing states in GetContainersStatus', function () { + $getContainersStatusFile = file_get_contents(__DIR__.'/../../app/Actions/Docker/GetContainersStatus.php'); + + // Verify tracking variable exists + expect($getContainersStatusFile) + ->toContain('$hasDead = false;'); + + // Verify detection for dead/removing states + expect($getContainersStatusFile) + ->toContain("} elseif (str(\$status)->contains('dead') || str(\$status)->contains('removing')) {") + ->toContain('$hasDead = true;'); + + // Verify aggregation returns degraded status + expect($getContainersStatusFile) + ->toContain('if ($hasDead) {') + ->toContain("return 'degraded (unhealthy)';"); +}); + +it('handles edge case states in ComplexStatusCheck for non-excluded containers', function () { + $complexStatusCheckFile = file_get_contents(__DIR__.'/../../app/Actions/Shared/ComplexStatusCheck.php'); + + // Verify tracking variables exist + expect($complexStatusCheckFile) + ->toContain('$hasStarting = false;') + ->toContain('$hasPaused = false;') + ->toContain('$hasDead = false;'); + + // Verify detection for created/starting + expect($complexStatusCheckFile) + ->toContain("} elseif (\$containerStatus === 'created' || \$containerStatus === 'starting') {") + ->toContain('$hasStarting = true;'); + + // Verify detection for paused + expect($complexStatusCheckFile) + ->toContain("} elseif (\$containerStatus === 'paused') {") + ->toContain('$hasPaused = true;'); + + // Verify detection for dead/removing + expect($complexStatusCheckFile) + ->toContain("} elseif (\$containerStatus === 'dead' || \$containerStatus === 'removing') {") + ->toContain('$hasDead = true;'); +}); + +it('handles edge case states in ComplexStatusCheck aggregation', function () { + $complexStatusCheckFile = file_get_contents(__DIR__.'/../../app/Actions/Shared/ComplexStatusCheck.php'); + + // Verify aggregation logic for edge cases + expect($complexStatusCheckFile) + ->toContain('if ($hasDead) {') + ->toContain("return 'degraded:unhealthy';") + ->toContain('if ($hasPaused) {') + ->toContain("return 'paused:unknown';") + ->toContain('if ($hasStarting) {') + ->toContain("return 'starting:unknown';"); +}); + +it('handles edge case states in Service model for all 4 locations', function () { + $serviceFile = file_get_contents(__DIR__.'/../../app/Models/Service.php'); + + // Check for created/starting handling pattern + $createdStartingCount = substr_count($serviceFile, "\$status->startsWith('created') || \$status->startsWith('starting')"); + expect($createdStartingCount)->toBe(4, 'created/starting handling should appear in all 4 locations'); + + // Check for paused handling pattern + $pausedCount = substr_count($serviceFile, "\$status->startsWith('paused')"); + expect($pausedCount)->toBe(4, 'paused handling should appear in all 4 locations'); + + // Check for dead/removing handling pattern + $deadRemovingCount = substr_count($serviceFile, "\$status->startsWith('dead') || \$status->startsWith('removing')"); + expect($deadRemovingCount)->toBe(4, 'dead/removing handling should appear in all 4 locations'); +}); + +it('appends :excluded suffix to excluded container statuses in GetContainersStatus', function () { + $getContainersStatusFile = file_get_contents(__DIR__.'/../../app/Actions/Docker/GetContainersStatus.php'); + + // Verify that we check for exclude_from_hc flag + expect($getContainersStatusFile) + ->toContain('$excludeFromHc = data_get($serviceConfig, \'exclude_from_hc\', false);'); + + // Verify that we append :excluded suffix + expect($getContainersStatusFile) + ->toContain('$containerStatus = str_replace(\')\', \':excluded)\', $containerStatus);'); +}); + +it('skips containers with :excluded suffix in Service model non-excluded sections', function () { + $serviceFile = file_get_contents(__DIR__.'/../../app/Models/Service.php'); + + // Verify that we skip :excluded containers in non-excluded sections + // This should appear twice (once for applications, once for databases) + $skipExcludedCount = substr_count($serviceFile, "if (\$health->contains(':excluded')) {"); + expect($skipExcludedCount)->toBeGreaterThanOrEqual(2, 'Should skip :excluded containers in non-excluded sections'); +}); + +it('processes containers with :excluded suffix in Service model excluded sections', function () { + $serviceFile = file_get_contents(__DIR__.'/../../app/Models/Service.php'); + + // Verify that we process :excluded containers in excluded sections + $processExcludedCount = substr_count($serviceFile, "if (! \$health->contains(':excluded') && !"); + expect($processExcludedCount)->toBeGreaterThanOrEqual(2, 'Should process :excluded containers in excluded sections'); + + // Verify that we strip :excluded suffix before health comparison + $stripExcludedCount = substr_count($serviceFile, "\$health = str(\$health)->replace(':excluded', '');"); + expect($stripExcludedCount)->toBeGreaterThanOrEqual(2, 'Should strip :excluded suffix in excluded sections'); +});