2024-02-07 13:55:06 +00:00
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
namespace App\Actions\Shared;
|
|
|
|
|
|
|
|
|
|
use App\Models\Application;
|
|
|
|
|
use Lorisleiva\Actions\Concerns\AsAction;
|
|
|
|
|
|
|
|
|
|
class ComplexStatusCheck
|
|
|
|
|
{
|
|
|
|
|
use AsAction;
|
2024-06-10 20:43:34 +00:00
|
|
|
|
2024-02-07 13:55:06 +00:00
|
|
|
public function handle(Application $application)
|
|
|
|
|
{
|
|
|
|
|
$servers = $application->additional_servers;
|
|
|
|
|
$servers->push($application->destination->server);
|
|
|
|
|
foreach ($servers as $server) {
|
|
|
|
|
$is_main_server = $application->destination->server->id === $server->id;
|
2024-06-10 20:43:34 +00:00
|
|
|
if (! $server->isFunctional()) {
|
2024-02-07 13:55:06 +00:00
|
|
|
if ($is_main_server) {
|
|
|
|
|
$application->update(['status' => 'exited:unhealthy']);
|
2024-06-10 20:43:34 +00:00
|
|
|
|
2024-02-07 13:55:06 +00:00
|
|
|
continue;
|
|
|
|
|
} else {
|
|
|
|
|
$application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited:unhealthy']);
|
2024-06-10 20:43:34 +00:00
|
|
|
|
2024-02-07 13:55:06 +00:00
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
}
|
2025-09-18 14:51:08 +00:00
|
|
|
$containers = instant_remote_process(["docker container inspect $(docker container ls -q --filter 'label=coolify.applicationId={$application->id}' --filter 'label=coolify.pullRequestId=0') --format '{{json .}}'"], $server, false);
|
|
|
|
|
$containers = format_docker_command_output_to_json($containers);
|
|
|
|
|
|
|
|
|
|
if ($containers->count() > 0) {
|
|
|
|
|
$statusToSet = $this->aggregateContainerStatuses($application, $containers);
|
|
|
|
|
|
2024-02-07 13:55:06 +00:00
|
|
|
if ($is_main_server) {
|
|
|
|
|
$statusFromDb = $application->status;
|
2025-09-18 14:51:08 +00:00
|
|
|
if ($statusFromDb !== $statusToSet) {
|
|
|
|
|
$application->update(['status' => $statusToSet]);
|
2024-02-07 13:55:06 +00:00
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
$additional_server = $application->additional_servers()->wherePivot('server_id', $server->id);
|
|
|
|
|
$statusFromDb = $additional_server->first()->pivot->status;
|
2025-09-18 14:51:08 +00:00
|
|
|
if ($statusFromDb !== $statusToSet) {
|
|
|
|
|
$additional_server->updateExistingPivot($server->id, ['status' => $statusToSet]);
|
2024-02-07 13:55:06 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if ($is_main_server) {
|
|
|
|
|
$application->update(['status' => 'exited:unhealthy']);
|
2024-06-10 20:43:34 +00:00
|
|
|
|
2024-02-07 13:55:06 +00:00
|
|
|
continue;
|
|
|
|
|
} else {
|
|
|
|
|
$application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited:unhealthy']);
|
2024-06-10 20:43:34 +00:00
|
|
|
|
2024-02-07 13:55:06 +00:00
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2025-09-18 14:51:08 +00:00
|
|
|
|
|
|
|
|
private function aggregateContainerStatuses($application, $containers)
|
|
|
|
|
{
|
|
|
|
|
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
|
|
|
|
|
$excludedContainers = collect();
|
|
|
|
|
|
|
|
|
|
if ($dockerComposeRaw) {
|
|
|
|
|
try {
|
|
|
|
|
$dockerCompose = \Symfony\Component\Yaml\Yaml::parse($dockerComposeRaw);
|
|
|
|
|
$services = data_get($dockerCompose, 'services', []);
|
|
|
|
|
|
|
|
|
|
foreach ($services as $serviceName => $serviceConfig) {
|
|
|
|
|
$excludeFromHc = data_get($serviceConfig, 'exclude_from_hc', false);
|
|
|
|
|
$restartPolicy = data_get($serviceConfig, 'restart', 'always');
|
|
|
|
|
|
|
|
|
|
if ($excludeFromHc || $restartPolicy === 'no') {
|
|
|
|
|
$excludedContainers->push($serviceName);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (\Exception $e) {
|
|
|
|
|
// If we can't parse, treat all containers as included
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$hasRunning = false;
|
|
|
|
|
$hasRestarting = false;
|
|
|
|
|
$hasUnhealthy = false;
|
2025-11-19 12:19:25 +00:00
|
|
|
$hasUnknown = false;
|
2025-09-18 14:51:08 +00:00
|
|
|
$hasExited = false;
|
2025-11-19 12:19:25 +00:00
|
|
|
$hasStarting = false;
|
|
|
|
|
$hasPaused = false;
|
|
|
|
|
$hasDead = false;
|
2025-09-18 14:51:08 +00:00
|
|
|
$relevantContainerCount = 0;
|
|
|
|
|
|
|
|
|
|
foreach ($containers as $container) {
|
|
|
|
|
$labels = data_get($container, 'Config.Labels', []);
|
|
|
|
|
$serviceName = data_get($labels, 'com.docker.compose.service');
|
|
|
|
|
|
|
|
|
|
if ($serviceName && $excludedContainers->contains($serviceName)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$relevantContainerCount++;
|
|
|
|
|
$containerStatus = data_get($container, 'State.Status');
|
2025-11-19 09:54:51 +00:00
|
|
|
$containerHealth = data_get($container, 'State.Health.Status');
|
2025-09-18 14:51:08 +00:00
|
|
|
|
|
|
|
|
if ($containerStatus === 'restarting') {
|
|
|
|
|
$hasRestarting = true;
|
|
|
|
|
$hasUnhealthy = true;
|
|
|
|
|
} elseif ($containerStatus === 'running') {
|
|
|
|
|
$hasRunning = true;
|
2025-11-19 12:19:25 +00:00
|
|
|
if ($containerHealth === 'unhealthy') {
|
2025-09-18 14:51:08 +00:00
|
|
|
$hasUnhealthy = true;
|
2025-11-19 12:19:25 +00:00
|
|
|
} elseif ($containerHealth === null) {
|
|
|
|
|
$hasUnknown = true;
|
2025-09-18 14:51:08 +00:00
|
|
|
}
|
|
|
|
|
} elseif ($containerStatus === 'exited') {
|
|
|
|
|
$hasExited = true;
|
|
|
|
|
$hasUnhealthy = true;
|
2025-11-19 12:19:25 +00:00
|
|
|
} elseif ($containerStatus === 'created' || $containerStatus === 'starting') {
|
|
|
|
|
$hasStarting = true;
|
|
|
|
|
} elseif ($containerStatus === 'paused') {
|
|
|
|
|
$hasPaused = true;
|
|
|
|
|
} elseif ($containerStatus === 'dead' || $containerStatus === 'removing') {
|
|
|
|
|
$hasDead = true;
|
2025-09-18 14:51:08 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2025-11-19 09:54:51 +00:00
|
|
|
// If all containers are excluded, calculate status from excluded containers
|
|
|
|
|
// but mark it with :excluded to indicate monitoring is disabled
|
2025-09-18 14:51:08 +00:00
|
|
|
if ($relevantContainerCount === 0) {
|
2025-11-19 09:54:51 +00:00
|
|
|
$excludedHasRunning = false;
|
|
|
|
|
$excludedHasRestarting = false;
|
|
|
|
|
$excludedHasUnhealthy = false;
|
2025-11-19 12:19:25 +00:00
|
|
|
$excludedHasUnknown = false;
|
2025-11-19 09:54:51 +00:00
|
|
|
$excludedHasExited = false;
|
2025-11-19 12:19:25 +00:00
|
|
|
$excludedHasStarting = false;
|
|
|
|
|
$excludedHasPaused = false;
|
|
|
|
|
$excludedHasDead = false;
|
2025-11-19 09:54:51 +00:00
|
|
|
|
|
|
|
|
foreach ($containers as $container) {
|
|
|
|
|
$labels = data_get($container, 'Config.Labels', []);
|
|
|
|
|
$serviceName = data_get($labels, 'com.docker.compose.service');
|
|
|
|
|
|
|
|
|
|
// Only process excluded containers
|
|
|
|
|
if (! $serviceName || ! $excludedContainers->contains($serviceName)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$containerStatus = data_get($container, 'State.Status');
|
|
|
|
|
$containerHealth = data_get($container, 'State.Health.Status');
|
|
|
|
|
|
|
|
|
|
if ($containerStatus === 'restarting') {
|
|
|
|
|
$excludedHasRestarting = true;
|
|
|
|
|
$excludedHasUnhealthy = true;
|
|
|
|
|
} elseif ($containerStatus === 'running') {
|
|
|
|
|
$excludedHasRunning = true;
|
2025-11-19 12:19:25 +00:00
|
|
|
if ($containerHealth === 'unhealthy') {
|
2025-11-19 09:54:51 +00:00
|
|
|
$excludedHasUnhealthy = true;
|
2025-11-19 12:19:25 +00:00
|
|
|
} elseif ($containerHealth === null) {
|
|
|
|
|
$excludedHasUnknown = true;
|
2025-11-19 09:54:51 +00:00
|
|
|
}
|
|
|
|
|
} elseif ($containerStatus === 'exited') {
|
|
|
|
|
$excludedHasExited = true;
|
|
|
|
|
$excludedHasUnhealthy = true;
|
2025-11-19 12:19:25 +00:00
|
|
|
} elseif ($containerStatus === 'created' || $containerStatus === 'starting') {
|
|
|
|
|
$excludedHasStarting = true;
|
|
|
|
|
} elseif ($containerStatus === 'paused') {
|
|
|
|
|
$excludedHasPaused = true;
|
|
|
|
|
} elseif ($containerStatus === 'dead' || $containerStatus === 'removing') {
|
|
|
|
|
$excludedHasDead = true;
|
2025-11-19 09:54:51 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($excludedHasRestarting) {
|
|
|
|
|
return 'degraded:excluded';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($excludedHasRunning && $excludedHasExited) {
|
|
|
|
|
return 'degraded:excluded';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($excludedHasRunning) {
|
2025-11-19 12:19:25 +00:00
|
|
|
if ($excludedHasUnhealthy) {
|
|
|
|
|
return 'running:unhealthy:excluded';
|
|
|
|
|
} elseif ($excludedHasUnknown) {
|
|
|
|
|
return 'running:unknown:excluded';
|
|
|
|
|
} else {
|
|
|
|
|
return 'running:healthy:excluded';
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($excludedHasDead) {
|
|
|
|
|
return 'degraded:excluded';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($excludedHasPaused) {
|
|
|
|
|
return 'paused:excluded';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($excludedHasStarting) {
|
|
|
|
|
return 'starting:excluded';
|
2025-11-19 09:54:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 'exited:excluded';
|
2025-09-18 14:51:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($hasRestarting) {
|
|
|
|
|
return 'degraded:unhealthy';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($hasRunning && $hasExited) {
|
|
|
|
|
return 'degraded:unhealthy';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($hasRunning) {
|
2025-11-19 12:19:25 +00:00
|
|
|
if ($hasUnhealthy) {
|
|
|
|
|
return 'running:unhealthy';
|
|
|
|
|
} elseif ($hasUnknown) {
|
|
|
|
|
return 'running:unknown';
|
|
|
|
|
} else {
|
|
|
|
|
return 'running:healthy';
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($hasDead) {
|
|
|
|
|
return 'degraded:unhealthy';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($hasPaused) {
|
|
|
|
|
return 'paused:unknown';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($hasStarting) {
|
|
|
|
|
return 'starting:unknown';
|
2025-09-18 14:51:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 'exited:unhealthy';
|
|
|
|
|
}
|
2024-02-07 13:55:06 +00:00
|
|
|
}
|