From 67b33c27a9eb6718e965ff2cd58cd5645b322350 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:13:26 +0200 Subject: [PATCH] fix(applications): persist restart limit state for stopped containers (#11566) --- app/Actions/Application/StopApplication.php | 24 +- .../Application/StopApplicationPreview.php | 36 +++ app/Actions/Database/StartDatabase.php | 1 + app/Actions/Database/StopDatabase.php | 28 ++- app/Actions/Docker/GetContainersStatus.php | 179 ++++++++------ app/Actions/Service/StartService.php | 2 + app/Actions/Service/StopService.php | 10 +- .../Service/StopServiceApplication.php | 17 +- .../Api/NotificationsController.php | 10 +- .../Controllers/Api/SentinelController.php | 3 +- app/Jobs/ApplicationDeploymentJob.php | 14 +- app/Jobs/PushServerUpdateJob.php | 184 ++++++++++++++- app/Livewire/Notifications/Discord.php | 5 + app/Livewire/Notifications/Email.php | 5 + app/Livewire/Notifications/Pushover.php | 5 + app/Livewire/Notifications/Slack.php | 5 + app/Livewire/Notifications/Telegram.php | 38 +-- app/Livewire/Notifications/Webhook.php | 5 + app/Livewire/Project/Resource/Index.php | 5 + app/Livewire/Project/Service/Heading.php | 26 ++ app/Livewire/Project/Service/Status.php | 14 +- app/Models/Application.php | 10 +- app/Models/ApplicationPreview.php | 8 +- app/Models/DiscordNotificationSettings.php | 2 + app/Models/EmailNotificationSettings.php | 2 + app/Models/PushoverNotificationSettings.php | 2 + app/Models/ServiceApplication.php | 3 +- app/Models/ServiceDatabase.php | 3 +- app/Models/SlackNotificationSettings.php | 2 + app/Models/StandaloneClickhouse.php | 3 +- app/Models/StandaloneDragonfly.php | 3 +- app/Models/StandaloneKeydb.php | 3 +- app/Models/StandaloneMariadb.php | 3 +- app/Models/StandaloneMongodb.php | 3 +- app/Models/StandaloneMysql.php | 3 +- app/Models/StandalonePostgresql.php | 3 +- app/Models/StandaloneRedis.php | 3 +- app/Models/TelegramNotificationSettings.php | 5 + app/Models/WebhookNotificationSettings.php | 2 + .../Application/RestartLimitReached.php | 48 ++-- .../Channels/TelegramChannel.php | 45 ++-- .../Container/ContainerStopped.php | 123 ---------- app/Services/ContainerStatusAggregator.php | 34 ++- app/Services/RestartCountTracker.php | 31 +++ app/Traits/HasRestartLimit.php | 73 ++++++ ...ontainer_present_to_applications_table.php | 28 +++ ...rt_limit_reached_to_applications_table.php | 37 +++ ..._restart_limit_to_application_previews.php | 32 +++ ..._restart_limit_to_service_applications.php | 32 +++ ...add_restart_limit_to_service_databases.php | 32 +++ ...estart_limit_to_standalone_postgresqls.php | 23 ++ ..._add_restart_limit_to_standalone_redis.php | 23 ++ ...d_restart_limit_to_standalone_mongodbs.php | 23 ++ ...add_restart_limit_to_standalone_mysqls.php | 23 ++ ...d_restart_limit_to_standalone_mariadbs.php | 23 ++ ...add_restart_limit_to_standalone_keydbs.php | 23 ++ ...estart_limit_to_standalone_dragonflies.php | 23 ++ ...estart_limit_to_standalone_clickhouses.php | 23 ++ ...s_to_email_notification_settings_table.php | 28 +++ ...to_discord_notification_settings_table.php | 28 +++ ...o_telegram_notification_settings_table.php | 30 +++ ...s_to_slack_notification_settings_table.php | 28 +++ ...o_pushover_notification_settings_table.php | 28 +++ ...to_webhook_notification_settings_table.php | 28 +++ .../restart-limit-warning.blade.php | 10 + .../notification/event-grid.blade.php | 11 +- .../views/components/status-summary.blade.php | 3 +- .../views/components/status/index.blade.php | 4 +- .../views/emails/container-stopped.blade.php | 7 - .../livewire/notifications/email.blade.php | 6 +- .../project/application/heading.blade.php | 54 +++-- .../project/application/previews.blade.php | 1 + .../project/application/status.blade.php | 1 + .../project/database/heading.blade.php | 7 +- .../project/database/status.blade.php | 1 + .../livewire/project/resource/index.blade.php | 27 ++- .../project/service/configuration.blade.php | 6 +- .../project/service/heading.blade.php | 44 +++- .../project/service/resource-card.blade.php | 30 ++- .../livewire/project/service/status.blade.php | 7 +- .../project/shared/destination.blade.php | 4 + .../views/livewire/server/resources.blade.php | 12 +- .../Feature/AllResourceRestartLimitsTest.php | 223 ++++++++++++++++++ .../ApplicationContainerPresenceTest.php | 56 +++++ ...pplicationStoppedAfterRestartLimitTest.php | 167 ++++++++++++- .../NotificationAuthorizationTest.php | 74 ++++++ ...bleLivewireComponentsAuthorizationTest.php | 16 ++ .../NotificationRestartLimitSettingTest.php | 48 ++++ tests/Feature/PreviewStatusSummaryTest.php | 12 +- .../ResourceHeadingUnifiedNavbarTest.php | 19 ++ .../Feature/SentinelPushDeduplicationTest.php | 10 + tests/Unit/ContainerStatusAggregatorTest.php | 32 ++- .../PushServerUpdateJobReturnTypesTest.php | 9 + ...emovedContainerStoppedNotificationTest.php | 11 + tests/Unit/RestartCountTrackerTest.php | 95 ++++++++ tests/Unit/StopActionsPersistStatusTest.php | 11 +- tests/Unit/StopDatabaseTypesTest.php | 12 + 97 files changed, 2181 insertions(+), 397 deletions(-) create mode 100644 app/Actions/Application/StopApplicationPreview.php delete mode 100644 app/Notifications/Container/ContainerStopped.php create mode 100644 app/Services/RestartCountTracker.php create mode 100644 app/Traits/HasRestartLimit.php create mode 100644 database/migrations/2026_08_30_193506_add_container_present_to_applications_table.php create mode 100644 database/migrations/2026_08_30_220617_add_restart_limit_reached_to_applications_table.php create mode 100644 database/migrations/2026_08_31_073116_add_restart_limit_to_application_previews.php create mode 100644 database/migrations/2026_08_31_073117_add_restart_limit_to_service_applications.php create mode 100644 database/migrations/2026_08_31_073118_add_restart_limit_to_service_databases.php create mode 100644 database/migrations/2026_08_31_073119_add_restart_limit_to_standalone_postgresqls.php create mode 100644 database/migrations/2026_08_31_073120_add_restart_limit_to_standalone_redis.php create mode 100644 database/migrations/2026_08_31_073121_add_restart_limit_to_standalone_mongodbs.php create mode 100644 database/migrations/2026_08_31_073122_add_restart_limit_to_standalone_mysqls.php create mode 100644 database/migrations/2026_08_31_073123_add_restart_limit_to_standalone_mariadbs.php create mode 100644 database/migrations/2026_08_31_073124_add_restart_limit_to_standalone_keydbs.php create mode 100644 database/migrations/2026_08_31_073125_add_restart_limit_to_standalone_dragonflies.php create mode 100644 database/migrations/2026_08_31_073126_add_restart_limit_to_standalone_clickhouses.php create mode 100644 database/migrations/2026_08_31_092837_add_restart_limit_reached_notifications_to_email_notification_settings_table.php create mode 100644 database/migrations/2026_08_31_092838_add_restart_limit_reached_notifications_to_discord_notification_settings_table.php create mode 100644 database/migrations/2026_08_31_092840_add_restart_limit_reached_notifications_to_telegram_notification_settings_table.php create mode 100644 database/migrations/2026_08_31_092841_add_restart_limit_reached_notifications_to_slack_notification_settings_table.php create mode 100644 database/migrations/2026_08_31_092842_add_restart_limit_reached_notifications_to_pushover_notification_settings_table.php create mode 100644 database/migrations/2026_08_31_092843_add_restart_limit_reached_notifications_to_webhook_notification_settings_table.php create mode 100644 resources/views/components/application/restart-limit-warning.blade.php delete mode 100644 resources/views/emails/container-stopped.blade.php create mode 100644 tests/Feature/AllResourceRestartLimitsTest.php create mode 100644 tests/Feature/ApplicationContainerPresenceTest.php create mode 100644 tests/Feature/NotificationRestartLimitSettingTest.php create mode 100644 tests/Unit/PushServerUpdateJobReturnTypesTest.php create mode 100644 tests/Unit/RemovedContainerStoppedNotificationTest.php create mode 100644 tests/Unit/RestartCountTrackerTest.php create mode 100644 tests/Unit/StopDatabaseTypesTest.php diff --git a/app/Actions/Application/StopApplication.php b/app/Actions/Application/StopApplication.php index 66ceb95f6..3feb5117d 100644 --- a/app/Actions/Application/StopApplication.php +++ b/app/Actions/Application/StopApplication.php @@ -13,8 +13,9 @@ class StopApplication public string $jobQueue = 'high'; - public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true, bool $resetRestartCount = true) + public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true, bool $resetRestartCount = true, bool $removeContainers = true): ?string { + $containerPresent = ! $removeContainers; $servers = collect([$application->destination->server]); if ($application?->additional_servers?->count() > 0) { $servers = $servers->merge($application->additional_servers); @@ -26,6 +27,7 @@ public function handle(Application $application, bool $previewDeployments = fals } if ($server->isSwarm()) { + $containerPresent = false; instant_remote_process(["docker stack rm {$application->uuid}"], $server); continue; @@ -39,13 +41,17 @@ public function handle(Application $application, bool $previewDeployments = fals $timeout = $application->settings->stopGracePeriodSeconds(); foreach ($containersToStop as $containerName) { - instant_remote_process(command: [ - dockerStopCommand($timeout, $containerName, $server), - "docker rm -f $containerName", - ], server: $server, throwError: false); + $commands = [dockerStopCommand($timeout, $containerName, $server)]; + if ($removeContainers) { + $commands[] = "docker rm -f $containerName"; + } else { + array_unshift($commands, "docker update --restart=no $containerName"); + } + + instant_remote_process(command: $commands, server: $server, throwError: false); } - if ($application->build_pack === 'dockercompose') { + if ($removeContainers && $application->build_pack === 'dockercompose') { $application->deleteConnectedNetworks(); } @@ -57,12 +63,16 @@ public function handle(Application $application, bool $previewDeployments = fals } } - $status = ['status' => 'exited']; + $status = [ + 'status' => 'exited', + 'container_present' => $containerPresent, + ]; if ($resetRestartCount) { $status = array_merge($status, [ 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null, + 'restart_limit_reached' => false, ]); } $application->update($status); diff --git a/app/Actions/Application/StopApplicationPreview.php b/app/Actions/Application/StopApplicationPreview.php new file mode 100644 index 000000000..af5f3fc0f --- /dev/null +++ b/app/Actions/Application/StopApplicationPreview.php @@ -0,0 +1,36 @@ +application; + $server = $application->destination->server; + $containers = getCurrentApplicationContainerStatus($server, $application->id, $preview->pull_request_id); + + foreach ($containers->pluck('Names') as $containerName) { + $commands = [dockerStopCommand($application->settings->stopGracePeriodSeconds(), $containerName, $server)]; + if ($removeContainer) { + $commands[] = "docker rm -f $containerName"; + } else { + array_unshift($commands, "docker update --restart=no $containerName"); + } + instant_remote_process($commands, $server, false); + } + + $preview->update(['status' => 'exited']); + if ($resetRestartCount) { + $preview->resetRestartLimit(); + } + + ServiceStatusChanged::dispatch($application->environment->project->team->id); + } +} diff --git a/app/Actions/Database/StartDatabase.php b/app/Actions/Database/StartDatabase.php index 4b55b0c1d..7735c77d8 100644 --- a/app/Actions/Database/StartDatabase.php +++ b/app/Actions/Database/StartDatabase.php @@ -28,6 +28,7 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St if (! $server->isFunctional()) { return 'Server is not functional'; } + $database->resetRestartLimit(); switch ($database->getMorphClass()) { case StandalonePostgresql::class: $activity = StartPostgresql::run($database); diff --git a/app/Actions/Database/StopDatabase.php b/app/Actions/Database/StopDatabase.php index a3a7f16ef..f3c591acf 100644 --- a/app/Actions/Database/StopDatabase.php +++ b/app/Actions/Database/StopDatabase.php @@ -4,6 +4,7 @@ use App\Actions\Server\CleanupDocker; use App\Events\ServiceStatusChanged; +use App\Models\BaseModel; use App\Models\StandaloneClickhouse; use App\Models\StandaloneDragonfly; use App\Models\StandaloneKeydb; @@ -18,7 +19,7 @@ class StopDatabase { use AsAction; - public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, bool $dockerCleanup = true) + public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, bool $dockerCleanup = true, bool $resetRestartCount = true, bool $removeContainer = true): string { try { $server = $database->destination->server; @@ -26,15 +27,13 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St return 'Server is not functional'; } - $this->stopContainer($database, $database->uuid, 30); + $this->stopContainer($database, $database->uuid, 30, $removeContainer); // Reset restart tracking when database is manually stopped - $database->update([ - 'status' => 'exited', - 'restart_count' => 0, - 'last_restart_at' => null, - 'last_restart_type' => null, - ]); + $database->update(['status' => 'exited']); + if ($resetRestartCount) { + $database->resetRestartLimit(); + } if ($dockerCleanup) { CleanupDocker::dispatch($server, false, false); @@ -53,12 +52,15 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St } - private function stopContainer($database, string $containerName, int $timeout = 30): void + private function stopContainer(BaseModel $database, string $containerName, int $timeout = 30, bool $removeContainer = true): void { $server = $database->destination->server; - instant_remote_process(command: [ - dockerStopCommand($timeout, $containerName, $server), - "docker rm -f $containerName", - ], server: $server, throwError: false); + $commands = [dockerStopCommand($timeout, $containerName, $server)]; + if ($removeContainer) { + $commands[] = "docker rm -f $containerName"; + } else { + array_unshift($commands, "docker update --restart=no $containerName"); + } + instant_remote_process(command: $commands, server: $server, throwError: false); } } diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index 904885dfc..be098e481 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -3,15 +3,20 @@ namespace App\Actions\Docker; use App\Actions\Application\StopApplication; +use App\Actions\Application\StopApplicationPreview; use App\Actions\Database\StartDatabaseProxy; +use App\Actions\Database\StopDatabase; use App\Actions\Database\StopDatabaseProxy; +use App\Actions\Service\StopServiceApplication; use App\Actions\Shared\ComplexStatusCheck; use App\Events\ServiceChecked; +use App\Models\Application; use App\Models\ApplicationPreview; use App\Models\Server; use App\Models\ServiceDatabase; use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached; use App\Services\ContainerStatusAggregator; +use App\Services\RestartCountTracker; use App\Traits\CalculatesExcludedStatus; use Illuminate\Support\Arr; use Illuminate\Support\Collection; @@ -37,8 +42,12 @@ class GetContainersStatus protected ?Collection $applicationContainerRestartCounts; + protected ?Collection $previewContainerRestartCounts; + protected ?Collection $serviceContainerStatuses; + protected ?Collection $serviceContainerRestartCounts; + public function handle(Server $server, ?Collection $containers = null, ?Collection $containerReplicates = null) { $this->containers = $containers; @@ -117,6 +126,9 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti $containerStatus = "$containerStatus:$healthSuffix"; } $labels = Arr::undot(format_docker_labels_to_json($labels)); + if (filter_var(data_get($labels, 'com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)) { + continue; + } $applicationId = data_get($labels, 'coolify.applicationId'); if ($applicationId) { $pullRequestId = data_get($labels, 'coolify.pullRequestId'); @@ -133,6 +145,12 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti } else { $preview->update(['last_online_at' => now()]); } + $key = $applicationId.':'.$pullRequestId; + $this->previewContainerRestartCounts ??= collect(); + $this->previewContainerRestartCounts->push([ + 'key' => $key, + 'count' => (int) data_get($container, 'RestartCount', 0), + ]); } else { // Notify user that this container should not be there. } @@ -140,6 +158,9 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti $application = $this->applications->where('id', $applicationId)->first(); if ($application) { $foundApplications[] = $application->id; + if ($application->container_present !== true) { + $application->update(['container_present' => true]); + } // Store container status for aggregation if (! isset($this->applicationContainerStatuses)) { $this->applicationContainerStatuses = collect(); @@ -220,23 +241,19 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti // Track restart count for databases (single-container) $restartCount = data_get($container, 'RestartCount', 0); - $previousRestartCount = $database->restart_count ?? 0; - if ($statusFromDb !== $containerStatus) { $updateData = ['status' => $containerStatus]; } else { $updateData = ['last_online_at' => now()]; } - // Update restart tracking if restart count increased - if ($restartCount > $previousRestartCount) { - $updateData['restart_count'] = $restartCount; - $updateData['last_restart_at'] = now(); - $updateData['last_restart_type'] = 'crash'; - } - $database->update($updateData); + if ($database->trackRestartCount((int) $restartCount)) { + StopDatabase::dispatch($database, false, false, false); + $database->team()?->notify(new ApplicationRestartLimitReached($database)); + } + if ($isPublic) { $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { if ($this->server->isSwarm()) { @@ -292,6 +309,11 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti $containerName = data_get($labels, 'com.docker.compose.service'); if ($containerName) { $this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus); + $this->serviceContainerRestartCounts ??= collect(); + if (! $this->serviceContainerRestartCounts->has($key)) { + $this->serviceContainerRestartCounts->put($key, collect()); + } + $this->serviceContainerRestartCounts->get($key)->put($containerName, (int) data_get($container, 'RestartCount', 0)); } // Mark service as found @@ -335,46 +357,35 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti continue; } - $name = data_get($exitedService, 'name'); - $fqdn = data_get($exitedService, 'fqdn'); - if ($name) { - if ($fqdn) { - $containerName = "$name, available at $fqdn"; - } else { - $containerName = $name; - } - } else { - if ($fqdn) { - $containerName = $fqdn; - } else { - $containerName = null; - } + if (! $exitedService->stoppedAfterRestartLimit()) { + $exitedService->update([ + 'status' => 'exited', + 'restart_count' => 0, + 'restart_limit_reached' => false, + 'last_restart_at' => null, + 'last_restart_type' => null, + ]); } - $projectUuid = data_get($service, 'environment.project.uuid'); - $serviceUuid = data_get($service, 'uuid'); - $environmentName = data_get($service, 'environment.name'); - - if ($projectUuid && $serviceUuid && $environmentName) { - $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/service/'.$serviceUuid; - } else { - $url = null; - } - // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); - $exitedService->update(['status' => 'exited']); } $notRunningApplications = $this->applications->pluck('id')->diff($foundApplications); foreach ($notRunningApplications as $applicationId) { $application = $this->applications->where('id', $applicationId)->first(); - if (str($application->status)->startsWith('exited')) { - continue; - } // Only protection: If no containers at all, Docker query might have failed if ($this->containers->isEmpty()) { continue; } + if (str($application->status)->startsWith('exited')) { + $application->update([ + 'container_present' => false, + 'restart_limit_reached' => false, + ]); + + continue; + } + // If container was recently restarting (crash loop), keep it as degraded for a grace period // This prevents false "exited" status during the brief moment between container removal and recreation $recentlyRestarted = $application->restart_count > 0 && @@ -388,9 +399,11 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti // Reset restart count when application exits completely $application->update([ 'status' => 'exited', + 'container_present' => false, 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null, + 'restart_limit_reached' => false, ]); } } @@ -411,6 +424,9 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); foreach ($notRunningDatabases as $database) { $database = $databases->where('id', $database)->first(); + if ($database->stoppedAfterRestartLimit()) { + continue; + } if (str($database->status)->startsWith('exited')) { continue; } @@ -426,6 +442,7 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null, + 'restart_limit_reached' => false, ]); // Stop proxy if database was public @@ -433,23 +450,10 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti StopDatabaseProxy::run($database); } - $name = data_get($database, 'name'); - $fqdn = data_get($database, 'fqdn'); - - $containerName = $name; - - $projectUuid = data_get($database, 'environment.project.uuid'); - $environmentName = data_get($database, 'environment.name'); - $databaseUuid = data_get($database, 'uuid'); - - if ($projectUuid && $databaseUuid && $environmentName) { - $url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/database/'.$databaseUuid; - } else { - $url = null; - } - // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); } + $this->trackPreviewRestartCounts($previews); + // Aggregate multi-container application statuses if (isset($this->applicationContainerStatuses) && $this->applicationContainerStatuses->isNotEmpty()) { foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) { @@ -470,21 +474,21 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti DB::transaction(function () use ($application, $maxRestartCount, $containerStatuses, &$restartLimitReached) { $previousRestartCount = $application->restart_count ?? 0; + $restartState = (new RestartCountTracker)->evaluate( + previousRestartCount: $previousRestartCount, + observedRestartCount: $maxRestartCount, + maxRestartCount: $application->max_restart_count ?? 0, + ); - if ($maxRestartCount > $previousRestartCount) { - // Restart count increased - this is a crash restart + if ($restartState['restart_count_changed']) { + $hasCrashRestarts = $restartState['restart_count'] > 0; $application->update([ - 'restart_count' => $maxRestartCount, - 'last_restart_at' => now(), - 'last_restart_type' => 'crash', + 'restart_count' => $restartState['restart_count'], + 'last_restart_at' => $hasCrashRestarts ? now() : null, + 'last_restart_type' => $hasCrashRestarts ? 'crash' : null, ]); - - // Check if restart limit has been reached - $maxAllowedRestarts = $application->max_restart_count ?? 0; - if ($maxAllowedRestarts > 0 && $maxRestartCount >= $maxAllowedRestarts && $previousRestartCount < $maxAllowedRestarts) { - $restartLimitReached = true; - } } + $restartLimitReached = $restartState['restart_limit_reached']; // Aggregate status after tracking restart counts $aggregatedStatus = $this->aggregateApplicationStatus($application, $containerStatuses, $maxRestartCount); @@ -499,9 +503,22 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti }); if ($restartLimitReached) { - $application->refresh(); - StopApplication::dispatch($application, false, true, false); - $application->environment->project->team?->notify(new ApplicationRestartLimitReached($application)); + $restartLimitClaimed = Application::query() + ->whereKey($application->getKey()) + ->where('restart_limit_reached', false) + ->update(['restart_limit_reached' => true]) === 1; + + if ($restartLimitClaimed) { + $application->refresh(); + StopApplication::dispatch( + application: $application, + previewDeployments: false, + dockerCleanup: false, + resetRestartCount: false, + removeContainers: false, + ); + $application->environment->project->team?->notify(new ApplicationRestartLimitReached($application)); + } } } } @@ -562,6 +579,16 @@ private function aggregateServiceContainerStatuses($services) continue; } + $restartCount = isset($this->serviceContainerRestartCounts) + ? ($this->serviceContainerRestartCounts->get($key)?->max() ?? 0) + : 0; + if ($subResource->trackRestartCount($restartCount)) { + StopServiceApplication::dispatch($subResource, false, false); + $subResource->team()?->notify(new ApplicationRestartLimitReached($subResource)); + + continue; + } + // Parse docker compose from service to check for excluded containers $dockerComposeRaw = data_get($service, 'docker_compose_raw'); $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw); @@ -602,4 +629,24 @@ private function aggregateServiceContainerStatuses($services) } } } + + private function trackPreviewRestartCounts(Collection $previews): void + { + if (! isset($this->previewContainerRestartCounts)) { + return; + } + + $this->previewContainerRestartCounts + ->groupBy('key') + ->each(function (Collection $counts, string $key) use ($previews): void { + [$applicationId, $pullRequestId] = explode(':', $key); + $preview = $previews->first(fn (ApplicationPreview $preview): bool => (string) $preview->application_id === $applicationId + && (string) $preview->pull_request_id === $pullRequestId + ); + if ($preview?->trackRestartCount((int) $counts->max('count'))) { + StopApplicationPreview::dispatch($preview, false, false); + $preview->application->environment->project->team?->notify(new ApplicationRestartLimitReached($preview)); + } + }); + } } diff --git a/app/Actions/Service/StartService.php b/app/Actions/Service/StartService.php index 463a8ad5b..13371d126 100644 --- a/app/Actions/Service/StartService.php +++ b/app/Actions/Service/StartService.php @@ -24,6 +24,8 @@ public function handle(Service $service, bool $pullLatestImages = false, bool $s } $service->saveComposeConfigs(); $service->isConfigurationChanged(save: true); + $service->applications()->get()->each->resetRestartLimit(); + $service->databases()->get()->each->resetRestartLimit(); $workdir = $service->workdir(); // $commands[] = "cd {$workdir}"; $commands[] = "echo 'Saved configuration files to {$workdir}.'"; diff --git a/app/Actions/Service/StopService.php b/app/Actions/Service/StopService.php index 5e34c8e6a..341687d0d 100644 --- a/app/Actions/Service/StopService.php +++ b/app/Actions/Service/StopService.php @@ -49,8 +49,14 @@ public function handle(Service $service, bool $deleteConnectedNetworks = false, $this->stopContainersInParallel($containersToStop, $server); } - $applications->each->update(['status' => 'exited']); - $dbs->each->update(['status' => 'exited']); + $applications->each(function ($application): void { + $application->update(['status' => 'exited']); + $application->resetRestartLimit(); + }); + $dbs->each(function ($database): void { + $database->update(['status' => 'exited']); + $database->resetRestartLimit(); + }); if ($deleteConnectedNetworks) { $service->deleteConnectedNetworks(); diff --git a/app/Actions/Service/StopServiceApplication.php b/app/Actions/Service/StopServiceApplication.php index 184dcb491..fa93a7880 100644 --- a/app/Actions/Service/StopServiceApplication.php +++ b/app/Actions/Service/StopServiceApplication.php @@ -13,17 +13,26 @@ class StopServiceApplication public string $jobQueue = 'high'; - public function handle(ServiceApplication|ServiceDatabase $serviceApplication): void + public function handle(ServiceApplication|ServiceDatabase $serviceApplication, bool $resetRestartCount = true, bool $removeContainer = false): void { $service = $serviceApplication->service; $server = $service->destination->server; $containerName = escapeshellarg($serviceApplication->name.'-'.$service->uuid); - instant_remote_process([ - "docker stop {$containerName}", - ], $server); + if ($removeContainer) { + $commands = ["docker rm -f {$containerName}"]; + } else { + $commands = [ + "docker update --restart=no {$containerName}", + "docker stop {$containerName}", + ]; + } + instant_remote_process($commands, $server, throwError: ! $removeContainer); $serviceApplication->update(['status' => 'exited']); + if ($resetRestartCount) { + $serviceApplication->resetRestartLimit(); + } ServiceStatusChanged::dispatch($service->environment->project->team->id); } } diff --git a/app/Http/Controllers/Api/NotificationsController.php b/app/Http/Controllers/Api/NotificationsController.php index f5493d024..1cca69a66 100644 --- a/app/Http/Controllers/Api/NotificationsController.php +++ b/app/Http/Controllers/Api/NotificationsController.php @@ -15,6 +15,7 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Http\JsonResponse; use Illuminate\Http\Request; +use Illuminate\Support\Facades\Validator; use OpenApi\Attributes as OA; class NotificationsController extends Controller @@ -45,6 +46,7 @@ private function channelConfig(string $channel): array 'deployment_success_email_notifications' => 'sometimes|boolean', 'deployment_failure_email_notifications' => 'sometimes|boolean', 'status_change_email_notifications' => 'sometimes|boolean', + 'restart_limit_reached_email_notifications' => 'sometimes|boolean', 'backup_success_email_notifications' => 'sometimes|boolean', 'backup_failure_email_notifications' => 'sometimes|boolean', 'scheduled_task_success_email_notifications' => 'sometimes|boolean', @@ -66,6 +68,7 @@ private function channelConfig(string $channel): array 'deployment_success_discord_notifications' => 'sometimes|boolean', 'deployment_failure_discord_notifications' => 'sometimes|boolean', 'status_change_discord_notifications' => 'sometimes|boolean', + 'restart_limit_reached_discord_notifications' => 'sometimes|boolean', 'backup_success_discord_notifications' => 'sometimes|boolean', 'backup_failure_discord_notifications' => 'sometimes|boolean', 'scheduled_task_success_discord_notifications' => 'sometimes|boolean', @@ -88,6 +91,7 @@ private function channelConfig(string $channel): array 'deployment_success_slack_notifications' => 'sometimes|boolean', 'deployment_failure_slack_notifications' => 'sometimes|boolean', 'status_change_slack_notifications' => 'sometimes|boolean', + 'restart_limit_reached_slack_notifications' => 'sometimes|boolean', 'backup_success_slack_notifications' => 'sometimes|boolean', 'backup_failure_slack_notifications' => 'sometimes|boolean', 'scheduled_task_success_slack_notifications' => 'sometimes|boolean', @@ -110,6 +114,7 @@ private function channelConfig(string $channel): array 'deployment_success_telegram_notifications' => 'sometimes|boolean', 'deployment_failure_telegram_notifications' => 'sometimes|boolean', 'status_change_telegram_notifications' => 'sometimes|boolean', + 'restart_limit_reached_telegram_notifications' => 'sometimes|boolean', 'backup_success_telegram_notifications' => 'sometimes|boolean', 'backup_failure_telegram_notifications' => 'sometimes|boolean', 'scheduled_task_success_telegram_notifications' => 'sometimes|boolean', @@ -124,6 +129,7 @@ private function channelConfig(string $channel): array 'telegram_notifications_deployment_success_thread_id' => 'sometimes|nullable|string|max:255', 'telegram_notifications_deployment_failure_thread_id' => 'sometimes|nullable|string|max:255', 'telegram_notifications_status_change_thread_id' => 'sometimes|nullable|string|max:255', + 'telegram_notifications_restart_limit_reached_thread_id' => 'sometimes|nullable|string|max:255', 'telegram_notifications_backup_success_thread_id' => 'sometimes|nullable|string|max:255', 'telegram_notifications_backup_failure_thread_id' => 'sometimes|nullable|string|max:255', 'telegram_notifications_scheduled_task_success_thread_id' => 'sometimes|nullable|string|max:255', @@ -146,6 +152,7 @@ private function channelConfig(string $channel): array 'deployment_success_pushover_notifications' => 'sometimes|boolean', 'deployment_failure_pushover_notifications' => 'sometimes|boolean', 'status_change_pushover_notifications' => 'sometimes|boolean', + 'restart_limit_reached_pushover_notifications' => 'sometimes|boolean', 'backup_success_pushover_notifications' => 'sometimes|boolean', 'backup_failure_pushover_notifications' => 'sometimes|boolean', 'scheduled_task_success_pushover_notifications' => 'sometimes|boolean', @@ -167,6 +174,7 @@ private function channelConfig(string $channel): array 'deployment_success_webhook_notifications' => 'sometimes|boolean', 'deployment_failure_webhook_notifications' => 'sometimes|boolean', 'status_change_webhook_notifications' => 'sometimes|boolean', + 'restart_limit_reached_webhook_notifications' => 'sometimes|boolean', 'backup_success_webhook_notifications' => 'sometimes|boolean', 'backup_failure_webhook_notifications' => 'sometimes|boolean', 'scheduled_task_success_webhook_notifications' => 'sometimes|boolean', @@ -249,7 +257,7 @@ private function updateChannel(Request $request, string $channel): JsonResponse $body = $request->json()->all(); $config = $this->channelConfig($channel); - $validator = customApiValidator($body, $config['rules']); + $validator = Validator::make($body, $config['rules']); $extraFields = array_diff(array_keys($body), $allowedFields); if ($validator->fails() || ! empty($extraFields)) { diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php index b3685daa4..81b932365 100644 --- a/app/Http/Controllers/Api/SentinelController.php +++ b/app/Http/Controllers/Api/SentinelController.php @@ -138,7 +138,7 @@ private function shouldDispatchUpdate(Server $server, array $data): bool /** * Build a stable hash of container state. * - * Covers [name, state] only — metrics, filesystem_usage_root, and + * Covers [name, state, restart_count] only — metrics, filesystem_usage_root, and * health_status are excluded on purpose. Disk % churns constantly, and * health checks can flap between starting/healthy/unhealthy while the * container lifecycle state remains unchanged. Both would otherwise defeat @@ -153,6 +153,7 @@ private function containerStateHash(array $data): string ->map(fn ($c) => [ 'name' => data_get($c, 'name'), 'state' => data_get($c, 'state'), + 'restart_count' => data_get($c, 'restart_count'), ]) ->sortBy('name') ->values() diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 48547c4d9..6d68160ce 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -4972,11 +4972,21 @@ private function handleSuccessfulDeployment(): void // Reset restart count after successful deployment // This is done here (not in Livewire) to avoid race conditions // with GetContainersStatus reading old container restart counts - $this->application->update([ + $restartState = [ 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null, - ]); + ]; + + if ($this->pull_request_id === 0) { + $restartState['restart_limit_reached'] = false; + } + + if ($this->pull_request_id === 0) { + $this->application->update($restartState); + } else { + $this->preview?->resetRestartLimit(); + } try { $this->application->markDeploymentConfigurationApplied($this->application_deployment_queue); diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 9c4a2531a..0e73ee41b 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -2,11 +2,15 @@ namespace App\Jobs; +use App\Actions\Application\StopApplication; +use App\Actions\Application\StopApplicationPreview; use App\Actions\Database\StartDatabaseProxy; +use App\Actions\Database\StopDatabase; use App\Actions\Database\StopDatabaseProxy; use App\Actions\Proxy\CheckProxy; use App\Actions\Proxy\StartProxy; use App\Actions\Server\StartLogDrain; +use App\Actions\Service\StopServiceApplication; use App\Actions\Shared\ComplexStatusCheck; use App\Models\Application; use App\Models\ApplicationPreview; @@ -23,8 +27,10 @@ use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; use App\Models\SwarmDocker; +use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached; use App\Notifications\Container\ContainerRestarted; use App\Services\ContainerStatusAggregator; +use App\Services\RestartCountTracker; use App\Traits\CalculatesExcludedStatus; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; @@ -95,8 +101,14 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced public Collection $applicationContainerStatuses; + public Collection $applicationContainerRestartCounts; + public Collection $serviceContainerStatuses; + public Collection $previewContainerRestartCounts; + + public Collection $serviceContainerRestartCounts; + public bool $foundProxy = false; public bool $foundLogDrainContainer = false; @@ -122,7 +134,10 @@ public function __construct(public Server $server, public $data) $this->foundApplicationPreviewsIds = collect(); $this->foundServiceDatabaseIds = collect(); $this->applicationContainerStatuses = collect(); + $this->applicationContainerRestartCounts = collect(); $this->serviceContainerStatuses = collect(); + $this->previewContainerRestartCounts = collect(); + $this->serviceContainerRestartCounts = collect(); $this->allApplicationIds = collect(); $this->allDatabaseUuids = collect(); $this->allTcpProxyUuids = collect(); @@ -140,7 +155,10 @@ public function handle() { // Defensive initialization for Collection properties to handle queue deserialization edge cases $this->serviceContainerStatuses ??= collect(); + $this->previewContainerRestartCounts ??= collect(); + $this->serviceContainerRestartCounts ??= collect(); $this->applicationContainerStatuses ??= collect(); + $this->applicationContainerRestartCounts ??= collect(); $this->foundApplicationIds ??= collect(); $this->foundDatabaseUuids ??= collect(); $this->foundServiceApplicationIds ??= collect(); @@ -231,6 +249,9 @@ public function handle() if (! $coolify_managed) { continue; } + if (filter_var($labels->get('com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)) { + continue; + } $name = data_get($container, 'name'); if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) { @@ -241,6 +262,10 @@ public function handle() $pullRequestId = $labels->get('coolify.pullRequestId', '0'); try { if ($pullRequestId === '0') { + $application = $this->applicationsById->get((string) $applicationId); + if ($application && $application->container_present !== true) { + $application->update(['container_present' => true]); + } if ($this->allApplicationIds->contains($applicationId)) { $this->foundApplicationIds->push($applicationId); } @@ -251,6 +276,13 @@ public function handle() $containerName = $labels->get('com.docker.compose.service'); if ($containerName) { $this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus); + $restartCount = data_get($container, 'restart_count'); + if (is_numeric($restartCount)) { + if (! $this->applicationContainerRestartCounts->has($applicationId)) { + $this->applicationContainerRestartCounts->put($applicationId, collect()); + } + $this->applicationContainerRestartCounts->get($applicationId)->put($containerName, (int) $restartCount); + } } } else { $previewKey = $applicationId.':'.$pullRequestId; @@ -258,6 +290,13 @@ public function handle() $this->foundApplicationPreviewsIds->push($previewKey); } $this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus); + $restartCount = data_get($container, 'restart_count'); + if (is_numeric($restartCount)) { + $this->previewContainerRestartCounts->push([ + 'key' => $previewKey, + 'count' => (int) $restartCount, + ]); + } } } catch (\Exception $e) { } @@ -278,6 +317,7 @@ public function handle() $containerName = $labels->get('com.docker.compose.service'); if ($containerName) { $this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus); + $this->storeServiceRestartCount($key, $containerName, data_get($container, 'restart_count')); } } elseif ($subType === 'database') { $this->foundServiceDatabaseIds->push($subId); @@ -289,6 +329,7 @@ public function handle() $containerName = $labels->get('com.docker.compose.service'); if ($containerName) { $this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus); + $this->storeServiceRestartCount($key, $containerName, data_get($container, 'restart_count')); } } } else { @@ -302,9 +343,9 @@ public function handle() $this->foundDatabaseUuids->push($uuid); // TCP proxy should only be started/managed when database is actually running if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) { - $this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true); + $this->updateDatabaseStatus($uuid, $containerStatus, data_get($container, 'restart_count'), tcpProxy: true); } else { - $this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false); + $this->updateDatabaseStatus($uuid, $containerStatus, data_get($container, 'restart_count'), tcpProxy: false); } } } @@ -317,6 +358,9 @@ public function handle() $this->updateProxyStatus(); + Application::whereIn('id', $this->foundApplicationIds->unique()) + ->update(['container_present' => true]); + $this->updateNotFoundApplicationStatus(); $this->updateNotFoundApplicationPreviewStatus(); $this->updateNotFoundDatabaseStatus(); @@ -324,6 +368,8 @@ public function handle() $this->updateAdditionalServersStatus(); + $this->trackPreviewRestartCounts(); + // Aggregate multi-container application statuses $this->aggregateMultiContainerStatuses(); @@ -349,11 +395,18 @@ private function loadApplications(): Collection 'uuid', 'name', 'status', + 'container_present', 'build_pack', 'docker_compose_raw', + 'environment_id', 'destination_id', 'destination_type', 'last_online_at', + 'restart_count', + 'max_restart_count', + 'restart_limit_reached', + 'last_restart_at', + 'last_restart_type', ]) ->withCount('additional_servers') ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds)) @@ -372,11 +425,18 @@ private function loadApplications(): Collection 'uuid', 'name', 'status', + 'container_present', 'build_pack', 'docker_compose_raw', + 'environment_id', 'destination_id', 'destination_type', 'last_online_at', + 'restart_count', + 'max_restart_count', + 'restart_limit_reached', + 'last_restart_at', + 'last_restart_type', ]) ->withCount('additional_servers') ->whereIn('id', $additionalApplicationIds) @@ -402,6 +462,11 @@ private function loadPreviews(): Collection 'pull_request_id', 'status', 'last_online_at', + 'restart_count', + 'max_restart_count', + 'restart_limit_reached', + 'last_restart_at', + 'last_restart_type', ]) ->whereIn('application_id', $applicationIds) ->get(); @@ -417,8 +482,8 @@ private function loadServices(): Collection 'docker_compose_raw', ]) ->with([ - 'applications:id,service_id,status,last_online_at', - 'databases:id,service_id,status,last_online_at,is_public,name', + 'applications:id,service_id,status,last_online_at,restart_count,max_restart_count,restart_limit_reached,last_restart_at,last_restart_type', + 'databases:id,service_id,status,last_online_at,is_public,name,restart_count,max_restart_count,restart_limit_reached,last_restart_at,last_restart_type', ]) ->get(); } @@ -441,6 +506,8 @@ private function loadDatabases(): Collection 'restart_count', 'last_restart_at', 'last_restart_type', + 'max_restart_count', + 'restart_limit_reached', ]; return collect([ @@ -495,6 +562,53 @@ private function aggregateMultiContainerStatuses() continue; } + $maxRestartCount = 0; + $restartCountsAvailable = $this->applicationContainerRestartCounts->has($applicationId); + if ($restartCountsAvailable) { + $maxRestartCount = $this->applicationContainerRestartCounts->get($applicationId)->max() ?? 0; + $restartState = (new RestartCountTracker)->evaluate( + previousRestartCount: $application->restart_count ?? 0, + observedRestartCount: $maxRestartCount, + maxRestartCount: $application->max_restart_count ?? 0, + ); + + if ($restartState['restart_count_changed']) { + $hasCrashRestarts = $restartState['restart_count'] > 0; + $application->update([ + 'restart_count' => $restartState['restart_count'], + 'last_restart_at' => $hasCrashRestarts ? now() : null, + 'last_restart_type' => $hasCrashRestarts ? 'crash' : null, + ]); + } + + if ($restartState['restart_limit_reached']) { + $restartLimitClaimed = Application::query() + ->whereKey($application->getKey()) + ->where('restart_limit_reached', false) + ->update(['restart_limit_reached' => true]) === 1; + + if ($restartLimitClaimed) { + $application->refresh(); + StopApplication::dispatch( + application: $application, + previewDeployments: false, + dockerCleanup: false, + resetRestartCount: false, + removeContainers: false, + ); + $application->environment->project->team?->notify(new ApplicationRestartLimitReached($application)); + } + } + } + + if ($application->stoppedAfterRestartLimit() && $containerStatuses->every( + fn (string $status): bool => str($status)->contains('exited') + )) { + $application->update(['status' => 'exited']); + + continue; + } + // Parse docker compose to check for excluded containers $dockerComposeRaw = data_get($application, 'docker_compose_raw'); $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw); @@ -519,7 +633,7 @@ private function aggregateMultiContainerStatuses() // Use ContainerStatusAggregator service for state machine logic // Use preserveRestarting: true so applications show "Restarting" instead of "Degraded" $aggregator = new ContainerStatusAggregator; - $aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0, preserveRestarting: true); + $aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, $maxRestartCount, preserveRestarting: true); // Update application status with aggregated result if ($aggregatedStatus && $application->status !== $aggregatedStatus) { @@ -560,6 +674,14 @@ private function aggregateServiceContainerStatuses() continue; } + $restartCount = $this->serviceContainerRestartCounts->get($key)?->max() ?? 0; + if ($subResource->trackRestartCount($restartCount)) { + StopServiceApplication::dispatch($subResource, false, false); + $subResource->team()?->notify(new ApplicationRestartLimitReached($subResource)); + + continue; + } + // Parse docker compose from service to check for excluded containers $dockerComposeRaw = data_get($service, 'docker_compose_raw'); $excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw); @@ -581,10 +703,9 @@ private function aggregateServiceContainerStatuses() } // Use ContainerStatusAggregator service for state machine logic - // NOTE: Sentinel does NOT provide restart count data, so maxRestartCount is always 0 // Use preserveRestarting: true so individual sub-resources show "Restarting" instead of "Degraded" $aggregator = new ContainerStatusAggregator; - $aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0, preserveRestarting: true); + $aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, $restartCount, preserveRestarting: true); // Update service sub-resource status with aggregated result if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) { @@ -627,8 +748,11 @@ private function updateNotFoundApplicationStatus() // Batch update: mark all not-found applications as exited (excluding already exited ones) Application::whereIn('id', $notFoundApplicationIds) - ->where('status', 'not like', 'exited%') - ->update(['status' => 'exited']); + ->update([ + 'status' => 'exited', + 'container_present' => false, + 'restart_limit_reached' => false, + ]); } private function updateNotFoundApplicationPreviewStatus() @@ -687,7 +811,7 @@ private function updateProxyStatus() } } - private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false) + private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, mixed $restartCount = null, bool $tcpProxy = false): void { $database = $this->databasesByUuid->get($databaseUuid); if (! $database) { @@ -697,6 +821,12 @@ private function updateDatabaseStatus(string $databaseUuid, string $containerSta $database->status = $containerStatus; $database->save(); } + if (is_numeric($restartCount) && $database->trackRestartCount((int) $restartCount)) { + StopDatabase::dispatch($database, false, false, false); + $database->team()?->notify(new ApplicationRestartLimitReached($database)); + + return; + } if (! $this->isCompleteSnapshot()) { return; } @@ -719,6 +849,30 @@ private function updateDatabaseStatus(string $databaseUuid, string $containerSta } } + private function storeServiceRestartCount(string $key, string $containerName, mixed $restartCount): void + { + if (! is_numeric($restartCount)) { + return; + } + if (! $this->serviceContainerRestartCounts->has($key)) { + $this->serviceContainerRestartCounts->put($key, collect()); + } + $this->serviceContainerRestartCounts->get($key)->put($containerName, (int) $restartCount); + } + + private function trackPreviewRestartCounts(): void + { + $this->previewContainerRestartCounts + ->groupBy('key') + ->each(function (Collection $counts, string $key): void { + $preview = $this->previewsByKey->get($key); + if ($preview?->trackRestartCount((int) $counts->max('count'))) { + StopApplicationPreview::dispatch($preview, false, false); + $preview->application->environment->project->team?->notify(new ApplicationRestartLimitReached($preview)); + } + }); + } + private function updateNotFoundDatabaseStatus() { $notFoundDatabaseUuids = $this->allDatabaseUuids->diff($this->foundDatabaseUuids); @@ -729,12 +883,16 @@ private function updateNotFoundDatabaseStatus() $notFoundDatabaseUuids->each(function ($databaseUuid) { $database = $this->databasesByUuid->get($databaseUuid); if ($database) { + if ($database->stoppedAfterRestartLimit()) { + return; + } if (! str($database->status)->startsWith('exited')) { $database->update([ 'status' => 'exited', 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null, + 'restart_limit_reached' => false, ]); } if ($database->is_public) { @@ -752,15 +910,17 @@ private function updateNotFoundServiceStatus() // Batch update service applications if ($notFoundServiceApplicationIds->isNotEmpty()) { ServiceApplication::whereIn('id', $notFoundServiceApplicationIds) + ->where('restart_limit_reached', false) ->where('status', '!=', 'exited') - ->update(['status' => 'exited']); + ->update(['status' => 'exited', 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null]); } // Batch update service databases if ($notFoundServiceDatabaseIds->isNotEmpty()) { ServiceDatabase::whereIn('id', $notFoundServiceDatabaseIds) + ->where('restart_limit_reached', false) ->where('status', '!=', 'exited') - ->update(['status' => 'exited']); + ->update(['status' => 'exited', 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null]); } } diff --git a/app/Livewire/Notifications/Discord.php b/app/Livewire/Notifications/Discord.php index 797db8362..71c24e238 100644 --- a/app/Livewire/Notifications/Discord.php +++ b/app/Livewire/Notifications/Discord.php @@ -34,6 +34,9 @@ class Discord extends Component #[Validate(['boolean'])] public bool $statusChangeDiscordNotifications = false; + #[Validate(['boolean'])] + public bool $restartLimitReachedDiscordNotifications = true; + #[Validate(['boolean'])] public bool $backupSuccessDiscordNotifications = false; @@ -93,6 +96,7 @@ public function syncData(bool $toModel = false) $this->settings->deployment_success_discord_notifications = $this->deploymentSuccessDiscordNotifications; $this->settings->deployment_failure_discord_notifications = $this->deploymentFailureDiscordNotifications; $this->settings->status_change_discord_notifications = $this->statusChangeDiscordNotifications; + $this->settings->restart_limit_reached_discord_notifications = $this->restartLimitReachedDiscordNotifications; $this->settings->backup_success_discord_notifications = $this->backupSuccessDiscordNotifications; $this->settings->backup_failure_discord_notifications = $this->backupFailureDiscordNotifications; $this->settings->scheduled_task_success_discord_notifications = $this->scheduledTaskSuccessDiscordNotifications; @@ -118,6 +122,7 @@ public function syncData(bool $toModel = false) $this->deploymentSuccessDiscordNotifications = $this->settings->deployment_success_discord_notifications; $this->deploymentFailureDiscordNotifications = $this->settings->deployment_failure_discord_notifications; $this->statusChangeDiscordNotifications = $this->settings->status_change_discord_notifications; + $this->restartLimitReachedDiscordNotifications = $this->settings->restart_limit_reached_discord_notifications; $this->backupSuccessDiscordNotifications = $this->settings->backup_success_discord_notifications; $this->backupFailureDiscordNotifications = $this->settings->backup_failure_discord_notifications; $this->scheduledTaskSuccessDiscordNotifications = $this->settings->scheduled_task_success_discord_notifications; diff --git a/app/Livewire/Notifications/Email.php b/app/Livewire/Notifications/Email.php index 3d95668b9..530583db5 100644 --- a/app/Livewire/Notifications/Email.php +++ b/app/Livewire/Notifications/Email.php @@ -79,6 +79,9 @@ class Email extends Component #[Validate(['boolean'])] public bool $statusChangeEmailNotifications = false; + #[Validate(['boolean'])] + public bool $restartLimitReachedEmailNotifications = true; + #[Validate(['boolean'])] public bool $backupSuccessEmailNotifications = false; @@ -155,6 +158,7 @@ public function syncData(bool $toModel = false) $this->settings->deployment_success_email_notifications = $this->deploymentSuccessEmailNotifications; $this->settings->deployment_failure_email_notifications = $this->deploymentFailureEmailNotifications; $this->settings->status_change_email_notifications = $this->statusChangeEmailNotifications; + $this->settings->restart_limit_reached_email_notifications = $this->restartLimitReachedEmailNotifications; $this->settings->backup_success_email_notifications = $this->backupSuccessEmailNotifications; $this->settings->backup_failure_email_notifications = $this->backupFailureEmailNotifications; $this->settings->scheduled_task_success_email_notifications = $this->scheduledTaskSuccessEmailNotifications; @@ -193,6 +197,7 @@ public function syncData(bool $toModel = false) $this->deploymentSuccessEmailNotifications = $this->settings->deployment_success_email_notifications; $this->deploymentFailureEmailNotifications = $this->settings->deployment_failure_email_notifications; $this->statusChangeEmailNotifications = $this->settings->status_change_email_notifications; + $this->restartLimitReachedEmailNotifications = $this->settings->restart_limit_reached_email_notifications; $this->backupSuccessEmailNotifications = $this->settings->backup_success_email_notifications; $this->backupFailureEmailNotifications = $this->settings->backup_failure_email_notifications; $this->scheduledTaskSuccessEmailNotifications = $this->settings->scheduled_task_success_email_notifications; diff --git a/app/Livewire/Notifications/Pushover.php b/app/Livewire/Notifications/Pushover.php index 3b7c3c6ae..04cd82756 100644 --- a/app/Livewire/Notifications/Pushover.php +++ b/app/Livewire/Notifications/Pushover.php @@ -41,6 +41,9 @@ class Pushover extends Component #[Validate(['boolean'])] public bool $statusChangePushoverNotifications = false; + #[Validate(['boolean'])] + public bool $restartLimitReachedPushoverNotifications = true; + #[Validate(['boolean'])] public bool $backupSuccessPushoverNotifications = false; @@ -98,6 +101,7 @@ public function syncData(bool $toModel = false) $this->settings->deployment_success_pushover_notifications = $this->deploymentSuccessPushoverNotifications; $this->settings->deployment_failure_pushover_notifications = $this->deploymentFailurePushoverNotifications; $this->settings->status_change_pushover_notifications = $this->statusChangePushoverNotifications; + $this->settings->restart_limit_reached_pushover_notifications = $this->restartLimitReachedPushoverNotifications; $this->settings->backup_success_pushover_notifications = $this->backupSuccessPushoverNotifications; $this->settings->backup_failure_pushover_notifications = $this->backupFailurePushoverNotifications; $this->settings->scheduled_task_success_pushover_notifications = $this->scheduledTaskSuccessPushoverNotifications; @@ -125,6 +129,7 @@ public function syncData(bool $toModel = false) $this->deploymentSuccessPushoverNotifications = $this->settings->deployment_success_pushover_notifications; $this->deploymentFailurePushoverNotifications = $this->settings->deployment_failure_pushover_notifications; $this->statusChangePushoverNotifications = $this->settings->status_change_pushover_notifications; + $this->restartLimitReachedPushoverNotifications = $this->settings->restart_limit_reached_pushover_notifications; $this->backupSuccessPushoverNotifications = $this->settings->backup_success_pushover_notifications; $this->backupFailurePushoverNotifications = $this->settings->backup_failure_pushover_notifications; $this->scheduledTaskSuccessPushoverNotifications = $this->settings->scheduled_task_success_pushover_notifications; diff --git a/app/Livewire/Notifications/Slack.php b/app/Livewire/Notifications/Slack.php index 9ee362402..f4323875f 100644 --- a/app/Livewire/Notifications/Slack.php +++ b/app/Livewire/Notifications/Slack.php @@ -39,6 +39,9 @@ class Slack extends Component #[Validate(['boolean'])] public bool $statusChangeSlackNotifications = false; + #[Validate(['boolean'])] + public bool $restartLimitReachedSlackNotifications = true; + #[Validate(['boolean'])] public bool $backupSuccessSlackNotifications = false; @@ -95,6 +98,7 @@ public function syncData(bool $toModel = false) $this->settings->deployment_success_slack_notifications = $this->deploymentSuccessSlackNotifications; $this->settings->deployment_failure_slack_notifications = $this->deploymentFailureSlackNotifications; $this->settings->status_change_slack_notifications = $this->statusChangeSlackNotifications; + $this->settings->restart_limit_reached_slack_notifications = $this->restartLimitReachedSlackNotifications; $this->settings->backup_success_slack_notifications = $this->backupSuccessSlackNotifications; $this->settings->backup_failure_slack_notifications = $this->backupFailureSlackNotifications; $this->settings->scheduled_task_success_slack_notifications = $this->scheduledTaskSuccessSlackNotifications; @@ -118,6 +122,7 @@ public function syncData(bool $toModel = false) $this->deploymentSuccessSlackNotifications = $this->settings->deployment_success_slack_notifications; $this->deploymentFailureSlackNotifications = $this->settings->deployment_failure_slack_notifications; $this->statusChangeSlackNotifications = $this->settings->status_change_slack_notifications; + $this->restartLimitReachedSlackNotifications = $this->settings->restart_limit_reached_slack_notifications; $this->backupSuccessSlackNotifications = $this->settings->backup_success_slack_notifications; $this->backupFailureSlackNotifications = $this->settings->backup_failure_slack_notifications; $this->scheduledTaskSuccessSlackNotifications = $this->settings->scheduled_task_success_slack_notifications; diff --git a/app/Livewire/Notifications/Telegram.php b/app/Livewire/Notifications/Telegram.php index b04d2c73d..f77b3c3a8 100644 --- a/app/Livewire/Notifications/Telegram.php +++ b/app/Livewire/Notifications/Telegram.php @@ -41,6 +41,9 @@ class Telegram extends Component #[Validate(['boolean'])] public bool $statusChangeTelegramNotifications = false; + #[Validate(['boolean'])] + public bool $restartLimitReachedTelegramNotifications = true; + #[Validate(['boolean'])] public bool $backupSuccessTelegramNotifications = false; @@ -83,6 +86,9 @@ class Telegram extends Component #[Validate(['nullable', 'string'])] public ?string $telegramNotificationsStatusChangeThreadId = null; + #[Validate(['nullable', 'string', 'max:255'])] + public ?string $telegramNotificationsRestartLimitReachedThreadId = null; + #[Validate(['nullable', 'string'])] public ?string $telegramNotificationsBackupSuccessThreadId = null; @@ -140,6 +146,7 @@ public function syncData(bool $toModel = false) $this->settings->deployment_success_telegram_notifications = $this->deploymentSuccessTelegramNotifications; $this->settings->deployment_failure_telegram_notifications = $this->deploymentFailureTelegramNotifications; $this->settings->status_change_telegram_notifications = $this->statusChangeTelegramNotifications; + $this->settings->restart_limit_reached_telegram_notifications = $this->restartLimitReachedTelegramNotifications; $this->settings->backup_success_telegram_notifications = $this->backupSuccessTelegramNotifications; $this->settings->backup_failure_telegram_notifications = $this->backupFailureTelegramNotifications; $this->settings->scheduled_task_success_telegram_notifications = $this->scheduledTaskSuccessTelegramNotifications; @@ -155,6 +162,7 @@ public function syncData(bool $toModel = false) $this->settings->telegram_notifications_deployment_success_thread_id = $this->telegramNotificationsDeploymentSuccessThreadId; $this->settings->telegram_notifications_deployment_failure_thread_id = $this->telegramNotificationsDeploymentFailureThreadId; $this->settings->telegram_notifications_status_change_thread_id = $this->telegramNotificationsStatusChangeThreadId; + $this->settings->telegram_notifications_restart_limit_reached_thread_id = $this->telegramNotificationsRestartLimitReachedThreadId; $this->settings->telegram_notifications_backup_success_thread_id = $this->telegramNotificationsBackupSuccessThreadId; $this->settings->telegram_notifications_backup_failure_thread_id = $this->telegramNotificationsBackupFailureThreadId; $this->settings->telegram_notifications_scheduled_task_success_thread_id = $this->telegramNotificationsScheduledTaskSuccessThreadId; @@ -173,6 +181,21 @@ public function syncData(bool $toModel = false) if (auth()->user()->can('update', $this->settings)) { $this->telegramToken = $this->settings->telegram_token; $this->telegramChatId = $this->settings->telegram_chat_id; + $this->telegramNotificationsDeploymentSuccessThreadId = $this->settings->telegram_notifications_deployment_success_thread_id; + $this->telegramNotificationsDeploymentFailureThreadId = $this->settings->telegram_notifications_deployment_failure_thread_id; + $this->telegramNotificationsStatusChangeThreadId = $this->settings->telegram_notifications_status_change_thread_id; + $this->telegramNotificationsRestartLimitReachedThreadId = $this->settings->telegram_notifications_restart_limit_reached_thread_id; + $this->telegramNotificationsBackupSuccessThreadId = $this->settings->telegram_notifications_backup_success_thread_id; + $this->telegramNotificationsBackupFailureThreadId = $this->settings->telegram_notifications_backup_failure_thread_id; + $this->telegramNotificationsScheduledTaskSuccessThreadId = $this->settings->telegram_notifications_scheduled_task_success_thread_id; + $this->telegramNotificationsScheduledTaskFailureThreadId = $this->settings->telegram_notifications_scheduled_task_failure_thread_id; + $this->telegramNotificationsDockerCleanupSuccessThreadId = $this->settings->telegram_notifications_docker_cleanup_success_thread_id; + $this->telegramNotificationsDockerCleanupFailureThreadId = $this->settings->telegram_notifications_docker_cleanup_failure_thread_id; + $this->telegramNotificationsServerDiskUsageThreadId = $this->settings->telegram_notifications_server_disk_usage_thread_id; + $this->telegramNotificationsServerReachableThreadId = $this->settings->telegram_notifications_server_reachable_thread_id; + $this->telegramNotificationsServerUnreachableThreadId = $this->settings->telegram_notifications_server_unreachable_thread_id; + $this->telegramNotificationsServerPatchThreadId = $this->settings->telegram_notifications_server_patch_thread_id; + $this->telegramNotificationsTraefikOutdatedThreadId = $this->settings->telegram_notifications_traefik_outdated_thread_id; } else { $this->telegramToken = null; $this->telegramChatId = null; @@ -181,6 +204,7 @@ public function syncData(bool $toModel = false) $this->deploymentSuccessTelegramNotifications = $this->settings->deployment_success_telegram_notifications; $this->deploymentFailureTelegramNotifications = $this->settings->deployment_failure_telegram_notifications; $this->statusChangeTelegramNotifications = $this->settings->status_change_telegram_notifications; + $this->restartLimitReachedTelegramNotifications = $this->settings->restart_limit_reached_telegram_notifications; $this->backupSuccessTelegramNotifications = $this->settings->backup_success_telegram_notifications; $this->backupFailureTelegramNotifications = $this->settings->backup_failure_telegram_notifications; $this->scheduledTaskSuccessTelegramNotifications = $this->settings->scheduled_task_success_telegram_notifications; @@ -193,20 +217,6 @@ public function syncData(bool $toModel = false) $this->serverPatchTelegramNotifications = $this->settings->server_patch_telegram_notifications; $this->traefikOutdatedTelegramNotifications = $this->settings->traefik_outdated_telegram_notifications; - $this->telegramNotificationsDeploymentSuccessThreadId = $this->settings->telegram_notifications_deployment_success_thread_id; - $this->telegramNotificationsDeploymentFailureThreadId = $this->settings->telegram_notifications_deployment_failure_thread_id; - $this->telegramNotificationsStatusChangeThreadId = $this->settings->telegram_notifications_status_change_thread_id; - $this->telegramNotificationsBackupSuccessThreadId = $this->settings->telegram_notifications_backup_success_thread_id; - $this->telegramNotificationsBackupFailureThreadId = $this->settings->telegram_notifications_backup_failure_thread_id; - $this->telegramNotificationsScheduledTaskSuccessThreadId = $this->settings->telegram_notifications_scheduled_task_success_thread_id; - $this->telegramNotificationsScheduledTaskFailureThreadId = $this->settings->telegram_notifications_scheduled_task_failure_thread_id; - $this->telegramNotificationsDockerCleanupSuccessThreadId = $this->settings->telegram_notifications_docker_cleanup_success_thread_id; - $this->telegramNotificationsDockerCleanupFailureThreadId = $this->settings->telegram_notifications_docker_cleanup_failure_thread_id; - $this->telegramNotificationsServerDiskUsageThreadId = $this->settings->telegram_notifications_server_disk_usage_thread_id; - $this->telegramNotificationsServerReachableThreadId = $this->settings->telegram_notifications_server_reachable_thread_id; - $this->telegramNotificationsServerUnreachableThreadId = $this->settings->telegram_notifications_server_unreachable_thread_id; - $this->telegramNotificationsServerPatchThreadId = $this->settings->telegram_notifications_server_patch_thread_id; - $this->telegramNotificationsTraefikOutdatedThreadId = $this->settings->telegram_notifications_traefik_outdated_thread_id; } } diff --git a/app/Livewire/Notifications/Webhook.php b/app/Livewire/Notifications/Webhook.php index fcf110778..3461503d4 100644 --- a/app/Livewire/Notifications/Webhook.php +++ b/app/Livewire/Notifications/Webhook.php @@ -34,6 +34,9 @@ class Webhook extends Component #[Validate(['boolean'])] public bool $statusChangeWebhookNotifications = false; + #[Validate(['boolean'])] + public bool $restartLimitReachedWebhookNotifications = true; + #[Validate(['boolean'])] public bool $backupSuccessWebhookNotifications = false; @@ -90,6 +93,7 @@ public function syncData(bool $toModel = false) $this->settings->deployment_success_webhook_notifications = $this->deploymentSuccessWebhookNotifications; $this->settings->deployment_failure_webhook_notifications = $this->deploymentFailureWebhookNotifications; $this->settings->status_change_webhook_notifications = $this->statusChangeWebhookNotifications; + $this->settings->restart_limit_reached_webhook_notifications = $this->restartLimitReachedWebhookNotifications; $this->settings->backup_success_webhook_notifications = $this->backupSuccessWebhookNotifications; $this->settings->backup_failure_webhook_notifications = $this->backupFailureWebhookNotifications; $this->settings->scheduled_task_success_webhook_notifications = $this->scheduledTaskSuccessWebhookNotifications; @@ -113,6 +117,7 @@ public function syncData(bool $toModel = false) $this->deploymentSuccessWebhookNotifications = $this->settings->deployment_success_webhook_notifications; $this->deploymentFailureWebhookNotifications = $this->settings->deployment_failure_webhook_notifications; $this->statusChangeWebhookNotifications = $this->settings->status_change_webhook_notifications; + $this->restartLimitReachedWebhookNotifications = $this->settings->restart_limit_reached_webhook_notifications; $this->backupSuccessWebhookNotifications = $this->settings->backup_success_webhook_notifications; $this->backupFailureWebhookNotifications = $this->settings->backup_failure_webhook_notifications; $this->scheduledTaskSuccessWebhookNotifications = $this->settings->scheduled_task_success_webhook_notifications; diff --git a/app/Livewire/Project/Resource/Index.php b/app/Livewire/Project/Resource/Index.php index 93633246a..7c375d0e0 100644 --- a/app/Livewire/Project/Resource/Index.php +++ b/app/Livewire/Project/Resource/Index.php @@ -187,6 +187,11 @@ private function toSearchableArray(Collection $items, string $type, string $type 'fqdn' => $item->fqdn ?? null, 'description' => $item->description ?? null, 'status' => $item->status ?? '', + 'restartLimitReached' => method_exists($item, 'stoppedAfterRestartLimit') && $item->stoppedAfterRestartLimit(), + 'restartCount' => method_exists($item, 'stoppedAfterRestartLimit') && $item->stoppedAfterRestartLimit() + ? max($item->restart_count ?? 0, $item->max_restart_count ?? 0) + : ($item->restart_count ?? 0), + 'maxRestartCount' => $item->max_restart_count ?? 0, 'server_status' => $item->server_status ?? null, 'hrefLink' => $item->hrefLink ?? '', 'destination' => [ diff --git a/app/Livewire/Project/Service/Heading.php b/app/Livewire/Project/Service/Heading.php index 34bb46ff1..0e7fed960 100644 --- a/app/Livewire/Project/Service/Heading.php +++ b/app/Livewire/Project/Service/Heading.php @@ -5,8 +5,11 @@ use App\Actions\Docker\GetContainersStatus; use App\Actions\Service\StartService; use App\Actions\Service\StopService; +use App\Actions\Service\StopServiceApplication; use App\Enums\ProcessStatus; use App\Models\Service; +use App\Models\ServiceApplication; +use App\Models\ServiceDatabase; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Facades\Auth; use Livewire\Component; @@ -169,6 +172,29 @@ public function restart() } } + public function removeSelectedResourceContainer(): void + { + $resource = $this->selectedResource(); + if (! $resource) { + return; + } + + $this->authorize('update', $resource); + StopServiceApplication::run($resource, true, true); + $this->dispatch('success', 'Container removed.'); + } + + private function selectedResource(): ServiceApplication|ServiceDatabase|null + { + $uuid = data_get($this->parameters, 'stack_service_uuid'); + if (! $uuid) { + return null; + } + + return $this->service->applications()->whereUuid($uuid)->first() + ?? $this->service->databases()->whereUuid($uuid)->first(); + } + public function pullAndRestartEvent() { try { diff --git a/app/Livewire/Project/Service/Status.php b/app/Livewire/Project/Service/Status.php index 192d7ca80..27919f1ae 100644 --- a/app/Livewire/Project/Service/Status.php +++ b/app/Livewire/Project/Service/Status.php @@ -10,6 +10,13 @@ class Status extends Component { public Service $service; + public ?string $selectedResourceUuid = null; + + public function mount(): void + { + $this->selectedResourceUuid = request()->route('stack_service_uuid'); + } + public function getListeners(): array { $teamId = auth()->user()->currentTeam()->id; @@ -27,6 +34,11 @@ public function refreshStatus(): void public function render(): View { - return view('livewire.project.service.status'); + $selectedResource = $this->selectedResourceUuid + ? $this->service->applications->firstWhere('uuid', $this->selectedResourceUuid) + ?? $this->service->databases->firstWhere('uuid', $this->selectedResourceUuid) + : null; + + return view('livewire.project.service.status', compact('selectedResource')); } } diff --git a/app/Models/Application.php b/app/Models/Application.php index af95cd4db..4873a9146 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -213,6 +213,8 @@ class Application extends BaseModel 'last_online_at', 'restart_count', 'max_restart_count', + 'restart_limit_reached', + 'container_present', 'last_restart_at', 'last_restart_type', 'uuid', @@ -258,6 +260,8 @@ protected function casts(): array 'domain_dns_statuses' => 'array', 'restart_count' => 'integer', 'max_restart_count' => 'integer', + 'restart_limit_reached' => 'boolean', + 'container_present' => 'boolean', 'last_restart_at' => 'datetime', ]; } @@ -605,10 +609,8 @@ public function link() public function stoppedAfterRestartLimit(): bool { return str($this->status)->startsWith('exited') - && ($this->restart_count ?? 0) > 0 - && ($this->max_restart_count ?? 0) > 0 - && $this->restart_count >= $this->max_restart_count - && $this->last_restart_type === 'crash'; + && $this->container_present === true + && $this->restart_limit_reached === true; } public function taskLink($task_uuid) diff --git a/app/Models/ApplicationPreview.php b/app/Models/ApplicationPreview.php index d9c981c59..381e71817 100644 --- a/app/Models/ApplicationPreview.php +++ b/app/Models/ApplicationPreview.php @@ -3,13 +3,14 @@ namespace App\Models; use App\Support\ValidationPatterns; +use App\Traits\HasRestartLimit; use Illuminate\Database\Eloquent\SoftDeletes; use RuntimeException; use Spatie\Url\Url; class ApplicationPreview extends BaseModel { - use SoftDeletes; + use HasRestartLimit, SoftDeletes; protected $fillable = [ 'uuid', @@ -102,6 +103,11 @@ public function application() return $this->belongsTo(Application::class); } + public function restartLimitMaximum(): int + { + return $this->application->max_restart_count ?? $this->max_restart_count ?? 0; + } + public function persistentStorages() { return $this->morphMany(LocalPersistentVolume::class, 'resource'); diff --git a/app/Models/DiscordNotificationSettings.php b/app/Models/DiscordNotificationSettings.php index 135c921f6..48d5b5d29 100644 --- a/app/Models/DiscordNotificationSettings.php +++ b/app/Models/DiscordNotificationSettings.php @@ -20,6 +20,7 @@ class DiscordNotificationSettings extends Model 'deployment_success_discord_notifications', 'deployment_failure_discord_notifications', 'status_change_discord_notifications', + 'restart_limit_reached_discord_notifications', 'backup_success_discord_notifications', 'backup_failure_discord_notifications', 'scheduled_task_success_discord_notifications', @@ -45,6 +46,7 @@ class DiscordNotificationSettings extends Model 'deployment_success_discord_notifications' => 'boolean', 'deployment_failure_discord_notifications' => 'boolean', 'status_change_discord_notifications' => 'boolean', + 'restart_limit_reached_discord_notifications' => 'boolean', 'backup_success_discord_notifications' => 'boolean', 'backup_failure_discord_notifications' => 'boolean', 'scheduled_task_success_discord_notifications' => 'boolean', diff --git a/app/Models/EmailNotificationSettings.php b/app/Models/EmailNotificationSettings.php index 814d05339..3b04b482a 100644 --- a/app/Models/EmailNotificationSettings.php +++ b/app/Models/EmailNotificationSettings.php @@ -31,6 +31,7 @@ class EmailNotificationSettings extends Model 'deployment_success_email_notifications', 'deployment_failure_email_notifications', 'status_change_email_notifications', + 'restart_limit_reached_email_notifications', 'backup_success_email_notifications', 'backup_failure_email_notifications', 'scheduled_task_success_email_notifications', @@ -73,6 +74,7 @@ class EmailNotificationSettings extends Model 'deployment_success_email_notifications' => 'boolean', 'deployment_failure_email_notifications' => 'boolean', 'status_change_email_notifications' => 'boolean', + 'restart_limit_reached_email_notifications' => 'boolean', 'backup_success_email_notifications' => 'boolean', 'backup_failure_email_notifications' => 'boolean', 'scheduled_task_success_email_notifications' => 'boolean', diff --git a/app/Models/PushoverNotificationSettings.php b/app/Models/PushoverNotificationSettings.php index dd0d81cc0..2ab669314 100644 --- a/app/Models/PushoverNotificationSettings.php +++ b/app/Models/PushoverNotificationSettings.php @@ -21,6 +21,7 @@ class PushoverNotificationSettings extends Model 'deployment_success_pushover_notifications', 'deployment_failure_pushover_notifications', 'status_change_pushover_notifications', + 'restart_limit_reached_pushover_notifications', 'backup_success_pushover_notifications', 'backup_failure_pushover_notifications', 'scheduled_task_success_pushover_notifications', @@ -47,6 +48,7 @@ class PushoverNotificationSettings extends Model 'deployment_success_pushover_notifications' => 'boolean', 'deployment_failure_pushover_notifications' => 'boolean', 'status_change_pushover_notifications' => 'boolean', + 'restart_limit_reached_pushover_notifications' => 'boolean', 'backup_success_pushover_notifications' => 'boolean', 'backup_failure_pushover_notifications' => 'boolean', 'scheduled_task_success_pushover_notifications' => 'boolean', diff --git a/app/Models/ServiceApplication.php b/app/Models/ServiceApplication.php index 9763fa894..f6284ecbf 100644 --- a/app/Models/ServiceApplication.php +++ b/app/Models/ServiceApplication.php @@ -3,6 +3,7 @@ namespace App\Models; use App\Traits\HasNoindexDomains; +use App\Traits\HasRestartLimit; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\SoftDeletes; @@ -10,7 +11,7 @@ class ServiceApplication extends BaseModel { - use HasFactory, HasNoindexDomains, SoftDeletes; + use HasFactory, HasNoindexDomains, HasRestartLimit, SoftDeletes; protected $fillable = [ 'service_id', diff --git a/app/Models/ServiceDatabase.php b/app/Models/ServiceDatabase.php index 603d11a7f..c932791a7 100644 --- a/app/Models/ServiceDatabase.php +++ b/app/Models/ServiceDatabase.php @@ -2,12 +2,13 @@ namespace App\Models; +use App\Traits\HasRestartLimit; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\SoftDeletes; class ServiceDatabase extends BaseModel { - use HasFactory, SoftDeletes; + use HasFactory, HasRestartLimit, SoftDeletes; protected $fillable = [ 'service_id', diff --git a/app/Models/SlackNotificationSettings.php b/app/Models/SlackNotificationSettings.php index 62603685e..648869baf 100644 --- a/app/Models/SlackNotificationSettings.php +++ b/app/Models/SlackNotificationSettings.php @@ -20,6 +20,7 @@ class SlackNotificationSettings extends Model 'deployment_success_slack_notifications', 'deployment_failure_slack_notifications', 'status_change_slack_notifications', + 'restart_limit_reached_slack_notifications', 'backup_success_slack_notifications', 'backup_failure_slack_notifications', 'scheduled_task_success_slack_notifications', @@ -44,6 +45,7 @@ class SlackNotificationSettings extends Model 'deployment_success_slack_notifications' => 'boolean', 'deployment_failure_slack_notifications' => 'boolean', 'status_change_slack_notifications' => 'boolean', + 'restart_limit_reached_slack_notifications' => 'boolean', 'backup_success_slack_notifications' => 'boolean', 'backup_failure_slack_notifications' => 'boolean', 'scheduled_task_success_slack_notifications' => 'boolean', diff --git a/app/Models/StandaloneClickhouse.php b/app/Models/StandaloneClickhouse.php index 7ca45cc3b..ee8558bb0 100644 --- a/app/Models/StandaloneClickhouse.php +++ b/app/Models/StandaloneClickhouse.php @@ -5,6 +5,7 @@ use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; use App\Traits\HasMetrics; +use App\Traits\HasRestartLimit; use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; @@ -12,7 +13,7 @@ class StandaloneClickhouse extends BaseModel { - use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes; + use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneDragonfly.php b/app/Models/StandaloneDragonfly.php index 769d9f00c..c72841955 100644 --- a/app/Models/StandaloneDragonfly.php +++ b/app/Models/StandaloneDragonfly.php @@ -5,6 +5,7 @@ use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; use App\Traits\HasMetrics; +use App\Traits\HasRestartLimit; use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; @@ -12,7 +13,7 @@ class StandaloneDragonfly extends BaseModel { - use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes; + use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneKeydb.php b/app/Models/StandaloneKeydb.php index 15a1fe2f8..9c4e4aa66 100644 --- a/app/Models/StandaloneKeydb.php +++ b/app/Models/StandaloneKeydb.php @@ -5,6 +5,7 @@ use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; use App\Traits\HasMetrics; +use App\Traits\HasRestartLimit; use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; @@ -12,7 +13,7 @@ class StandaloneKeydb extends BaseModel { - use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes; + use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index 378d36395..4eada8a00 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -5,6 +5,7 @@ use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; use App\Traits\HasMetrics; +use App\Traits\HasRestartLimit; use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; @@ -13,7 +14,7 @@ class StandaloneMariadb extends BaseModel { - use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes; + use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index 1010ca5f3..d0a68d4e7 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -5,6 +5,7 @@ use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; use App\Traits\HasMetrics; +use App\Traits\HasRestartLimit; use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; @@ -12,7 +13,7 @@ class StandaloneMongodb extends BaseModel { - use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes; + use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index 90828bf01..4f9e73719 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -5,6 +5,7 @@ use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; use App\Traits\HasMetrics; +use App\Traits\HasRestartLimit; use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; @@ -12,7 +13,7 @@ class StandaloneMysql extends BaseModel { - use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes; + use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index e7db81285..602728855 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -5,6 +5,7 @@ use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; use App\Traits\HasMetrics; +use App\Traits\HasRestartLimit; use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; @@ -12,7 +13,7 @@ class StandalonePostgresql extends BaseModel { - use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes; + use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index 326261190..8a74824be 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -5,6 +5,7 @@ use App\Traits\ClearsGlobalSearchCache; use App\Traits\HasDatabaseHealthCheck; use App\Traits\HasMetrics; +use App\Traits\HasRestartLimit; use App\Traits\HasSafeStringAttribute; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; @@ -12,7 +13,7 @@ class StandaloneRedis extends BaseModel { - use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes; + use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes; protected $fillable = [ 'uuid', diff --git a/app/Models/TelegramNotificationSettings.php b/app/Models/TelegramNotificationSettings.php index 8c644f9bc..3376e239d 100644 --- a/app/Models/TelegramNotificationSettings.php +++ b/app/Models/TelegramNotificationSettings.php @@ -21,6 +21,7 @@ class TelegramNotificationSettings extends Model 'deployment_success_telegram_notifications', 'deployment_failure_telegram_notifications', 'status_change_telegram_notifications', + 'restart_limit_reached_telegram_notifications', 'backup_success_telegram_notifications', 'backup_failure_telegram_notifications', 'scheduled_task_success_telegram_notifications', @@ -36,6 +37,7 @@ class TelegramNotificationSettings extends Model 'telegram_notifications_deployment_success_thread_id', 'telegram_notifications_deployment_failure_thread_id', 'telegram_notifications_status_change_thread_id', + 'telegram_notifications_restart_limit_reached_thread_id', 'telegram_notifications_backup_success_thread_id', 'telegram_notifications_backup_failure_thread_id', 'telegram_notifications_scheduled_task_success_thread_id', @@ -55,6 +57,7 @@ class TelegramNotificationSettings extends Model 'telegram_notifications_deployment_success_thread_id', 'telegram_notifications_deployment_failure_thread_id', 'telegram_notifications_status_change_thread_id', + 'telegram_notifications_restart_limit_reached_thread_id', 'telegram_notifications_backup_success_thread_id', 'telegram_notifications_backup_failure_thread_id', 'telegram_notifications_scheduled_task_success_thread_id', @@ -76,6 +79,7 @@ class TelegramNotificationSettings extends Model 'deployment_success_telegram_notifications' => 'boolean', 'deployment_failure_telegram_notifications' => 'boolean', 'status_change_telegram_notifications' => 'boolean', + 'restart_limit_reached_telegram_notifications' => 'boolean', 'backup_success_telegram_notifications' => 'boolean', 'backup_failure_telegram_notifications' => 'boolean', 'scheduled_task_success_telegram_notifications' => 'boolean', @@ -90,6 +94,7 @@ class TelegramNotificationSettings extends Model 'telegram_notifications_deployment_success_thread_id' => 'encrypted', 'telegram_notifications_deployment_failure_thread_id' => 'encrypted', 'telegram_notifications_status_change_thread_id' => 'encrypted', + 'telegram_notifications_restart_limit_reached_thread_id' => 'encrypted', 'telegram_notifications_backup_success_thread_id' => 'encrypted', 'telegram_notifications_backup_failure_thread_id' => 'encrypted', 'telegram_notifications_scheduled_task_success_thread_id' => 'encrypted', diff --git a/app/Models/WebhookNotificationSettings.php b/app/Models/WebhookNotificationSettings.php index c6a81b50a..7ffd20a8c 100644 --- a/app/Models/WebhookNotificationSettings.php +++ b/app/Models/WebhookNotificationSettings.php @@ -20,6 +20,7 @@ class WebhookNotificationSettings extends Model 'deployment_success_webhook_notifications', 'deployment_failure_webhook_notifications', 'status_change_webhook_notifications', + 'restart_limit_reached_webhook_notifications', 'backup_success_webhook_notifications', 'backup_failure_webhook_notifications', 'scheduled_task_success_webhook_notifications', @@ -46,6 +47,7 @@ protected function casts(): array 'deployment_success_webhook_notifications' => 'boolean', 'deployment_failure_webhook_notifications' => 'boolean', 'status_change_webhook_notifications' => 'boolean', + 'restart_limit_reached_webhook_notifications' => 'boolean', 'backup_success_webhook_notifications' => 'boolean', 'backup_failure_webhook_notifications' => 'boolean', 'scheduled_task_success_webhook_notifications' => 'boolean', diff --git a/app/Notifications/Application/RestartLimitReached.php b/app/Notifications/Application/RestartLimitReached.php index 635dfdbdc..687fd3086 100644 --- a/app/Notifications/Application/RestartLimitReached.php +++ b/app/Notifications/Application/RestartLimitReached.php @@ -2,7 +2,8 @@ namespace App\Notifications\Application; -use App\Models\Application; +use App\Models\ApplicationPreview; +use App\Models\BaseModel; use App\Notifications\CustomEmailNotification; use App\Notifications\Dto\DiscordMessage; use App\Notifications\Dto\PushoverMessage; @@ -27,26 +28,40 @@ class RestartLimitReached extends CustomEmailNotification public int $max_restart_count; - public function __construct(public Application $resource) + public function __construct(public BaseModel $resource) { $this->onQueue('high'); $this->afterCommit(); - $this->resource_name = data_get($resource, 'name'); - $this->project_uuid = data_get($resource, 'environment.project.uuid'); - $this->environment_uuid = data_get($resource, 'environment.uuid'); - $this->environment_name = data_get($resource, 'environment.name'); + $environment = data_get($resource, 'environment') + ?? data_get($resource, 'application.environment') + ?? data_get($resource, 'service.environment'); + $this->resource_name = $resource instanceof ApplicationPreview + ? data_get($resource, 'application.name').' PR #'.$resource->pull_request_id + : data_get($resource, 'name'); + $this->project_uuid = data_get($environment, 'project.uuid'); + $this->environment_uuid = data_get($environment, 'uuid'); + $this->environment_name = data_get($environment, 'name'); $this->fqdn = data_get($resource, 'fqdn', null); $this->restart_count = $resource->restart_count; - $this->max_restart_count = $resource->max_restart_count; + $this->max_restart_count = method_exists($resource, 'restartLimitMaximum') + ? $resource->restartLimitMaximum() + : $resource->max_restart_count; if (str($this->fqdn)->explode(',')->count() > 1) { $this->fqdn = str($this->fqdn)->explode(',')->first(); } - $this->resource_url = $this->resource->link() ?? base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}/application/{$this->resource->uuid}"; + $service = data_get($resource, 'service'); + $this->resource_url = match (true) { + method_exists($this->resource, 'link') => $this->resource->link(), + $resource instanceof ApplicationPreview => $resource->application->link(), + is_object($service) && method_exists($service, 'link') => $service->link(), + default => null, + }; + $this->resource_url ??= base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}"; } public function via(object $notifiable): array { - return $notifiable->getEnabledChannels('status_change'); + return $notifiable->getEnabledChannels('restart_limit_reached'); } public function toMail(): MailMessage @@ -68,7 +83,7 @@ public function toDiscord(): DiscordMessage { return new DiscordMessage( title: ':warning: Restart limit reached', - description: "{$this->resource_name} has been stopped after {$this->restart_count} restarts (limit: {$this->max_restart_count}).\n\n[Open Application in Coolify]({$this->resource_url})", + description: "{$this->resource_name} has been stopped after {$this->restart_count} restarts (limit: {$this->max_restart_count}).\n\n[Open Resource in Coolify]({$this->resource_url})", color: DiscordMessage::errorColor(), isCritical: true, ); @@ -82,7 +97,7 @@ public function toTelegram(): array 'message' => $message, 'buttons' => [ [ - 'text' => 'Open Application in Coolify', + 'text' => 'Open Resource in Coolify', 'url' => $this->resource_url, ], ], @@ -99,7 +114,7 @@ public function toPushover(): PushoverMessage message: $message, buttons: [ [ - 'text' => 'Open Application in Coolify', + 'text' => 'Open Resource in Coolify', 'url' => $this->resource_url, ], ], @@ -110,10 +125,13 @@ public function toSlack(): SlackMessage { $title = 'Restart limit reached'; $description = "{$this->resource_name} has been stopped after {$this->restart_count} restarts (limit: {$this->max_restart_count})"; + $environment = data_get($this->resource, 'environment') + ?? data_get($this->resource, 'application.environment') + ?? data_get($this->resource, 'service.environment'); - $description .= "\n\n*Project:* ".data_get($this->resource, 'environment.project.name'); + $description .= "\n\n*Project:* ".data_get($environment, 'project.name'); $description .= "\n*Environment:* {$this->environment_name}"; - $description .= "\n*Application URL:* {$this->resource_url}"; + $description .= "\n*Resource URL:* {$this->resource_url}"; return new SlackMessage( title: $title, @@ -130,6 +148,8 @@ public function toWebhook(): array 'event' => 'restart_limit_reached', 'application_name' => $this->resource_name, 'application_uuid' => $this->resource->uuid, + 'resource_name' => $this->resource_name, + 'resource_uuid' => $this->resource->uuid, 'restart_count' => $this->restart_count, 'max_restart_count' => $this->max_restart_count, 'url' => $this->resource_url, diff --git a/app/Notifications/Channels/TelegramChannel.php b/app/Notifications/Channels/TelegramChannel.php index c2fa3ff10..4f311bf68 100644 --- a/app/Notifications/Channels/TelegramChannel.php +++ b/app/Notifications/Channels/TelegramChannel.php @@ -3,6 +3,21 @@ namespace App\Notifications\Channels; use App\Jobs\SendMessageToTelegramJob; +use App\Notifications\Application\DeploymentFailed; +use App\Notifications\Application\DeploymentSuccess; +use App\Notifications\Application\RestartLimitReached; +use App\Notifications\Application\StatusChanged; +use App\Notifications\Container\ContainerRestarted; +use App\Notifications\Database\BackupFailed; +use App\Notifications\Database\BackupSuccess; +use App\Notifications\ScheduledTask\TaskFailed; +use App\Notifications\ScheduledTask\TaskSuccess; +use App\Notifications\Server\DockerCleanupFailed; +use App\Notifications\Server\DockerCleanupSuccess; +use App\Notifications\Server\HighDiskUsage; +use App\Notifications\Server\Reachable; +use App\Notifications\Server\ServerPatchCheck; +use App\Notifications\Server\Unreachable; class TelegramChannel { @@ -17,24 +32,24 @@ public function send($notifiable, $notification): void $chatId = $settings->telegram_chat_id; $threadId = match (get_class($notification)) { - \App\Notifications\Application\DeploymentSuccess::class => $settings->telegram_notifications_deployment_success_thread_id, - \App\Notifications\Application\DeploymentFailed::class => $settings->telegram_notifications_deployment_failure_thread_id, - \App\Notifications\Application\StatusChanged::class, - \App\Notifications\Container\ContainerRestarted::class, - \App\Notifications\Container\ContainerStopped::class => $settings->telegram_notifications_status_change_thread_id, + DeploymentSuccess::class => $settings->telegram_notifications_deployment_success_thread_id, + DeploymentFailed::class => $settings->telegram_notifications_deployment_failure_thread_id, + StatusChanged::class, + ContainerRestarted::class => $settings->telegram_notifications_status_change_thread_id, + RestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id, - \App\Notifications\Database\BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id, - \App\Notifications\Database\BackupFailed::class => $settings->telegram_notifications_backup_failure_thread_id, + BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id, + BackupFailed::class => $settings->telegram_notifications_backup_failure_thread_id, - \App\Notifications\ScheduledTask\TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id, - \App\Notifications\ScheduledTask\TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id, + TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id, + TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id, - \App\Notifications\Server\DockerCleanupSuccess::class => $settings->telegram_notifications_docker_cleanup_success_thread_id, - \App\Notifications\Server\DockerCleanupFailed::class => $settings->telegram_notifications_docker_cleanup_failure_thread_id, - \App\Notifications\Server\HighDiskUsage::class => $settings->telegram_notifications_server_disk_usage_thread_id, - \App\Notifications\Server\Unreachable::class => $settings->telegram_notifications_server_unreachable_thread_id, - \App\Notifications\Server\Reachable::class => $settings->telegram_notifications_server_reachable_thread_id, - \App\Notifications\Server\ServerPatchCheck::class => $settings->telegram_notifications_server_patch_thread_id, + DockerCleanupSuccess::class => $settings->telegram_notifications_docker_cleanup_success_thread_id, + DockerCleanupFailed::class => $settings->telegram_notifications_docker_cleanup_failure_thread_id, + HighDiskUsage::class => $settings->telegram_notifications_server_disk_usage_thread_id, + Unreachable::class => $settings->telegram_notifications_server_unreachable_thread_id, + Reachable::class => $settings->telegram_notifications_server_reachable_thread_id, + ServerPatchCheck::class => $settings->telegram_notifications_server_patch_thread_id, default => null, }; diff --git a/app/Notifications/Container/ContainerStopped.php b/app/Notifications/Container/ContainerStopped.php deleted file mode 100644 index f518cd2fd..000000000 --- a/app/Notifications/Container/ContainerStopped.php +++ /dev/null @@ -1,123 +0,0 @@ -onQueue('high'); - } - - public function via(object $notifiable): array - { - return $notifiable->getEnabledChannels('status_change'); - } - - public function toMail(): MailMessage - { - $mail = new MailMessage; - $mail->subject("Coolify: A resource has been stopped unexpectedly on {$this->server->name}"); - $mail->view('emails.container-stopped', [ - 'containerName' => $this->name, - 'serverName' => $this->server->name, - 'url' => $this->url, - ]); - - return $mail; - } - - public function toDiscord(): DiscordMessage - { - $message = new DiscordMessage( - title: ':cross_mark: Resource stopped', - description: "{$this->name} has been stopped unexpectedly on {$this->server->name}.", - color: DiscordMessage::errorColor(), - ); - - if ($this->url) { - $message->addField('Resource', '[Link]('.$this->url.')'); - } - - return $message; - } - - public function toTelegram(): array - { - $message = "Coolify: A resource ($this->name) has been stopped unexpectedly on {$this->server->name}"; - $payload = [ - 'message' => $message, - ]; - if ($this->url) { - $payload['buttons'] = [ - [ - [ - 'text' => 'Open Application in Coolify', - 'url' => $this->url, - ], - ], - ]; - } - - return $payload; - } - - public function toPushover(): PushoverMessage - { - $buttons = []; - if ($this->url) { - $buttons[] = [ - 'text' => 'Open Application in Coolify', - 'url' => $this->url, - ]; - } - - return new PushoverMessage( - title: 'Resource stopped', - level: 'error', - message: "A resource ({$this->name}) has been stopped unexpectedly on {$this->server->name}", - buttons: $buttons, - ); - } - - public function toSlack(): SlackMessage - { - $title = 'Resource stopped'; - $description = "A resource ({$this->name}) has been stopped unexpectedly on {$this->server->name}"; - - if ($this->url) { - $description .= "\n*Resource URL:* {$this->url}"; - } - - return new SlackMessage( - title: $title, - description: $description, - color: SlackMessage::errorColor() - ); - } - - public function toWebhook(): array - { - $data = [ - 'success' => false, - 'message' => 'Resource stopped unexpectedly', - 'event' => 'container_stopped', - 'container_name' => $this->name, - 'server_name' => $this->server->name, - 'server_uuid' => $this->server->uuid, - ]; - - if ($this->url) { - $data['url'] = $this->url; - } - - return $data; - } -} diff --git a/app/Services/ContainerStatusAggregator.php b/app/Services/ContainerStatusAggregator.php index 8859a9980..3a59ad58f 100644 --- a/app/Services/ContainerStatusAggregator.php +++ b/app/Services/ContainerStatusAggregator.php @@ -18,14 +18,13 @@ * State Priority (highest to lowest): * 1. Degraded (from sub-resources) → degraded:unhealthy * 2. Restarting → degraded:unhealthy (or restarting:unknown if preserveRestarting=true) - * 3. Crash Loop (exited with restarts) → degraded:unhealthy - * 4. Mixed (running + exited) → degraded:unhealthy - * 5. Mixed (running + starting) → starting:unknown - * 6. Running → running:healthy/unhealthy/unknown - * 7. Dead/Removing → degraded:unhealthy - * 8. Paused → paused:unknown - * 9. Starting/Created → starting:unknown - * 10. Exited → exited + * 3. Mixed (running + exited) → degraded:unhealthy + * 4. Mixed (running + starting) → starting:unknown + * 5. Running → running:healthy/unhealthy/unknown + * 6. Dead/Removing → degraded:unhealthy + * 7. Paused → paused:unknown + * 8. Starting/Created → starting:unknown + * 9. Exited → exited * * The $preserveRestarting parameter controls whether "restarting" containers should be * reported as "restarting:unknown" (true) or "degraded:unhealthy" (false, default). @@ -228,23 +227,18 @@ private function resolveStatus( return $preserveRestarting ? 'restarting:unknown' : 'degraded:unhealthy'; } - // Priority 3: Crash loop detection (exited with restart count > 0) - if ($hasExited && $maxRestartCount > 0) { - return 'degraded:unhealthy'; - } - - // Priority 4: Mixed state (some running, some exited = degraded) + // Priority 3: Mixed state (some running, some exited = degraded) if ($hasRunning && $hasExited) { return 'degraded:unhealthy'; } - // Priority 5: Mixed state (some running, some starting = still starting) + // Priority 4: Mixed state (some running, some starting = still starting) // If any component is still starting, the entire service stack is not fully ready if ($hasRunning && $hasStarting) { return 'starting:unknown'; } - // Priority 6: Running containers (check health status) + // Priority 5: Running containers (check health status) if ($hasRunning) { if ($hasUnhealthy) { return 'running:unhealthy'; @@ -255,22 +249,22 @@ private function resolveStatus( } } - // Priority 7: Dead or removing containers + // Priority 6: Dead or removing containers if ($hasDead) { return 'degraded:unhealthy'; } - // Priority 8: Paused containers + // Priority 7: Paused containers if ($hasPaused) { return 'paused:unknown'; } - // Priority 9: Starting/created containers + // Priority 8: Starting/created containers if ($hasStarting) { return 'starting:unknown'; } - // Priority 10: All containers exited (no restart count = truly stopped) + // Priority 9: All containers exited return 'exited'; } } diff --git a/app/Services/RestartCountTracker.php b/app/Services/RestartCountTracker.php new file mode 100644 index 000000000..e67deacb3 --- /dev/null +++ b/app/Services/RestartCountTracker.php @@ -0,0 +1,31 @@ + $previousRestartCount; + $restartCountChanged = $newGeneration || $restartCountIncreased; + + $restartLimitReached = $maxRestartCount > 0 + && $observedRestartCount >= $maxRestartCount; + + return [ + 'restart_count' => $restartCountChanged ? $observedRestartCount : $previousRestartCount, + 'restart_count_changed' => $restartCountChanged, + 'restart_limit_reached' => $restartLimitReached, + 'new_generation' => $newGeneration, + ]; + } +} diff --git a/app/Traits/HasRestartLimit.php b/app/Traits/HasRestartLimit.php new file mode 100644 index 000000000..edb461cdc --- /dev/null +++ b/app/Traits/HasRestartLimit.php @@ -0,0 +1,73 @@ +mergeFillable(['restart_count', 'max_restart_count', 'restart_limit_reached', 'last_restart_at', 'last_restart_type']); + $this->mergeCasts([ + 'restart_count' => 'integer', + 'max_restart_count' => 'integer', + 'restart_limit_reached' => 'boolean', + 'last_restart_at' => 'datetime', + 'last_restart_type' => 'string', + ]); + } + + public function stoppedAfterRestartLimit(): bool + { + return str($this->status)->startsWith('exited') && $this->restart_limit_reached === true; + } + + public function trackRestartCount(int $observedRestartCount): bool + { + $state = (new RestartCountTracker)->evaluate( + previousRestartCount: $this->restart_count ?? 0, + observedRestartCount: $observedRestartCount, + maxRestartCount: $this->restartLimitMaximum(), + ); + + if ($state['restart_count_changed']) { + $hasCrashRestarts = $state['restart_count'] > 0; + $this->update([ + 'restart_count' => $state['restart_count'], + 'last_restart_at' => $hasCrashRestarts ? now() : null, + 'last_restart_type' => $hasCrashRestarts ? 'crash' : null, + ]); + } + + if (! $state['restart_limit_reached']) { + return false; + } + + $claimed = $this->newQuery() + ->whereKey($this->getKey()) + ->where('restart_limit_reached', false) + ->update(['restart_limit_reached' => true]) === 1; + + if ($claimed) { + $this->restart_limit_reached = true; + } + + return $claimed; + } + + public function resetRestartLimit(): void + { + $this->update([ + 'restart_count' => 0, + 'restart_limit_reached' => false, + 'last_restart_at' => null, + 'last_restart_type' => null, + ]); + } + + public function restartLimitMaximum(): int + { + return $this->max_restart_count ?? 0; + } +} diff --git a/database/migrations/2026_08_30_193506_add_container_present_to_applications_table.php b/database/migrations/2026_08_30_193506_add_container_present_to_applications_table.php new file mode 100644 index 000000000..fc59fb587 --- /dev/null +++ b/database/migrations/2026_08_30_193506_add_container_present_to_applications_table.php @@ -0,0 +1,28 @@ +boolean('container_present')->nullable()->after('status'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('applications', function (Blueprint $table) { + $table->dropColumn('container_present'); + }); + } +}; diff --git a/database/migrations/2026_08_30_220617_add_restart_limit_reached_to_applications_table.php b/database/migrations/2026_08_30_220617_add_restart_limit_reached_to_applications_table.php new file mode 100644 index 000000000..80603ee97 --- /dev/null +++ b/database/migrations/2026_08_30_220617_add_restart_limit_reached_to_applications_table.php @@ -0,0 +1,37 @@ +boolean('restart_limit_reached')->default(false)->after('max_restart_count'); + }); + + DB::table('applications') + ->where('status', 'like', 'exited%') + ->where('restart_count', '>', 0) + ->where('max_restart_count', '>', 0) + ->whereColumn('restart_count', '>=', 'max_restart_count') + ->where('last_restart_type', 'crash') + ->update(['restart_limit_reached' => true]); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('applications', function (Blueprint $table) { + $table->dropColumn('restart_limit_reached'); + }); + } +}; diff --git a/database/migrations/2026_08_31_073116_add_restart_limit_to_application_previews.php b/database/migrations/2026_08_31_073116_add_restart_limit_to_application_previews.php new file mode 100644 index 000000000..adf119a6e --- /dev/null +++ b/database/migrations/2026_08_31_073116_add_restart_limit_to_application_previews.php @@ -0,0 +1,32 @@ +integer('restart_count')->default(0); + $table->integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + $table->timestamp('last_restart_at')->nullable(); + $table->string('last_restart_type', 10)->nullable(); + }); + } + + public function down(): void + { + Schema::table('application_previews', function (Blueprint $table) { + $table->dropColumn([ + 'restart_count', + 'max_restart_count', + 'restart_limit_reached', + 'last_restart_at', + 'last_restart_type', + ]); + }); + } +}; diff --git a/database/migrations/2026_08_31_073117_add_restart_limit_to_service_applications.php b/database/migrations/2026_08_31_073117_add_restart_limit_to_service_applications.php new file mode 100644 index 000000000..0838ad4bd --- /dev/null +++ b/database/migrations/2026_08_31_073117_add_restart_limit_to_service_applications.php @@ -0,0 +1,32 @@ +integer('restart_count')->default(0); + $table->integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + $table->timestamp('last_restart_at')->nullable(); + $table->string('last_restart_type', 10)->nullable(); + }); + } + + public function down(): void + { + Schema::table('service_applications', function (Blueprint $table) { + $table->dropColumn([ + 'restart_count', + 'max_restart_count', + 'restart_limit_reached', + 'last_restart_at', + 'last_restart_type', + ]); + }); + } +}; diff --git a/database/migrations/2026_08_31_073118_add_restart_limit_to_service_databases.php b/database/migrations/2026_08_31_073118_add_restart_limit_to_service_databases.php new file mode 100644 index 000000000..046b02960 --- /dev/null +++ b/database/migrations/2026_08_31_073118_add_restart_limit_to_service_databases.php @@ -0,0 +1,32 @@ +integer('restart_count')->default(0); + $table->integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + $table->timestamp('last_restart_at')->nullable(); + $table->string('last_restart_type', 10)->nullable(); + }); + } + + public function down(): void + { + Schema::table('service_databases', function (Blueprint $table) { + $table->dropColumn([ + 'restart_count', + 'max_restart_count', + 'restart_limit_reached', + 'last_restart_at', + 'last_restart_type', + ]); + }); + } +}; diff --git a/database/migrations/2026_08_31_073119_add_restart_limit_to_standalone_postgresqls.php b/database/migrations/2026_08_31_073119_add_restart_limit_to_standalone_postgresqls.php new file mode 100644 index 000000000..641da5b75 --- /dev/null +++ b/database/migrations/2026_08_31_073119_add_restart_limit_to_standalone_postgresqls.php @@ -0,0 +1,23 @@ +integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + }); + } + + public function down(): void + { + Schema::table('standalone_postgresqls', function (Blueprint $table) { + $table->dropColumn(['max_restart_count', 'restart_limit_reached']); + }); + } +}; diff --git a/database/migrations/2026_08_31_073120_add_restart_limit_to_standalone_redis.php b/database/migrations/2026_08_31_073120_add_restart_limit_to_standalone_redis.php new file mode 100644 index 000000000..24329da9f --- /dev/null +++ b/database/migrations/2026_08_31_073120_add_restart_limit_to_standalone_redis.php @@ -0,0 +1,23 @@ +integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + }); + } + + public function down(): void + { + Schema::table('standalone_redis', function (Blueprint $table) { + $table->dropColumn(['max_restart_count', 'restart_limit_reached']); + }); + } +}; diff --git a/database/migrations/2026_08_31_073121_add_restart_limit_to_standalone_mongodbs.php b/database/migrations/2026_08_31_073121_add_restart_limit_to_standalone_mongodbs.php new file mode 100644 index 000000000..08980a7cb --- /dev/null +++ b/database/migrations/2026_08_31_073121_add_restart_limit_to_standalone_mongodbs.php @@ -0,0 +1,23 @@ +integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + }); + } + + public function down(): void + { + Schema::table('standalone_mongodbs', function (Blueprint $table) { + $table->dropColumn(['max_restart_count', 'restart_limit_reached']); + }); + } +}; diff --git a/database/migrations/2026_08_31_073122_add_restart_limit_to_standalone_mysqls.php b/database/migrations/2026_08_31_073122_add_restart_limit_to_standalone_mysqls.php new file mode 100644 index 000000000..729f85273 --- /dev/null +++ b/database/migrations/2026_08_31_073122_add_restart_limit_to_standalone_mysqls.php @@ -0,0 +1,23 @@ +integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + }); + } + + public function down(): void + { + Schema::table('standalone_mysqls', function (Blueprint $table) { + $table->dropColumn(['max_restart_count', 'restart_limit_reached']); + }); + } +}; diff --git a/database/migrations/2026_08_31_073123_add_restart_limit_to_standalone_mariadbs.php b/database/migrations/2026_08_31_073123_add_restart_limit_to_standalone_mariadbs.php new file mode 100644 index 000000000..6bada2326 --- /dev/null +++ b/database/migrations/2026_08_31_073123_add_restart_limit_to_standalone_mariadbs.php @@ -0,0 +1,23 @@ +integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + }); + } + + public function down(): void + { + Schema::table('standalone_mariadbs', function (Blueprint $table) { + $table->dropColumn(['max_restart_count', 'restart_limit_reached']); + }); + } +}; diff --git a/database/migrations/2026_08_31_073124_add_restart_limit_to_standalone_keydbs.php b/database/migrations/2026_08_31_073124_add_restart_limit_to_standalone_keydbs.php new file mode 100644 index 000000000..41a983924 --- /dev/null +++ b/database/migrations/2026_08_31_073124_add_restart_limit_to_standalone_keydbs.php @@ -0,0 +1,23 @@ +integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + }); + } + + public function down(): void + { + Schema::table('standalone_keydbs', function (Blueprint $table) { + $table->dropColumn(['max_restart_count', 'restart_limit_reached']); + }); + } +}; diff --git a/database/migrations/2026_08_31_073125_add_restart_limit_to_standalone_dragonflies.php b/database/migrations/2026_08_31_073125_add_restart_limit_to_standalone_dragonflies.php new file mode 100644 index 000000000..23d8ccf2c --- /dev/null +++ b/database/migrations/2026_08_31_073125_add_restart_limit_to_standalone_dragonflies.php @@ -0,0 +1,23 @@ +integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + }); + } + + public function down(): void + { + Schema::table('standalone_dragonflies', function (Blueprint $table) { + $table->dropColumn(['max_restart_count', 'restart_limit_reached']); + }); + } +}; diff --git a/database/migrations/2026_08_31_073126_add_restart_limit_to_standalone_clickhouses.php b/database/migrations/2026_08_31_073126_add_restart_limit_to_standalone_clickhouses.php new file mode 100644 index 000000000..5ec8d4522 --- /dev/null +++ b/database/migrations/2026_08_31_073126_add_restart_limit_to_standalone_clickhouses.php @@ -0,0 +1,23 @@ +integer('max_restart_count')->default(10); + $table->boolean('restart_limit_reached')->default(false); + }); + } + + public function down(): void + { + Schema::table('standalone_clickhouses', function (Blueprint $table) { + $table->dropColumn(['max_restart_count', 'restart_limit_reached']); + }); + } +}; diff --git a/database/migrations/2026_08_31_092837_add_restart_limit_reached_notifications_to_email_notification_settings_table.php b/database/migrations/2026_08_31_092837_add_restart_limit_reached_notifications_to_email_notification_settings_table.php new file mode 100644 index 000000000..e4db4b7e8 --- /dev/null +++ b/database/migrations/2026_08_31_092837_add_restart_limit_reached_notifications_to_email_notification_settings_table.php @@ -0,0 +1,28 @@ +boolean('restart_limit_reached_email_notifications')->default(true); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('email_notification_settings', function (Blueprint $table) { + $table->dropColumn('restart_limit_reached_email_notifications'); + }); + } +}; diff --git a/database/migrations/2026_08_31_092838_add_restart_limit_reached_notifications_to_discord_notification_settings_table.php b/database/migrations/2026_08_31_092838_add_restart_limit_reached_notifications_to_discord_notification_settings_table.php new file mode 100644 index 000000000..b7803e633 --- /dev/null +++ b/database/migrations/2026_08_31_092838_add_restart_limit_reached_notifications_to_discord_notification_settings_table.php @@ -0,0 +1,28 @@ +boolean('restart_limit_reached_discord_notifications')->default(true); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('discord_notification_settings', function (Blueprint $table) { + $table->dropColumn('restart_limit_reached_discord_notifications'); + }); + } +}; diff --git a/database/migrations/2026_08_31_092840_add_restart_limit_reached_notifications_to_telegram_notification_settings_table.php b/database/migrations/2026_08_31_092840_add_restart_limit_reached_notifications_to_telegram_notification_settings_table.php new file mode 100644 index 000000000..51880ceb8 --- /dev/null +++ b/database/migrations/2026_08_31_092840_add_restart_limit_reached_notifications_to_telegram_notification_settings_table.php @@ -0,0 +1,30 @@ +boolean('restart_limit_reached_telegram_notifications')->default(true); + $table->text('telegram_notifications_restart_limit_reached_thread_id')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('telegram_notification_settings', function (Blueprint $table) { + $table->dropColumn('restart_limit_reached_telegram_notifications'); + $table->dropColumn('telegram_notifications_restart_limit_reached_thread_id'); + }); + } +}; diff --git a/database/migrations/2026_08_31_092841_add_restart_limit_reached_notifications_to_slack_notification_settings_table.php b/database/migrations/2026_08_31_092841_add_restart_limit_reached_notifications_to_slack_notification_settings_table.php new file mode 100644 index 000000000..0b82b423f --- /dev/null +++ b/database/migrations/2026_08_31_092841_add_restart_limit_reached_notifications_to_slack_notification_settings_table.php @@ -0,0 +1,28 @@ +boolean('restart_limit_reached_slack_notifications')->default(true); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('slack_notification_settings', function (Blueprint $table) { + $table->dropColumn('restart_limit_reached_slack_notifications'); + }); + } +}; diff --git a/database/migrations/2026_08_31_092842_add_restart_limit_reached_notifications_to_pushover_notification_settings_table.php b/database/migrations/2026_08_31_092842_add_restart_limit_reached_notifications_to_pushover_notification_settings_table.php new file mode 100644 index 000000000..596af0b5c --- /dev/null +++ b/database/migrations/2026_08_31_092842_add_restart_limit_reached_notifications_to_pushover_notification_settings_table.php @@ -0,0 +1,28 @@ +boolean('restart_limit_reached_pushover_notifications')->default(true); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('pushover_notification_settings', function (Blueprint $table) { + $table->dropColumn('restart_limit_reached_pushover_notifications'); + }); + } +}; diff --git a/database/migrations/2026_08_31_092843_add_restart_limit_reached_notifications_to_webhook_notification_settings_table.php b/database/migrations/2026_08_31_092843_add_restart_limit_reached_notifications_to_webhook_notification_settings_table.php new file mode 100644 index 000000000..cc03cf0f5 --- /dev/null +++ b/database/migrations/2026_08_31_092843_add_restart_limit_reached_notifications_to_webhook_notification_settings_table.php @@ -0,0 +1,28 @@ +boolean('restart_limit_reached_webhook_notifications')->default(true); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('webhook_notification_settings', function (Blueprint $table) { + $table->dropColumn('restart_limit_reached_webhook_notifications'); + }); + } +}; diff --git a/resources/views/components/application/restart-limit-warning.blade.php b/resources/views/components/application/restart-limit-warning.blade.php new file mode 100644 index 000000000..3d08c803d --- /dev/null +++ b/resources/views/components/application/restart-limit-warning.blade.php @@ -0,0 +1,10 @@ +@props(['application']) + +@if ($application->stoppedAfterRestartLimit()) + @php($restartLimit = method_exists($application, 'restartLimitMaximum') ? $application->restartLimitMaximum() : ($application->max_restart_count ?? 0)) + @php($displayRestartCount = max($application->restart_count ?? 0, $restartLimit)) + +@endif diff --git a/resources/views/components/notification/event-grid.blade.php b/resources/views/components/notification/event-grid.blade.php index 2ce6a4a7f..55b452cf5 100644 --- a/resources/views/components/notification/event-grid.blade.php +++ b/resources/views/components/notification/event-grid.blade.php @@ -9,10 +9,17 @@ 'Deployments' => [ ['key' => 'deploymentSuccess', 'label' => 'Deployment success'], ['key' => 'deploymentFailure', 'label' => 'Deployment failure'], + ], + 'Resources' => [ [ 'key' => 'statusChange', - 'label' => 'Container status changes', - 'helper' => 'Notify when a container stops or restarts.', + 'label' => 'Resource status changes', + 'helper' => 'Notify when a resource stops or Coolify automatically restarts it.', + ], + [ + 'key' => 'restartLimitReached', + 'label' => 'Restart limit reached', + 'helper' => 'Notify when a resource is stopped after reaching its restart limit.', ], ], 'Backups' => [ diff --git a/resources/views/components/status-summary.blade.php b/resources/views/components/status-summary.blade.php index aee198a52..88102f57e 100644 --- a/resources/views/components/status-summary.blade.php +++ b/resources/views/components/status-summary.blade.php @@ -23,7 +23,7 @@ $containerType = match (true) { str($containerStatus)->startsWith('running') => 'success', - str($containerStatus)->startsWith(['starting', 'restarting']) => 'warning', + str($containerStatus)->startsWith(['starting', 'restarting', 'degraded']) => 'warning', default => 'error', }; @@ -36,6 +36,7 @@ [$summaryLabel, $summaryType] = match (true) { $containerType === 'error' => [$containerLabel, 'error'], + str($containerStatus)->startsWith('degraded') => ['Degraded', 'warning'], $healthType === 'error' => ['Degraded', 'error'], $containerType === 'warning' => [$containerLabel, 'warning'], $monitoringExcluded => ["{$containerLabel} (monitoring disabled)", 'warning'], diff --git a/resources/views/components/status/index.blade.php b/resources/views/components/status/index.blade.php index d774fa94c..aafa106a7 100644 --- a/resources/views/components/status/index.blade.php +++ b/resources/views/components/status/index.blade.php @@ -22,9 +22,7 @@ title="Container has restarted {{ $resource->restart_count }} time{{ $resource->restart_count > 1 ? 's' : '' }}. Last restart: {{ $resource->last_restart_at?->diffForHumans() }}" /> @endif @if ($stoppedAfterRestartLimit) - + @endif @if (!str($resource->status)->contains('exited') && $showRefreshButton) -A resource ({{ $containerName }}) has been stopped unexpectedly on {{ $serverName }}. - -@if ($url) -Please check what is going on [here]({{ $url }}). -@endif - diff --git a/resources/views/livewire/notifications/email.blade.php b/resources/views/livewire/notifications/email.blade.php index a41e72cb7..fdc2ac672 100644 --- a/resources/views/livewire/notifications/email.blade.php +++ b/resources/views/livewire/notifications/email.blade.php @@ -149,7 +149,11 @@ class="button button-highlighted"> :events="[ ['property' => 'deploymentSuccessEmailNotifications', 'label' => 'Deployment success', 'enabled' => $deploymentSuccessEmailNotifications], ['property' => 'deploymentFailureEmailNotifications', 'label' => 'Deployment failure', 'enabled' => $deploymentFailureEmailNotifications], - ['property' => 'statusChangeEmailNotifications', 'label' => 'Container status changes', 'enabled' => $statusChangeEmailNotifications], + ]" /> + status" /> +
+ +
@@ -103,32 +106,18 @@ class="listbox-panel top-full! left-0! right-0! mt-1! w-full! min-w-0!" role="me @endcan @endif @endif - @can('deploy', $application) - - @else - - @endcan @else @can('deploy', $application) @else @endcan @endif @@ -138,25 +127,36 @@ class="listbox-panel top-full! left-0! right-0! mt-1! w-full! min-w-0!" role="me wire:click="{{ $application->status === 'running' ? 'force_deploy_without_cache' : 'deploy(true)' }}" @click="open = false" role="menuitem"> - Deploy (without cache) + {{ $application->stoppedAfterRestartLimit() ? 'Retry deployment (without cache)' : 'Deploy (without cache)' }} @else @endcan @endif + @if (!str($application->status)->startsWith('exited') || $application->container_present !== false) + + @endif @endcan @endif diff --git a/resources/views/livewire/project/application/status.blade.php b/resources/views/livewire/project/application/status.blade.php index 2aa68fd77..6d692d36b 100644 --- a/resources/views/livewire/project/application/status.blade.php +++ b/resources/views/livewire/project/application/status.blade.php @@ -1,3 +1,4 @@
+
diff --git a/resources/views/livewire/project/database/heading.blade.php b/resources/views/livewire/project/database/heading.blade.php index bcdf1cf85..ad1c43889 100644 --- a/resources/views/livewire/project/database/heading.blade.php +++ b/resources/views/livewire/project/database/heading.blade.php @@ -63,7 +63,12 @@

{{ $database->name }}

- +
+ +
+
+ +
diff --git a/resources/views/livewire/project/database/status.blade.php b/resources/views/livewire/project/database/status.blade.php index 8f8a72067..cf2d32c7a 100644 --- a/resources/views/livewire/project/database/status.blade.php +++ b/resources/views/livewire/project/database/status.blade.php @@ -1,3 +1,4 @@
+
diff --git a/resources/views/livewire/project/resource/index.blade.php b/resources/views/livewire/project/resource/index.blade.php index 3ab0ac633..772a01fdc 100644 --- a/resources/views/livewire/project/resource/index.blade.php +++ b/resources/views/livewire/project/resource/index.blade.php @@ -220,7 +220,7 @@ class="relative z-10 block truncate text-[11px] text-neutral-500 hover:underline x-text="item.typeLabel">
- + @@ -299,7 +299,7 @@ class="truncate text-[13px]! leading-4! font-semibold! text-black dark:text-fg"

- + @@ -518,13 +518,32 @@ function resourceIndex() { localStorage.setItem('environment-resource-view', mode); }, statusState(item) { + if (item.restartLimitReached) { + return 'restart-limit'; + } + return String(item.status || 'unknown').split(':')[0].toLowerCase(); }, statusLabel(item) { + if (item.restartLimitReached) { + return 'Restart limit reached'; + } + const state = this.statusState(item); return state.charAt(0).toUpperCase() + state.slice(1); }, + statusTitle(item) { + if (item.restartLimitReached) { + return `${item.restartCount}/${item.maxRestartCount} restarts. Container preserved.`; + } + + return this.statusLabel(item); + }, statusTone(item) { + if (item.restartLimitReached) { + return 'warning'; + } + const state = this.statusState(item); if (state === 'running') { return 'success'; @@ -539,6 +558,10 @@ function resourceIndex() { return 'neutral'; }, statusDotClass(item) { + if (item.restartLimitReached) { + return 'bg-warning'; + } + return { success: 'bg-emerald-500', warning: 'bg-warning', diff --git a/resources/views/livewire/project/service/configuration.blade.php b/resources/views/livewire/project/service/configuration.blade.php index beb6df6e2..4afbcbc6a 100644 --- a/resources/views/livewire/project/service/configuration.blade.php +++ b/resources/views/livewire/project/service/configuration.blade.php @@ -143,12 +143,12 @@ class="flex size-7.5 items-center justify-center rounded-md transition-colors"
+ : 'overflow-x-auto rounded-xl border border-neutral-200 bg-white shadow-sm dark:border-white/[0.08] dark:bg-white/[0.025]'"> @if ($applications->isNotEmpty() || $databases->isNotEmpty())
+ class="grid min-w-[48rem] grid-cols-[minmax(14rem,1fr)_minmax(12rem,1fr)_12rem_5rem] gap-3 border-b border-neutral-200 bg-neutral-50 px-4 py-2.5 text-[11px] font-medium text-neutral-500 dark:border-white/[0.08] dark:bg-white/[0.025] dark:text-fg-faint">
Resource
- +
Image
Status
diff --git a/resources/views/livewire/project/service/heading.blade.php b/resources/views/livewire/project/service/heading.blade.php index b022eef10..e7da50c80 100644 --- a/resources/views/livewire/project/service/heading.blade.php +++ b/resources/views/livewire/project/service/heading.blade.php @@ -42,6 +42,13 @@ )); $serviceStatus = str($service->status ?? 'exited'); + $selectedResourceUuid = data_get($parameters, 'stack_service_uuid'); + $selectedResource = $selectedResourceUuid + ? $service->applications->firstWhere('uuid', $selectedResourceUuid) + ?? $service->databases->firstWhere('uuid', $selectedResourceUuid) + : null; + $displayStatus = $selectedResource?->status ?? $service->status; + $selectedResourceStatus = str($selectedResource?->status ?? ''); $environmentVariablesUrl = route('project.service.environment-variables', [ 'project_uuid' => $service->environment->project->uuid, 'environment_uuid' => $service->environment->uuid, @@ -65,9 +72,15 @@ {{ $service->name }}
- +
+
+ @if ($selectedResource) + + @endif +
@@ -91,7 +104,14 @@ @endcan @@ -281,6 +310,17 @@ class="listbox-panel top-full! right-0! left-auto! mt-1! w-64! min-w-0!" role="m + @if ($selectedResource) + + + + + + @endif @endif diff --git a/resources/views/livewire/project/service/resource-card.blade.php b/resources/views/livewire/project/service/resource-card.blade.php index fe976180d..c9106b0fc 100644 --- a/resources/views/livewire/project/service/resource-card.blade.php +++ b/resources/views/livewire/project/service/resource-card.blade.php @@ -1,4 +1,12 @@ -
+
@php [$statusType, $statusLabel] = match (true) { str($resource->status)->contains('running') => ['success', formatContainerStatus($resource->status)], @@ -26,6 +34,7 @@ class="flex size-9 shrink-0 items-center justify-center rounded-lg bg-neutral-10
+ @if ($resource->configuration_required) @@ -76,7 +85,9 @@ class="flex items-center justify-end gap-1 border-t border-neutral-200 bg-neutra
+ x-on:click="openSettings($event)" x-on:keydown.enter="openSettings($event)" + role="link" tabindex="0" aria-label="Open {{ $resourceName }} settings" + class="grid min-h-14 min-w-[48rem] cursor-pointer grid-cols-[minmax(14rem,1fr)_minmax(12rem,1fr)_12rem_5rem] items-center gap-3 border-b border-neutral-200 px-4 py-2.5 last:border-b-0 hover:bg-neutral-50 focus-visible:outline-2 focus-visible:outline-offset-[-2px] focus-visible:outline-primary dark:border-white/[0.07] dark:hover:bg-white/[0.025]">
@@ -86,14 +97,14 @@ class="flex size-8 shrink-0 items-center justify-center rounded-lg bg-neutral-10
{{ $resourceName }}
- diff --git a/resources/views/livewire/project/service/status.blade.php b/resources/views/livewire/project/service/status.blade.php index dd7a0bd8c..2bdf531b0 100644 --- a/resources/views/livewire/project/service/status.blade.php +++ b/resources/views/livewire/project/service/status.blade.php @@ -1,3 +1,8 @@
- + @php($displayStatus = $selectedResource?->status ?? $service->status) + + @if ($selectedResource) + + @endif
diff --git a/resources/views/livewire/project/shared/destination.blade.php b/resources/views/livewire/project/shared/destination.blade.php index a1f7923d3..5aba6c9ec 100644 --- a/resources/views/livewire/project/shared/destination.blade.php +++ b/resources/views/livewire/project/shared/destination.blade.php @@ -35,6 +35,7 @@ class="rounded bg-neutral-100 px-1.5 py-0.5 font-mono text-xs text-neutral-700 d
Open server + @if ($hasAdditionalDestinations) Open server + @if (method_exists($resource, 'stoppedAfterRestartLimit')) + + @endif @if ($primaryStatus->startsWith('running')) @elseif ($primaryStatus->startsWith(['starting', 'restarting'])) diff --git a/resources/views/livewire/server/resources.blade.php b/resources/views/livewire/server/resources.blade.php index 491535dd6..9fb2ca2fc 100644 --- a/resources/views/livewire/server/resources.blade.php +++ b/resources/views/livewire/server/resources.blade.php @@ -61,10 +61,14 @@ class="data-table-row server-resources-managed-table-grid border-b border-neutra {{ str($resource->type())->headline() }}
- + @if (method_exists($resource, 'stoppedAfterRestartLimit') && $resource->stoppedAfterRestartLimit()) + + @else + + @endif
@endforeach diff --git a/tests/Feature/AllResourceRestartLimitsTest.php b/tests/Feature/AllResourceRestartLimitsTest.php new file mode 100644 index 000000000..8418b552c --- /dev/null +++ b/tests/Feature/AllResourceRestartLimitsTest.php @@ -0,0 +1,223 @@ +toContain(HasRestartLimit::class); + + $resource = new $modelClass; + + expect($resource->getFillable())->toContain( + 'restart_count', + 'max_restart_count', + 'restart_limit_reached', + 'last_restart_at', + 'last_restart_type', + )->and($resource->getCasts())->toMatchArray([ + 'restart_count' => 'integer', + 'max_restart_count' => 'integer', + 'restart_limit_reached' => 'boolean', + 'last_restart_at' => 'datetime', + ]); +})->with([ + ApplicationPreview::class, + ServiceApplication::class, + ServiceDatabase::class, + StandaloneClickhouse::class, + StandaloneDragonfly::class, + StandaloneKeydb::class, + StandaloneMariadb::class, + StandaloneMongodb::class, + StandaloneMysql::class, + StandalonePostgresql::class, + StandaloneRedis::class, +]); + +it('collects restart counts for preview and service containers from both status sources', function () { + $dockerStatus = file_get_contents(app_path('Actions/Docker/GetContainersStatus.php')); + $sentinelStatus = file_get_contents(app_path('Jobs/PushServerUpdateJob.php')); + + expect($dockerStatus) + ->toContain('previewContainerRestartCounts') + ->toContain('serviceContainerRestartCounts') + ->and($sentinelStatus) + ->toContain('previewContainerRestartCounts') + ->toContain('serviceContainerRestartCounts'); +}); + +it('ignores docker compose one-off job containers in both status sources', function () { + $dockerStatus = file_get_contents(app_path('Actions/Docker/GetContainersStatus.php')); + $sentinelStatus = file_get_contents(app_path('Jobs/PushServerUpdateJob.php')); + + expect($dockerStatus) + ->toContain("filter_var(data_get(\$labels, 'com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)") + ->and($sentinelStatus) + ->toContain("filter_var(\$labels->get('com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)"); +}); + +it('shows restart limit warnings for every resource family', function () { + $previews = file_get_contents(resource_path('views/livewire/project/application/previews.blade.php')); + $serviceCard = file_get_contents(resource_path('views/livewire/project/service/resource-card.blade.php')); + $applicationStatus = file_get_contents(resource_path('views/livewire/project/application/status.blade.php')); + $databaseStatus = file_get_contents(resource_path('views/livewire/project/database/status.blade.php')); + + expect($previews)->toContain('') + ->and($serviceCard)->toContain('') + ->and($applicationStatus)->toContain('') + ->and($databaseStatus)->toContain(''); + + $serviceStatus = file_get_contents(resource_path('views/livewire/project/service/status.blade.php')); + $serviceHeading = file_get_contents(resource_path('views/livewire/project/service/heading.blade.php')); + expect($serviceStatus)->toContain('') + ->and($serviceStatus)->toContain('$selectedResource?->status ?? $service->status') + ->and($serviceHeading)->toContain('') + ->and($serviceHeading)->toContain('$selectedResource?->status ?? $service->status'); +}); + +it('matches application restart badge layout on mobile resource headings', function () { + $applicationHeading = file_get_contents(resource_path('views/livewire/project/application/heading.blade.php')); + $databaseHeading = file_get_contents(resource_path('views/livewire/project/database/heading.blade.php')); + $serviceHeading = file_get_contents(resource_path('views/livewire/project/service/heading.blade.php')); + + foreach ([$applicationHeading, $databaseHeading, $serviceHeading] as $heading) { + expect($heading) + ->toContain('class="relative flex w-full min-w-0 items-center gap-2"') + ->toContain('class="flex w-full flex-wrap gap-1"'); + } +}); + +it('shows database restart limits in shared resource listings', function () { + $resourceIndex = file_get_contents(app_path('Livewire/Project/Resource/Index.php')); + $serverResources = file_get_contents(resource_path('views/livewire/server/resources.blade.php')); + $destination = file_get_contents(resource_path('views/livewire/project/shared/destination.blade.php')); + + expect($resourceIndex) + ->toContain("method_exists(\$item, 'stoppedAfterRestartLimit')") + ->not->toContain("\$type === 'application' && \$item->stoppedAfterRestartLimit()") + ->and($serverResources) + ->toContain('') + ->and($destination) + ->toContain(''); +}); + +it('keeps the service resource table readable with horizontal scrolling on mobile', function () { + $configuration = file_get_contents(resource_path('views/livewire/project/service/configuration.blade.php')); + $resourceCard = file_get_contents(resource_path('views/livewire/project/service/resource-card.blade.php')); + + expect($configuration) + ->toContain("'overflow-x-auto rounded-xl") + ->toContain('min-w-[48rem]') + ->and($resourceCard) + ->toContain('min-w-[48rem]') + ->not->toContain('