fix(applications): persist restart limit state for stopped containers (#11566)

This commit is contained in:
Andras Bacsai 2026-08-31 13:13:26 +02:00 committed by GitHub
parent e514a2d61f
commit 67b33c27a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
97 changed files with 2181 additions and 397 deletions

View file

@ -13,8 +13,9 @@ class StopApplication
public string $jobQueue = 'high';
public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true, bool $resetRestartCount = true)
public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true, bool $resetRestartCount = true, bool $removeContainers = true): ?string
{
$containerPresent = ! $removeContainers;
$servers = collect([$application->destination->server]);
if ($application?->additional_servers?->count() > 0) {
$servers = $servers->merge($application->additional_servers);
@ -26,6 +27,7 @@ public function handle(Application $application, bool $previewDeployments = fals
}
if ($server->isSwarm()) {
$containerPresent = false;
instant_remote_process(["docker stack rm {$application->uuid}"], $server);
continue;
@ -39,13 +41,17 @@ public function handle(Application $application, bool $previewDeployments = fals
$timeout = $application->settings->stopGracePeriodSeconds();
foreach ($containersToStop as $containerName) {
instant_remote_process(command: [
dockerStopCommand($timeout, $containerName, $server),
"docker rm -f $containerName",
], server: $server, throwError: false);
$commands = [dockerStopCommand($timeout, $containerName, $server)];
if ($removeContainers) {
$commands[] = "docker rm -f $containerName";
} else {
array_unshift($commands, "docker update --restart=no $containerName");
}
if ($application->build_pack === 'dockercompose') {
instant_remote_process(command: $commands, server: $server, throwError: false);
}
if ($removeContainers && $application->build_pack === 'dockercompose') {
$application->deleteConnectedNetworks();
}
@ -57,12 +63,16 @@ public function handle(Application $application, bool $previewDeployments = fals
}
}
$status = ['status' => 'exited'];
$status = [
'status' => 'exited',
'container_present' => $containerPresent,
];
if ($resetRestartCount) {
$status = array_merge($status, [
'restart_count' => 0,
'last_restart_at' => null,
'last_restart_type' => null,
'restart_limit_reached' => false,
]);
}
$application->update($status);

View file

@ -0,0 +1,36 @@
<?php
namespace App\Actions\Application;
use App\Events\ServiceStatusChanged;
use App\Models\ApplicationPreview;
use Lorisleiva\Actions\Concerns\AsAction;
class StopApplicationPreview
{
use AsAction;
public function handle(ApplicationPreview $preview, bool $resetRestartCount = true, bool $removeContainer = true): void
{
$application = $preview->application;
$server = $application->destination->server;
$containers = getCurrentApplicationContainerStatus($server, $application->id, $preview->pull_request_id);
foreach ($containers->pluck('Names') as $containerName) {
$commands = [dockerStopCommand($application->settings->stopGracePeriodSeconds(), $containerName, $server)];
if ($removeContainer) {
$commands[] = "docker rm -f $containerName";
} else {
array_unshift($commands, "docker update --restart=no $containerName");
}
instant_remote_process($commands, $server, false);
}
$preview->update(['status' => 'exited']);
if ($resetRestartCount) {
$preview->resetRestartLimit();
}
ServiceStatusChanged::dispatch($application->environment->project->team->id);
}
}

View file

@ -28,6 +28,7 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St
if (! $server->isFunctional()) {
return 'Server is not functional';
}
$database->resetRestartLimit();
switch ($database->getMorphClass()) {
case StandalonePostgresql::class:
$activity = StartPostgresql::run($database);

View file

@ -4,6 +4,7 @@
use App\Actions\Server\CleanupDocker;
use App\Events\ServiceStatusChanged;
use App\Models\BaseModel;
use App\Models\StandaloneClickhouse;
use App\Models\StandaloneDragonfly;
use App\Models\StandaloneKeydb;
@ -18,7 +19,7 @@ class StopDatabase
{
use AsAction;
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, bool $dockerCleanup = true)
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $database, bool $dockerCleanup = true, bool $resetRestartCount = true, bool $removeContainer = true): string
{
try {
$server = $database->destination->server;
@ -26,15 +27,13 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St
return 'Server is not functional';
}
$this->stopContainer($database, $database->uuid, 30);
$this->stopContainer($database, $database->uuid, 30, $removeContainer);
// Reset restart tracking when database is manually stopped
$database->update([
'status' => 'exited',
'restart_count' => 0,
'last_restart_at' => null,
'last_restart_type' => null,
]);
$database->update(['status' => 'exited']);
if ($resetRestartCount) {
$database->resetRestartLimit();
}
if ($dockerCleanup) {
CleanupDocker::dispatch($server, false, false);
@ -53,12 +52,15 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St
}
private function stopContainer($database, string $containerName, int $timeout = 30): void
private function stopContainer(BaseModel $database, string $containerName, int $timeout = 30, bool $removeContainer = true): void
{
$server = $database->destination->server;
instant_remote_process(command: [
dockerStopCommand($timeout, $containerName, $server),
"docker rm -f $containerName",
], server: $server, throwError: false);
$commands = [dockerStopCommand($timeout, $containerName, $server)];
if ($removeContainer) {
$commands[] = "docker rm -f $containerName";
} else {
array_unshift($commands, "docker update --restart=no $containerName");
}
instant_remote_process(command: $commands, server: $server, throwError: false);
}
}

View file

@ -3,15 +3,20 @@
namespace App\Actions\Docker;
use App\Actions\Application\StopApplication;
use App\Actions\Application\StopApplicationPreview;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Database\StopDatabase;
use App\Actions\Database\StopDatabaseProxy;
use App\Actions\Service\StopServiceApplication;
use App\Actions\Shared\ComplexStatusCheck;
use App\Events\ServiceChecked;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\Server;
use App\Models\ServiceDatabase;
use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;
use App\Services\ContainerStatusAggregator;
use App\Services\RestartCountTracker;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
@ -37,8 +42,12 @@ class GetContainersStatus
protected ?Collection $applicationContainerRestartCounts;
protected ?Collection $previewContainerRestartCounts;
protected ?Collection $serviceContainerStatuses;
protected ?Collection $serviceContainerRestartCounts;
public function handle(Server $server, ?Collection $containers = null, ?Collection $containerReplicates = null)
{
$this->containers = $containers;
@ -117,6 +126,9 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
$containerStatus = "$containerStatus:$healthSuffix";
}
$labels = Arr::undot(format_docker_labels_to_json($labels));
if (filter_var(data_get($labels, 'com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)) {
continue;
}
$applicationId = data_get($labels, 'coolify.applicationId');
if ($applicationId) {
$pullRequestId = data_get($labels, 'coolify.pullRequestId');
@ -133,6 +145,12 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
} else {
$preview->update(['last_online_at' => now()]);
}
$key = $applicationId.':'.$pullRequestId;
$this->previewContainerRestartCounts ??= collect();
$this->previewContainerRestartCounts->push([
'key' => $key,
'count' => (int) data_get($container, 'RestartCount', 0),
]);
} else {
// Notify user that this container should not be there.
}
@ -140,6 +158,9 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
$application = $this->applications->where('id', $applicationId)->first();
if ($application) {
$foundApplications[] = $application->id;
if ($application->container_present !== true) {
$application->update(['container_present' => true]);
}
// Store container status for aggregation
if (! isset($this->applicationContainerStatuses)) {
$this->applicationContainerStatuses = collect();
@ -220,23 +241,19 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
// Track restart count for databases (single-container)
$restartCount = data_get($container, 'RestartCount', 0);
$previousRestartCount = $database->restart_count ?? 0;
if ($statusFromDb !== $containerStatus) {
$updateData = ['status' => $containerStatus];
} else {
$updateData = ['last_online_at' => now()];
}
// Update restart tracking if restart count increased
if ($restartCount > $previousRestartCount) {
$updateData['restart_count'] = $restartCount;
$updateData['last_restart_at'] = now();
$updateData['last_restart_type'] = 'crash';
}
$database->update($updateData);
if ($database->trackRestartCount((int) $restartCount)) {
StopDatabase::dispatch($database, false, false, false);
$database->team()?->notify(new ApplicationRestartLimitReached($database));
}
if ($isPublic) {
$foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) {
if ($this->server->isSwarm()) {
@ -292,6 +309,11 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
$containerName = data_get($labels, 'com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
$this->serviceContainerRestartCounts ??= collect();
if (! $this->serviceContainerRestartCounts->has($key)) {
$this->serviceContainerRestartCounts->put($key, collect());
}
$this->serviceContainerRestartCounts->get($key)->put($containerName, (int) data_get($container, 'RestartCount', 0));
}
// Mark service as found
@ -335,46 +357,35 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
continue;
}
$name = data_get($exitedService, 'name');
$fqdn = data_get($exitedService, 'fqdn');
if ($name) {
if ($fqdn) {
$containerName = "$name, available at $fqdn";
} else {
$containerName = $name;
if (! $exitedService->stoppedAfterRestartLimit()) {
$exitedService->update([
'status' => 'exited',
'restart_count' => 0,
'restart_limit_reached' => false,
'last_restart_at' => null,
'last_restart_type' => null,
]);
}
} else {
if ($fqdn) {
$containerName = $fqdn;
} else {
$containerName = null;
}
}
$projectUuid = data_get($service, 'environment.project.uuid');
$serviceUuid = data_get($service, 'uuid');
$environmentName = data_get($service, 'environment.name');
if ($projectUuid && $serviceUuid && $environmentName) {
$url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/service/'.$serviceUuid;
} else {
$url = null;
}
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
$exitedService->update(['status' => 'exited']);
}
$notRunningApplications = $this->applications->pluck('id')->diff($foundApplications);
foreach ($notRunningApplications as $applicationId) {
$application = $this->applications->where('id', $applicationId)->first();
if (str($application->status)->startsWith('exited')) {
continue;
}
// Only protection: If no containers at all, Docker query might have failed
if ($this->containers->isEmpty()) {
continue;
}
if (str($application->status)->startsWith('exited')) {
$application->update([
'container_present' => false,
'restart_limit_reached' => false,
]);
continue;
}
// If container was recently restarting (crash loop), keep it as degraded for a grace period
// This prevents false "exited" status during the brief moment between container removal and recreation
$recentlyRestarted = $application->restart_count > 0 &&
@ -388,9 +399,11 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
// Reset restart count when application exits completely
$application->update([
'status' => 'exited',
'container_present' => false,
'restart_count' => 0,
'last_restart_at' => null,
'last_restart_type' => null,
'restart_limit_reached' => false,
]);
}
}
@ -411,6 +424,9 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
$notRunningDatabases = $databases->pluck('id')->diff($foundDatabases);
foreach ($notRunningDatabases as $database) {
$database = $databases->where('id', $database)->first();
if ($database->stoppedAfterRestartLimit()) {
continue;
}
if (str($database->status)->startsWith('exited')) {
continue;
}
@ -426,6 +442,7 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
'restart_count' => 0,
'last_restart_at' => null,
'last_restart_type' => null,
'restart_limit_reached' => false,
]);
// Stop proxy if database was public
@ -433,23 +450,10 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
StopDatabaseProxy::run($database);
}
$name = data_get($database, 'name');
$fqdn = data_get($database, 'fqdn');
$containerName = $name;
$projectUuid = data_get($database, 'environment.project.uuid');
$environmentName = data_get($database, 'environment.name');
$databaseUuid = data_get($database, 'uuid');
if ($projectUuid && $databaseUuid && $environmentName) {
$url = base_url().'/project/'.$projectUuid.'/'.$environmentName.'/database/'.$databaseUuid;
} else {
$url = null;
}
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
}
$this->trackPreviewRestartCounts($previews);
// Aggregate multi-container application statuses
if (isset($this->applicationContainerStatuses) && $this->applicationContainerStatuses->isNotEmpty()) {
foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
@ -470,21 +474,21 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
DB::transaction(function () use ($application, $maxRestartCount, $containerStatuses, &$restartLimitReached) {
$previousRestartCount = $application->restart_count ?? 0;
$restartState = (new RestartCountTracker)->evaluate(
previousRestartCount: $previousRestartCount,
observedRestartCount: $maxRestartCount,
maxRestartCount: $application->max_restart_count ?? 0,
);
if ($maxRestartCount > $previousRestartCount) {
// Restart count increased - this is a crash restart
if ($restartState['restart_count_changed']) {
$hasCrashRestarts = $restartState['restart_count'] > 0;
$application->update([
'restart_count' => $maxRestartCount,
'last_restart_at' => now(),
'last_restart_type' => 'crash',
'restart_count' => $restartState['restart_count'],
'last_restart_at' => $hasCrashRestarts ? now() : null,
'last_restart_type' => $hasCrashRestarts ? 'crash' : null,
]);
// Check if restart limit has been reached
$maxAllowedRestarts = $application->max_restart_count ?? 0;
if ($maxAllowedRestarts > 0 && $maxRestartCount >= $maxAllowedRestarts && $previousRestartCount < $maxAllowedRestarts) {
$restartLimitReached = true;
}
}
$restartLimitReached = $restartState['restart_limit_reached'];
// Aggregate status after tracking restart counts
$aggregatedStatus = $this->aggregateApplicationStatus($application, $containerStatuses, $maxRestartCount);
@ -499,12 +503,25 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
});
if ($restartLimitReached) {
$restartLimitClaimed = Application::query()
->whereKey($application->getKey())
->where('restart_limit_reached', false)
->update(['restart_limit_reached' => true]) === 1;
if ($restartLimitClaimed) {
$application->refresh();
StopApplication::dispatch($application, false, true, false);
StopApplication::dispatch(
application: $application,
previewDeployments: false,
dockerCleanup: false,
resetRestartCount: false,
removeContainers: false,
);
$application->environment->project->team?->notify(new ApplicationRestartLimitReached($application));
}
}
}
}
// Aggregate multi-container service statuses
$this->aggregateServiceContainerStatuses($services);
@ -562,6 +579,16 @@ private function aggregateServiceContainerStatuses($services)
continue;
}
$restartCount = isset($this->serviceContainerRestartCounts)
? ($this->serviceContainerRestartCounts->get($key)?->max() ?? 0)
: 0;
if ($subResource->trackRestartCount($restartCount)) {
StopServiceApplication::dispatch($subResource, false, false);
$subResource->team()?->notify(new ApplicationRestartLimitReached($subResource));
continue;
}
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
@ -602,4 +629,24 @@ private function aggregateServiceContainerStatuses($services)
}
}
}
private function trackPreviewRestartCounts(Collection $previews): void
{
if (! isset($this->previewContainerRestartCounts)) {
return;
}
$this->previewContainerRestartCounts
->groupBy('key')
->each(function (Collection $counts, string $key) use ($previews): void {
[$applicationId, $pullRequestId] = explode(':', $key);
$preview = $previews->first(fn (ApplicationPreview $preview): bool => (string) $preview->application_id === $applicationId
&& (string) $preview->pull_request_id === $pullRequestId
);
if ($preview?->trackRestartCount((int) $counts->max('count'))) {
StopApplicationPreview::dispatch($preview, false, false);
$preview->application->environment->project->team?->notify(new ApplicationRestartLimitReached($preview));
}
});
}
}

View file

@ -24,6 +24,8 @@ public function handle(Service $service, bool $pullLatestImages = false, bool $s
}
$service->saveComposeConfigs();
$service->isConfigurationChanged(save: true);
$service->applications()->get()->each->resetRestartLimit();
$service->databases()->get()->each->resetRestartLimit();
$workdir = $service->workdir();
// $commands[] = "cd {$workdir}";
$commands[] = "echo 'Saved configuration files to {$workdir}.'";

View file

@ -49,8 +49,14 @@ public function handle(Service $service, bool $deleteConnectedNetworks = false,
$this->stopContainersInParallel($containersToStop, $server);
}
$applications->each->update(['status' => 'exited']);
$dbs->each->update(['status' => 'exited']);
$applications->each(function ($application): void {
$application->update(['status' => 'exited']);
$application->resetRestartLimit();
});
$dbs->each(function ($database): void {
$database->update(['status' => 'exited']);
$database->resetRestartLimit();
});
if ($deleteConnectedNetworks) {
$service->deleteConnectedNetworks();

View file

@ -13,17 +13,26 @@ class StopServiceApplication
public string $jobQueue = 'high';
public function handle(ServiceApplication|ServiceDatabase $serviceApplication): void
public function handle(ServiceApplication|ServiceDatabase $serviceApplication, bool $resetRestartCount = true, bool $removeContainer = false): void
{
$service = $serviceApplication->service;
$server = $service->destination->server;
$containerName = escapeshellarg($serviceApplication->name.'-'.$service->uuid);
instant_remote_process([
if ($removeContainer) {
$commands = ["docker rm -f {$containerName}"];
} else {
$commands = [
"docker update --restart=no {$containerName}",
"docker stop {$containerName}",
], $server);
];
}
instant_remote_process($commands, $server, throwError: ! $removeContainer);
$serviceApplication->update(['status' => 'exited']);
if ($resetRestartCount) {
$serviceApplication->resetRestartLimit();
}
ServiceStatusChanged::dispatch($service->environment->project->team->id);
}
}

View file

@ -15,6 +15,7 @@
use Illuminate\Database\Eloquent\Model;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Validator;
use OpenApi\Attributes as OA;
class NotificationsController extends Controller
@ -45,6 +46,7 @@ private function channelConfig(string $channel): array
'deployment_success_email_notifications' => 'sometimes|boolean',
'deployment_failure_email_notifications' => 'sometimes|boolean',
'status_change_email_notifications' => 'sometimes|boolean',
'restart_limit_reached_email_notifications' => 'sometimes|boolean',
'backup_success_email_notifications' => 'sometimes|boolean',
'backup_failure_email_notifications' => 'sometimes|boolean',
'scheduled_task_success_email_notifications' => 'sometimes|boolean',
@ -66,6 +68,7 @@ private function channelConfig(string $channel): array
'deployment_success_discord_notifications' => 'sometimes|boolean',
'deployment_failure_discord_notifications' => 'sometimes|boolean',
'status_change_discord_notifications' => 'sometimes|boolean',
'restart_limit_reached_discord_notifications' => 'sometimes|boolean',
'backup_success_discord_notifications' => 'sometimes|boolean',
'backup_failure_discord_notifications' => 'sometimes|boolean',
'scheduled_task_success_discord_notifications' => 'sometimes|boolean',
@ -88,6 +91,7 @@ private function channelConfig(string $channel): array
'deployment_success_slack_notifications' => 'sometimes|boolean',
'deployment_failure_slack_notifications' => 'sometimes|boolean',
'status_change_slack_notifications' => 'sometimes|boolean',
'restart_limit_reached_slack_notifications' => 'sometimes|boolean',
'backup_success_slack_notifications' => 'sometimes|boolean',
'backup_failure_slack_notifications' => 'sometimes|boolean',
'scheduled_task_success_slack_notifications' => 'sometimes|boolean',
@ -110,6 +114,7 @@ private function channelConfig(string $channel): array
'deployment_success_telegram_notifications' => 'sometimes|boolean',
'deployment_failure_telegram_notifications' => 'sometimes|boolean',
'status_change_telegram_notifications' => 'sometimes|boolean',
'restart_limit_reached_telegram_notifications' => 'sometimes|boolean',
'backup_success_telegram_notifications' => 'sometimes|boolean',
'backup_failure_telegram_notifications' => 'sometimes|boolean',
'scheduled_task_success_telegram_notifications' => 'sometimes|boolean',
@ -124,6 +129,7 @@ private function channelConfig(string $channel): array
'telegram_notifications_deployment_success_thread_id' => 'sometimes|nullable|string|max:255',
'telegram_notifications_deployment_failure_thread_id' => 'sometimes|nullable|string|max:255',
'telegram_notifications_status_change_thread_id' => 'sometimes|nullable|string|max:255',
'telegram_notifications_restart_limit_reached_thread_id' => 'sometimes|nullable|string|max:255',
'telegram_notifications_backup_success_thread_id' => 'sometimes|nullable|string|max:255',
'telegram_notifications_backup_failure_thread_id' => 'sometimes|nullable|string|max:255',
'telegram_notifications_scheduled_task_success_thread_id' => 'sometimes|nullable|string|max:255',
@ -146,6 +152,7 @@ private function channelConfig(string $channel): array
'deployment_success_pushover_notifications' => 'sometimes|boolean',
'deployment_failure_pushover_notifications' => 'sometimes|boolean',
'status_change_pushover_notifications' => 'sometimes|boolean',
'restart_limit_reached_pushover_notifications' => 'sometimes|boolean',
'backup_success_pushover_notifications' => 'sometimes|boolean',
'backup_failure_pushover_notifications' => 'sometimes|boolean',
'scheduled_task_success_pushover_notifications' => 'sometimes|boolean',
@ -167,6 +174,7 @@ private function channelConfig(string $channel): array
'deployment_success_webhook_notifications' => 'sometimes|boolean',
'deployment_failure_webhook_notifications' => 'sometimes|boolean',
'status_change_webhook_notifications' => 'sometimes|boolean',
'restart_limit_reached_webhook_notifications' => 'sometimes|boolean',
'backup_success_webhook_notifications' => 'sometimes|boolean',
'backup_failure_webhook_notifications' => 'sometimes|boolean',
'scheduled_task_success_webhook_notifications' => 'sometimes|boolean',
@ -249,7 +257,7 @@ private function updateChannel(Request $request, string $channel): JsonResponse
$body = $request->json()->all();
$config = $this->channelConfig($channel);
$validator = customApiValidator($body, $config['rules']);
$validator = Validator::make($body, $config['rules']);
$extraFields = array_diff(array_keys($body), $allowedFields);
if ($validator->fails() || ! empty($extraFields)) {

View file

@ -138,7 +138,7 @@ private function shouldDispatchUpdate(Server $server, array $data): bool
/**
* Build a stable hash of container state.
*
* Covers [name, state] only metrics, filesystem_usage_root, and
* Covers [name, state, restart_count] only metrics, filesystem_usage_root, and
* health_status are excluded on purpose. Disk % churns constantly, and
* health checks can flap between starting/healthy/unhealthy while the
* container lifecycle state remains unchanged. Both would otherwise defeat
@ -153,6 +153,7 @@ private function containerStateHash(array $data): string
->map(fn ($c) => [
'name' => data_get($c, 'name'),
'state' => data_get($c, 'state'),
'restart_count' => data_get($c, 'restart_count'),
])
->sortBy('name')
->values()

View file

@ -4972,11 +4972,21 @@ private function handleSuccessfulDeployment(): void
// Reset restart count after successful deployment
// This is done here (not in Livewire) to avoid race conditions
// with GetContainersStatus reading old container restart counts
$this->application->update([
$restartState = [
'restart_count' => 0,
'last_restart_at' => null,
'last_restart_type' => null,
]);
];
if ($this->pull_request_id === 0) {
$restartState['restart_limit_reached'] = false;
}
if ($this->pull_request_id === 0) {
$this->application->update($restartState);
} else {
$this->preview?->resetRestartLimit();
}
try {
$this->application->markDeploymentConfigurationApplied($this->application_deployment_queue);

View file

@ -2,11 +2,15 @@
namespace App\Jobs;
use App\Actions\Application\StopApplication;
use App\Actions\Application\StopApplicationPreview;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Database\StopDatabase;
use App\Actions\Database\StopDatabaseProxy;
use App\Actions\Proxy\CheckProxy;
use App\Actions\Proxy\StartProxy;
use App\Actions\Server\StartLogDrain;
use App\Actions\Service\StopServiceApplication;
use App\Actions\Shared\ComplexStatusCheck;
use App\Models\Application;
use App\Models\ApplicationPreview;
@ -23,8 +27,10 @@
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use App\Models\SwarmDocker;
use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;
use App\Notifications\Container\ContainerRestarted;
use App\Services\ContainerStatusAggregator;
use App\Services\RestartCountTracker;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@ -95,8 +101,14 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
public Collection $applicationContainerStatuses;
public Collection $applicationContainerRestartCounts;
public Collection $serviceContainerStatuses;
public Collection $previewContainerRestartCounts;
public Collection $serviceContainerRestartCounts;
public bool $foundProxy = false;
public bool $foundLogDrainContainer = false;
@ -122,7 +134,10 @@ public function __construct(public Server $server, public $data)
$this->foundApplicationPreviewsIds = collect();
$this->foundServiceDatabaseIds = collect();
$this->applicationContainerStatuses = collect();
$this->applicationContainerRestartCounts = collect();
$this->serviceContainerStatuses = collect();
$this->previewContainerRestartCounts = collect();
$this->serviceContainerRestartCounts = collect();
$this->allApplicationIds = collect();
$this->allDatabaseUuids = collect();
$this->allTcpProxyUuids = collect();
@ -140,7 +155,10 @@ public function handle()
{
// Defensive initialization for Collection properties to handle queue deserialization edge cases
$this->serviceContainerStatuses ??= collect();
$this->previewContainerRestartCounts ??= collect();
$this->serviceContainerRestartCounts ??= collect();
$this->applicationContainerStatuses ??= collect();
$this->applicationContainerRestartCounts ??= collect();
$this->foundApplicationIds ??= collect();
$this->foundDatabaseUuids ??= collect();
$this->foundServiceApplicationIds ??= collect();
@ -231,6 +249,9 @@ public function handle()
if (! $coolify_managed) {
continue;
}
if (filter_var($labels->get('com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)) {
continue;
}
$name = data_get($container, 'name');
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
@ -241,6 +262,10 @@ public function handle()
$pullRequestId = $labels->get('coolify.pullRequestId', '0');
try {
if ($pullRequestId === '0') {
$application = $this->applicationsById->get((string) $applicationId);
if ($application && $application->container_present !== true) {
$application->update(['container_present' => true]);
}
if ($this->allApplicationIds->contains($applicationId)) {
$this->foundApplicationIds->push($applicationId);
}
@ -251,6 +276,13 @@ public function handle()
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
$restartCount = data_get($container, 'restart_count');
if (is_numeric($restartCount)) {
if (! $this->applicationContainerRestartCounts->has($applicationId)) {
$this->applicationContainerRestartCounts->put($applicationId, collect());
}
$this->applicationContainerRestartCounts->get($applicationId)->put($containerName, (int) $restartCount);
}
}
} else {
$previewKey = $applicationId.':'.$pullRequestId;
@ -258,6 +290,13 @@ public function handle()
$this->foundApplicationPreviewsIds->push($previewKey);
}
$this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
$restartCount = data_get($container, 'restart_count');
if (is_numeric($restartCount)) {
$this->previewContainerRestartCounts->push([
'key' => $previewKey,
'count' => (int) $restartCount,
]);
}
}
} catch (\Exception $e) {
}
@ -278,6 +317,7 @@ public function handle()
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
$this->storeServiceRestartCount($key, $containerName, data_get($container, 'restart_count'));
}
} elseif ($subType === 'database') {
$this->foundServiceDatabaseIds->push($subId);
@ -289,6 +329,7 @@ public function handle()
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
$this->storeServiceRestartCount($key, $containerName, data_get($container, 'restart_count'));
}
}
} else {
@ -302,9 +343,9 @@ public function handle()
$this->foundDatabaseUuids->push($uuid);
// TCP proxy should only be started/managed when database is actually running
if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true);
$this->updateDatabaseStatus($uuid, $containerStatus, data_get($container, 'restart_count'), tcpProxy: true);
} else {
$this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: false);
$this->updateDatabaseStatus($uuid, $containerStatus, data_get($container, 'restart_count'), tcpProxy: false);
}
}
}
@ -317,6 +358,9 @@ public function handle()
$this->updateProxyStatus();
Application::whereIn('id', $this->foundApplicationIds->unique())
->update(['container_present' => true]);
$this->updateNotFoundApplicationStatus();
$this->updateNotFoundApplicationPreviewStatus();
$this->updateNotFoundDatabaseStatus();
@ -324,6 +368,8 @@ public function handle()
$this->updateAdditionalServersStatus();
$this->trackPreviewRestartCounts();
// Aggregate multi-container application statuses
$this->aggregateMultiContainerStatuses();
@ -349,11 +395,18 @@ private function loadApplications(): Collection
'uuid',
'name',
'status',
'container_present',
'build_pack',
'docker_compose_raw',
'environment_id',
'destination_id',
'destination_type',
'last_online_at',
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
])
->withCount('additional_servers')
->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
@ -372,11 +425,18 @@ private function loadApplications(): Collection
'uuid',
'name',
'status',
'container_present',
'build_pack',
'docker_compose_raw',
'environment_id',
'destination_id',
'destination_type',
'last_online_at',
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
])
->withCount('additional_servers')
->whereIn('id', $additionalApplicationIds)
@ -402,6 +462,11 @@ private function loadPreviews(): Collection
'pull_request_id',
'status',
'last_online_at',
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
])
->whereIn('application_id', $applicationIds)
->get();
@ -417,8 +482,8 @@ private function loadServices(): Collection
'docker_compose_raw',
])
->with([
'applications:id,service_id,status,last_online_at',
'databases:id,service_id,status,last_online_at,is_public,name',
'applications:id,service_id,status,last_online_at,restart_count,max_restart_count,restart_limit_reached,last_restart_at,last_restart_type',
'databases:id,service_id,status,last_online_at,is_public,name,restart_count,max_restart_count,restart_limit_reached,last_restart_at,last_restart_type',
])
->get();
}
@ -441,6 +506,8 @@ private function loadDatabases(): Collection
'restart_count',
'last_restart_at',
'last_restart_type',
'max_restart_count',
'restart_limit_reached',
];
return collect([
@ -495,6 +562,53 @@ private function aggregateMultiContainerStatuses()
continue;
}
$maxRestartCount = 0;
$restartCountsAvailable = $this->applicationContainerRestartCounts->has($applicationId);
if ($restartCountsAvailable) {
$maxRestartCount = $this->applicationContainerRestartCounts->get($applicationId)->max() ?? 0;
$restartState = (new RestartCountTracker)->evaluate(
previousRestartCount: $application->restart_count ?? 0,
observedRestartCount: $maxRestartCount,
maxRestartCount: $application->max_restart_count ?? 0,
);
if ($restartState['restart_count_changed']) {
$hasCrashRestarts = $restartState['restart_count'] > 0;
$application->update([
'restart_count' => $restartState['restart_count'],
'last_restart_at' => $hasCrashRestarts ? now() : null,
'last_restart_type' => $hasCrashRestarts ? 'crash' : null,
]);
}
if ($restartState['restart_limit_reached']) {
$restartLimitClaimed = Application::query()
->whereKey($application->getKey())
->where('restart_limit_reached', false)
->update(['restart_limit_reached' => true]) === 1;
if ($restartLimitClaimed) {
$application->refresh();
StopApplication::dispatch(
application: $application,
previewDeployments: false,
dockerCleanup: false,
resetRestartCount: false,
removeContainers: false,
);
$application->environment->project->team?->notify(new ApplicationRestartLimitReached($application));
}
}
}
if ($application->stoppedAfterRestartLimit() && $containerStatuses->every(
fn (string $status): bool => str($status)->contains('exited')
)) {
$application->update(['status' => 'exited']);
continue;
}
// Parse docker compose to check for excluded containers
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
@ -519,7 +633,7 @@ private function aggregateMultiContainerStatuses()
// Use ContainerStatusAggregator service for state machine logic
// Use preserveRestarting: true so applications show "Restarting" instead of "Degraded"
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0, preserveRestarting: true);
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, $maxRestartCount, preserveRestarting: true);
// Update application status with aggregated result
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
@ -560,6 +674,14 @@ private function aggregateServiceContainerStatuses()
continue;
}
$restartCount = $this->serviceContainerRestartCounts->get($key)?->max() ?? 0;
if ($subResource->trackRestartCount($restartCount)) {
StopServiceApplication::dispatch($subResource, false, false);
$subResource->team()?->notify(new ApplicationRestartLimitReached($subResource));
continue;
}
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
@ -581,10 +703,9 @@ private function aggregateServiceContainerStatuses()
}
// Use ContainerStatusAggregator service for state machine logic
// NOTE: Sentinel does NOT provide restart count data, so maxRestartCount is always 0
// Use preserveRestarting: true so individual sub-resources show "Restarting" instead of "Degraded"
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, 0, preserveRestarting: true);
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, $restartCount, preserveRestarting: true);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
@ -627,8 +748,11 @@ private function updateNotFoundApplicationStatus()
// Batch update: mark all not-found applications as exited (excluding already exited ones)
Application::whereIn('id', $notFoundApplicationIds)
->where('status', 'not like', 'exited%')
->update(['status' => 'exited']);
->update([
'status' => 'exited',
'container_present' => false,
'restart_limit_reached' => false,
]);
}
private function updateNotFoundApplicationPreviewStatus()
@ -687,7 +811,7 @@ private function updateProxyStatus()
}
}
private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false)
private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, mixed $restartCount = null, bool $tcpProxy = false): void
{
$database = $this->databasesByUuid->get($databaseUuid);
if (! $database) {
@ -697,6 +821,12 @@ private function updateDatabaseStatus(string $databaseUuid, string $containerSta
$database->status = $containerStatus;
$database->save();
}
if (is_numeric($restartCount) && $database->trackRestartCount((int) $restartCount)) {
StopDatabase::dispatch($database, false, false, false);
$database->team()?->notify(new ApplicationRestartLimitReached($database));
return;
}
if (! $this->isCompleteSnapshot()) {
return;
}
@ -719,6 +849,30 @@ private function updateDatabaseStatus(string $databaseUuid, string $containerSta
}
}
private function storeServiceRestartCount(string $key, string $containerName, mixed $restartCount): void
{
if (! is_numeric($restartCount)) {
return;
}
if (! $this->serviceContainerRestartCounts->has($key)) {
$this->serviceContainerRestartCounts->put($key, collect());
}
$this->serviceContainerRestartCounts->get($key)->put($containerName, (int) $restartCount);
}
private function trackPreviewRestartCounts(): void
{
$this->previewContainerRestartCounts
->groupBy('key')
->each(function (Collection $counts, string $key): void {
$preview = $this->previewsByKey->get($key);
if ($preview?->trackRestartCount((int) $counts->max('count'))) {
StopApplicationPreview::dispatch($preview, false, false);
$preview->application->environment->project->team?->notify(new ApplicationRestartLimitReached($preview));
}
});
}
private function updateNotFoundDatabaseStatus()
{
$notFoundDatabaseUuids = $this->allDatabaseUuids->diff($this->foundDatabaseUuids);
@ -729,12 +883,16 @@ private function updateNotFoundDatabaseStatus()
$notFoundDatabaseUuids->each(function ($databaseUuid) {
$database = $this->databasesByUuid->get($databaseUuid);
if ($database) {
if ($database->stoppedAfterRestartLimit()) {
return;
}
if (! str($database->status)->startsWith('exited')) {
$database->update([
'status' => 'exited',
'restart_count' => 0,
'last_restart_at' => null,
'last_restart_type' => null,
'restart_limit_reached' => false,
]);
}
if ($database->is_public) {
@ -752,15 +910,17 @@ private function updateNotFoundServiceStatus()
// Batch update service applications
if ($notFoundServiceApplicationIds->isNotEmpty()) {
ServiceApplication::whereIn('id', $notFoundServiceApplicationIds)
->where('restart_limit_reached', false)
->where('status', '!=', 'exited')
->update(['status' => 'exited']);
->update(['status' => 'exited', 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null]);
}
// Batch update service databases
if ($notFoundServiceDatabaseIds->isNotEmpty()) {
ServiceDatabase::whereIn('id', $notFoundServiceDatabaseIds)
->where('restart_limit_reached', false)
->where('status', '!=', 'exited')
->update(['status' => 'exited']);
->update(['status' => 'exited', 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null]);
}
}

View file

@ -34,6 +34,9 @@ class Discord extends Component
#[Validate(['boolean'])]
public bool $statusChangeDiscordNotifications = false;
#[Validate(['boolean'])]
public bool $restartLimitReachedDiscordNotifications = true;
#[Validate(['boolean'])]
public bool $backupSuccessDiscordNotifications = false;
@ -93,6 +96,7 @@ public function syncData(bool $toModel = false)
$this->settings->deployment_success_discord_notifications = $this->deploymentSuccessDiscordNotifications;
$this->settings->deployment_failure_discord_notifications = $this->deploymentFailureDiscordNotifications;
$this->settings->status_change_discord_notifications = $this->statusChangeDiscordNotifications;
$this->settings->restart_limit_reached_discord_notifications = $this->restartLimitReachedDiscordNotifications;
$this->settings->backup_success_discord_notifications = $this->backupSuccessDiscordNotifications;
$this->settings->backup_failure_discord_notifications = $this->backupFailureDiscordNotifications;
$this->settings->scheduled_task_success_discord_notifications = $this->scheduledTaskSuccessDiscordNotifications;
@ -118,6 +122,7 @@ public function syncData(bool $toModel = false)
$this->deploymentSuccessDiscordNotifications = $this->settings->deployment_success_discord_notifications;
$this->deploymentFailureDiscordNotifications = $this->settings->deployment_failure_discord_notifications;
$this->statusChangeDiscordNotifications = $this->settings->status_change_discord_notifications;
$this->restartLimitReachedDiscordNotifications = $this->settings->restart_limit_reached_discord_notifications;
$this->backupSuccessDiscordNotifications = $this->settings->backup_success_discord_notifications;
$this->backupFailureDiscordNotifications = $this->settings->backup_failure_discord_notifications;
$this->scheduledTaskSuccessDiscordNotifications = $this->settings->scheduled_task_success_discord_notifications;

View file

@ -79,6 +79,9 @@ class Email extends Component
#[Validate(['boolean'])]
public bool $statusChangeEmailNotifications = false;
#[Validate(['boolean'])]
public bool $restartLimitReachedEmailNotifications = true;
#[Validate(['boolean'])]
public bool $backupSuccessEmailNotifications = false;
@ -155,6 +158,7 @@ public function syncData(bool $toModel = false)
$this->settings->deployment_success_email_notifications = $this->deploymentSuccessEmailNotifications;
$this->settings->deployment_failure_email_notifications = $this->deploymentFailureEmailNotifications;
$this->settings->status_change_email_notifications = $this->statusChangeEmailNotifications;
$this->settings->restart_limit_reached_email_notifications = $this->restartLimitReachedEmailNotifications;
$this->settings->backup_success_email_notifications = $this->backupSuccessEmailNotifications;
$this->settings->backup_failure_email_notifications = $this->backupFailureEmailNotifications;
$this->settings->scheduled_task_success_email_notifications = $this->scheduledTaskSuccessEmailNotifications;
@ -193,6 +197,7 @@ public function syncData(bool $toModel = false)
$this->deploymentSuccessEmailNotifications = $this->settings->deployment_success_email_notifications;
$this->deploymentFailureEmailNotifications = $this->settings->deployment_failure_email_notifications;
$this->statusChangeEmailNotifications = $this->settings->status_change_email_notifications;
$this->restartLimitReachedEmailNotifications = $this->settings->restart_limit_reached_email_notifications;
$this->backupSuccessEmailNotifications = $this->settings->backup_success_email_notifications;
$this->backupFailureEmailNotifications = $this->settings->backup_failure_email_notifications;
$this->scheduledTaskSuccessEmailNotifications = $this->settings->scheduled_task_success_email_notifications;

View file

@ -41,6 +41,9 @@ class Pushover extends Component
#[Validate(['boolean'])]
public bool $statusChangePushoverNotifications = false;
#[Validate(['boolean'])]
public bool $restartLimitReachedPushoverNotifications = true;
#[Validate(['boolean'])]
public bool $backupSuccessPushoverNotifications = false;
@ -98,6 +101,7 @@ public function syncData(bool $toModel = false)
$this->settings->deployment_success_pushover_notifications = $this->deploymentSuccessPushoverNotifications;
$this->settings->deployment_failure_pushover_notifications = $this->deploymentFailurePushoverNotifications;
$this->settings->status_change_pushover_notifications = $this->statusChangePushoverNotifications;
$this->settings->restart_limit_reached_pushover_notifications = $this->restartLimitReachedPushoverNotifications;
$this->settings->backup_success_pushover_notifications = $this->backupSuccessPushoverNotifications;
$this->settings->backup_failure_pushover_notifications = $this->backupFailurePushoverNotifications;
$this->settings->scheduled_task_success_pushover_notifications = $this->scheduledTaskSuccessPushoverNotifications;
@ -125,6 +129,7 @@ public function syncData(bool $toModel = false)
$this->deploymentSuccessPushoverNotifications = $this->settings->deployment_success_pushover_notifications;
$this->deploymentFailurePushoverNotifications = $this->settings->deployment_failure_pushover_notifications;
$this->statusChangePushoverNotifications = $this->settings->status_change_pushover_notifications;
$this->restartLimitReachedPushoverNotifications = $this->settings->restart_limit_reached_pushover_notifications;
$this->backupSuccessPushoverNotifications = $this->settings->backup_success_pushover_notifications;
$this->backupFailurePushoverNotifications = $this->settings->backup_failure_pushover_notifications;
$this->scheduledTaskSuccessPushoverNotifications = $this->settings->scheduled_task_success_pushover_notifications;

View file

@ -39,6 +39,9 @@ class Slack extends Component
#[Validate(['boolean'])]
public bool $statusChangeSlackNotifications = false;
#[Validate(['boolean'])]
public bool $restartLimitReachedSlackNotifications = true;
#[Validate(['boolean'])]
public bool $backupSuccessSlackNotifications = false;
@ -95,6 +98,7 @@ public function syncData(bool $toModel = false)
$this->settings->deployment_success_slack_notifications = $this->deploymentSuccessSlackNotifications;
$this->settings->deployment_failure_slack_notifications = $this->deploymentFailureSlackNotifications;
$this->settings->status_change_slack_notifications = $this->statusChangeSlackNotifications;
$this->settings->restart_limit_reached_slack_notifications = $this->restartLimitReachedSlackNotifications;
$this->settings->backup_success_slack_notifications = $this->backupSuccessSlackNotifications;
$this->settings->backup_failure_slack_notifications = $this->backupFailureSlackNotifications;
$this->settings->scheduled_task_success_slack_notifications = $this->scheduledTaskSuccessSlackNotifications;
@ -118,6 +122,7 @@ public function syncData(bool $toModel = false)
$this->deploymentSuccessSlackNotifications = $this->settings->deployment_success_slack_notifications;
$this->deploymentFailureSlackNotifications = $this->settings->deployment_failure_slack_notifications;
$this->statusChangeSlackNotifications = $this->settings->status_change_slack_notifications;
$this->restartLimitReachedSlackNotifications = $this->settings->restart_limit_reached_slack_notifications;
$this->backupSuccessSlackNotifications = $this->settings->backup_success_slack_notifications;
$this->backupFailureSlackNotifications = $this->settings->backup_failure_slack_notifications;
$this->scheduledTaskSuccessSlackNotifications = $this->settings->scheduled_task_success_slack_notifications;

View file

@ -41,6 +41,9 @@ class Telegram extends Component
#[Validate(['boolean'])]
public bool $statusChangeTelegramNotifications = false;
#[Validate(['boolean'])]
public bool $restartLimitReachedTelegramNotifications = true;
#[Validate(['boolean'])]
public bool $backupSuccessTelegramNotifications = false;
@ -83,6 +86,9 @@ class Telegram extends Component
#[Validate(['nullable', 'string'])]
public ?string $telegramNotificationsStatusChangeThreadId = null;
#[Validate(['nullable', 'string', 'max:255'])]
public ?string $telegramNotificationsRestartLimitReachedThreadId = null;
#[Validate(['nullable', 'string'])]
public ?string $telegramNotificationsBackupSuccessThreadId = null;
@ -140,6 +146,7 @@ public function syncData(bool $toModel = false)
$this->settings->deployment_success_telegram_notifications = $this->deploymentSuccessTelegramNotifications;
$this->settings->deployment_failure_telegram_notifications = $this->deploymentFailureTelegramNotifications;
$this->settings->status_change_telegram_notifications = $this->statusChangeTelegramNotifications;
$this->settings->restart_limit_reached_telegram_notifications = $this->restartLimitReachedTelegramNotifications;
$this->settings->backup_success_telegram_notifications = $this->backupSuccessTelegramNotifications;
$this->settings->backup_failure_telegram_notifications = $this->backupFailureTelegramNotifications;
$this->settings->scheduled_task_success_telegram_notifications = $this->scheduledTaskSuccessTelegramNotifications;
@ -155,6 +162,7 @@ public function syncData(bool $toModel = false)
$this->settings->telegram_notifications_deployment_success_thread_id = $this->telegramNotificationsDeploymentSuccessThreadId;
$this->settings->telegram_notifications_deployment_failure_thread_id = $this->telegramNotificationsDeploymentFailureThreadId;
$this->settings->telegram_notifications_status_change_thread_id = $this->telegramNotificationsStatusChangeThreadId;
$this->settings->telegram_notifications_restart_limit_reached_thread_id = $this->telegramNotificationsRestartLimitReachedThreadId;
$this->settings->telegram_notifications_backup_success_thread_id = $this->telegramNotificationsBackupSuccessThreadId;
$this->settings->telegram_notifications_backup_failure_thread_id = $this->telegramNotificationsBackupFailureThreadId;
$this->settings->telegram_notifications_scheduled_task_success_thread_id = $this->telegramNotificationsScheduledTaskSuccessThreadId;
@ -173,6 +181,21 @@ public function syncData(bool $toModel = false)
if (auth()->user()->can('update', $this->settings)) {
$this->telegramToken = $this->settings->telegram_token;
$this->telegramChatId = $this->settings->telegram_chat_id;
$this->telegramNotificationsDeploymentSuccessThreadId = $this->settings->telegram_notifications_deployment_success_thread_id;
$this->telegramNotificationsDeploymentFailureThreadId = $this->settings->telegram_notifications_deployment_failure_thread_id;
$this->telegramNotificationsStatusChangeThreadId = $this->settings->telegram_notifications_status_change_thread_id;
$this->telegramNotificationsRestartLimitReachedThreadId = $this->settings->telegram_notifications_restart_limit_reached_thread_id;
$this->telegramNotificationsBackupSuccessThreadId = $this->settings->telegram_notifications_backup_success_thread_id;
$this->telegramNotificationsBackupFailureThreadId = $this->settings->telegram_notifications_backup_failure_thread_id;
$this->telegramNotificationsScheduledTaskSuccessThreadId = $this->settings->telegram_notifications_scheduled_task_success_thread_id;
$this->telegramNotificationsScheduledTaskFailureThreadId = $this->settings->telegram_notifications_scheduled_task_failure_thread_id;
$this->telegramNotificationsDockerCleanupSuccessThreadId = $this->settings->telegram_notifications_docker_cleanup_success_thread_id;
$this->telegramNotificationsDockerCleanupFailureThreadId = $this->settings->telegram_notifications_docker_cleanup_failure_thread_id;
$this->telegramNotificationsServerDiskUsageThreadId = $this->settings->telegram_notifications_server_disk_usage_thread_id;
$this->telegramNotificationsServerReachableThreadId = $this->settings->telegram_notifications_server_reachable_thread_id;
$this->telegramNotificationsServerUnreachableThreadId = $this->settings->telegram_notifications_server_unreachable_thread_id;
$this->telegramNotificationsServerPatchThreadId = $this->settings->telegram_notifications_server_patch_thread_id;
$this->telegramNotificationsTraefikOutdatedThreadId = $this->settings->telegram_notifications_traefik_outdated_thread_id;
} else {
$this->telegramToken = null;
$this->telegramChatId = null;
@ -181,6 +204,7 @@ public function syncData(bool $toModel = false)
$this->deploymentSuccessTelegramNotifications = $this->settings->deployment_success_telegram_notifications;
$this->deploymentFailureTelegramNotifications = $this->settings->deployment_failure_telegram_notifications;
$this->statusChangeTelegramNotifications = $this->settings->status_change_telegram_notifications;
$this->restartLimitReachedTelegramNotifications = $this->settings->restart_limit_reached_telegram_notifications;
$this->backupSuccessTelegramNotifications = $this->settings->backup_success_telegram_notifications;
$this->backupFailureTelegramNotifications = $this->settings->backup_failure_telegram_notifications;
$this->scheduledTaskSuccessTelegramNotifications = $this->settings->scheduled_task_success_telegram_notifications;
@ -193,20 +217,6 @@ public function syncData(bool $toModel = false)
$this->serverPatchTelegramNotifications = $this->settings->server_patch_telegram_notifications;
$this->traefikOutdatedTelegramNotifications = $this->settings->traefik_outdated_telegram_notifications;
$this->telegramNotificationsDeploymentSuccessThreadId = $this->settings->telegram_notifications_deployment_success_thread_id;
$this->telegramNotificationsDeploymentFailureThreadId = $this->settings->telegram_notifications_deployment_failure_thread_id;
$this->telegramNotificationsStatusChangeThreadId = $this->settings->telegram_notifications_status_change_thread_id;
$this->telegramNotificationsBackupSuccessThreadId = $this->settings->telegram_notifications_backup_success_thread_id;
$this->telegramNotificationsBackupFailureThreadId = $this->settings->telegram_notifications_backup_failure_thread_id;
$this->telegramNotificationsScheduledTaskSuccessThreadId = $this->settings->telegram_notifications_scheduled_task_success_thread_id;
$this->telegramNotificationsScheduledTaskFailureThreadId = $this->settings->telegram_notifications_scheduled_task_failure_thread_id;
$this->telegramNotificationsDockerCleanupSuccessThreadId = $this->settings->telegram_notifications_docker_cleanup_success_thread_id;
$this->telegramNotificationsDockerCleanupFailureThreadId = $this->settings->telegram_notifications_docker_cleanup_failure_thread_id;
$this->telegramNotificationsServerDiskUsageThreadId = $this->settings->telegram_notifications_server_disk_usage_thread_id;
$this->telegramNotificationsServerReachableThreadId = $this->settings->telegram_notifications_server_reachable_thread_id;
$this->telegramNotificationsServerUnreachableThreadId = $this->settings->telegram_notifications_server_unreachable_thread_id;
$this->telegramNotificationsServerPatchThreadId = $this->settings->telegram_notifications_server_patch_thread_id;
$this->telegramNotificationsTraefikOutdatedThreadId = $this->settings->telegram_notifications_traefik_outdated_thread_id;
}
}

View file

@ -34,6 +34,9 @@ class Webhook extends Component
#[Validate(['boolean'])]
public bool $statusChangeWebhookNotifications = false;
#[Validate(['boolean'])]
public bool $restartLimitReachedWebhookNotifications = true;
#[Validate(['boolean'])]
public bool $backupSuccessWebhookNotifications = false;
@ -90,6 +93,7 @@ public function syncData(bool $toModel = false)
$this->settings->deployment_success_webhook_notifications = $this->deploymentSuccessWebhookNotifications;
$this->settings->deployment_failure_webhook_notifications = $this->deploymentFailureWebhookNotifications;
$this->settings->status_change_webhook_notifications = $this->statusChangeWebhookNotifications;
$this->settings->restart_limit_reached_webhook_notifications = $this->restartLimitReachedWebhookNotifications;
$this->settings->backup_success_webhook_notifications = $this->backupSuccessWebhookNotifications;
$this->settings->backup_failure_webhook_notifications = $this->backupFailureWebhookNotifications;
$this->settings->scheduled_task_success_webhook_notifications = $this->scheduledTaskSuccessWebhookNotifications;
@ -113,6 +117,7 @@ public function syncData(bool $toModel = false)
$this->deploymentSuccessWebhookNotifications = $this->settings->deployment_success_webhook_notifications;
$this->deploymentFailureWebhookNotifications = $this->settings->deployment_failure_webhook_notifications;
$this->statusChangeWebhookNotifications = $this->settings->status_change_webhook_notifications;
$this->restartLimitReachedWebhookNotifications = $this->settings->restart_limit_reached_webhook_notifications;
$this->backupSuccessWebhookNotifications = $this->settings->backup_success_webhook_notifications;
$this->backupFailureWebhookNotifications = $this->settings->backup_failure_webhook_notifications;
$this->scheduledTaskSuccessWebhookNotifications = $this->settings->scheduled_task_success_webhook_notifications;

View file

@ -187,6 +187,11 @@ private function toSearchableArray(Collection $items, string $type, string $type
'fqdn' => $item->fqdn ?? null,
'description' => $item->description ?? null,
'status' => $item->status ?? '',
'restartLimitReached' => method_exists($item, 'stoppedAfterRestartLimit') && $item->stoppedAfterRestartLimit(),
'restartCount' => method_exists($item, 'stoppedAfterRestartLimit') && $item->stoppedAfterRestartLimit()
? max($item->restart_count ?? 0, $item->max_restart_count ?? 0)
: ($item->restart_count ?? 0),
'maxRestartCount' => $item->max_restart_count ?? 0,
'server_status' => $item->server_status ?? null,
'hrefLink' => $item->hrefLink ?? '',
'destination' => [

View file

@ -5,8 +5,11 @@
use App\Actions\Docker\GetContainersStatus;
use App\Actions\Service\StartService;
use App\Actions\Service\StopService;
use App\Actions\Service\StopServiceApplication;
use App\Enums\ProcessStatus;
use App\Models\Service;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Facades\Auth;
use Livewire\Component;
@ -169,6 +172,29 @@ public function restart()
}
}
public function removeSelectedResourceContainer(): void
{
$resource = $this->selectedResource();
if (! $resource) {
return;
}
$this->authorize('update', $resource);
StopServiceApplication::run($resource, true, true);
$this->dispatch('success', 'Container removed.');
}
private function selectedResource(): ServiceApplication|ServiceDatabase|null
{
$uuid = data_get($this->parameters, 'stack_service_uuid');
if (! $uuid) {
return null;
}
return $this->service->applications()->whereUuid($uuid)->first()
?? $this->service->databases()->whereUuid($uuid)->first();
}
public function pullAndRestartEvent()
{
try {

View file

@ -10,6 +10,13 @@ class Status extends Component
{
public Service $service;
public ?string $selectedResourceUuid = null;
public function mount(): void
{
$this->selectedResourceUuid = request()->route('stack_service_uuid');
}
public function getListeners(): array
{
$teamId = auth()->user()->currentTeam()->id;
@ -27,6 +34,11 @@ public function refreshStatus(): void
public function render(): View
{
return view('livewire.project.service.status');
$selectedResource = $this->selectedResourceUuid
? $this->service->applications->firstWhere('uuid', $this->selectedResourceUuid)
?? $this->service->databases->firstWhere('uuid', $this->selectedResourceUuid)
: null;
return view('livewire.project.service.status', compact('selectedResource'));
}
}

View file

@ -213,6 +213,8 @@ class Application extends BaseModel
'last_online_at',
'restart_count',
'max_restart_count',
'restart_limit_reached',
'container_present',
'last_restart_at',
'last_restart_type',
'uuid',
@ -258,6 +260,8 @@ protected function casts(): array
'domain_dns_statuses' => 'array',
'restart_count' => 'integer',
'max_restart_count' => 'integer',
'restart_limit_reached' => 'boolean',
'container_present' => 'boolean',
'last_restart_at' => 'datetime',
];
}
@ -605,10 +609,8 @@ public function link()
public function stoppedAfterRestartLimit(): bool
{
return str($this->status)->startsWith('exited')
&& ($this->restart_count ?? 0) > 0
&& ($this->max_restart_count ?? 0) > 0
&& $this->restart_count >= $this->max_restart_count
&& $this->last_restart_type === 'crash';
&& $this->container_present === true
&& $this->restart_limit_reached === true;
}
public function taskLink($task_uuid)

View file

@ -3,13 +3,14 @@
namespace App\Models;
use App\Support\ValidationPatterns;
use App\Traits\HasRestartLimit;
use Illuminate\Database\Eloquent\SoftDeletes;
use RuntimeException;
use Spatie\Url\Url;
class ApplicationPreview extends BaseModel
{
use SoftDeletes;
use HasRestartLimit, SoftDeletes;
protected $fillable = [
'uuid',
@ -102,6 +103,11 @@ public function application()
return $this->belongsTo(Application::class);
}
public function restartLimitMaximum(): int
{
return $this->application->max_restart_count ?? $this->max_restart_count ?? 0;
}
public function persistentStorages()
{
return $this->morphMany(LocalPersistentVolume::class, 'resource');

View file

@ -20,6 +20,7 @@ class DiscordNotificationSettings extends Model
'deployment_success_discord_notifications',
'deployment_failure_discord_notifications',
'status_change_discord_notifications',
'restart_limit_reached_discord_notifications',
'backup_success_discord_notifications',
'backup_failure_discord_notifications',
'scheduled_task_success_discord_notifications',
@ -45,6 +46,7 @@ class DiscordNotificationSettings extends Model
'deployment_success_discord_notifications' => 'boolean',
'deployment_failure_discord_notifications' => 'boolean',
'status_change_discord_notifications' => 'boolean',
'restart_limit_reached_discord_notifications' => 'boolean',
'backup_success_discord_notifications' => 'boolean',
'backup_failure_discord_notifications' => 'boolean',
'scheduled_task_success_discord_notifications' => 'boolean',

View file

@ -31,6 +31,7 @@ class EmailNotificationSettings extends Model
'deployment_success_email_notifications',
'deployment_failure_email_notifications',
'status_change_email_notifications',
'restart_limit_reached_email_notifications',
'backup_success_email_notifications',
'backup_failure_email_notifications',
'scheduled_task_success_email_notifications',
@ -73,6 +74,7 @@ class EmailNotificationSettings extends Model
'deployment_success_email_notifications' => 'boolean',
'deployment_failure_email_notifications' => 'boolean',
'status_change_email_notifications' => 'boolean',
'restart_limit_reached_email_notifications' => 'boolean',
'backup_success_email_notifications' => 'boolean',
'backup_failure_email_notifications' => 'boolean',
'scheduled_task_success_email_notifications' => 'boolean',

View file

@ -21,6 +21,7 @@ class PushoverNotificationSettings extends Model
'deployment_success_pushover_notifications',
'deployment_failure_pushover_notifications',
'status_change_pushover_notifications',
'restart_limit_reached_pushover_notifications',
'backup_success_pushover_notifications',
'backup_failure_pushover_notifications',
'scheduled_task_success_pushover_notifications',
@ -47,6 +48,7 @@ class PushoverNotificationSettings extends Model
'deployment_success_pushover_notifications' => 'boolean',
'deployment_failure_pushover_notifications' => 'boolean',
'status_change_pushover_notifications' => 'boolean',
'restart_limit_reached_pushover_notifications' => 'boolean',
'backup_success_pushover_notifications' => 'boolean',
'backup_failure_pushover_notifications' => 'boolean',
'scheduled_task_success_pushover_notifications' => 'boolean',

View file

@ -3,6 +3,7 @@
namespace App\Models;
use App\Traits\HasNoindexDomains;
use App\Traits\HasRestartLimit;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\SoftDeletes;
@ -10,7 +11,7 @@
class ServiceApplication extends BaseModel
{
use HasFactory, HasNoindexDomains, SoftDeletes;
use HasFactory, HasNoindexDomains, HasRestartLimit, SoftDeletes;
protected $fillable = [
'service_id',

View file

@ -2,12 +2,13 @@
namespace App\Models;
use App\Traits\HasRestartLimit;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\SoftDeletes;
class ServiceDatabase extends BaseModel
{
use HasFactory, SoftDeletes;
use HasFactory, HasRestartLimit, SoftDeletes;
protected $fillable = [
'service_id',

View file

@ -20,6 +20,7 @@ class SlackNotificationSettings extends Model
'deployment_success_slack_notifications',
'deployment_failure_slack_notifications',
'status_change_slack_notifications',
'restart_limit_reached_slack_notifications',
'backup_success_slack_notifications',
'backup_failure_slack_notifications',
'scheduled_task_success_slack_notifications',
@ -44,6 +45,7 @@ class SlackNotificationSettings extends Model
'deployment_success_slack_notifications' => 'boolean',
'deployment_failure_slack_notifications' => 'boolean',
'status_change_slack_notifications' => 'boolean',
'restart_limit_reached_slack_notifications' => 'boolean',
'backup_success_slack_notifications' => 'boolean',
'backup_failure_slack_notifications' => 'boolean',
'scheduled_task_success_slack_notifications' => 'boolean',

View file

@ -5,6 +5,7 @@
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasDatabaseHealthCheck;
use App\Traits\HasMetrics;
use App\Traits\HasRestartLimit;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
@ -12,7 +13,7 @@
class StandaloneClickhouse extends BaseModel
{
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes;
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes;
protected $fillable = [
'uuid',

View file

@ -5,6 +5,7 @@
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasDatabaseHealthCheck;
use App\Traits\HasMetrics;
use App\Traits\HasRestartLimit;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
@ -12,7 +13,7 @@
class StandaloneDragonfly extends BaseModel
{
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes;
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes;
protected $fillable = [
'uuid',

View file

@ -5,6 +5,7 @@
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasDatabaseHealthCheck;
use App\Traits\HasMetrics;
use App\Traits\HasRestartLimit;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
@ -12,7 +13,7 @@
class StandaloneKeydb extends BaseModel
{
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes;
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes;
protected $fillable = [
'uuid',

View file

@ -5,6 +5,7 @@
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasDatabaseHealthCheck;
use App\Traits\HasMetrics;
use App\Traits\HasRestartLimit;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
@ -13,7 +14,7 @@
class StandaloneMariadb extends BaseModel
{
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes;
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes;
protected $fillable = [
'uuid',

View file

@ -5,6 +5,7 @@
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasDatabaseHealthCheck;
use App\Traits\HasMetrics;
use App\Traits\HasRestartLimit;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
@ -12,7 +13,7 @@
class StandaloneMongodb extends BaseModel
{
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes;
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes;
protected $fillable = [
'uuid',

View file

@ -5,6 +5,7 @@
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasDatabaseHealthCheck;
use App\Traits\HasMetrics;
use App\Traits\HasRestartLimit;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
@ -12,7 +13,7 @@
class StandaloneMysql extends BaseModel
{
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes;
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes;
protected $fillable = [
'uuid',

View file

@ -5,6 +5,7 @@
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasDatabaseHealthCheck;
use App\Traits\HasMetrics;
use App\Traits\HasRestartLimit;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
@ -12,7 +13,7 @@
class StandalonePostgresql extends BaseModel
{
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes;
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes;
protected $fillable = [
'uuid',

View file

@ -5,6 +5,7 @@
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasDatabaseHealthCheck;
use App\Traits\HasMetrics;
use App\Traits\HasRestartLimit;
use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
@ -12,7 +13,7 @@
class StandaloneRedis extends BaseModel
{
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasSafeStringAttribute, SoftDeletes;
use ClearsGlobalSearchCache, HasDatabaseHealthCheck, HasFactory, HasMetrics, HasRestartLimit, HasSafeStringAttribute, SoftDeletes;
protected $fillable = [
'uuid',

View file

@ -21,6 +21,7 @@ class TelegramNotificationSettings extends Model
'deployment_success_telegram_notifications',
'deployment_failure_telegram_notifications',
'status_change_telegram_notifications',
'restart_limit_reached_telegram_notifications',
'backup_success_telegram_notifications',
'backup_failure_telegram_notifications',
'scheduled_task_success_telegram_notifications',
@ -36,6 +37,7 @@ class TelegramNotificationSettings extends Model
'telegram_notifications_deployment_success_thread_id',
'telegram_notifications_deployment_failure_thread_id',
'telegram_notifications_status_change_thread_id',
'telegram_notifications_restart_limit_reached_thread_id',
'telegram_notifications_backup_success_thread_id',
'telegram_notifications_backup_failure_thread_id',
'telegram_notifications_scheduled_task_success_thread_id',
@ -55,6 +57,7 @@ class TelegramNotificationSettings extends Model
'telegram_notifications_deployment_success_thread_id',
'telegram_notifications_deployment_failure_thread_id',
'telegram_notifications_status_change_thread_id',
'telegram_notifications_restart_limit_reached_thread_id',
'telegram_notifications_backup_success_thread_id',
'telegram_notifications_backup_failure_thread_id',
'telegram_notifications_scheduled_task_success_thread_id',
@ -76,6 +79,7 @@ class TelegramNotificationSettings extends Model
'deployment_success_telegram_notifications' => 'boolean',
'deployment_failure_telegram_notifications' => 'boolean',
'status_change_telegram_notifications' => 'boolean',
'restart_limit_reached_telegram_notifications' => 'boolean',
'backup_success_telegram_notifications' => 'boolean',
'backup_failure_telegram_notifications' => 'boolean',
'scheduled_task_success_telegram_notifications' => 'boolean',
@ -90,6 +94,7 @@ class TelegramNotificationSettings extends Model
'telegram_notifications_deployment_success_thread_id' => 'encrypted',
'telegram_notifications_deployment_failure_thread_id' => 'encrypted',
'telegram_notifications_status_change_thread_id' => 'encrypted',
'telegram_notifications_restart_limit_reached_thread_id' => 'encrypted',
'telegram_notifications_backup_success_thread_id' => 'encrypted',
'telegram_notifications_backup_failure_thread_id' => 'encrypted',
'telegram_notifications_scheduled_task_success_thread_id' => 'encrypted',

View file

@ -20,6 +20,7 @@ class WebhookNotificationSettings extends Model
'deployment_success_webhook_notifications',
'deployment_failure_webhook_notifications',
'status_change_webhook_notifications',
'restart_limit_reached_webhook_notifications',
'backup_success_webhook_notifications',
'backup_failure_webhook_notifications',
'scheduled_task_success_webhook_notifications',
@ -46,6 +47,7 @@ protected function casts(): array
'deployment_success_webhook_notifications' => 'boolean',
'deployment_failure_webhook_notifications' => 'boolean',
'status_change_webhook_notifications' => 'boolean',
'restart_limit_reached_webhook_notifications' => 'boolean',
'backup_success_webhook_notifications' => 'boolean',
'backup_failure_webhook_notifications' => 'boolean',
'scheduled_task_success_webhook_notifications' => 'boolean',

View file

@ -2,7 +2,8 @@
namespace App\Notifications\Application;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\BaseModel;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
@ -27,26 +28,40 @@ class RestartLimitReached extends CustomEmailNotification
public int $max_restart_count;
public function __construct(public Application $resource)
public function __construct(public BaseModel $resource)
{
$this->onQueue('high');
$this->afterCommit();
$this->resource_name = data_get($resource, 'name');
$this->project_uuid = data_get($resource, 'environment.project.uuid');
$this->environment_uuid = data_get($resource, 'environment.uuid');
$this->environment_name = data_get($resource, 'environment.name');
$environment = data_get($resource, 'environment')
?? data_get($resource, 'application.environment')
?? data_get($resource, 'service.environment');
$this->resource_name = $resource instanceof ApplicationPreview
? data_get($resource, 'application.name').' PR #'.$resource->pull_request_id
: data_get($resource, 'name');
$this->project_uuid = data_get($environment, 'project.uuid');
$this->environment_uuid = data_get($environment, 'uuid');
$this->environment_name = data_get($environment, 'name');
$this->fqdn = data_get($resource, 'fqdn', null);
$this->restart_count = $resource->restart_count;
$this->max_restart_count = $resource->max_restart_count;
$this->max_restart_count = method_exists($resource, 'restartLimitMaximum')
? $resource->restartLimitMaximum()
: $resource->max_restart_count;
if (str($this->fqdn)->explode(',')->count() > 1) {
$this->fqdn = str($this->fqdn)->explode(',')->first();
}
$this->resource_url = $this->resource->link() ?? base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}/application/{$this->resource->uuid}";
$service = data_get($resource, 'service');
$this->resource_url = match (true) {
method_exists($this->resource, 'link') => $this->resource->link(),
$resource instanceof ApplicationPreview => $resource->application->link(),
is_object($service) && method_exists($service, 'link') => $service->link(),
default => null,
};
$this->resource_url ??= base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}";
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels('status_change');
return $notifiable->getEnabledChannels('restart_limit_reached');
}
public function toMail(): MailMessage
@ -68,7 +83,7 @@ public function toDiscord(): DiscordMessage
{
return new DiscordMessage(
title: ':warning: Restart limit reached',
description: "{$this->resource_name} has been stopped after {$this->restart_count} restarts (limit: {$this->max_restart_count}).\n\n[Open Application in Coolify]({$this->resource_url})",
description: "{$this->resource_name} has been stopped after {$this->restart_count} restarts (limit: {$this->max_restart_count}).\n\n[Open Resource in Coolify]({$this->resource_url})",
color: DiscordMessage::errorColor(),
isCritical: true,
);
@ -82,7 +97,7 @@ public function toTelegram(): array
'message' => $message,
'buttons' => [
[
'text' => 'Open Application in Coolify',
'text' => 'Open Resource in Coolify',
'url' => $this->resource_url,
],
],
@ -99,7 +114,7 @@ public function toPushover(): PushoverMessage
message: $message,
buttons: [
[
'text' => 'Open Application in Coolify',
'text' => 'Open Resource in Coolify',
'url' => $this->resource_url,
],
],
@ -110,10 +125,13 @@ public function toSlack(): SlackMessage
{
$title = 'Restart limit reached';
$description = "{$this->resource_name} has been stopped after {$this->restart_count} restarts (limit: {$this->max_restart_count})";
$environment = data_get($this->resource, 'environment')
?? data_get($this->resource, 'application.environment')
?? data_get($this->resource, 'service.environment');
$description .= "\n\n*Project:* ".data_get($this->resource, 'environment.project.name');
$description .= "\n\n*Project:* ".data_get($environment, 'project.name');
$description .= "\n*Environment:* {$this->environment_name}";
$description .= "\n*Application URL:* {$this->resource_url}";
$description .= "\n*Resource URL:* {$this->resource_url}";
return new SlackMessage(
title: $title,
@ -130,6 +148,8 @@ public function toWebhook(): array
'event' => 'restart_limit_reached',
'application_name' => $this->resource_name,
'application_uuid' => $this->resource->uuid,
'resource_name' => $this->resource_name,
'resource_uuid' => $this->resource->uuid,
'restart_count' => $this->restart_count,
'max_restart_count' => $this->max_restart_count,
'url' => $this->resource_url,

View file

@ -3,6 +3,21 @@
namespace App\Notifications\Channels;
use App\Jobs\SendMessageToTelegramJob;
use App\Notifications\Application\DeploymentFailed;
use App\Notifications\Application\DeploymentSuccess;
use App\Notifications\Application\RestartLimitReached;
use App\Notifications\Application\StatusChanged;
use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Database\BackupFailed;
use App\Notifications\Database\BackupSuccess;
use App\Notifications\ScheduledTask\TaskFailed;
use App\Notifications\ScheduledTask\TaskSuccess;
use App\Notifications\Server\DockerCleanupFailed;
use App\Notifications\Server\DockerCleanupSuccess;
use App\Notifications\Server\HighDiskUsage;
use App\Notifications\Server\Reachable;
use App\Notifications\Server\ServerPatchCheck;
use App\Notifications\Server\Unreachable;
class TelegramChannel
{
@ -17,24 +32,24 @@ public function send($notifiable, $notification): void
$chatId = $settings->telegram_chat_id;
$threadId = match (get_class($notification)) {
\App\Notifications\Application\DeploymentSuccess::class => $settings->telegram_notifications_deployment_success_thread_id,
\App\Notifications\Application\DeploymentFailed::class => $settings->telegram_notifications_deployment_failure_thread_id,
\App\Notifications\Application\StatusChanged::class,
\App\Notifications\Container\ContainerRestarted::class,
\App\Notifications\Container\ContainerStopped::class => $settings->telegram_notifications_status_change_thread_id,
DeploymentSuccess::class => $settings->telegram_notifications_deployment_success_thread_id,
DeploymentFailed::class => $settings->telegram_notifications_deployment_failure_thread_id,
StatusChanged::class,
ContainerRestarted::class => $settings->telegram_notifications_status_change_thread_id,
RestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id,
\App\Notifications\Database\BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id,
\App\Notifications\Database\BackupFailed::class => $settings->telegram_notifications_backup_failure_thread_id,
BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id,
BackupFailed::class => $settings->telegram_notifications_backup_failure_thread_id,
\App\Notifications\ScheduledTask\TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id,
\App\Notifications\ScheduledTask\TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id,
TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id,
TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id,
\App\Notifications\Server\DockerCleanupSuccess::class => $settings->telegram_notifications_docker_cleanup_success_thread_id,
\App\Notifications\Server\DockerCleanupFailed::class => $settings->telegram_notifications_docker_cleanup_failure_thread_id,
\App\Notifications\Server\HighDiskUsage::class => $settings->telegram_notifications_server_disk_usage_thread_id,
\App\Notifications\Server\Unreachable::class => $settings->telegram_notifications_server_unreachable_thread_id,
\App\Notifications\Server\Reachable::class => $settings->telegram_notifications_server_reachable_thread_id,
\App\Notifications\Server\ServerPatchCheck::class => $settings->telegram_notifications_server_patch_thread_id,
DockerCleanupSuccess::class => $settings->telegram_notifications_docker_cleanup_success_thread_id,
DockerCleanupFailed::class => $settings->telegram_notifications_docker_cleanup_failure_thread_id,
HighDiskUsage::class => $settings->telegram_notifications_server_disk_usage_thread_id,
Unreachable::class => $settings->telegram_notifications_server_unreachable_thread_id,
Reachable::class => $settings->telegram_notifications_server_reachable_thread_id,
ServerPatchCheck::class => $settings->telegram_notifications_server_patch_thread_id,
default => null,
};

View file

@ -1,123 +0,0 @@
<?php
namespace App\Notifications\Container;
use App\Models\Server;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Notifications\Messages\MailMessage;
class ContainerStopped extends CustomEmailNotification
{
public function __construct(public string $name, public Server $server, public ?string $url = null)
{
$this->onQueue('high');
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels('status_change');
}
public function toMail(): MailMessage
{
$mail = new MailMessage;
$mail->subject("Coolify: A resource has been stopped unexpectedly on {$this->server->name}");
$mail->view('emails.container-stopped', [
'containerName' => $this->name,
'serverName' => $this->server->name,
'url' => $this->url,
]);
return $mail;
}
public function toDiscord(): DiscordMessage
{
$message = new DiscordMessage(
title: ':cross_mark: Resource stopped',
description: "{$this->name} has been stopped unexpectedly on {$this->server->name}.",
color: DiscordMessage::errorColor(),
);
if ($this->url) {
$message->addField('Resource', '[Link]('.$this->url.')');
}
return $message;
}
public function toTelegram(): array
{
$message = "Coolify: A resource ($this->name) has been stopped unexpectedly on {$this->server->name}";
$payload = [
'message' => $message,
];
if ($this->url) {
$payload['buttons'] = [
[
[
'text' => 'Open Application in Coolify',
'url' => $this->url,
],
],
];
}
return $payload;
}
public function toPushover(): PushoverMessage
{
$buttons = [];
if ($this->url) {
$buttons[] = [
'text' => 'Open Application in Coolify',
'url' => $this->url,
];
}
return new PushoverMessage(
title: 'Resource stopped',
level: 'error',
message: "A resource ({$this->name}) has been stopped unexpectedly on {$this->server->name}",
buttons: $buttons,
);
}
public function toSlack(): SlackMessage
{
$title = 'Resource stopped';
$description = "A resource ({$this->name}) has been stopped unexpectedly on {$this->server->name}";
if ($this->url) {
$description .= "\n*Resource URL:* {$this->url}";
}
return new SlackMessage(
title: $title,
description: $description,
color: SlackMessage::errorColor()
);
}
public function toWebhook(): array
{
$data = [
'success' => false,
'message' => 'Resource stopped unexpectedly',
'event' => 'container_stopped',
'container_name' => $this->name,
'server_name' => $this->server->name,
'server_uuid' => $this->server->uuid,
];
if ($this->url) {
$data['url'] = $this->url;
}
return $data;
}
}

View file

@ -18,14 +18,13 @@
* State Priority (highest to lowest):
* 1. Degraded (from sub-resources) degraded:unhealthy
* 2. Restarting degraded:unhealthy (or restarting:unknown if preserveRestarting=true)
* 3. Crash Loop (exited with restarts) degraded:unhealthy
* 4. Mixed (running + exited) degraded:unhealthy
* 5. Mixed (running + starting) starting:unknown
* 6. Running running:healthy/unhealthy/unknown
* 7. Dead/Removing degraded:unhealthy
* 8. Paused paused:unknown
* 9. Starting/Created starting:unknown
* 10. Exited exited
* 3. Mixed (running + exited) degraded:unhealthy
* 4. Mixed (running + starting) starting:unknown
* 5. Running running:healthy/unhealthy/unknown
* 6. Dead/Removing degraded:unhealthy
* 7. Paused paused:unknown
* 8. Starting/Created starting:unknown
* 9. Exited exited
*
* The $preserveRestarting parameter controls whether "restarting" containers should be
* reported as "restarting:unknown" (true) or "degraded:unhealthy" (false, default).
@ -228,23 +227,18 @@ private function resolveStatus(
return $preserveRestarting ? 'restarting:unknown' : 'degraded:unhealthy';
}
// Priority 3: Crash loop detection (exited with restart count > 0)
if ($hasExited && $maxRestartCount > 0) {
return 'degraded:unhealthy';
}
// Priority 4: Mixed state (some running, some exited = degraded)
// Priority 3: Mixed state (some running, some exited = degraded)
if ($hasRunning && $hasExited) {
return 'degraded:unhealthy';
}
// Priority 5: Mixed state (some running, some starting = still starting)
// Priority 4: Mixed state (some running, some starting = still starting)
// If any component is still starting, the entire service stack is not fully ready
if ($hasRunning && $hasStarting) {
return 'starting:unknown';
}
// Priority 6: Running containers (check health status)
// Priority 5: Running containers (check health status)
if ($hasRunning) {
if ($hasUnhealthy) {
return 'running:unhealthy';
@ -255,22 +249,22 @@ private function resolveStatus(
}
}
// Priority 7: Dead or removing containers
// Priority 6: Dead or removing containers
if ($hasDead) {
return 'degraded:unhealthy';
}
// Priority 8: Paused containers
// Priority 7: Paused containers
if ($hasPaused) {
return 'paused:unknown';
}
// Priority 9: Starting/created containers
// Priority 8: Starting/created containers
if ($hasStarting) {
return 'starting:unknown';
}
// Priority 10: All containers exited (no restart count = truly stopped)
// Priority 9: All containers exited
return 'exited';
}
}

View file

@ -0,0 +1,31 @@
<?php
namespace App\Services;
class RestartCountTracker
{
/**
* @return array{restart_count: int, restart_count_changed: bool, restart_limit_reached: bool, new_generation: bool}
*/
public function evaluate(
int $previousRestartCount,
int $observedRestartCount,
int $maxRestartCount,
bool $newGenerationConfirmed = false,
): array {
$newGeneration = $newGenerationConfirmed
&& $observedRestartCount < $previousRestartCount;
$restartCountIncreased = $observedRestartCount > $previousRestartCount;
$restartCountChanged = $newGeneration || $restartCountIncreased;
$restartLimitReached = $maxRestartCount > 0
&& $observedRestartCount >= $maxRestartCount;
return [
'restart_count' => $restartCountChanged ? $observedRestartCount : $previousRestartCount,
'restart_count_changed' => $restartCountChanged,
'restart_limit_reached' => $restartLimitReached,
'new_generation' => $newGeneration,
];
}
}

View file

@ -0,0 +1,73 @@
<?php
namespace App\Traits;
use App\Services\RestartCountTracker;
trait HasRestartLimit
{
public function initializeHasRestartLimit(): void
{
$this->mergeFillable(['restart_count', 'max_restart_count', 'restart_limit_reached', 'last_restart_at', 'last_restart_type']);
$this->mergeCasts([
'restart_count' => 'integer',
'max_restart_count' => 'integer',
'restart_limit_reached' => 'boolean',
'last_restart_at' => 'datetime',
'last_restart_type' => 'string',
]);
}
public function stoppedAfterRestartLimit(): bool
{
return str($this->status)->startsWith('exited') && $this->restart_limit_reached === true;
}
public function trackRestartCount(int $observedRestartCount): bool
{
$state = (new RestartCountTracker)->evaluate(
previousRestartCount: $this->restart_count ?? 0,
observedRestartCount: $observedRestartCount,
maxRestartCount: $this->restartLimitMaximum(),
);
if ($state['restart_count_changed']) {
$hasCrashRestarts = $state['restart_count'] > 0;
$this->update([
'restart_count' => $state['restart_count'],
'last_restart_at' => $hasCrashRestarts ? now() : null,
'last_restart_type' => $hasCrashRestarts ? 'crash' : null,
]);
}
if (! $state['restart_limit_reached']) {
return false;
}
$claimed = $this->newQuery()
->whereKey($this->getKey())
->where('restart_limit_reached', false)
->update(['restart_limit_reached' => true]) === 1;
if ($claimed) {
$this->restart_limit_reached = true;
}
return $claimed;
}
public function resetRestartLimit(): void
{
$this->update([
'restart_count' => 0,
'restart_limit_reached' => false,
'last_restart_at' => null,
'last_restart_type' => null,
]);
}
public function restartLimitMaximum(): int
{
return $this->max_restart_count ?? 0;
}
}

View file

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('applications', function (Blueprint $table) {
$table->boolean('container_present')->nullable()->after('status');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('applications', function (Blueprint $table) {
$table->dropColumn('container_present');
});
}
};

View file

@ -0,0 +1,37 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('applications', function (Blueprint $table) {
$table->boolean('restart_limit_reached')->default(false)->after('max_restart_count');
});
DB::table('applications')
->where('status', 'like', 'exited%')
->where('restart_count', '>', 0)
->where('max_restart_count', '>', 0)
->whereColumn('restart_count', '>=', 'max_restart_count')
->where('last_restart_type', 'crash')
->update(['restart_limit_reached' => true]);
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('applications', function (Blueprint $table) {
$table->dropColumn('restart_limit_reached');
});
}
};

View file

@ -0,0 +1,32 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('application_previews', function (Blueprint $table) {
$table->integer('restart_count')->default(0);
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
$table->timestamp('last_restart_at')->nullable();
$table->string('last_restart_type', 10)->nullable();
});
}
public function down(): void
{
Schema::table('application_previews', function (Blueprint $table) {
$table->dropColumn([
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
]);
});
}
};

View file

@ -0,0 +1,32 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('service_applications', function (Blueprint $table) {
$table->integer('restart_count')->default(0);
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
$table->timestamp('last_restart_at')->nullable();
$table->string('last_restart_type', 10)->nullable();
});
}
public function down(): void
{
Schema::table('service_applications', function (Blueprint $table) {
$table->dropColumn([
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
]);
});
}
};

View file

@ -0,0 +1,32 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('service_databases', function (Blueprint $table) {
$table->integer('restart_count')->default(0);
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
$table->timestamp('last_restart_at')->nullable();
$table->string('last_restart_type', 10)->nullable();
});
}
public function down(): void
{
Schema::table('service_databases', function (Blueprint $table) {
$table->dropColumn([
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
]);
});
}
};

View file

@ -0,0 +1,23 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('standalone_postgresqls', function (Blueprint $table) {
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
});
}
public function down(): void
{
Schema::table('standalone_postgresqls', function (Blueprint $table) {
$table->dropColumn(['max_restart_count', 'restart_limit_reached']);
});
}
};

View file

@ -0,0 +1,23 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('standalone_redis', function (Blueprint $table) {
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
});
}
public function down(): void
{
Schema::table('standalone_redis', function (Blueprint $table) {
$table->dropColumn(['max_restart_count', 'restart_limit_reached']);
});
}
};

View file

@ -0,0 +1,23 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('standalone_mongodbs', function (Blueprint $table) {
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
});
}
public function down(): void
{
Schema::table('standalone_mongodbs', function (Blueprint $table) {
$table->dropColumn(['max_restart_count', 'restart_limit_reached']);
});
}
};

View file

@ -0,0 +1,23 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('standalone_mysqls', function (Blueprint $table) {
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
});
}
public function down(): void
{
Schema::table('standalone_mysqls', function (Blueprint $table) {
$table->dropColumn(['max_restart_count', 'restart_limit_reached']);
});
}
};

View file

@ -0,0 +1,23 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('standalone_mariadbs', function (Blueprint $table) {
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
});
}
public function down(): void
{
Schema::table('standalone_mariadbs', function (Blueprint $table) {
$table->dropColumn(['max_restart_count', 'restart_limit_reached']);
});
}
};

View file

@ -0,0 +1,23 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('standalone_keydbs', function (Blueprint $table) {
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
});
}
public function down(): void
{
Schema::table('standalone_keydbs', function (Blueprint $table) {
$table->dropColumn(['max_restart_count', 'restart_limit_reached']);
});
}
};

View file

@ -0,0 +1,23 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('standalone_dragonflies', function (Blueprint $table) {
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
});
}
public function down(): void
{
Schema::table('standalone_dragonflies', function (Blueprint $table) {
$table->dropColumn(['max_restart_count', 'restart_limit_reached']);
});
}
};

View file

@ -0,0 +1,23 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('standalone_clickhouses', function (Blueprint $table) {
$table->integer('max_restart_count')->default(10);
$table->boolean('restart_limit_reached')->default(false);
});
}
public function down(): void
{
Schema::table('standalone_clickhouses', function (Blueprint $table) {
$table->dropColumn(['max_restart_count', 'restart_limit_reached']);
});
}
};

View file

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('email_notification_settings', function (Blueprint $table) {
$table->boolean('restart_limit_reached_email_notifications')->default(true);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('email_notification_settings', function (Blueprint $table) {
$table->dropColumn('restart_limit_reached_email_notifications');
});
}
};

View file

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('discord_notification_settings', function (Blueprint $table) {
$table->boolean('restart_limit_reached_discord_notifications')->default(true);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('discord_notification_settings', function (Blueprint $table) {
$table->dropColumn('restart_limit_reached_discord_notifications');
});
}
};

View file

@ -0,0 +1,30 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('telegram_notification_settings', function (Blueprint $table) {
$table->boolean('restart_limit_reached_telegram_notifications')->default(true);
$table->text('telegram_notifications_restart_limit_reached_thread_id')->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('telegram_notification_settings', function (Blueprint $table) {
$table->dropColumn('restart_limit_reached_telegram_notifications');
$table->dropColumn('telegram_notifications_restart_limit_reached_thread_id');
});
}
};

View file

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('slack_notification_settings', function (Blueprint $table) {
$table->boolean('restart_limit_reached_slack_notifications')->default(true);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('slack_notification_settings', function (Blueprint $table) {
$table->dropColumn('restart_limit_reached_slack_notifications');
});
}
};

View file

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('pushover_notification_settings', function (Blueprint $table) {
$table->boolean('restart_limit_reached_pushover_notifications')->default(true);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('pushover_notification_settings', function (Blueprint $table) {
$table->dropColumn('restart_limit_reached_pushover_notifications');
});
}
};

View file

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('webhook_notification_settings', function (Blueprint $table) {
$table->boolean('restart_limit_reached_webhook_notifications')->default(true);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('webhook_notification_settings', function (Blueprint $table) {
$table->dropColumn('restart_limit_reached_webhook_notifications');
});
}
};

View file

@ -0,0 +1,10 @@
@props(['application'])
@if ($application->stoppedAfterRestartLimit())
@php($restartLimit = method_exists($application, 'restartLimitMaximum') ? $application->restartLimitMaximum() : ($application->max_restart_count ?? 0))
@php($displayRestartCount = max($application->restart_count ?? 0, $restartLimit))
<x-status-badge
status="Restart limit reached"
type="warning"
title="Container has crashed and Coolify stopped it after {{ $displayRestartCount }} restart attempts." />
@endif

View file

@ -9,10 +9,17 @@
'Deployments' => [
['key' => 'deploymentSuccess', 'label' => 'Deployment success'],
['key' => 'deploymentFailure', 'label' => 'Deployment failure'],
],
'Resources' => [
[
'key' => 'statusChange',
'label' => 'Container status changes',
'helper' => 'Notify when a container stops or restarts.',
'label' => 'Resource status changes',
'helper' => 'Notify when a resource stops or Coolify automatically restarts it.',
],
[
'key' => 'restartLimitReached',
'label' => 'Restart limit reached',
'helper' => 'Notify when a resource is stopped after reaching its restart limit.',
],
],
'Backups' => [

View file

@ -23,7 +23,7 @@
$containerType = match (true) {
str($containerStatus)->startsWith('running') => 'success',
str($containerStatus)->startsWith(['starting', 'restarting']) => 'warning',
str($containerStatus)->startsWith(['starting', 'restarting', 'degraded']) => 'warning',
default => 'error',
};
@ -36,6 +36,7 @@
[$summaryLabel, $summaryType] = match (true) {
$containerType === 'error' => [$containerLabel, 'error'],
str($containerStatus)->startsWith('degraded') => ['Degraded', 'warning'],
$healthType === 'error' => ['Degraded', 'error'],
$containerType === 'warning' => [$containerLabel, 'warning'],
$monitoringExcluded => ["{$containerLabel} (monitoring disabled)", 'warning'],

View file

@ -22,9 +22,7 @@
title="Container has restarted {{ $resource->restart_count }} time{{ $resource->restart_count > 1 ? 's' : '' }}. Last restart: {{ $resource->last_restart_at?->diffForHumans() }}" />
@endif
@if ($stoppedAfterRestartLimit)
<x-status-badge status="Stopped after reaching restart limit ({{ $resource->restart_count }}/{{ $resource->max_restart_count }})."
type="warning"
title="Container has crashed and Coolify stopped it after {{ $resource->restart_count }} restart attempts." />
<x-application.restart-limit-warning :application="$resource" />
@endif
@if (!str($resource->status)->contains('exited') && $showRefreshButton)
<x-status-badge as="button" wire:target="manualCheckStatus" wire:loading.attr="disabled"

View file

@ -1,7 +0,0 @@
<x-emails.layout>
A resource ({{ $containerName }}) has been stopped unexpectedly on {{ $serverName }}.
@if ($url)
Please check what is going on [here]({{ $url }}).
@endif
</x-emails.layout>

View file

@ -149,7 +149,11 @@ class="button button-highlighted">
:events="[
['property' => 'deploymentSuccessEmailNotifications', 'label' => 'Deployment success', 'enabled' => $deploymentSuccessEmailNotifications],
['property' => 'deploymentFailureEmailNotifications', 'label' => 'Deployment failure', 'enabled' => $deploymentFailureEmailNotifications],
['property' => 'statusChangeEmailNotifications', 'label' => 'Container status changes', 'enabled' => $statusChangeEmailNotifications],
]" />
<x-notification.event-multiselect :settings="$settings" id="resource-email-events" label="Resources"
:events="[
['property' => 'statusChangeEmailNotifications', 'label' => 'Resource status changes', 'enabled' => $statusChangeEmailNotifications],
['property' => 'restartLimitReachedEmailNotifications', 'label' => 'Restart limit reached', 'enabled' => $restartLimitReachedEmailNotifications],
]" />
<x-notification.event-multiselect :settings="$settings" id="backup-email-events" label="Backups"
:events="[

View file

@ -34,6 +34,9 @@
<x-status-summary :status="$application->status" />
<x-applications.links :application="$application" compact />
</div>
<div class="flex w-full flex-wrap gap-1">
<x-application.restart-limit-warning :application="$application" />
</div>
</div>
</div>
@ -103,32 +106,18 @@ class="listbox-panel top-full! left-0! right-0! mt-1! w-full! min-w-0!" role="me
@endcan
@endif
@endif
@can('deploy', $application)
<button type="button" class="listbox-option justify-start! gap-2.5!"
@click="open = false; document.getElementById('application-mobile-stop-trigger')?.click()"
role="menuitem">
<x-reicon name="stop-circle" class="size-3.5 text-error" />
Stop
</button>
@else
<button type="button" class="listbox-option justify-start! gap-2.5!" disabled
role="menuitem">
<x-reicon name="stop-circle" class="size-3.5 opacity-70" />
Stop
</button>
@endcan
@else
@can('deploy', $application)
<button type="button" class="listbox-option justify-start! gap-2.5!"
wire:click="deploy" @click="open = false" role="menuitem">
<x-reicon name="play-circle" class="size-3.5 opacity-70" />
Deploy
{{ $application->stoppedAfterRestartLimit() ? 'Retry deployment' : 'Deploy' }}
</button>
@else
<button type="button" class="listbox-option justify-start! gap-2.5!" disabled
role="menuitem">
<x-reicon name="play-circle" class="size-3.5 opacity-70" />
Deploy
{{ $application->stoppedAfterRestartLimit() ? 'Retry deployment' : 'Deploy' }}
</button>
@endcan
@endif
@ -138,25 +127,36 @@ class="listbox-panel top-full! left-0! right-0! mt-1! w-full! min-w-0!" role="me
wire:click="{{ $application->status === 'running' ? 'force_deploy_without_cache' : 'deploy(true)' }}"
@click="open = false" role="menuitem">
<x-reicon name="refresh" class="size-3.5 opacity-70" />
Deploy (without cache)
{{ $application->stoppedAfterRestartLimit() ? 'Retry deployment (without cache)' : 'Deploy (without cache)' }}
</button>
@else
<button type="button" class="listbox-option justify-start! gap-2.5!" disabled
role="menuitem">
<x-reicon name="refresh" class="size-3.5 opacity-70" />
Deploy (without cache)
{{ $application->stoppedAfterRestartLimit() ? 'Retry deployment (without cache)' : 'Deploy (without cache)' }}
</button>
@endcan
@endif
@if (!str($application->status)->startsWith('exited') || $application->container_present !== false)
<button type="button" class="listbox-option justify-start! gap-2.5!"
@click="open = false; document.getElementById('application-mobile-stop-trigger')?.click()"
role="menuitem">
<x-reicon name="stop-circle" class="size-3.5 text-error" />
{{ str($application->status)->startsWith('exited') ? 'Remove container' : 'Stop' }}
</button>
@endif
</div>
</div>
@endcan
@endif
<div class="hidden" aria-hidden="true">
<x-modal-confirmation title="Confirm Application Stopping?" buttonTitle="Stop"
<x-modal-confirmation
canGate="deploy" :canResource="$application"
title="{{ str($application->status)->startsWith('exited') ? 'Confirm Container Removal?' : 'Confirm Application Stopping?' }}"
buttonTitle="{{ str($application->status)->startsWith('exited') ? 'Remove container' : 'Stop' }}"
submitAction="stop" :checkboxes="$checkboxes" :actions="[
'This application will be stopped.',
'All non-persistent data of this application will be deleted.',
str($application->status)->startsWith('exited') ? 'The exited application container will be removed.' : 'This application will be stopped.',
str($application->status)->startsWith('exited') ? 'Anonymous volumes may become eligible for Docker cleanup.' : 'All non-persistent data of this application will be deleted.',
]" :confirmWithText="false" :confirmWithPassword="false"
step1ButtonText="Continue" step2ButtonText="Confirm">
<x-slot:trigger>
@ -204,14 +204,22 @@ class="listbox-panel top-full! right-0! left-auto! mt-1! w-60! min-w-0!" role="m
@disabled(!auth()->user()->can('deploy', $application))
wire:click="deploy" @click="open = false" role="menuitem">
<x-reicon name="play-circle" class="size-3.5 opacity-70" />
Deploy
{{ $application->stoppedAfterRestartLimit() ? 'Retry deployment' : 'Deploy' }}
</button>
@if (!$application->destination->server->isSwarm())
<button type="button" class="listbox-option justify-start! gap-2.5!"
@disabled(!auth()->user()->can('deploy', $application))
wire:click="deploy(true)" @click="open = false" role="menuitem">
<x-reicon name="refresh" class="size-3.5 opacity-70" />
Deploy (without cache)
{{ $application->stoppedAfterRestartLimit() ? 'Retry deployment (without cache)' : 'Deploy (without cache)' }}
</button>
@endif
@if ($application->container_present !== false)
<button type="button" class="listbox-option justify-start! gap-2.5!"
@click="open = false; document.getElementById('application-mobile-stop-trigger')?.click()"
role="menuitem">
<x-reicon name="stop-circle" class="size-3.5 text-error" />
Remove container
</button>
@endif
@else

View file

@ -103,6 +103,7 @@ class="flex size-9 shrink-0 items-center justify-center rounded-lg bg-neutral-10
Preview #{{ data_get($preview, 'pull_request_id') }}
</h4>
<x-status-summary :status="data_get($preview, 'status')" title="Preview status" />
<x-application.restart-limit-warning :application="$preview" />
</div>
</div>
</div>

View file

@ -1,3 +1,4 @@
<div wire:poll.10000ms="refreshStatus">
<x-status-summary :status="$application->status" />
<x-application.restart-limit-warning :application="$application" />
</div>

View file

@ -63,8 +63,13 @@
<h1 class="min-w-0 max-w-full truncate text-[24px]! leading-7! font-semibold! tracking-tight! text-black dark:text-fg">
{{ $database->name }}
</h1>
<div class="relative flex w-full min-w-0 items-center gap-2">
<x-status-summary :status="$database->status" title="Database status" />
</div>
<div class="flex w-full flex-wrap gap-1">
<x-application.restart-limit-warning :application="$database" />
</div>
</div>
</div>
<div class="w-full xl:hidden">

View file

@ -1,3 +1,4 @@
<div wire:poll.10000ms="refreshStatus">
<x-status-summary :status="$database->status" title="Database status" />
<x-application.restart-limit-warning :application="$database" />
</div>

View file

@ -220,7 +220,7 @@ class="relative z-10 block truncate text-[11px] text-neutral-500 hover:underline
x-text="item.typeLabel"></div>
<div>
<x-status-badge dynamic>
<x-status-badge dynamic x-bind:title="statusTitle(item)">
<span class="size-1.5 shrink-0 rounded-full"
x-bind:class="statusDotClass(item)"></span>
<span class="truncate" x-text="statusLabel(item)"></span>
@ -299,7 +299,7 @@ class="truncate text-[13px]! leading-4! font-semibold! text-black dark:text-fg"
<p class="mt-0.5 text-[11px] text-neutral-500 dark:text-fg-faint"
x-text="item.typeLabel"></p>
</div>
<x-status-badge dynamic>
<x-status-badge dynamic x-bind:title="statusTitle(item)">
<span class="size-1.5 shrink-0 rounded-full"
x-bind:class="statusDotClass(item)"></span>
<span class="truncate" x-text="statusLabel(item)"></span>
@ -518,13 +518,32 @@ function resourceIndex() {
localStorage.setItem('environment-resource-view', mode);
},
statusState(item) {
if (item.restartLimitReached) {
return 'restart-limit';
}
return String(item.status || 'unknown').split(':')[0].toLowerCase();
},
statusLabel(item) {
if (item.restartLimitReached) {
return 'Restart limit reached';
}
const state = this.statusState(item);
return state.charAt(0).toUpperCase() + state.slice(1);
},
statusTitle(item) {
if (item.restartLimitReached) {
return `${item.restartCount}/${item.maxRestartCount} restarts. Container preserved.`;
}
return this.statusLabel(item);
},
statusTone(item) {
if (item.restartLimitReached) {
return 'warning';
}
const state = this.statusState(item);
if (state === 'running') {
return 'success';
@ -539,6 +558,10 @@ function resourceIndex() {
return 'neutral';
},
statusDotClass(item) {
if (item.restartLimitReached) {
return 'bg-warning';
}
return {
success: 'bg-emerald-500',
warning: 'bg-warning',

View file

@ -143,12 +143,12 @@ class="flex size-7.5 items-center justify-center rounded-md transition-colors"
<div :class="viewMode === 'grid'
? 'grid grid-cols-1 gap-3 sm:grid-cols-2'
: 'overflow-hidden rounded-xl border border-neutral-200 bg-white shadow-sm dark:border-white/[0.08] dark:bg-white/[0.025]'">
: 'overflow-x-auto rounded-xl border border-neutral-200 bg-white shadow-sm dark:border-white/[0.08] dark:bg-white/[0.025]'">
@if ($applications->isNotEmpty() || $databases->isNotEmpty())
<div x-cloak x-show="viewMode === 'table'"
class="grid grid-cols-[minmax(0,1fr)_auto] gap-3 border-b border-neutral-200 bg-neutral-50 px-4 py-2.5 text-[11px] font-medium text-neutral-500 sm:grid-cols-[minmax(0,1fr)_minmax(0,1fr)_8rem_5rem] dark:border-white/[0.08] dark:bg-white/[0.025] dark:text-fg-faint">
class="grid min-w-[48rem] grid-cols-[minmax(14rem,1fr)_minmax(12rem,1fr)_12rem_5rem] gap-3 border-b border-neutral-200 bg-neutral-50 px-4 py-2.5 text-[11px] font-medium text-neutral-500 dark:border-white/[0.08] dark:bg-white/[0.025] dark:text-fg-faint">
<div>Resource</div>
<div class="hidden sm:block">Image</div>
<div>Image</div>
<div class="justify-self-start">Status</div>
<div></div>
</div>

View file

@ -42,6 +42,13 @@
));
$serviceStatus = str($service->status ?? 'exited');
$selectedResourceUuid = data_get($parameters, 'stack_service_uuid');
$selectedResource = $selectedResourceUuid
? $service->applications->firstWhere('uuid', $selectedResourceUuid)
?? $service->databases->firstWhere('uuid', $selectedResourceUuid)
: null;
$displayStatus = $selectedResource?->status ?? $service->status;
$selectedResourceStatus = str($selectedResource?->status ?? '');
$environmentVariablesUrl = route('project.service.environment-variables', [
'project_uuid' => $service->environment->project->uuid,
'environment_uuid' => $service->environment->uuid,
@ -65,9 +72,15 @@
{{ $service->name }}
</h1>
<div class="relative flex w-full min-w-0 items-center gap-2">
<x-status-summary :status="$service->status" title="Service status" container-name="Containers" />
<x-status-summary :status="$displayStatus" :title="$selectedResource ? 'Resource status' : 'Service status'"
:container-name="$selectedResource ? 'Container' : 'Containers'" />
<x-services.links :service="$service" compact />
</div>
<div class="flex w-full flex-wrap gap-1">
@if ($selectedResource)
<x-application.restart-limit-warning :application="$selectedResource" />
@endif
</div>
</div>
</div>
@ -91,7 +104,14 @@
<div x-cloak x-show="open" x-transition.origin.top.left
class="listbox-panel top-full! left-0! right-0! mt-1! w-full! min-w-0!" role="menu">
@if ($serviceStatus->contains('running') || $serviceStatus->contains('degraded'))
@if ($selectedResource && $selectedResource->container_present !== false && $selectedResourceStatus->startsWith('exited'))
<button type="button" class="listbox-option justify-start! gap-2.5!"
@click="open = false; document.getElementById('selected-resource-remove-trigger')?.click()"
role="menuitem">
<x-reicon name="trash" class="size-3.5 text-error" />
Remove container
</button>
@elseif ($serviceStatus->contains('running') || $serviceStatus->contains('degraded'))
@can('deploy', $service)
<button type="button" class="listbox-option justify-start! gap-2.5!"
@click="open = false; document.getElementById('service-restart-trigger')?.click()"
@ -195,6 +215,14 @@ class="resource-heading-navbar application-heading-actions flex w-auto min-w-0 i
</button>
<div x-cloak x-show="open" x-transition.origin.top.right
class="listbox-panel top-full! right-0! left-auto! mt-1! w-64! min-w-0!" role="menu">
@if ($selectedResource && $selectedResource->container_present !== false && $selectedResourceStatus->startsWith('exited'))
<button type="button" class="listbox-option justify-start! gap-2.5!"
@click="open = false; document.getElementById('selected-resource-remove-trigger')?.click()"
role="menuitem">
<x-reicon name="trash" class="size-3.5 text-error" />
Remove container
</button>
@else
@if ($serviceStatus->contains('running') || $serviceStatus->contains('degraded'))
<button type="button" class="listbox-option justify-start! gap-2.5!"
@disabled(!auth()->user()->can('deploy', $service))
@ -248,6 +276,7 @@ class="listbox-panel top-full! right-0! left-auto! mt-1! w-64! min-w-0!" role="m
Force Cleanup Containers
</button>
@endif
@endif
</div>
</div>
@endcan
@ -281,6 +310,17 @@ class="listbox-panel top-full! right-0! left-auto! mt-1! w-64! min-w-0!" role="m
<button id="service-stop-trigger" type="button">Stop</button>
</x-slot:trigger>
</x-modal-confirmation>
@if ($selectedResource)
<x-modal-confirmation title="Confirm Container Removal?" buttonTitle="Remove container"
canGate="deploy" :canResource="$service" submitAction="removeSelectedResourceContainer"
:actions="['The exited service resource container will be removed.', __('resource.non_persistent')]"
:confirmWithText="false" :confirmWithPassword="false" step1ButtonText="Continue"
step2ButtonText="Confirm">
<x-slot:trigger>
<button id="selected-resource-remove-trigger" type="button">Remove container</button>
</x-slot:trigger>
</x-modal-confirmation>
@endif
</div>
@endif

View file

@ -1,4 +1,12 @@
<div>
<div x-data="{
settingsUrl: @js(route('project.service.index', [...$parameters, 'stack_service_uuid' => $resource->uuid])),
openSettings(event) {
if (event.target.closest('a, button')) {
return;
}
Livewire.navigate(this.settingsUrl);
}
}">
@php
[$statusType, $statusLabel] = match (true) {
str($resource->status)->contains('running') => ['success', formatContainerStatus($resource->status)],
@ -26,6 +34,7 @@ class="flex size-9 shrink-0 items-center justify-center rounded-lg bg-neutral-10
</div>
</div>
<x-status-badge :status="$statusLabel" :type="$statusType" />
<x-application.restart-limit-warning :application="$resource" />
</div>
@if ($resource->configuration_required)
@ -76,7 +85,9 @@ class="flex items-center justify-end gap-1 border-t border-neutral-200 bg-neutra
</div>
<div x-cloak x-show="viewMode === 'table'"
class="grid min-h-14 grid-cols-[minmax(0,1fr)_auto] items-center gap-3 border-b border-neutral-200 px-4 py-2.5 last:border-b-0 hover:bg-neutral-50 sm:grid-cols-[minmax(0,1fr)_minmax(0,1fr)_8rem_5rem] dark:border-white/[0.07] dark:hover:bg-white/[0.025]">
x-on:click="openSettings($event)" x-on:keydown.enter="openSettings($event)"
role="link" tabindex="0" aria-label="Open {{ $resourceName }} settings"
class="grid min-h-14 min-w-[48rem] cursor-pointer grid-cols-[minmax(14rem,1fr)_minmax(12rem,1fr)_12rem_5rem] items-center gap-3 border-b border-neutral-200 px-4 py-2.5 last:border-b-0 hover:bg-neutral-50 focus-visible:outline-2 focus-visible:outline-offset-[-2px] focus-visible:outline-primary dark:border-white/[0.07] dark:hover:bg-white/[0.025]">
<div class="flex min-w-0 items-center gap-3">
<div
class="flex size-8 shrink-0 items-center justify-center rounded-lg bg-neutral-100 text-neutral-500 dark:bg-white/[0.06] dark:text-fg-dim">
@ -86,12 +97,12 @@ class="flex size-8 shrink-0 items-center justify-center rounded-lg bg-neutral-10
<div class="truncate text-[13px] font-semibold text-black dark:text-fg">{{ $resourceName }}</div>
</div>
</div>
<div class="hidden truncate font-mono text-xs text-neutral-500 sm:block dark:text-fg-faint">
<div class="truncate font-mono text-xs text-neutral-500 dark:text-fg-faint">
{{ $resource->image }}
</div>
<div class="flex flex-wrap items-center justify-end gap-1 sm:contents">
<div class="justify-self-start">
<div class="flex flex-wrap items-center gap-1">
<x-status-badge :status="$statusLabel" :type="$statusType" />
<x-application.restart-limit-warning :application="$resource" />
</div>
<div class="flex items-center justify-end gap-1">
@if ($isDatabase && ($resource->isBackupSolutionAvailable() || $resource->is_migrated))
@ -114,5 +125,4 @@ class="flex size-8 shrink-0 items-center justify-center rounded-lg bg-neutral-10
</a>
</div>
</div>
</div>
</div>

View file

@ -1,3 +1,8 @@
<div wire:poll.10000ms="refreshStatus">
<x-status-summary :status="$service->status" title="Service status" container-name="Containers" />
@php($displayStatus = $selectedResource?->status ?? $service->status)
<x-status-summary :status="$displayStatus" :title="$selectedResource ? 'Resource status' : 'Service status'"
:container-name="$selectedResource ? 'Container' : 'Containers'" />
@if ($selectedResource)
<x-application.restart-limit-warning :application="$selectedResource" />
@endif
</div>

View file

@ -35,6 +35,7 @@ class="rounded bg-neutral-100 px-1.5 py-0.5 font-mono text-xs text-neutral-700 d
<div class="flex flex-wrap items-center gap-2 sm:justify-end">
<a href="{{ route('server.show', ['server_uuid' => data_get($resource, 'destination.server.uuid')]) }}"
{{ wireNavigate() }} class="button">Open server</a>
<x-application.restart-limit-warning :application="$resource" />
<x-status-summary :status="$resource->status" />
@if ($hasAdditionalDestinations)
<x-forms.button canGate="deploy" :canResource="$resource"
@ -217,6 +218,9 @@ class="rounded bg-neutral-100 px-1.5 py-0.5 font-mono text-xs text-neutral-700 d
<div class="flex flex-wrap items-center gap-2 sm:justify-end">
<a href="{{ route('server.show', ['server_uuid' => data_get($resource, 'destination.server.uuid')]) }}"
{{ wireNavigate() }} class="button">Open server</a>
@if (method_exists($resource, 'stoppedAfterRestartLimit'))
<x-application.restart-limit-warning :application="$resource" />
@endif
@if ($primaryStatus->startsWith('running'))
<x-status.running :status="$primaryStatus->value()" />
@elseif ($primaryStatus->startsWith(['starting', 'restarting']))

View file

@ -61,10 +61,14 @@ class="data-table-row server-resources-managed-table-grid border-b border-neutra
{{ str($resource->type())->headline() }}
</div>
<div>
@if (method_exists($resource, 'stoppedAfterRestartLimit') && $resource->stoppedAfterRestartLimit())
<x-application.restart-limit-warning :application="$resource" />
@else
<x-status-badge :status="str($resourceStatus)->headline()"
:type="str($resourceStatus)->contains('running')
? 'success'
: (str($resourceStatus)->contains(['failed', 'exited']) ? 'error' : 'neutral')" />
@endif
</div>
</div>
@endforeach

View file

@ -0,0 +1,223 @@
<?php
use App\Models\ApplicationPreview;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneClickhouse;
use App\Models\StandaloneDragonfly;
use App\Models\StandaloneKeydb;
use App\Models\StandaloneMariadb;
use App\Models\StandaloneMongodb;
use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use App\Traits\HasRestartLimit;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
it('gives every independently runnable non-application resource restart limit state', function (string $modelClass) {
expect(class_uses_recursive($modelClass))->toContain(HasRestartLimit::class);
$resource = new $modelClass;
expect($resource->getFillable())->toContain(
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
)->and($resource->getCasts())->toMatchArray([
'restart_count' => 'integer',
'max_restart_count' => 'integer',
'restart_limit_reached' => 'boolean',
'last_restart_at' => 'datetime',
]);
})->with([
ApplicationPreview::class,
ServiceApplication::class,
ServiceDatabase::class,
StandaloneClickhouse::class,
StandaloneDragonfly::class,
StandaloneKeydb::class,
StandaloneMariadb::class,
StandaloneMongodb::class,
StandaloneMysql::class,
StandalonePostgresql::class,
StandaloneRedis::class,
]);
it('collects restart counts for preview and service containers from both status sources', function () {
$dockerStatus = file_get_contents(app_path('Actions/Docker/GetContainersStatus.php'));
$sentinelStatus = file_get_contents(app_path('Jobs/PushServerUpdateJob.php'));
expect($dockerStatus)
->toContain('previewContainerRestartCounts')
->toContain('serviceContainerRestartCounts')
->and($sentinelStatus)
->toContain('previewContainerRestartCounts')
->toContain('serviceContainerRestartCounts');
});
it('ignores docker compose one-off job containers in both status sources', function () {
$dockerStatus = file_get_contents(app_path('Actions/Docker/GetContainersStatus.php'));
$sentinelStatus = file_get_contents(app_path('Jobs/PushServerUpdateJob.php'));
expect($dockerStatus)
->toContain("filter_var(data_get(\$labels, 'com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)")
->and($sentinelStatus)
->toContain("filter_var(\$labels->get('com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)");
});
it('shows restart limit warnings for every resource family', function () {
$previews = file_get_contents(resource_path('views/livewire/project/application/previews.blade.php'));
$serviceCard = file_get_contents(resource_path('views/livewire/project/service/resource-card.blade.php'));
$applicationStatus = file_get_contents(resource_path('views/livewire/project/application/status.blade.php'));
$databaseStatus = file_get_contents(resource_path('views/livewire/project/database/status.blade.php'));
expect($previews)->toContain('<x-application.restart-limit-warning :application="$preview" />')
->and($serviceCard)->toContain('<x-application.restart-limit-warning :application="$resource" />')
->and($applicationStatus)->toContain('<x-application.restart-limit-warning :application="$application" />')
->and($databaseStatus)->toContain('<x-application.restart-limit-warning :application="$database" />');
$serviceStatus = file_get_contents(resource_path('views/livewire/project/service/status.blade.php'));
$serviceHeading = file_get_contents(resource_path('views/livewire/project/service/heading.blade.php'));
expect($serviceStatus)->toContain('<x-application.restart-limit-warning :application="$selectedResource" />')
->and($serviceStatus)->toContain('$selectedResource?->status ?? $service->status')
->and($serviceHeading)->toContain('<x-application.restart-limit-warning :application="$selectedResource" />')
->and($serviceHeading)->toContain('$selectedResource?->status ?? $service->status');
});
it('matches application restart badge layout on mobile resource headings', function () {
$applicationHeading = file_get_contents(resource_path('views/livewire/project/application/heading.blade.php'));
$databaseHeading = file_get_contents(resource_path('views/livewire/project/database/heading.blade.php'));
$serviceHeading = file_get_contents(resource_path('views/livewire/project/service/heading.blade.php'));
foreach ([$applicationHeading, $databaseHeading, $serviceHeading] as $heading) {
expect($heading)
->toContain('class="relative flex w-full min-w-0 items-center gap-2"')
->toContain('class="flex w-full flex-wrap gap-1"');
}
});
it('shows database restart limits in shared resource listings', function () {
$resourceIndex = file_get_contents(app_path('Livewire/Project/Resource/Index.php'));
$serverResources = file_get_contents(resource_path('views/livewire/server/resources.blade.php'));
$destination = file_get_contents(resource_path('views/livewire/project/shared/destination.blade.php'));
expect($resourceIndex)
->toContain("method_exists(\$item, 'stoppedAfterRestartLimit')")
->not->toContain("\$type === 'application' && \$item->stoppedAfterRestartLimit()")
->and($serverResources)
->toContain('<x-application.restart-limit-warning :application="$resource" />')
->and($destination)
->toContain('<x-application.restart-limit-warning :application="$resource" />');
});
it('keeps the service resource table readable with horizontal scrolling on mobile', function () {
$configuration = file_get_contents(resource_path('views/livewire/project/service/configuration.blade.php'));
$resourceCard = file_get_contents(resource_path('views/livewire/project/service/resource-card.blade.php'));
expect($configuration)
->toContain("'overflow-x-auto rounded-xl")
->toContain('min-w-[48rem]')
->and($resourceCard)
->toContain('min-w-[48rem]')
->not->toContain('<div class="hidden truncate font-mono');
});
it('opens service resource settings when a table row is clicked', function () {
$resourceCard = file_get_contents(resource_path('views/livewire/project/service/resource-card.blade.php'));
expect($resourceCard)
->toContain('x-on:click="openSettings($event)"')
->toContain('x-on:keydown.enter="openSettings($event)"')
->toContain("closest('a, button')")
->toContain('role="link"')
->toContain('tabindex="0"');
});
it('uses selected service resource actions instead of parent complex status actions', function () {
$heading = file_get_contents(resource_path('views/livewire/project/service/heading.blade.php'));
$headingClass = file_get_contents(app_path('Livewire/Project/Service/Heading.php'));
expect(substr_count($heading, "\$selectedResource && \$selectedResource->container_present !== false && \$selectedResourceStatus->startsWith('exited')"))->toBe(2)
->and($heading)
->toContain('Remove container')
->toContain('removeSelectedResourceContainer')
->and($headingClass)
->toContain('public function removeSelectedResourceContainer(): void');
});
it('imports the application model used when claiming a restart limit', function () {
$statusAction = file_get_contents(app_path('Actions/Docker/GetContainersStatus.php'));
expect($statusAction)
->toContain('use App\\Models\\Application;')
->toContain('Application::query()');
});
it('adds restart limit columns to previews services and standalone databases', function () {
$migrations = collect(glob(database_path('migrations/*.php')))
->map(fn (string $path): string => file_get_contents($path))
->implode("\n");
expect($migrations)
->toContain("'application_previews'")
->toContain("'service_applications'")
->toContain("'service_databases'")
->toContain("'max_restart_count'")
->toContain("'restart_limit_reached'");
$restartLimitMigrations = collect(glob(database_path('migrations/*_add_restart_limit_to_*.php')));
expect($restartLimitMigrations)->toHaveCount(11);
expect($restartLimitMigrations->map(
fn (string $path): string => substr(basename($path), 0, 17)
)->unique())->toHaveCount(11);
$restartLimitMigrations->each(function (string $path): void {
expect(file_get_contents($path))->not->toContain('foreach (');
});
});
it('atomically claims a resource restart limit once and can reset it', function () {
Schema::create('restart_limit_test_resources', function (Blueprint $table): void {
$table->id();
$table->string('status')->default('running');
$table->integer('restart_count')->default(0);
$table->integer('max_restart_count')->default(2);
$table->boolean('restart_limit_reached')->default(false);
$table->timestamp('last_restart_at')->nullable();
$table->string('last_restart_type')->nullable();
$table->timestamps();
});
$resource = new class extends Model
{
use HasRestartLimit;
protected $table = 'restart_limit_test_resources';
};
$resource->save();
$resource->refresh();
expect($resource->trackRestartCount(2))->toBeTrue()
->and($resource->fresh()->restart_limit_reached)->toBeTrue()
->and($resource->trackRestartCount(2))->toBeFalse();
$resource->resetRestartLimit();
expect($resource->fresh()->restart_count)->toBe(0)
->and($resource->restart_limit_reached)->toBeFalse();
$resourceWithExistingRestarts = $resource->newInstance();
$resourceWithExistingRestarts->max_restart_count = 0;
$resourceWithExistingRestarts->save();
expect($resourceWithExistingRestarts->trackRestartCount(17))->toBeFalse();
$resourceWithExistingRestarts->update(['max_restart_count' => 10]);
expect($resourceWithExistingRestarts->trackRestartCount(17))->toBeTrue()
->and($resourceWithExistingRestarts->fresh()->restart_limit_reached)->toBeTrue();
Schema::drop('restart_limit_test_resources');
});

View file

@ -0,0 +1,56 @@
<?php
use App\Models\Application;
it('stores nullable application container presence as a boolean', function () {
$application = new Application;
$application->forceFill(['container_present' => 1]);
$migration = file_get_contents(base_path('database/migrations/2026_08_30_193506_add_container_present_to_applications_table.php'));
expect($application->container_present)->toBeTrue()
->and($migration)->toContain("boolean('container_present')->nullable()");
});
it('updates container presence at application lifecycle boundaries', function () {
$stopAction = file_get_contents(app_path('Actions/Application/StopApplication.php'));
$dockerStatus = file_get_contents(app_path('Actions/Docker/GetContainersStatus.php'));
$sentinelStatus = file_get_contents(app_path('Jobs/PushServerUpdateJob.php'));
expect($stopAction)->toContain('$containerPresent = ! $removeContainers;')
->and($stopAction)->toMatch('/if \(\$server->isSwarm\(\)\).*?\$containerPresent = false;.*?docker stack rm/s')
->and($stopAction)->toContain("'container_present' => \$containerPresent")
->and($dockerStatus)->toContain("'container_present' => true")
->and($dockerStatus)->toContain("'container_present' => false")
->and($sentinelStatus)->toContain("'container_present' => true")
->and($sentinelStatus)->toContain("'container_present' => false");
});
it('shows accurate destructive actions and the restart warning on mobile', function () {
$heading = file_get_contents(resource_path('views/livewire/project/application/heading.blade.php'));
$mobileActions = str($heading)
->after('id="application-mobile-actions"')
->before('<div class="hidden" aria-hidden="true">')
->toString();
expect($heading)->toContain('<x-application.restart-limit-warning :application="$application" />')
->and(substr_count($heading, '$application->container_present !== false'))->toBe(2)
->and($heading)->toContain("\$application->stoppedAfterRestartLimit() ? 'Retry deployment' : 'Deploy'")
->and($heading)->toContain("\$application->stoppedAfterRestartLimit() ? 'Retry deployment (without cache)' : 'Deploy (without cache)'")
->and($heading)->toContain('Remove container')
->and($mobileActions)->toContain('Deploy (without cache)')
->and($mobileActions)->toContain('Remove container')
->and(strrpos($mobileActions, 'Remove container'))->toBeGreaterThan(strrpos($mobileActions, 'Deploy (without cache)'));
});
it('shows restart limit reached as a yellow state in environment resource lists', function () {
$indexClass = file_get_contents(app_path('Livewire/Project/Resource/Index.php'));
$indexView = file_get_contents(resource_path('views/livewire/project/resource/index.blade.php'));
expect($indexClass)->toContain("'restartLimitReached' => \$type === 'application' && \$item->stoppedAfterRestartLimit()")
->and($indexClass)->toContain('? max($item->restart_count ?? 0, $item->max_restart_count ?? 0)')
->and($indexClass)->toContain("'maxRestartCount' => \$item->max_restart_count ?? 0")
->and($indexView)->toContain("if (item.restartLimitReached) {\n return 'restart-limit';")
->and($indexView)->toContain("return 'Restart limit reached';")
->and($indexView)->toContain("if (item.restartLimitReached) {\n return 'bg-warning';")
->and($indexView)->toContain('x-bind:title="statusTitle(item)"');
});

View file

@ -1,16 +1,24 @@
<?php
use App\Actions\Application\StopApplication;
use App\Actions\Docker\GetContainersStatus;
use App\Jobs\ApplicationDeploymentJob;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\BaseModel;
use App\Models\Server;
use App\Notifications\Application\RestartLimitReached;
use Mockery\MockInterface;
function applicationWithRestartState(array $attributes = []): Application
{
$application = new Application;
$application->forceFill(array_merge([
'status' => 'exited:unhealthy',
'container_present' => true,
'restart_count' => 2,
'max_restart_count' => 2,
'restart_limit_reached' => true,
'last_restart_type' => 'crash',
'last_restart_at' => now(),
], $attributes));
@ -21,9 +29,51 @@ function applicationWithRestartState(array $attributes = []): Application
it('detects applications stopped after reaching the crash restart limit', function () {
expect(applicationWithRestartState()->stoppedAfterRestartLimit())->toBeTrue()
->and(applicationWithRestartState(['status' => 'running:unhealthy'])->stoppedAfterRestartLimit())->toBeFalse()
->and(applicationWithRestartState(['restart_count' => 1])->stoppedAfterRestartLimit())->toBeFalse()
->and(applicationWithRestartState(['max_restart_count' => 0])->stoppedAfterRestartLimit())->toBeFalse()
->and(applicationWithRestartState(['last_restart_type' => null])->stoppedAfterRestartLimit())->toBeFalse();
->and(applicationWithRestartState(['restart_limit_reached' => false])->stoppedAfterRestartLimit())->toBeFalse();
});
it('keeps the restart limit state after Docker resets its counter', function () {
expect(applicationWithRestartState([
'restart_count' => 0,
'last_restart_type' => null,
'last_restart_at' => null,
])->stoppedAfterRestartLimit())->toBeTrue();
});
it('preserves exited application state when the container snapshot is empty', function () {
$application = Mockery::mock(Application::class)->makePartial();
$application->setRelation('additional_servers', collect());
$application->forceFill([
'id' => 1,
'status' => 'exited:unhealthy',
'container_present' => true,
'restart_limit_reached' => true,
]);
$application->shouldNotReceive('update');
$services = Mockery::mock();
$services->shouldReceive('get')->once()->andReturn(collect());
$server = Mockery::mock(Server::class, function (MockInterface $mock) use ($application, $services) {
$mock->shouldReceive('isFunctional')->once()->andReturnTrue();
$mock->shouldReceive('applications')->once()->andReturn(collect([$application]));
$mock->shouldReceive('databases')->once()->andReturn(collect());
$mock->shouldReceive('services')->once()->andReturn($services);
$mock->shouldReceive('previews')->once()->andReturn(collect());
})->makePartial();
$server->setRelation('team', (object) ['id' => 1]);
GetContainersStatus::run($server, collect(), collect());
expect($application->container_present)->toBeTrue()
->and($application->restart_limit_reached)->toBeTrue();
});
it('does not infer the restart limit from an exited existing container', function () {
expect(applicationWithRestartState([
'container_present' => true,
'restart_limit_reached' => false,
])->stoppedAfterRestartLimit())->toBeFalse();
});
it('shows a stopped after restart limit warning in the status badge', function () {
@ -32,7 +82,8 @@ function applicationWithRestartState(array $attributes = []): Application
'showRefreshButton' => false,
])->render();
expect($html)->toContain('Stopped after reaching restart limit (2/2).')
expect($html)->toContain('Restart limit reached')
->not->toContain('Stopped after reaching restart limit (2/2).')
->and($html)->toContain('Container has crashed and Coolify stopped it after 2 restart attempts.');
});
@ -41,11 +92,12 @@ function applicationWithRestartState(array $attributes = []): Application
'resource' => applicationWithRestartState([
'restart_count' => 0,
'last_restart_type' => null,
'restart_limit_reached' => false,
]),
'showRefreshButton' => false,
])->render();
expect($html)->not->toContain('Stopped after reaching restart limit');
expect($html)->not->toContain('Restart limit reached');
});
it('keeps restart tracking configurable when stopping an application', function () {
@ -56,6 +108,59 @@ function applicationWithRestartState(array $attributes = []): Application
->and($resetRestartCount->getDefaultValue())->toBeTrue();
});
it('can stop an application without removing its containers', function () {
$method = new ReflectionMethod(StopApplication::class, 'handle');
$removeContainers = collect($method->getParameters())->firstWhere('name', 'removeContainers');
$action = file_get_contents(app_path('Actions/Application/StopApplication.php'));
expect($removeContainers)->not->toBeNull()
->and($removeContainers->getDefaultValue())->toBeTrue()
->and($action)->toContain('docker update --restart=no')
->and($action)->toContain('if ($removeContainers)');
});
it('preserves containers and skips cleanup when the restart limit is reached', function () {
$statusAction = file_get_contents(app_path('Actions/Docker/GetContainersStatus.php'));
$sentinelJob = file_get_contents(app_path('Jobs/PushServerUpdateJob.php'));
expect($statusAction)->toContain('dockerCleanup: false')
->and($statusAction)->toContain('resetRestartCount: false')
->and($statusAction)->toContain('removeContainers: false')
->and($statusAction)->toContain("['restart_limit_reached' => true]")
->and($sentinelJob)->toContain("['restart_limit_reached' => true]");
});
it('atomically claims the restart limit transition before stopping and notifying', function () {
$statusAction = file_get_contents(app_path('Actions/Docker/GetContainersStatus.php'));
$sentinelJob = file_get_contents(app_path('Jobs/PushServerUpdateJob.php'));
foreach ([$statusAction, $sentinelJob] as $detector) {
expect($detector)
->toContain("->where('restart_limit_reached', false)")
->toContain("->update(['restart_limit_reached' => true]) === 1");
}
});
it('clears the explicit restart limit state only after a successful main deployment', function () {
$method = new ReflectionMethod(ApplicationDeploymentJob::class, 'handleSuccessfulDeployment');
$source = file($method->getFileName());
$deploymentJob = implode(array_slice($source, $method->getStartLine() - 1, $method->getEndLine() - $method->getStartLine() + 1));
expect(substr_count($deploymentJob, "'restart_limit_reached'] = false"))->toBe(1)
->and($deploymentJob)->toContain("if (\$this->pull_request_id === 0) {\n \$restartState['restart_limit_reached'] = false;\n }")
->and($deploymentJob)->toContain('$this->application->update($restartState);')
->and(substr_count($deploymentJob, '$this->application->update('))->toBe(1);
});
it('preserves restart-limit applications only while their exited container exists', function () {
$statusAction = file_get_contents(app_path('Actions/Docker/GetContainersStatus.php'));
$sentinelJob = file_get_contents(app_path('Jobs/PushServerUpdateJob.php'));
expect($statusAction)->toContain("'container_present' => false")
->and($statusAction)->toContain("'restart_limit_reached' => false")
->and($sentinelJob)->toContain('if ($application->stoppedAfterRestartLimit() && $containerStatuses->every(');
});
it('uses the application link for restart limit notifications', function () {
$application = new class extends Application
{
@ -80,3 +185,55 @@ public function link()
expect($notification->resource_url)->toBe('https://coolify.test/project/link-from-model');
});
it('uses the resolved environment project name in Slack restart limit notifications', function () {
$environment = (object) [
'uuid' => 'environment-uuid',
'name' => 'production',
'project' => (object) [
'uuid' => 'project-uuid',
'name' => 'Coolify',
],
];
$application = new class extends Application
{
public function link(): string
{
return 'https://coolify.test/application';
}
};
$application->forceFill(['name' => 'app']);
$application->setRelation('environment', $environment);
$preview = new ApplicationPreview;
$preview->forceFill([
'uuid' => 'preview-uuid',
'pull_request_id' => 42,
'restart_count' => 2,
'max_restart_count' => 2,
]);
$preview->setRelation('application', $application);
$serviceResource = new class extends BaseModel {};
$serviceResource->forceFill([
'name' => 'database',
'uuid' => 'service-resource-uuid',
'restart_count' => 2,
'max_restart_count' => 2,
]);
$serviceResource->setRelation('service', new class($environment)
{
public function __construct(public object $environment) {}
public function link(): string
{
return 'https://coolify.test/service';
}
});
expect((new RestartLimitReached($preview))->toSlack()->description)
->toContain('*Project:* Coolify')
->and((new RestartLimitReached($serviceResource))->toSlack()->description)
->toContain('*Project:* Coolify');
});

View file

@ -132,6 +132,80 @@
expect($this->admin->can('update', $settings))->toBeTrue();
});
test('telegram restart limit thread id accepts 255 characters', function () {
$this->actingAs($this->admin);
session(['currentTeam' => $this->team]);
Livewire::test(TelegramNotification::class)
->set('telegramNotificationsRestartLimitReachedThreadId', str_repeat('a', 255))
->call('syncData', true)
->assertHasNoErrors(['telegramNotificationsRestartLimitReachedThreadId']);
expect($this->team->telegramNotificationSettings->fresh()->telegram_notifications_restart_limit_reached_thread_id)
->toBe(str_repeat('a', 255));
});
test('telegram restart limit thread id rejects 256 characters', function () {
$this->actingAs($this->admin);
session(['currentTeam' => $this->team]);
Livewire::test(TelegramNotification::class)
->set('telegramNotificationsRestartLimitReachedThreadId', str_repeat('a', 256))
->call('syncData', true)
->assertHasErrors(['telegramNotificationsRestartLimitReachedThreadId' => 'max']);
});
test('member cannot view telegram thread ids', function () {
$threadIds = [
'telegram_notifications_deployment_success_thread_id' => 'deployment-success-thread',
'telegram_notifications_deployment_failure_thread_id' => 'deployment-failure-thread',
'telegram_notifications_status_change_thread_id' => 'status-change-thread',
'telegram_notifications_restart_limit_reached_thread_id' => 'restart-limit-thread',
'telegram_notifications_backup_success_thread_id' => 'backup-success-thread',
'telegram_notifications_backup_failure_thread_id' => 'backup-failure-thread',
'telegram_notifications_scheduled_task_success_thread_id' => 'scheduled-task-success-thread',
'telegram_notifications_scheduled_task_failure_thread_id' => 'scheduled-task-failure-thread',
'telegram_notifications_docker_cleanup_success_thread_id' => 'docker-cleanup-success-thread',
'telegram_notifications_docker_cleanup_failure_thread_id' => 'docker-cleanup-failure-thread',
'telegram_notifications_server_disk_usage_thread_id' => 'server-disk-usage-thread',
'telegram_notifications_server_reachable_thread_id' => 'server-reachable-thread',
'telegram_notifications_server_unreachable_thread_id' => 'server-unreachable-thread',
'telegram_notifications_server_patch_thread_id' => 'server-patch-thread',
'telegram_notifications_traefik_outdated_thread_id' => 'traefik-outdated-thread',
];
$this->team->telegramNotificationSettings->update($threadIds);
$this->actingAs($this->member);
session(['currentTeam' => $this->team]);
$component = Livewire::test(TelegramNotification::class);
foreach ($threadIds as $column => $threadId) {
$component
->assertSet(str($column)->camel()->toString(), null)
->assertDontSee($threadId);
}
});
test('admin can view telegram thread ids', function () {
$threadIds = [
'telegram_notifications_deployment_success_thread_id' => 'deployment-success-thread',
'telegram_notifications_restart_limit_reached_thread_id' => 'restart-limit-thread',
];
$this->team->telegramNotificationSettings->update($threadIds);
$this->actingAs($this->admin);
session(['currentTeam' => $this->team]);
$component = Livewire::test(TelegramNotification::class);
foreach ($threadIds as $column => $threadId) {
$component->assertSet(str($column)->camel()->toString(), $threadId);
}
});
// --- Email ---
test('member cannot send test email notification', function () {

View file

@ -53,6 +53,22 @@
'server actions' => ['views/livewire/server/navbar.blade.php', 'manageProxy', 'server', 'server'],
]);
it('declares deploy authorization on the application stop confirmation', function () {
$source = file_get_contents(resource_path('views/livewire/project/application/heading.blade.php'));
expect($source)->toMatch(
'/<x-modal-confirmation\s+canGate="deploy" :canResource="\$application"/'
);
});
it('declares deploy authorization on the service container removal confirmation', function () {
$source = file_get_contents(resource_path('views/livewire/project/service/heading.blade.php'));
expect($source)->toMatch(
'/<x-modal-confirmation(?=[^>]*title="Confirm Container Removal\?")(?=[^>]*canGate="deploy")(?=[^>]*:canResource="\$service")[^>]*>/'
);
});
it('keeps mutable Livewire components behind authorization checks', function (string $path, array $requiredNeedles) {
$source = file_get_contents(base_path($path));

View file

@ -0,0 +1,48 @@
<?php
it('uses a dedicated notification event for restart limits', function () {
$notification = file_get_contents(app_path('Notifications/Application/RestartLimitReached.php'));
$telegramChannel = file_get_contents(app_path('Notifications/Channels/TelegramChannel.php'));
$eventGrid = file_get_contents(resource_path('views/components/notification/event-grid.blade.php'));
expect($notification)->toContain("getEnabledChannels('restart_limit_reached')")
->and($eventGrid)
->toContain("'Resources' => [")
->toContain("'key' => 'statusChange'")
->toContain("'helper' => 'Notify when a resource stops or Coolify automatically restarts it.'")
->toContain("'key' => 'restartLimitReached'")
->toContain("'label' => 'Restart limit reached'")
->and($telegramChannel)
->toContain('RestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id');
});
it('persists a restart limit notification preference for every channel', function (string $channel) {
$studly = Str::studly($channel);
$component = file_get_contents(app_path("Livewire/Notifications/{$studly}.php"));
$model = file_get_contents(app_path('Models/'.$studly.'NotificationSettings.php'));
$column = "restart_limit_reached_{$channel}_notifications";
$property = "restartLimitReached{$studly}Notifications";
expect($component)
->toContain("public bool \${$property} = true;")
->toContain("\$this->settings->{$column} = \$this->{$property};")
->toContain("\$this->{$property} = \$this->settings->{$column};")
->and($model)
->toContain("'{$column}'");
})->with(['email', 'discord', 'telegram', 'slack', 'pushover', 'webhook']);
it('enables restart limit notifications by default in every channel migration', function () {
$migrations = collect(glob(database_path('migrations/*_add_restart_limit_reached_notifications_to_*')));
expect($migrations)->toHaveCount(6);
$migrations->each(fn (string $migration) => expect(file_get_contents($migration))->toContain('->default(true)'));
});
it('uses the inline validator facade for notification API updates', function () {
$controller = file_get_contents(app_path('Http/Controllers/Api/NotificationsController.php'));
expect($controller)
->toContain('use Illuminate\\Support\\Facades\\Validator;')
->toContain("Validator::make(\$body, \$config['rules'])")
->not->toContain("customApiValidator(\$body, \$config['rules'])");
});

View file

@ -19,6 +19,16 @@
->toContain('w-[min(16rem,calc(100vw-1.5rem))]!');
});
it('shows degraded aggregate service status as a warning', function () {
$html = Blade::render('<x-status-summary status="degraded:unhealthy" title="Service status" container-name="Containers" />');
$summaryButton = str($html)->between('<button', '</button>')->toString();
expect($summaryButton)
->toContain('Degraded')
->toContain('bg-warning')
->not->toContain('bg-error');
});
it('uses the aggregated preview status in the previews list', function () {
$view = file_get_contents(resource_path('views/livewire/project/application/previews.blade.php'));
@ -42,7 +52,7 @@
expect($databaseStatus)
->toContain('<x-status-summary :status="$database->status" title="Database status" />')
->and($serviceStatus)
->toContain('<x-status-summary :status="$service->status" title="Service status" container-name="Containers" />');
->toContain('<x-status-summary :status="$displayStatus" :title="$selectedResource ? \'Resource status\' : \'Service status\'"');
});
it('groups preview deployment actions in a dropdown', function () {

View file

@ -255,6 +255,25 @@
->not->toContain('Force deploy without cache');
});
it('shows stop in application action menus when the application is exited', function () {
$heading = file_get_contents(resource_path('views/livewire/project/application/heading.blade.php'));
$desktopExitedActions = str($heading)
->after("@if (str(\$application->status)->startsWith('exited'))")
->before('@else')
->toString();
$mobileActions = str($heading)
->after('id="application-mobile-actions"')
->before('<div class="hidden" aria-hidden="true">')
->toString();
expect($mobileActions)->toContain('application-mobile-stop-trigger')
->and($desktopExitedActions)->toContain('application-mobile-stop-trigger')
->and($mobileActions)->toContain('Deploy (without cache)')
->and(strrpos($mobileActions, 'Deploy (without cache)'))
->toBeLessThan(strrpos($mobileActions, 'application-mobile-stop-trigger'));
});
it('places the state-aware no-cache action immediately after deploy or redeploy', function () {
$heading = file_get_contents(resource_path('views/livewire/project/application/heading.blade.php'));
$actions = str($heading)->after('id="application-desktop-actions"')->before('@endteleport')->toString();

View file

@ -147,6 +147,16 @@ function sentinelPayload(array $containers, ?float $diskPercentage = 42.0): arra
Queue::assertPushed(PushServerUpdateJob::class, 2);
});
it('dispatches the job when only the container restart count changes', function () {
$beforeRestart = [['name' => 'app-1', 'state' => 'running', 'restart_count' => 0]];
$afterRestart = [['name' => 'app-1', 'state' => 'running', 'restart_count' => 1]];
pushSentinel($this->token, sentinelPayload($beforeRestart))->assertOk();
pushSentinel($this->token, sentinelPayload($afterRestart))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 2);
});
it('ignores health status changes while container lifecycle state is unchanged', function () {
$healthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']];
$unhealthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'unhealthy']];

View file

@ -70,12 +70,12 @@
expect($result)->toBe('running:unknown');
});
test('returns degraded:unhealthy for crash loop (exited with restart count)', function () {
test('returns exited for an exited container with a restart count', function () {
$statuses = collect(['exited']);
$result = $this->aggregator->aggregateFromStrings($statuses, maxRestartCount: 5);
expect($result)->toBe('degraded:unhealthy');
expect($result)->toBe('exited');
});
test('returns exited for exited containers without restart count', function () {
@ -214,12 +214,12 @@
expect($result)->toBe('degraded:unhealthy');
});
test('prioritizes crash loop over running containers', function () {
test('returns exited when all containers are exited with restart counts', function () {
$statuses = collect(['exited', 'exited']);
$result = $this->aggregator->aggregateFromStrings($statuses, maxRestartCount: 3);
expect($result)->toBe('degraded:unhealthy');
expect($result)->toBe('exited');
});
test('prioritizes mixed state over healthy running', function () {
@ -238,12 +238,12 @@
expect($result)->toBe('starting:unknown');
});
test('prioritizes running over paused/exited when no starting', function () {
test('returns degraded for mixed running and exited containers', function () {
$statuses = collect(['running:healthy', 'paused', 'exited']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('running:healthy');
expect($result)->toBe('degraded:unhealthy');
});
test('prioritizes dead over paused/starting/exited', function () {
@ -357,7 +357,7 @@
expect($result)->toBe('degraded:unhealthy');
});
test('returns degraded:unhealthy for crash loop (exited with restart count)', function () {
test('returns exited for an exited container object with a restart count', function () {
$containers = collect([
(object) [
'State' => (object) [
@ -368,7 +368,7 @@
$result = $this->aggregator->aggregateFromContainers($containers, maxRestartCount: 5);
expect($result)->toBe('degraded:unhealthy');
expect($result)->toBe('exited');
});
test('returns exited for exited containers without restart count', function () {
@ -501,7 +501,7 @@
expect($result)->toBe('degraded:unhealthy');
});
test('crash loop has third highest priority', function () {
test('mixed running and exited containers are degraded before paused or starting states', function () {
$statuses = collect([
'exited',
'running:healthy',
@ -602,31 +602,29 @@
$result = $this->aggregator->aggregateFromStrings($statuses, maxRestartCount: 0);
// Zero is valid default - no crash loop detection
expect($result)->toBe('exited');
});
test('positive maxRestartCount works correctly', function () {
test('positive maxRestartCount does not override an exited state', function () {
$statuses = collect(['exited']);
$result = $this->aggregator->aggregateFromStrings($statuses, maxRestartCount: 5);
// Positive value enables crash loop detection
expect($result)->toBe('degraded:unhealthy');
expect($result)->toBe('exited');
});
test('crash loop detection still functions after validation', function () {
test('exited state is preserved for any positive restart count', function () {
$statuses = collect(['exited']);
// Test with various positive restart counts
expect($this->aggregator->aggregateFromStrings($statuses, maxRestartCount: 1))
->toBe('degraded:unhealthy');
->toBe('exited');
expect($this->aggregator->aggregateFromStrings($statuses, maxRestartCount: 100))
->toBe('degraded:unhealthy');
->toBe('exited');
expect($this->aggregator->aggregateFromStrings($statuses, maxRestartCount: 999))
->toBe('degraded:unhealthy');
->toBe('exited');
});
test('default maxRestartCount parameter works', function () {

View file

@ -0,0 +1,9 @@
<?php
use App\Jobs\PushServerUpdateJob;
test('database status update declares a void return type', function () {
$method = new ReflectionMethod(PushServerUpdateJob::class, 'updateDatabaseStatus');
expect($method->getReturnType()?->getName())->toBe('void');
});

View file

@ -0,0 +1,11 @@
<?php
it('does not keep the unreliable generic stopped container notification path', function () {
$statusAction = file_get_contents(__DIR__.'/../../app/Actions/Docker/GetContainersStatus.php');
$telegramChannel = file_get_contents(__DIR__.'/../../app/Notifications/Channels/TelegramChannel.php');
expect($statusAction)->not->toContain('ContainerStopped')
->and($telegramChannel)->not->toContain('ContainerStopped')
->and(file_exists(__DIR__.'/../../app/Notifications/Container/ContainerStopped.php'))->toBeFalse()
->and(file_exists(__DIR__.'/../../resources/views/emails/container-stopped.blade.php'))->toBeFalse();
});

View file

@ -0,0 +1,95 @@
<?php
use App\Services\RestartCountTracker;
it('starts a new generation when an active container restart count drops', function () {
$result = (new RestartCountTracker)->evaluate(
previousRestartCount: 11,
observedRestartCount: 0,
maxRestartCount: 2,
newGenerationConfirmed: true,
);
expect($result)->toMatchArray([
'restart_count' => 0,
'restart_count_changed' => true,
'restart_limit_reached' => false,
'new_generation' => true,
]);
});
it('evaluates the restart limit immediately in a new generation', function () {
$result = (new RestartCountTracker)->evaluate(
previousRestartCount: 11,
observedRestartCount: 3,
maxRestartCount: 2,
newGenerationConfirmed: true,
);
expect($result)->toMatchArray([
'restart_count' => 3,
'restart_count_changed' => true,
'restart_limit_reached' => true,
'new_generation' => true,
]);
});
it('does not reset the generation without explicit confirmation', function () {
$result = (new RestartCountTracker)->evaluate(
previousRestartCount: 11,
observedRestartCount: 0,
maxRestartCount: 2,
);
expect($result)->toMatchArray([
'restart_count' => 11,
'restart_count_changed' => false,
'restart_limit_reached' => false,
'new_generation' => false,
]);
});
it('preserves the previous count when an active payload omits the container with the previous maximum', function () {
$result = (new RestartCountTracker)->evaluate(
previousRestartCount: 11,
observedRestartCount: 3,
maxRestartCount: 20,
);
expect($result)->toMatchArray([
'restart_count' => 11,
'restart_count_changed' => false,
'restart_limit_reached' => false,
'new_generation' => false,
]);
});
it('detects a normal threshold crossing', function () {
$result = (new RestartCountTracker)->evaluate(
previousRestartCount: 1,
observedRestartCount: 2,
maxRestartCount: 2,
);
expect($result)->toMatchArray([
'restart_count' => 2,
'restart_count_changed' => true,
'restart_limit_reached' => true,
'new_generation' => false,
]);
});
it('detects a limit that is enabled below the current observed restart count', function () {
$result = (new RestartCountTracker)->evaluate(
previousRestartCount: 17,
observedRestartCount: 17,
maxRestartCount: 10,
);
expect($result)->toMatchArray([
'restart_count' => 17,
'restart_count_changed' => false,
'restart_limit_reached' => true,
'new_generation' => false,
]);
});

View file

@ -10,8 +10,7 @@
$action = file_get_contents(__DIR__.'/../../app/Actions/Application/StopApplication.php');
expect($action)
->toContain("\$status = ['status' => 'exited'];")
->toContain('$application->update($status);')
->toMatch('/\$status\s*=\s*\[\s*\'status\'\s*=>\s*\'exited\',.*?\];.*?\$application->update\(\$status\);/s')
->not->toMatch('/docker stack rm .*?return;/s');
});
@ -19,8 +18,10 @@
$action = file_get_contents(__DIR__.'/../../app/Actions/Service/StopService.php');
expect($action)
->toContain("\$applications->each->update(['status' => 'exited']);")
->toContain("\$dbs->each->update(['status' => 'exited']);");
->toContain("\$application->update(['status' => 'exited']);")
->toContain('$application->resetRestartLimit();')
->toContain("\$database->update(['status' => 'exited']);")
->toContain('$database->resetRestartLimit();');
});
it('persists exited status when stopping an individual service resource', function () {
@ -28,6 +29,8 @@
expect($action)
->toContain("\$serviceApplication->update(['status' => 'exited']);")
->toContain('$commands = ["docker rm -f {$containerName}"];')
->toContain('throwError: ! $removeContainer')
->toContain('ServiceStatusChanged::dispatch($service->environment->project->team->id);');
});

View file

@ -0,0 +1,12 @@
<?php
use App\Actions\Database\StopDatabase;
use App\Models\BaseModel;
it('declares strict method types', function () {
$handle = new ReflectionMethod(StopDatabase::class, 'handle');
$stopContainer = new ReflectionMethod(StopDatabase::class, 'stopContainer');
expect($handle->getReturnType()?->getName())->toBe('string')
->and($stopContainer->getParameters()[0]->getType()?->getName())->toBe(BaseModel::class);
});