feat(application): preserve crash restart limit status
Stop applications after they hit the crash restart limit without clearing restart tracking, surface the stopped-limit warning in status UI, and use the application link in restart limit notifications.
This commit is contained in:
parent
d581abb284
commit
aaa540421f
7 changed files with 128 additions and 13 deletions
|
|
@ -13,7 +13,7 @@ class StopApplication
|
||||||
|
|
||||||
public string $jobQueue = 'high';
|
public string $jobQueue = 'high';
|
||||||
|
|
||||||
public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true)
|
public function handle(Application $application, bool $previewDeployments = false, bool $dockerCleanup = true, bool $resetRestartCount = true)
|
||||||
{
|
{
|
||||||
$servers = collect([$application->destination->server]);
|
$servers = collect([$application->destination->server]);
|
||||||
if ($application?->additional_servers?->count() > 0) {
|
if ($application?->additional_servers?->count() > 0) {
|
||||||
|
|
@ -57,12 +57,17 @@ public function handle(Application $application, bool $previewDeployments = fals
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reset restart tracking when application is manually stopped
|
if ($resetRestartCount) {
|
||||||
$application->update([
|
$application->update([
|
||||||
'restart_count' => 0,
|
'restart_count' => 0,
|
||||||
'last_restart_at' => null,
|
'last_restart_at' => null,
|
||||||
'last_restart_type' => null,
|
'last_restart_type' => null,
|
||||||
]);
|
]);
|
||||||
|
} else {
|
||||||
|
$application->update([
|
||||||
|
'status' => 'exited',
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
ServiceStatusChanged::dispatch($application->environment->project->team->id);
|
ServiceStatusChanged::dispatch($application->environment->project->team->id);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -466,7 +466,9 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wrap all database updates in a transaction to ensure consistency
|
// Wrap all database updates in a transaction to ensure consistency
|
||||||
DB::transaction(function () use ($application, $maxRestartCount, $containerStatuses) {
|
$restartLimitReached = false;
|
||||||
|
|
||||||
|
DB::transaction(function () use ($application, $maxRestartCount, $containerStatuses, &$restartLimitReached) {
|
||||||
$previousRestartCount = $application->restart_count ?? 0;
|
$previousRestartCount = $application->restart_count ?? 0;
|
||||||
|
|
||||||
if ($maxRestartCount > $previousRestartCount) {
|
if ($maxRestartCount > $previousRestartCount) {
|
||||||
|
|
@ -480,8 +482,7 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
|
||||||
// Check if restart limit has been reached
|
// Check if restart limit has been reached
|
||||||
$maxAllowedRestarts = $application->max_restart_count ?? 0;
|
$maxAllowedRestarts = $application->max_restart_count ?? 0;
|
||||||
if ($maxAllowedRestarts > 0 && $maxRestartCount >= $maxAllowedRestarts && $previousRestartCount < $maxAllowedRestarts) {
|
if ($maxAllowedRestarts > 0 && $maxRestartCount >= $maxAllowedRestarts && $previousRestartCount < $maxAllowedRestarts) {
|
||||||
StopApplication::dispatch($application);
|
$restartLimitReached = true;
|
||||||
$application->environment->project->team?->notify(new ApplicationRestartLimitReached($application));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -496,6 +497,12 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if ($restartLimitReached) {
|
||||||
|
$application->refresh();
|
||||||
|
StopApplication::dispatch($application, false, true, false);
|
||||||
|
$application->environment->project->team?->notify(new ApplicationRestartLimitReached($application));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -572,6 +572,15 @@ public function link()
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function stoppedAfterRestartLimit(): bool
|
||||||
|
{
|
||||||
|
return str($this->status)->startsWith('exited')
|
||||||
|
&& ($this->restart_count ?? 0) > 0
|
||||||
|
&& ($this->max_restart_count ?? 0) > 0
|
||||||
|
&& $this->restart_count >= $this->max_restart_count
|
||||||
|
&& $this->last_restart_type === 'crash';
|
||||||
|
}
|
||||||
|
|
||||||
public function taskLink($task_uuid)
|
public function taskLink($task_uuid)
|
||||||
{
|
{
|
||||||
if (data_get($this, 'environment.project.uuid')) {
|
if (data_get($this, 'environment.project.uuid')) {
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,7 @@ class RestartLimitReached extends CustomEmailNotification
|
||||||
public function __construct(public Application $resource)
|
public function __construct(public Application $resource)
|
||||||
{
|
{
|
||||||
$this->onQueue('high');
|
$this->onQueue('high');
|
||||||
|
$this->afterCommit();
|
||||||
$this->resource_name = data_get($resource, 'name');
|
$this->resource_name = data_get($resource, 'name');
|
||||||
$this->project_uuid = data_get($resource, 'environment.project.uuid');
|
$this->project_uuid = data_get($resource, 'environment.project.uuid');
|
||||||
$this->environment_uuid = data_get($resource, 'environment.uuid');
|
$this->environment_uuid = data_get($resource, 'environment.uuid');
|
||||||
|
|
@ -40,7 +41,7 @@ public function __construct(public Application $resource)
|
||||||
if (str($this->fqdn)->explode(',')->count() > 1) {
|
if (str($this->fqdn)->explode(',')->count() > 1) {
|
||||||
$this->fqdn = str($this->fqdn)->explode(',')->first();
|
$this->fqdn = str($this->fqdn)->explode(',')->first();
|
||||||
}
|
}
|
||||||
$this->resource_url = base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}/application/{$this->resource->uuid}";
|
$this->resource_url = $this->resource->link() ?? base_url()."/project/{$this->project_uuid}/environment/{$this->environment_uuid}/application/{$this->resource->uuid}";
|
||||||
}
|
}
|
||||||
|
|
||||||
public function via(object $notifiable): array
|
public function via(object $notifiable): array
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,11 @@
|
||||||
'title' => null,
|
'title' => null,
|
||||||
'lastDeploymentLink' => null,
|
'lastDeploymentLink' => null,
|
||||||
'resource' => null,
|
'resource' => null,
|
||||||
|
'showRefreshButton' => true,
|
||||||
])
|
])
|
||||||
|
@php
|
||||||
|
$stoppedAfterRestartLimit = $resource && method_exists($resource, 'stoppedAfterRestartLimit') && $resource->stoppedAfterRestartLimit();
|
||||||
|
@endphp
|
||||||
<div class="flex flex-wrap items-center gap-1">
|
<div class="flex flex-wrap items-center gap-1">
|
||||||
@if (str($resource->status)->startsWith('running'))
|
@if (str($resource->status)->startsWith('running'))
|
||||||
<x-status.running :status="$resource->status" :title="$title" :lastDeploymentLink="$lastDeploymentLink" />
|
<x-status.running :status="$resource->status" :title="$title" :lastDeploymentLink="$lastDeploymentLink" />
|
||||||
|
|
@ -13,13 +17,20 @@
|
||||||
@else
|
@else
|
||||||
<x-status.stopped :status="$resource->status" />
|
<x-status.stopped :status="$resource->status" />
|
||||||
@endif
|
@endif
|
||||||
@if (isset($resource->restart_count) && $resource->restart_count > 0 && !str($resource->status)->startsWith('exited'))
|
@if (isset($resource->restart_count) && $resource->restart_count > 0 && (!str($resource->status)->startsWith('exited') || $stoppedAfterRestartLimit))
|
||||||
<div class="flex items-center">
|
<div class="flex items-center">
|
||||||
<span class="text-xs dark:text-warning" title="Container has restarted {{ $resource->restart_count }} time{{ $resource->restart_count > 1 ? 's' : '' }}. Last restart: {{ $resource->last_restart_at?->diffForHumans() }}">
|
<span class="text-xs dark:text-warning" title="Container has restarted {{ $resource->restart_count }} time{{ $resource->restart_count > 1 ? 's' : '' }}. Last restart: {{ $resource->last_restart_at?->diffForHumans() }}">
|
||||||
({{ $resource->restart_count }}x restarts)
|
({{ $resource->restart_count }}x restarts)
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
@endif
|
@endif
|
||||||
|
@if ($stoppedAfterRestartLimit)
|
||||||
|
<div class="flex items-center">
|
||||||
|
<span class="text-xs dark:text-warning" title="Container has crashed and Coolify stopped it after {{ $resource->restart_count }} restart attempts.">
|
||||||
|
Stopped after reaching restart limit ({{ $resource->restart_count }}/{{ $resource->max_restart_count }}).
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
@endif
|
||||||
@if (!str($resource->status)->contains('exited') && $showRefreshButton)
|
@if (!str($resource->status)->contains('exited') && $showRefreshButton)
|
||||||
<button wire:loading.remove.delay.shortest wire:target="manualCheckStatus" title="Refresh Status" wire:click='manualCheckStatus'
|
<button wire:loading.remove.delay.shortest wire:target="manualCheckStatus" title="Refresh Status" wire:click='manualCheckStatus'
|
||||||
class="dark:hover:fill-white fill-black dark:fill-warning">
|
class="dark:hover:fill-white fill-black dark:fill-warning">
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ class="scrollbar flex min-h-10 w-full flex-nowrap items-center gap-6 overflow-x-
|
||||||
href="{{ route('project.application.logs', $parameters) }}">
|
href="{{ route('project.application.logs', $parameters) }}">
|
||||||
<div class="flex items-center gap-1">
|
<div class="flex items-center gap-1">
|
||||||
Logs
|
Logs
|
||||||
@if ($application->restart_count > 0 && !str($application->status)->startsWith('exited'))
|
@if ($application->restart_count > 0 && (!str($application->status)->startsWith('exited') || $application->stoppedAfterRestartLimit()))
|
||||||
<svg class="w-4 h-4 dark:text-warning" viewBox="0 0 24 24" fill="currentColor" xmlns="http://www.w3.org/2000/svg" title="Container has restarted {{ $application->restart_count }} time{{ $application->restart_count > 1 ? 's' : '' }}">
|
<svg class="w-4 h-4 dark:text-warning" viewBox="0 0 24 24" fill="currentColor" xmlns="http://www.w3.org/2000/svg" title="Container has restarted {{ $application->restart_count }} time{{ $application->restart_count > 1 ? 's' : '' }}">
|
||||||
<path d="M12 2L1 21h22L12 2zm0 4l7.53 13H4.47L12 6zm-1 5v4h2v-4h-2zm0 5v2h2v-2h-2z"/>
|
<path d="M12 2L1 21h22L12 2zm0 4l7.53 13H4.47L12 6zm-1 5v4h2v-4h-2zm0 5v2h2v-2h-2z"/>
|
||||||
</svg>
|
</svg>
|
||||||
|
|
|
||||||
82
tests/Feature/ApplicationStoppedAfterRestartLimitTest.php
Normal file
82
tests/Feature/ApplicationStoppedAfterRestartLimitTest.php
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
use App\Actions\Application\StopApplication;
|
||||||
|
use App\Models\Application;
|
||||||
|
use App\Notifications\Application\RestartLimitReached;
|
||||||
|
|
||||||
|
function applicationWithRestartState(array $attributes = []): Application
|
||||||
|
{
|
||||||
|
$application = new Application;
|
||||||
|
$application->forceFill(array_merge([
|
||||||
|
'status' => 'exited:unhealthy',
|
||||||
|
'restart_count' => 2,
|
||||||
|
'max_restart_count' => 2,
|
||||||
|
'last_restart_type' => 'crash',
|
||||||
|
'last_restart_at' => now(),
|
||||||
|
], $attributes));
|
||||||
|
|
||||||
|
return $application;
|
||||||
|
}
|
||||||
|
|
||||||
|
it('detects applications stopped after reaching the crash restart limit', function () {
|
||||||
|
expect(applicationWithRestartState()->stoppedAfterRestartLimit())->toBeTrue()
|
||||||
|
->and(applicationWithRestartState(['status' => 'running:unhealthy'])->stoppedAfterRestartLimit())->toBeFalse()
|
||||||
|
->and(applicationWithRestartState(['restart_count' => 1])->stoppedAfterRestartLimit())->toBeFalse()
|
||||||
|
->and(applicationWithRestartState(['max_restart_count' => 0])->stoppedAfterRestartLimit())->toBeFalse()
|
||||||
|
->and(applicationWithRestartState(['last_restart_type' => null])->stoppedAfterRestartLimit())->toBeFalse();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shows a stopped after restart limit warning in the status badge', function () {
|
||||||
|
$html = view('components.status.index', [
|
||||||
|
'resource' => applicationWithRestartState(),
|
||||||
|
'showRefreshButton' => false,
|
||||||
|
])->render();
|
||||||
|
|
||||||
|
expect($html)->toContain('Stopped after reaching restart limit (2/2).')
|
||||||
|
->and($html)->toContain('Container has crashed and Coolify stopped it after 2 restart attempts.');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not show the restart limit warning for a normal manual stop', function () {
|
||||||
|
$html = view('components.status.index', [
|
||||||
|
'resource' => applicationWithRestartState([
|
||||||
|
'restart_count' => 0,
|
||||||
|
'last_restart_type' => null,
|
||||||
|
]),
|
||||||
|
'showRefreshButton' => false,
|
||||||
|
])->render();
|
||||||
|
|
||||||
|
expect($html)->not->toContain('Stopped after reaching restart limit');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps restart tracking configurable when stopping an application', function () {
|
||||||
|
$method = new ReflectionMethod(StopApplication::class, 'handle');
|
||||||
|
$resetRestartCount = collect($method->getParameters())->firstWhere('name', 'resetRestartCount');
|
||||||
|
|
||||||
|
expect($resetRestartCount)->not->toBeNull()
|
||||||
|
->and($resetRestartCount->getDefaultValue())->toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the application link for restart limit notifications', function () {
|
||||||
|
$application = new class extends Application
|
||||||
|
{
|
||||||
|
public function link()
|
||||||
|
{
|
||||||
|
return 'https://coolify.test/project/link-from-model';
|
||||||
|
}
|
||||||
|
};
|
||||||
|
$application->forceFill([
|
||||||
|
'name' => 'crashy-app',
|
||||||
|
'uuid' => 'application-uuid',
|
||||||
|
'restart_count' => 2,
|
||||||
|
'max_restart_count' => 2,
|
||||||
|
]);
|
||||||
|
$application->setRelation('environment', (object) [
|
||||||
|
'uuid' => 'environment-uuid',
|
||||||
|
'name' => 'production',
|
||||||
|
'project' => (object) ['uuid' => 'project-uuid'],
|
||||||
|
]);
|
||||||
|
|
||||||
|
$notification = new RestartLimitReached($application);
|
||||||
|
|
||||||
|
expect($notification->resource_url)->toBe('https://coolify.test/project/link-from-model');
|
||||||
|
});
|
||||||
Loading…
Reference in a new issue