rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(sentinel): reduce resource churn from health flaps

Browse source 
Ignore health status changes in Sentinel push deduplication when the container lifecycle state is unchanged.

Scope stale resource checks to Sentinel servers whose heartbeat is stale, and avoid refreshing resource last_online_at on unchanged statuses.
This commit is contained in:
Andras Bacsai 2026-05-27 16:48:38 +02:00
parent 1c5d5676ef
commit 626cfb4a22
8 changed files with 410 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
app/Actions/Server/ResourcesCheck.php
View file

@ -3,9 +3,11 @@
namespace App\Actions\Server;
use App\Models\Application;
use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneClickhouse;
use App\Models\StandaloneDocker;
use App\Models\StandaloneDragonfly;
use App\Models\StandaloneKeydb;
use App\Models\StandaloneMariadb;
@ -13,6 +15,8 @@
use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use App\Models\SwarmDocker;
use Illuminate\Support\Collection;
use Lorisleiva\Actions\Concerns\AsAction;
class ResourcesCheck
@ -21,21 +25,73 @@ class ResourcesCheck
public function handle()
{
$seconds = 60;
$seconds = config('constants.sentinel.resource_stale_seconds', 300);
$staleServerIds = $this->staleSentinelServerIds($seconds);
if ($staleServerIds->isEmpty()) {
return;
}
[$standaloneDockerIds, $swarmDockerIds] = $this->destinationIdsForServers($staleServerIds);
try {
Application::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
ServiceApplication::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
ServiceDatabase::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
StandalonePostgresql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
StandaloneRedis::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
StandaloneMongodb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
StandaloneMysql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
StandaloneMariadb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
StandaloneKeydb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
StandaloneDragonfly::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
StandaloneClickhouse::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
Application::where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
->where('status', 'not like', 'exited%')
->update(['status' => 'exited']);
ServiceApplication::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))
->where('status', 'not like', 'exited%')
->update(['status' => 'exited']);
ServiceDatabase::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))
->where('status', 'not like', 'exited%')
->update(['status' => 'exited']);
collect([
StandalonePostgresql::class,
StandaloneRedis::class,
StandaloneMongodb::class,
StandaloneMysql::class,
StandaloneMariadb::class,
StandaloneKeydb::class,
StandaloneDragonfly::class,
StandaloneClickhouse::class,
])->each(function (string $databaseClass) use ($standaloneDockerIds, $swarmDockerIds) {
$databaseClass::query()
->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
->where('status', 'not like', 'exited%')
->update(['status' => 'exited']);
});
} catch (\Throwable $e) {
return handleError($e);
}
}
private function staleSentinelServerIds(int $seconds): Collection
{
return Server::query()
->whereNotNull('sentinel_updated_at')
->where('sentinel_updated_at', '<', now()->subSeconds($seconds))
->whereHas('settings', fn ($query) => $query->where('is_sentinel_enabled', true))
->pluck('id');
}
private function destinationIdsForServers(Collection $serverIds): array
{
return [
StandaloneDocker::whereIn('server_id', $serverIds)->pluck('id'),
SwarmDocker::whereIn('server_id', $serverIds)->pluck('id'),
];
}
private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
{
$query->where(function ($query) use ($standaloneDockerIds) {
$query->where('destination_type', StandaloneDocker::class)
->whereIn('destination_id', $standaloneDockerIds);
})->orWhere(function ($query) use ($swarmDockerIds) {
$query->where('destination_type', SwarmDocker::class)
->whereIn('destination_id', $swarmDockerIds);
});
}
}

13
app/Http/Controllers/Api/SentinelController.php
View file

@ -143,11 +143,13 @@ private function shouldDispatchUpdate(Server $server, array $data): bool
/**
* Build a stable hash of container state.
*
* Covers [name, state, health_status] only — metrics and
* filesystem_usage_root are excluded on purpose (disk % churns constantly
* and would defeat the hash; the storage check is separately cache-gated
* inside PushServerUpdateJob). Sorted by name so container ordering from
* Sentinel does not affect the hash.
* Covers [name, state] only — metrics, filesystem_usage_root, and
* health_status are excluded on purpose. Disk % churns constantly, and
* health checks can flap between starting/healthy/unhealthy while the
* container lifecycle state remains unchanged. Both would otherwise defeat
* the hash and dispatch DB-heavy PushServerUpdateJob instances too often.
* The force window still refreshes full state periodically. Sorted by name
* so container ordering from Sentinel does not affect the hash.
*/
private function containerStateHash(array $data): string
{
@ -155,7 +157,6 @@ private function containerStateHash(array $data): string
->map(fn ($c) => [
'name' => data_get($c, 'name'),
'state' => data_get($c, 'state'),
'health_status' => data_get($c, 'health_status'),
])
->sortBy('name')
->values()

229
app/Jobs/PushServerUpdateJob.php
View file

@ -13,6 +13,16 @@
use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneClickhouse;
use App\Models\StandaloneDocker;
use App\Models\StandaloneDragonfly;
use App\Models\StandaloneKeydb;
use App\Models\StandaloneMariadb;
use App\Models\StandaloneMongodb;
use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use App\Models\SwarmDocker;
use App\Notifications\Container\ContainerRestarted;
use App\Services\ContainerStatusAggregator;
use App\Traits\CalculatesExcludedStatus;
@ -25,6 +35,7 @@
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Laravel\Horizon\Contracts\Silenced;
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@ -46,6 +57,18 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
public Collection $services;
public Collection $applicationsById;
public Collection $previewsByKey;
public Collection $databasesByUuid;
public Collection $servicesById;
public Collection $serviceApplicationsById;
public Collection $serviceDatabasesById;
public Collection $allApplicationIds;
public Collection $allDatabaseUuids;
@ -103,6 +126,12 @@ public function __construct(public Server $server, public $data)
$this->allTcpProxyUuids = collect();
$this->allServiceApplicationIds = collect();
$this->allServiceDatabaseIds = collect();
$this->applicationsById = collect();
$this->previewsByKey = collect();
$this->databasesByUuid = collect();
$this->servicesById = collect();
$this->serviceApplicationsById = collect();
$this->serviceDatabasesById = collect();
}
public function handle()
@ -120,6 +149,12 @@ public function handle()
$this->allTcpProxyUuids ??= collect();
$this->allServiceApplicationIds ??= collect();
$this->allServiceDatabaseIds ??= collect();
$this->applicationsById ??= collect();
$this->previewsByKey ??= collect();
$this->databasesByUuid ??= collect();
$this->servicesById ??= collect();
$this->serviceApplicationsById ??= collect();
$this->serviceDatabasesById ??= collect();
// TODO: Swarm is not supported yet
if (! $this->data) {
@ -151,13 +186,16 @@ public function handle()
return;
}
$this->applications = $this->server->applications();
$this->databases = $this->server->databases();
$this->previews = $this->server->previews();
// Eager load service applications and databases to avoid N+1 queries
$this->services = $this->server->services()
->with(['applications:id,service_id', 'databases:id,service_id'])
->get();
$this->applications = $this->loadApplications();
$this->databases = $this->loadDatabases();
$this->previews = $this->loadPreviews();
$this->services = $this->loadServices();
$this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id);
$this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id);
$this->databasesByUuid = $this->databases->keyBy('uuid');
$this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id);
$this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id);
$this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id);
$this->allApplicationIds = $this->applications->filter(function ($application) {
return $application->additional_servers_count === 0;
@ -170,9 +208,8 @@ public function handle()
});
$this->allDatabaseUuids = $this->databases->pluck('uuid');
$this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid');
// Use eager-loaded relationships instead of querying in loop
$this->allServiceApplicationIds = $this->services->flatMap(fn ($service) => $service->applications->pluck('id'));
$this->allServiceDatabaseIds = $this->services->flatMap(fn ($service) => $service->databases->pluck('id'));
$this->allServiceApplicationIds = $this->serviceApplicationsById->keys();
$this->allServiceDatabaseIds = $this->serviceDatabasesById->keys();
foreach ($this->containers as $container) {
$containerStatus = data_get($container, 'state', 'exited');
@ -286,6 +323,142 @@ public function handle()
$this->checkLogDrainContainer();
}
private function loadApplications(): Collection
{
[$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
$applications = Application::withoutGlobalScope('withRelations')
->select([
'id',
'uuid',
'name',
'status',
'build_pack',
'docker_compose_raw',
'destination_id',
'destination_type',
'last_online_at',
])
->withCount('additional_servers')
->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
->get();
$additionalApplicationIds = DB::table('additional_destinations')
->where('server_id', $this->server->id)
->pluck('application_id');
if ($additionalApplicationIds->isNotEmpty()) {
$applications = $applications->concat(
Application::withoutGlobalScope('withRelations')
->select([
'id',
'uuid',
'name',
'status',
'build_pack',
'docker_compose_raw',
'destination_id',
'destination_type',
'last_online_at',
])
->withCount('additional_servers')
->whereIn('id', $additionalApplicationIds)
->get()
);
}
return $applications->unique('id')->values();
}
private function loadPreviews(): Collection
{
$applicationIds = $this->applications->pluck('id');
if ($applicationIds->isEmpty()) {
return collect();
}
return ApplicationPreview::query()
->select([
'id',
'application_id',
'pull_request_id',
'status',
'last_online_at',
])
->whereIn('application_id', $applicationIds)
->get();
}
private function loadServices(): Collection
{
return $this->server->services()
->select([
'id',
'server_id',
'uuid',
'docker_compose_raw',
])
->with([
'applications:id,service_id,status,last_online_at',
'databases:id,service_id,status,last_online_at,is_public,name',
])
->get();
}
private function loadDatabases(): Collection
{
[$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
$databaseColumns = [
'id',
'uuid',
'name',
'status',
'is_public',
'destination_id',
'destination_type',
'last_online_at',
'restart_count',
'last_restart_at',
'last_restart_type',
];
return collect([
StandalonePostgresql::class,
StandaloneRedis::class,
StandaloneMongodb::class,
StandaloneMysql::class,
StandaloneMariadb::class,
StandaloneKeydb::class,
StandaloneDragonfly::class,
StandaloneClickhouse::class,
])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) {
return $databaseClass::query()
->select($databaseColumns)
->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
->get();
})->filter(fn ($database) => data_get($database, 'name') !== 'coolify-db')->values();
}
private function serverDestinationIds(): array
{
return [
StandaloneDocker::where('server_id', $this->server->id)->pluck('id'),
SwarmDocker::where('server_id', $this->server->id)->pluck('id'),
];
}
private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
{
$query->where(function ($query) use ($standaloneDockerIds) {
$query->where('destination_type', StandaloneDocker::class)
->whereIn('destination_id', $standaloneDockerIds);
})->orWhere(function ($query) use ($swarmDockerIds) {
$query->where('destination_type', SwarmDocker::class)
->whereIn('destination_id', $swarmDockerIds);
});
}
private function aggregateMultiContainerStatuses()
{
if ($this->applicationContainerStatuses->isEmpty()) {
@ -293,7 +466,7 @@ private function aggregateMultiContainerStatuses()
}
foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
$application = $this->applications->where('id', $applicationId)->first();
$application = $this->applicationsById->get((string) $applicationId);
if (! $application) {
continue;
}
@ -314,8 +487,6 @@ private function aggregateMultiContainerStatuses()
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
} elseif ($aggregatedStatus) {
$application->update(['last_online_at' => now()]);
}
continue;
@ -330,8 +501,6 @@ private function aggregateMultiContainerStatuses()
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
} elseif ($aggregatedStatus) {
$application->update(['last_online_at' => now()]);
}
}
}
@ -350,7 +519,7 @@ private function aggregateServiceContainerStatuses()
continue;
}
$service = $this->services->where('id', $serviceId)->first();
$service = $this->servicesById->get((string) $serviceId);
if (! $service) {
continue;
}
@ -358,9 +527,9 @@ private function aggregateServiceContainerStatuses()
// Get the service sub-resource (ServiceApplication or ServiceDatabase)
$subResource = null;
if ($subType === 'application') {
$subResource = $service->applications->where('id', $subId)->first();
$subResource = $this->serviceApplicationsById->get((string) $subId);
} elseif ($subType === 'database') {
$subResource = $service->databases->where('id', $subId)->first();
$subResource = $this->serviceDatabasesById->get((string) $subId);
}
if (! $subResource) {
@ -382,8 +551,6 @@ private function aggregateServiceContainerStatuses()
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
} elseif ($aggregatedStatus) {
$subResource->update(['last_online_at' => now()]);
}
continue;
@ -399,39 +566,31 @@ private function aggregateServiceContainerStatuses()
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
} elseif ($aggregatedStatus) {
$subResource->update(['last_online_at' => now()]);
}
}
}
private function updateApplicationStatus(string $applicationId, string $containerStatus)
{
$application = $this->applications->where('id', $applicationId)->first();
$application = $this->applicationsById->get((string) $applicationId);
if (! $application) {
return;
}
if ($application->status !== $containerStatus) {
$application->status = $containerStatus;
$application->save();
} else {
$application->update(['last_online_at' => now()]);
}
}
private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus)
{
$application = $this->previews->where('application_id', $applicationId)
->where('pull_request_id', $pullRequestId)
->first();
$application = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
if (! $application) {
return;
}
if ($application->status !== $containerStatus) {
$application->status = $containerStatus;
$application->save();
} else {
$application->update(['last_online_at' => now()]);
}
}
@ -479,9 +638,7 @@ private function updateNotFoundApplicationPreviewStatus()
$applicationId = $parts[0];
$pullRequestId = $parts[1];
$applicationPreview = $this->previews->where('application_id', $applicationId)
->where('pull_request_id', $pullRequestId)
->first();
$applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) {
$previewIdsToUpdate->push($applicationPreview->id);
@ -520,15 +677,13 @@ private function updateProxyStatus()
private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false)
{
$database = $this->databases->where('uuid', $databaseUuid)->first();
$database = $this->databasesByUuid->get($databaseUuid);
if (! $database) {
return;
}
if ($database->status !== $containerStatus) {
$database->status = $containerStatus;
$database->save();
} else {
$database->update(['last_online_at' => now()]);
}
if ($this->isRunning($containerStatus) && $tcpProxy) {
$tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
@ -563,7 +718,7 @@ private function updateNotFoundDatabaseStatus()
}
$notFoundDatabaseUuids->each(function ($databaseUuid) {
$database = $this->databases->where('uuid', $databaseUuid)->first();
$database = $this->databasesByUuid->get($databaseUuid);
if ($database) {
if (! str($database->status)->startsWith('exited')) {
$database->update([

10
config/constants.php
View file

@ -93,9 +93,15 @@
'sentinel' => [
// How often (seconds) PushServerUpdateJob is force-dispatched even when
// the container state hash is unchanged. Keeps last_online_at,
// exited-detection and storage checks from going stale.
// the container state hash is unchanged. Keeps exited-detection and
// storage checks from going stale without writing every resource row on
// every push.
'push_force_interval_seconds' => env('SENTINEL_PUSH_FORCE_INTERVAL_SECONDS', 300),
// How long a Sentinel-enabled server may go without a heartbeat before
// ResourcesCheck considers its resources stale. Per-resource
// last_online_at is only updated on real status changes, not every push.
'resource_stale_seconds' => env('SENTINEL_RESOURCE_STALE_SECONDS', 300),
],
'proxy' => [

45
tests/Feature/PushServerUpdateJobLastOnlineTest.php
View file

@ -1,17 +1,19 @@
<?php
use App\Jobs\PushServerUpdateJob;
use App\Models\Environment;
use App\Models\Project;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
test('database last_online_at is updated when status unchanged', function () {
test('database last_online_at is not updated when status is unchanged', function () {
$team = Team::factory()->create();
$database = StandalonePostgresql::factory()->create([
'team_id' => $team->id,
$database = createPushUpdatePostgresql($team, [
'status' => 'running:healthy',
'last_online_at' => now()->subMinutes(5),
]);
@ -40,15 +42,13 @@
$database->refresh();
// last_online_at should be updated even though status didn't change
expect($database->last_online_at->greaterThan($oldLastOnline))->toBeTrue();
expect((string) $database->last_online_at)->toBe((string) $oldLastOnline);
expect($database->status)->toBe('running:healthy');
});
test('database status is updated when container status changes', function () {
$team = Team::factory()->create();
$database = StandalonePostgresql::factory()->create([
'team_id' => $team->id,
$database = createPushUpdatePostgresql($team, [
'status' => 'exited',
]);
@ -79,8 +79,7 @@
test('database is not marked exited when containers list is empty', function () {
$team = Team::factory()->create();
$database = StandalonePostgresql::factory()->create([
'team_id' => $team->id,
$database = createPushUpdatePostgresql($team, [
'status' => 'running:healthy',
]);
@ -99,3 +98,31 @@
// Status should remain running, NOT be set to exited
expect($database->status)->toBe('running:healthy');
});
function createPushUpdatePostgresql(Team $team, array $attributes = []): StandalonePostgresql
{
$lastOnlineAt = $attributes['last_online_at'] ?? null;
unset($attributes['last_online_at']);
$server = Server::factory()->create(['team_id' => $team->id]);
$destination = StandaloneDocker::where('server_id', $server->id)->first()
?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
$project = Project::factory()->create(['team_id' => $team->id]);
$environment = Environment::factory()->create(['project_id' => $project->id]);
$database = StandalonePostgresql::create(array_merge([
'uuid' => (string) str()->uuid(),
'name' => 'postgres-'.str()->random(8),
'postgres_password' => 'secret',
'status' => 'exited',
'destination_id' => $destination->id,
'destination_type' => $destination->getMorphClass(),
'environment_id' => $environment->id,
], $attributes));
if ($lastOnlineAt !== null) {
$database->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();
}
return $database;
}

15
tests/Feature/PushServerUpdateJobTest.php
View file

@ -4,17 +4,21 @@
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceApplication;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
test('containers with empty service subId are skipped', function () {
$server = Server::factory()->create();
$team = Team::factory()->create();
$server = Server::factory()->create(['team_id' => $team->id]);
$service = Service::factory()->create([
'server_id' => $server->id,
]);
$serviceApp = ServiceApplication::factory()->create([
$serviceApp = ServiceApplication::create([
'service_id' => $service->id,
'uuid' => (string) str()->uuid(),
'name' => 'app-'.str()->random(8),
]);
$data = [
@ -44,12 +48,15 @@
});
test('containers with valid service subId are processed', function () {
$server = Server::factory()->create();
$team = Team::factory()->create();
$server = Server::factory()->create(['team_id' => $team->id]);
$service = Service::factory()->create([
'server_id' => $server->id,
]);
$serviceApp = ServiceApplication::factory()->create([
$serviceApp = ServiceApplication::create([
'service_id' => $service->id,
'uuid' => (string) str()->uuid(),
'name' => 'app-'.str()->random(8),
]);
$data = [

78
tests/Feature/ResourcesCheckTest.php Normal file
View file

@ -0,0 +1,78 @@
<?php
use App\Actions\Server\ResourcesCheck;
use App\Models\Application;
use App\Models\Environment;
use App\Models\Project;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
uses(RefreshDatabase::class);
it('does not mark resources exited when sentinel is still reporting for the server', function () {
Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));
config(['constants.sentinel.resource_stale_seconds' => 300]);
$application = resourcesCheckApplication([
'last_online_at' => now()->subHour(),
'status' => 'running:healthy',
], [
'sentinel_updated_at' => now()->subMinute(),
]);
ResourcesCheck::run();
$application->refresh();
expect($application->status)->toBe('running:healthy');
});
it('marks resources exited when their sentinel server is stale', function () {
Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));
config(['constants.sentinel.resource_stale_seconds' => 300]);
$application = resourcesCheckApplication([
'last_online_at' => now()->subHour(),
'status' => 'running:healthy',
], [
'sentinel_updated_at' => now()->subMinutes(10),
]);
ResourcesCheck::run();
$application->refresh();
expect($application->status)->toBe('exited:unhealthy');
});
function resourcesCheckApplication(array $applicationAttributes = [], array $serverAttributes = []): Application
{
$lastOnlineAt = $applicationAttributes['last_online_at'] ?? null;
unset($applicationAttributes['last_online_at']);
$team = Team::factory()->create();
$server = Server::factory()->create(array_merge([
'team_id' => $team->id,
], $serverAttributes));
$server->settings()->update(['is_sentinel_enabled' => true]);
$destination = StandaloneDocker::where('server_id', $server->id)->first()
?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
$project = Project::factory()->create(['team_id' => $team->id]);
$environment = Environment::factory()->create(['project_id' => $project->id]);
$application = Application::factory()->create(array_merge([
'environment_id' => $environment->id,
'destination_id' => $destination->id,
'destination_type' => $destination->getMorphClass(),
], $applicationAttributes));
if ($lastOnlineAt !== null) {
$application->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();
}
return $application;
}

10
tests/Feature/SentinelPushDeduplicationTest.php
View file

@ -138,6 +138,16 @@ function sentinelPayload(array $containers, ?float $diskPercentage = 42.0): arra
Queue::assertPushed(PushServerUpdateJob::class, 2);
});
it('ignores health status changes while container lifecycle state is unchanged', function () {
$healthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']];
$unhealthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'unhealthy']];
pushSentinel($this->token, sentinelPayload($healthy))->assertOk();
pushSentinel($this->token, sentinelPayload($unhealthy))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 1);
});
it('ignores disk percentage changes (excluded from the hash)', function () use ($running) {
pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 42.0))->assertOk();
pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 88.0))->assertOk();