From 626cfb4a2281ec0415abcb422e0e13d7759c1fba Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 27 May 2026 16:48:38 +0200 Subject: [PATCH] fix(sentinel): reduce resource churn from health flaps Ignore health status changes in Sentinel push deduplication when the container lifecycle state is unchanged. Scope stale resource checks to Sentinel servers whose heartbeat is stale, and avoid refreshing resource last_online_at on unchanged statuses. --- app/Actions/Server/ResourcesCheck.php | 80 +++++- .../Controllers/Api/SentinelController.php | 13 +- app/Jobs/PushServerUpdateJob.php | 229 +++++++++++++++--- config/constants.php | 10 +- .../PushServerUpdateJobLastOnlineTest.php | 45 +++- tests/Feature/PushServerUpdateJobTest.php | 15 +- tests/Feature/ResourcesCheckTest.php | 78 ++++++ .../Feature/SentinelPushDeduplicationTest.php | 10 + 8 files changed, 410 insertions(+), 70 deletions(-) create mode 100644 tests/Feature/ResourcesCheckTest.php diff --git a/app/Actions/Server/ResourcesCheck.php b/app/Actions/Server/ResourcesCheck.php index e6b90ba38..cc11a2e07 100644 --- a/app/Actions/Server/ResourcesCheck.php +++ b/app/Actions/Server/ResourcesCheck.php @@ -3,9 +3,11 @@ namespace App\Actions\Server; use App\Models\Application; +use App\Models\Server; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; use App\Models\StandaloneClickhouse; +use App\Models\StandaloneDocker; use App\Models\StandaloneDragonfly; use App\Models\StandaloneKeydb; use App\Models\StandaloneMariadb; @@ -13,6 +15,8 @@ use App\Models\StandaloneMysql; use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; +use App\Models\SwarmDocker; +use Illuminate\Support\Collection; use Lorisleiva\Actions\Concerns\AsAction; class ResourcesCheck @@ -21,21 +25,73 @@ class ResourcesCheck public function handle() { - $seconds = 60; + $seconds = config('constants.sentinel.resource_stale_seconds', 300); + $staleServerIds = $this->staleSentinelServerIds($seconds); + + if ($staleServerIds->isEmpty()) { + return; + } + + [$standaloneDockerIds, $swarmDockerIds] = $this->destinationIdsForServers($staleServerIds); + try { - Application::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - ServiceApplication::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - ServiceDatabase::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandalonePostgresql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneRedis::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneMongodb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneMysql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneMariadb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneKeydb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneDragonfly::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneClickhouse::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + Application::where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds)) + ->where('status', 'not like', 'exited%') + ->update(['status' => 'exited']); + + ServiceApplication::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds)) + ->where('status', 'not like', 'exited%') + ->update(['status' => 'exited']); + + ServiceDatabase::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds)) + ->where('status', 'not like', 'exited%') + ->update(['status' => 'exited']); + + collect([ + StandalonePostgresql::class, + StandaloneRedis::class, + StandaloneMongodb::class, + StandaloneMysql::class, + StandaloneMariadb::class, + StandaloneKeydb::class, + StandaloneDragonfly::class, + StandaloneClickhouse::class, + ])->each(function (string $databaseClass) use ($standaloneDockerIds, $swarmDockerIds) { + $databaseClass::query() + ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds)) + ->where('status', 'not like', 'exited%') + ->update(['status' => 'exited']); + }); } catch (\Throwable $e) { return handleError($e); } } + + private function staleSentinelServerIds(int $seconds): Collection + { + return Server::query() + ->whereNotNull('sentinel_updated_at') + ->where('sentinel_updated_at', '<', now()->subSeconds($seconds)) + ->whereHas('settings', fn ($query) => $query->where('is_sentinel_enabled', true)) + ->pluck('id'); + } + + private function destinationIdsForServers(Collection $serverIds): array + { + return [ + StandaloneDocker::whereIn('server_id', $serverIds)->pluck('id'), + SwarmDocker::whereIn('server_id', $serverIds)->pluck('id'), + ]; + } + + private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void + { + $query->where(function ($query) use ($standaloneDockerIds) { + $query->where('destination_type', StandaloneDocker::class) + ->whereIn('destination_id', $standaloneDockerIds); + })->orWhere(function ($query) use ($swarmDockerIds) { + $query->where('destination_type', SwarmDocker::class) + ->whereIn('destination_id', $swarmDockerIds); + }); + } } diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php index 53b611afa..df5c60d40 100644 --- a/app/Http/Controllers/Api/SentinelController.php +++ b/app/Http/Controllers/Api/SentinelController.php @@ -143,11 +143,13 @@ private function shouldDispatchUpdate(Server $server, array $data): bool /** * Build a stable hash of container state. * - * Covers [name, state, health_status] only — metrics and - * filesystem_usage_root are excluded on purpose (disk % churns constantly - * and would defeat the hash; the storage check is separately cache-gated - * inside PushServerUpdateJob). Sorted by name so container ordering from - * Sentinel does not affect the hash. + * Covers [name, state] only — metrics, filesystem_usage_root, and + * health_status are excluded on purpose. Disk % churns constantly, and + * health checks can flap between starting/healthy/unhealthy while the + * container lifecycle state remains unchanged. Both would otherwise defeat + * the hash and dispatch DB-heavy PushServerUpdateJob instances too often. + * The force window still refreshes full state periodically. Sorted by name + * so container ordering from Sentinel does not affect the hash. */ private function containerStateHash(array $data): string { @@ -155,7 +157,6 @@ private function containerStateHash(array $data): string ->map(fn ($c) => [ 'name' => data_get($c, 'name'), 'state' => data_get($c, 'state'), - 'health_status' => data_get($c, 'health_status'), ]) ->sortBy('name') ->values() diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index e75509f62..37c73f168 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -13,6 +13,16 @@ use App\Models\Server; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; +use App\Models\StandaloneClickhouse; +use App\Models\StandaloneDocker; +use App\Models\StandaloneDragonfly; +use App\Models\StandaloneKeydb; +use App\Models\StandaloneMariadb; +use App\Models\StandaloneMongodb; +use App\Models\StandaloneMysql; +use App\Models\StandalonePostgresql; +use App\Models\StandaloneRedis; +use App\Models\SwarmDocker; use App\Notifications\Container\ContainerRestarted; use App\Services\ContainerStatusAggregator; use App\Traits\CalculatesExcludedStatus; @@ -25,6 +35,7 @@ use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\DB; use Laravel\Horizon\Contracts\Silenced; class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced @@ -46,6 +57,18 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced public Collection $services; + public Collection $applicationsById; + + public Collection $previewsByKey; + + public Collection $databasesByUuid; + + public Collection $servicesById; + + public Collection $serviceApplicationsById; + + public Collection $serviceDatabasesById; + public Collection $allApplicationIds; public Collection $allDatabaseUuids; @@ -103,6 +126,12 @@ public function __construct(public Server $server, public $data) $this->allTcpProxyUuids = collect(); $this->allServiceApplicationIds = collect(); $this->allServiceDatabaseIds = collect(); + $this->applicationsById = collect(); + $this->previewsByKey = collect(); + $this->databasesByUuid = collect(); + $this->servicesById = collect(); + $this->serviceApplicationsById = collect(); + $this->serviceDatabasesById = collect(); } public function handle() @@ -120,6 +149,12 @@ public function handle() $this->allTcpProxyUuids ??= collect(); $this->allServiceApplicationIds ??= collect(); $this->allServiceDatabaseIds ??= collect(); + $this->applicationsById ??= collect(); + $this->previewsByKey ??= collect(); + $this->databasesByUuid ??= collect(); + $this->servicesById ??= collect(); + $this->serviceApplicationsById ??= collect(); + $this->serviceDatabasesById ??= collect(); // TODO: Swarm is not supported yet if (! $this->data) { @@ -151,13 +186,16 @@ public function handle() return; } - $this->applications = $this->server->applications(); - $this->databases = $this->server->databases(); - $this->previews = $this->server->previews(); - // Eager load service applications and databases to avoid N+1 queries - $this->services = $this->server->services() - ->with(['applications:id,service_id', 'databases:id,service_id']) - ->get(); + $this->applications = $this->loadApplications(); + $this->databases = $this->loadDatabases(); + $this->previews = $this->loadPreviews(); + $this->services = $this->loadServices(); + $this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id); + $this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id); + $this->databasesByUuid = $this->databases->keyBy('uuid'); + $this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id); + $this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id); + $this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id); $this->allApplicationIds = $this->applications->filter(function ($application) { return $application->additional_servers_count === 0; @@ -170,9 +208,8 @@ public function handle() }); $this->allDatabaseUuids = $this->databases->pluck('uuid'); $this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid'); - // Use eager-loaded relationships instead of querying in loop - $this->allServiceApplicationIds = $this->services->flatMap(fn ($service) => $service->applications->pluck('id')); - $this->allServiceDatabaseIds = $this->services->flatMap(fn ($service) => $service->databases->pluck('id')); + $this->allServiceApplicationIds = $this->serviceApplicationsById->keys(); + $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys(); foreach ($this->containers as $container) { $containerStatus = data_get($container, 'state', 'exited'); @@ -286,6 +323,142 @@ public function handle() $this->checkLogDrainContainer(); } + private function loadApplications(): Collection + { + [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds(); + + $applications = Application::withoutGlobalScope('withRelations') + ->select([ + 'id', + 'uuid', + 'name', + 'status', + 'build_pack', + 'docker_compose_raw', + 'destination_id', + 'destination_type', + 'last_online_at', + ]) + ->withCount('additional_servers') + ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds)) + ->get(); + + $additionalApplicationIds = DB::table('additional_destinations') + ->where('server_id', $this->server->id) + ->pluck('application_id'); + + if ($additionalApplicationIds->isNotEmpty()) { + $applications = $applications->concat( + Application::withoutGlobalScope('withRelations') + ->select([ + 'id', + 'uuid', + 'name', + 'status', + 'build_pack', + 'docker_compose_raw', + 'destination_id', + 'destination_type', + 'last_online_at', + ]) + ->withCount('additional_servers') + ->whereIn('id', $additionalApplicationIds) + ->get() + ); + } + + return $applications->unique('id')->values(); + } + + private function loadPreviews(): Collection + { + $applicationIds = $this->applications->pluck('id'); + + if ($applicationIds->isEmpty()) { + return collect(); + } + + return ApplicationPreview::query() + ->select([ + 'id', + 'application_id', + 'pull_request_id', + 'status', + 'last_online_at', + ]) + ->whereIn('application_id', $applicationIds) + ->get(); + } + + private function loadServices(): Collection + { + return $this->server->services() + ->select([ + 'id', + 'server_id', + 'uuid', + 'docker_compose_raw', + ]) + ->with([ + 'applications:id,service_id,status,last_online_at', + 'databases:id,service_id,status,last_online_at,is_public,name', + ]) + ->get(); + } + + private function loadDatabases(): Collection + { + [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds(); + $databaseColumns = [ + 'id', + 'uuid', + 'name', + 'status', + 'is_public', + 'destination_id', + 'destination_type', + 'last_online_at', + 'restart_count', + 'last_restart_at', + 'last_restart_type', + ]; + + return collect([ + StandalonePostgresql::class, + StandaloneRedis::class, + StandaloneMongodb::class, + StandaloneMysql::class, + StandaloneMariadb::class, + StandaloneKeydb::class, + StandaloneDragonfly::class, + StandaloneClickhouse::class, + ])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) { + return $databaseClass::query() + ->select($databaseColumns) + ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds)) + ->get(); + })->filter(fn ($database) => data_get($database, 'name') !== 'coolify-db')->values(); + } + + private function serverDestinationIds(): array + { + return [ + StandaloneDocker::where('server_id', $this->server->id)->pluck('id'), + SwarmDocker::where('server_id', $this->server->id)->pluck('id'), + ]; + } + + private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void + { + $query->where(function ($query) use ($standaloneDockerIds) { + $query->where('destination_type', StandaloneDocker::class) + ->whereIn('destination_id', $standaloneDockerIds); + })->orWhere(function ($query) use ($swarmDockerIds) { + $query->where('destination_type', SwarmDocker::class) + ->whereIn('destination_id', $swarmDockerIds); + }); + } + private function aggregateMultiContainerStatuses() { if ($this->applicationContainerStatuses->isEmpty()) { @@ -293,7 +466,7 @@ private function aggregateMultiContainerStatuses() } foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) { - $application = $this->applications->where('id', $applicationId)->first(); + $application = $this->applicationsById->get((string) $applicationId); if (! $application) { continue; } @@ -314,8 +487,6 @@ private function aggregateMultiContainerStatuses() if ($aggregatedStatus && $application->status !== $aggregatedStatus) { $application->status = $aggregatedStatus; $application->save(); - } elseif ($aggregatedStatus) { - $application->update(['last_online_at' => now()]); } continue; @@ -330,8 +501,6 @@ private function aggregateMultiContainerStatuses() if ($aggregatedStatus && $application->status !== $aggregatedStatus) { $application->status = $aggregatedStatus; $application->save(); - } elseif ($aggregatedStatus) { - $application->update(['last_online_at' => now()]); } } } @@ -350,7 +519,7 @@ private function aggregateServiceContainerStatuses() continue; } - $service = $this->services->where('id', $serviceId)->first(); + $service = $this->servicesById->get((string) $serviceId); if (! $service) { continue; } @@ -358,9 +527,9 @@ private function aggregateServiceContainerStatuses() // Get the service sub-resource (ServiceApplication or ServiceDatabase) $subResource = null; if ($subType === 'application') { - $subResource = $service->applications->where('id', $subId)->first(); + $subResource = $this->serviceApplicationsById->get((string) $subId); } elseif ($subType === 'database') { - $subResource = $service->databases->where('id', $subId)->first(); + $subResource = $this->serviceDatabasesById->get((string) $subId); } if (! $subResource) { @@ -382,8 +551,6 @@ private function aggregateServiceContainerStatuses() if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) { $subResource->status = $aggregatedStatus; $subResource->save(); - } elseif ($aggregatedStatus) { - $subResource->update(['last_online_at' => now()]); } continue; @@ -399,39 +566,31 @@ private function aggregateServiceContainerStatuses() if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) { $subResource->status = $aggregatedStatus; $subResource->save(); - } elseif ($aggregatedStatus) { - $subResource->update(['last_online_at' => now()]); } } } private function updateApplicationStatus(string $applicationId, string $containerStatus) { - $application = $this->applications->where('id', $applicationId)->first(); + $application = $this->applicationsById->get((string) $applicationId); if (! $application) { return; } if ($application->status !== $containerStatus) { $application->status = $containerStatus; $application->save(); - } else { - $application->update(['last_online_at' => now()]); } } private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus) { - $application = $this->previews->where('application_id', $applicationId) - ->where('pull_request_id', $pullRequestId) - ->first(); + $application = $this->previewsByKey->get($applicationId.':'.$pullRequestId); if (! $application) { return; } if ($application->status !== $containerStatus) { $application->status = $containerStatus; $application->save(); - } else { - $application->update(['last_online_at' => now()]); } } @@ -479,9 +638,7 @@ private function updateNotFoundApplicationPreviewStatus() $applicationId = $parts[0]; $pullRequestId = $parts[1]; - $applicationPreview = $this->previews->where('application_id', $applicationId) - ->where('pull_request_id', $pullRequestId) - ->first(); + $applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId); if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) { $previewIdsToUpdate->push($applicationPreview->id); @@ -520,15 +677,13 @@ private function updateProxyStatus() private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false) { - $database = $this->databases->where('uuid', $databaseUuid)->first(); + $database = $this->databasesByUuid->get($databaseUuid); if (! $database) { return; } if ($database->status !== $containerStatus) { $database->status = $containerStatus; $database->save(); - } else { - $database->update(['last_online_at' => now()]); } if ($this->isRunning($containerStatus) && $tcpProxy) { $tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) { @@ -563,7 +718,7 @@ private function updateNotFoundDatabaseStatus() } $notFoundDatabaseUuids->each(function ($databaseUuid) { - $database = $this->databases->where('uuid', $databaseUuid)->first(); + $database = $this->databasesByUuid->get($databaseUuid); if ($database) { if (! str($database->status)->startsWith('exited')) { $database->update([ diff --git a/config/constants.php b/config/constants.php index dfff17542..e967ca3bc 100644 --- a/config/constants.php +++ b/config/constants.php @@ -93,9 +93,15 @@ 'sentinel' => [ // How often (seconds) PushServerUpdateJob is force-dispatched even when - // the container state hash is unchanged. Keeps last_online_at, - // exited-detection and storage checks from going stale. + // the container state hash is unchanged. Keeps exited-detection and + // storage checks from going stale without writing every resource row on + // every push. 'push_force_interval_seconds' => env('SENTINEL_PUSH_FORCE_INTERVAL_SECONDS', 300), + + // How long a Sentinel-enabled server may go without a heartbeat before + // ResourcesCheck considers its resources stale. Per-resource + // last_online_at is only updated on real status changes, not every push. + 'resource_stale_seconds' => env('SENTINEL_RESOURCE_STALE_SECONDS', 300), ], 'proxy' => [ diff --git a/tests/Feature/PushServerUpdateJobLastOnlineTest.php b/tests/Feature/PushServerUpdateJobLastOnlineTest.php index 5d2fd6c6a..d986d421d 100644 --- a/tests/Feature/PushServerUpdateJobLastOnlineTest.php +++ b/tests/Feature/PushServerUpdateJobLastOnlineTest.php @@ -1,17 +1,19 @@ create(); - $database = StandalonePostgresql::factory()->create([ - 'team_id' => $team->id, + $database = createPushUpdatePostgresql($team, [ 'status' => 'running:healthy', 'last_online_at' => now()->subMinutes(5), ]); @@ -40,15 +42,13 @@ $database->refresh(); - // last_online_at should be updated even though status didn't change - expect($database->last_online_at->greaterThan($oldLastOnline))->toBeTrue(); + expect((string) $database->last_online_at)->toBe((string) $oldLastOnline); expect($database->status)->toBe('running:healthy'); }); test('database status is updated when container status changes', function () { $team = Team::factory()->create(); - $database = StandalonePostgresql::factory()->create([ - 'team_id' => $team->id, + $database = createPushUpdatePostgresql($team, [ 'status' => 'exited', ]); @@ -79,8 +79,7 @@ test('database is not marked exited when containers list is empty', function () { $team = Team::factory()->create(); - $database = StandalonePostgresql::factory()->create([ - 'team_id' => $team->id, + $database = createPushUpdatePostgresql($team, [ 'status' => 'running:healthy', ]); @@ -99,3 +98,31 @@ // Status should remain running, NOT be set to exited expect($database->status)->toBe('running:healthy'); }); + +function createPushUpdatePostgresql(Team $team, array $attributes = []): StandalonePostgresql +{ + $lastOnlineAt = $attributes['last_online_at'] ?? null; + unset($attributes['last_online_at']); + + $server = Server::factory()->create(['team_id' => $team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->first() + ?? StandaloneDocker::factory()->create(['server_id' => $server->id]); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + + $database = StandalonePostgresql::create(array_merge([ + 'uuid' => (string) str()->uuid(), + 'name' => 'postgres-'.str()->random(8), + 'postgres_password' => 'secret', + 'status' => 'exited', + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + 'environment_id' => $environment->id, + ], $attributes)); + + if ($lastOnlineAt !== null) { + $database->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly(); + } + + return $database; +} diff --git a/tests/Feature/PushServerUpdateJobTest.php b/tests/Feature/PushServerUpdateJobTest.php index d508d58ab..af667d023 100644 --- a/tests/Feature/PushServerUpdateJobTest.php +++ b/tests/Feature/PushServerUpdateJobTest.php @@ -4,17 +4,21 @@ use App\Models\Server; use App\Models\Service; use App\Models\ServiceApplication; +use App\Models\Team; use Illuminate\Foundation\Testing\RefreshDatabase; uses(RefreshDatabase::class); test('containers with empty service subId are skipped', function () { - $server = Server::factory()->create(); + $team = Team::factory()->create(); + $server = Server::factory()->create(['team_id' => $team->id]); $service = Service::factory()->create([ 'server_id' => $server->id, ]); - $serviceApp = ServiceApplication::factory()->create([ + $serviceApp = ServiceApplication::create([ 'service_id' => $service->id, + 'uuid' => (string) str()->uuid(), + 'name' => 'app-'.str()->random(8), ]); $data = [ @@ -44,12 +48,15 @@ }); test('containers with valid service subId are processed', function () { - $server = Server::factory()->create(); + $team = Team::factory()->create(); + $server = Server::factory()->create(['team_id' => $team->id]); $service = Service::factory()->create([ 'server_id' => $server->id, ]); - $serviceApp = ServiceApplication::factory()->create([ + $serviceApp = ServiceApplication::create([ 'service_id' => $service->id, + 'uuid' => (string) str()->uuid(), + 'name' => 'app-'.str()->random(8), ]); $data = [ diff --git a/tests/Feature/ResourcesCheckTest.php b/tests/Feature/ResourcesCheckTest.php new file mode 100644 index 000000000..9fbf0ce2d --- /dev/null +++ b/tests/Feature/ResourcesCheckTest.php @@ -0,0 +1,78 @@ + 300]); + + $application = resourcesCheckApplication([ + 'last_online_at' => now()->subHour(), + 'status' => 'running:healthy', + ], [ + 'sentinel_updated_at' => now()->subMinute(), + ]); + + ResourcesCheck::run(); + + $application->refresh(); + + expect($application->status)->toBe('running:healthy'); +}); + +it('marks resources exited when their sentinel server is stale', function () { + Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC')); + config(['constants.sentinel.resource_stale_seconds' => 300]); + + $application = resourcesCheckApplication([ + 'last_online_at' => now()->subHour(), + 'status' => 'running:healthy', + ], [ + 'sentinel_updated_at' => now()->subMinutes(10), + ]); + + ResourcesCheck::run(); + + $application->refresh(); + + expect($application->status)->toBe('exited:unhealthy'); +}); + +function resourcesCheckApplication(array $applicationAttributes = [], array $serverAttributes = []): Application +{ + $lastOnlineAt = $applicationAttributes['last_online_at'] ?? null; + unset($applicationAttributes['last_online_at']); + + $team = Team::factory()->create(); + $server = Server::factory()->create(array_merge([ + 'team_id' => $team->id, + ], $serverAttributes)); + $server->settings()->update(['is_sentinel_enabled' => true]); + + $destination = StandaloneDocker::where('server_id', $server->id)->first() + ?? StandaloneDocker::factory()->create(['server_id' => $server->id]); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + + $application = Application::factory()->create(array_merge([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ], $applicationAttributes)); + + if ($lastOnlineAt !== null) { + $application->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly(); + } + + return $application; +} diff --git a/tests/Feature/SentinelPushDeduplicationTest.php b/tests/Feature/SentinelPushDeduplicationTest.php index c14d5c67e..a7ecd5d4c 100644 --- a/tests/Feature/SentinelPushDeduplicationTest.php +++ b/tests/Feature/SentinelPushDeduplicationTest.php @@ -138,6 +138,16 @@ function sentinelPayload(array $containers, ?float $diskPercentage = 42.0): arra Queue::assertPushed(PushServerUpdateJob::class, 2); }); +it('ignores health status changes while container lifecycle state is unchanged', function () { + $healthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']]; + $unhealthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'unhealthy']]; + + pushSentinel($this->token, sentinelPayload($healthy))->assertOk(); + pushSentinel($this->token, sentinelPayload($unhealthy))->assertOk(); + + Queue::assertPushed(PushServerUpdateJob::class, 1); +}); + it('ignores disk percentage changes (excluded from the hash)', function () use ($running) { pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 42.0))->assertOk(); pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 88.0))->assertOk();