diff --git a/app/Actions/Server/ResourcesCheck.php b/app/Actions/Server/ResourcesCheck.php index e6b90ba38..cc11a2e07 100644 --- a/app/Actions/Server/ResourcesCheck.php +++ b/app/Actions/Server/ResourcesCheck.php @@ -3,9 +3,11 @@ namespace App\Actions\Server; use App\Models\Application; +use App\Models\Server; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; use App\Models\StandaloneClickhouse; +use App\Models\StandaloneDocker; use App\Models\StandaloneDragonfly; use App\Models\StandaloneKeydb; use App\Models\StandaloneMariadb; @@ -13,6 +15,8 @@ use App\Models\StandaloneMysql; use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; +use App\Models\SwarmDocker; +use Illuminate\Support\Collection; use Lorisleiva\Actions\Concerns\AsAction; class ResourcesCheck @@ -21,21 +25,73 @@ class ResourcesCheck public function handle() { - $seconds = 60; + $seconds = config('constants.sentinel.resource_stale_seconds', 300); + $staleServerIds = $this->staleSentinelServerIds($seconds); + + if ($staleServerIds->isEmpty()) { + return; + } + + [$standaloneDockerIds, $swarmDockerIds] = $this->destinationIdsForServers($staleServerIds); + try { - Application::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - ServiceApplication::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - ServiceDatabase::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandalonePostgresql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneRedis::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneMongodb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneMysql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneMariadb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneKeydb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneDragonfly::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); - StandaloneClickhouse::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']); + Application::where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds)) + ->where('status', 'not like', 'exited%') + ->update(['status' => 'exited']); + + ServiceApplication::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds)) + ->where('status', 'not like', 'exited%') + ->update(['status' => 'exited']); + + ServiceDatabase::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds)) + ->where('status', 'not like', 'exited%') + ->update(['status' => 'exited']); + + collect([ + StandalonePostgresql::class, + StandaloneRedis::class, + StandaloneMongodb::class, + StandaloneMysql::class, + StandaloneMariadb::class, + StandaloneKeydb::class, + StandaloneDragonfly::class, + StandaloneClickhouse::class, + ])->each(function (string $databaseClass) use ($standaloneDockerIds, $swarmDockerIds) { + $databaseClass::query() + ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds)) + ->where('status', 'not like', 'exited%') + ->update(['status' => 'exited']); + }); } catch (\Throwable $e) { return handleError($e); } } + + private function staleSentinelServerIds(int $seconds): Collection + { + return Server::query() + ->whereNotNull('sentinel_updated_at') + ->where('sentinel_updated_at', '<', now()->subSeconds($seconds)) + ->whereHas('settings', fn ($query) => $query->where('is_sentinel_enabled', true)) + ->pluck('id'); + } + + private function destinationIdsForServers(Collection $serverIds): array + { + return [ + StandaloneDocker::whereIn('server_id', $serverIds)->pluck('id'), + SwarmDocker::whereIn('server_id', $serverIds)->pluck('id'), + ]; + } + + private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void + { + $query->where(function ($query) use ($standaloneDockerIds) { + $query->where('destination_type', StandaloneDocker::class) + ->whereIn('destination_id', $standaloneDockerIds); + })->orWhere(function ($query) use ($swarmDockerIds) { + $query->where('destination_type', SwarmDocker::class) + ->whereIn('destination_id', $swarmDockerIds); + }); + } } diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php index 53b611afa..df5c60d40 100644 --- a/app/Http/Controllers/Api/SentinelController.php +++ b/app/Http/Controllers/Api/SentinelController.php @@ -143,11 +143,13 @@ private function shouldDispatchUpdate(Server $server, array $data): bool /** * Build a stable hash of container state. * - * Covers [name, state, health_status] only — metrics and - * filesystem_usage_root are excluded on purpose (disk % churns constantly - * and would defeat the hash; the storage check is separately cache-gated - * inside PushServerUpdateJob). Sorted by name so container ordering from - * Sentinel does not affect the hash. + * Covers [name, state] only — metrics, filesystem_usage_root, and + * health_status are excluded on purpose. Disk % churns constantly, and + * health checks can flap between starting/healthy/unhealthy while the + * container lifecycle state remains unchanged. Both would otherwise defeat + * the hash and dispatch DB-heavy PushServerUpdateJob instances too often. + * The force window still refreshes full state periodically. Sorted by name + * so container ordering from Sentinel does not affect the hash. */ private function containerStateHash(array $data): string { @@ -155,7 +157,6 @@ private function containerStateHash(array $data): string ->map(fn ($c) => [ 'name' => data_get($c, 'name'), 'state' => data_get($c, 'state'), - 'health_status' => data_get($c, 'health_status'), ]) ->sortBy('name') ->values() diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index e75509f62..37c73f168 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -13,6 +13,16 @@ use App\Models\Server; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; +use App\Models\StandaloneClickhouse; +use App\Models\StandaloneDocker; +use App\Models\StandaloneDragonfly; +use App\Models\StandaloneKeydb; +use App\Models\StandaloneMariadb; +use App\Models\StandaloneMongodb; +use App\Models\StandaloneMysql; +use App\Models\StandalonePostgresql; +use App\Models\StandaloneRedis; +use App\Models\SwarmDocker; use App\Notifications\Container\ContainerRestarted; use App\Services\ContainerStatusAggregator; use App\Traits\CalculatesExcludedStatus; @@ -25,6 +35,7 @@ use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\DB; use Laravel\Horizon\Contracts\Silenced; class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced @@ -46,6 +57,18 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced public Collection $services; + public Collection $applicationsById; + + public Collection $previewsByKey; + + public Collection $databasesByUuid; + + public Collection $servicesById; + + public Collection $serviceApplicationsById; + + public Collection $serviceDatabasesById; + public Collection $allApplicationIds; public Collection $allDatabaseUuids; @@ -103,6 +126,12 @@ public function __construct(public Server $server, public $data) $this->allTcpProxyUuids = collect(); $this->allServiceApplicationIds = collect(); $this->allServiceDatabaseIds = collect(); + $this->applicationsById = collect(); + $this->previewsByKey = collect(); + $this->databasesByUuid = collect(); + $this->servicesById = collect(); + $this->serviceApplicationsById = collect(); + $this->serviceDatabasesById = collect(); } public function handle() @@ -120,6 +149,12 @@ public function handle() $this->allTcpProxyUuids ??= collect(); $this->allServiceApplicationIds ??= collect(); $this->allServiceDatabaseIds ??= collect(); + $this->applicationsById ??= collect(); + $this->previewsByKey ??= collect(); + $this->databasesByUuid ??= collect(); + $this->servicesById ??= collect(); + $this->serviceApplicationsById ??= collect(); + $this->serviceDatabasesById ??= collect(); // TODO: Swarm is not supported yet if (! $this->data) { @@ -151,13 +186,16 @@ public function handle() return; } - $this->applications = $this->server->applications(); - $this->databases = $this->server->databases(); - $this->previews = $this->server->previews(); - // Eager load service applications and databases to avoid N+1 queries - $this->services = $this->server->services() - ->with(['applications:id,service_id', 'databases:id,service_id']) - ->get(); + $this->applications = $this->loadApplications(); + $this->databases = $this->loadDatabases(); + $this->previews = $this->loadPreviews(); + $this->services = $this->loadServices(); + $this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id); + $this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id); + $this->databasesByUuid = $this->databases->keyBy('uuid'); + $this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id); + $this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id); + $this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id); $this->allApplicationIds = $this->applications->filter(function ($application) { return $application->additional_servers_count === 0; @@ -170,9 +208,8 @@ public function handle() }); $this->allDatabaseUuids = $this->databases->pluck('uuid'); $this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid'); - // Use eager-loaded relationships instead of querying in loop - $this->allServiceApplicationIds = $this->services->flatMap(fn ($service) => $service->applications->pluck('id')); - $this->allServiceDatabaseIds = $this->services->flatMap(fn ($service) => $service->databases->pluck('id')); + $this->allServiceApplicationIds = $this->serviceApplicationsById->keys(); + $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys(); foreach ($this->containers as $container) { $containerStatus = data_get($container, 'state', 'exited'); @@ -286,6 +323,142 @@ public function handle() $this->checkLogDrainContainer(); } + private function loadApplications(): Collection + { + [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds(); + + $applications = Application::withoutGlobalScope('withRelations') + ->select([ + 'id', + 'uuid', + 'name', + 'status', + 'build_pack', + 'docker_compose_raw', + 'destination_id', + 'destination_type', + 'last_online_at', + ]) + ->withCount('additional_servers') + ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds)) + ->get(); + + $additionalApplicationIds = DB::table('additional_destinations') + ->where('server_id', $this->server->id) + ->pluck('application_id'); + + if ($additionalApplicationIds->isNotEmpty()) { + $applications = $applications->concat( + Application::withoutGlobalScope('withRelations') + ->select([ + 'id', + 'uuid', + 'name', + 'status', + 'build_pack', + 'docker_compose_raw', + 'destination_id', + 'destination_type', + 'last_online_at', + ]) + ->withCount('additional_servers') + ->whereIn('id', $additionalApplicationIds) + ->get() + ); + } + + return $applications->unique('id')->values(); + } + + private function loadPreviews(): Collection + { + $applicationIds = $this->applications->pluck('id'); + + if ($applicationIds->isEmpty()) { + return collect(); + } + + return ApplicationPreview::query() + ->select([ + 'id', + 'application_id', + 'pull_request_id', + 'status', + 'last_online_at', + ]) + ->whereIn('application_id', $applicationIds) + ->get(); + } + + private function loadServices(): Collection + { + return $this->server->services() + ->select([ + 'id', + 'server_id', + 'uuid', + 'docker_compose_raw', + ]) + ->with([ + 'applications:id,service_id,status,last_online_at', + 'databases:id,service_id,status,last_online_at,is_public,name', + ]) + ->get(); + } + + private function loadDatabases(): Collection + { + [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds(); + $databaseColumns = [ + 'id', + 'uuid', + 'name', + 'status', + 'is_public', + 'destination_id', + 'destination_type', + 'last_online_at', + 'restart_count', + 'last_restart_at', + 'last_restart_type', + ]; + + return collect([ + StandalonePostgresql::class, + StandaloneRedis::class, + StandaloneMongodb::class, + StandaloneMysql::class, + StandaloneMariadb::class, + StandaloneKeydb::class, + StandaloneDragonfly::class, + StandaloneClickhouse::class, + ])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) { + return $databaseClass::query() + ->select($databaseColumns) + ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds)) + ->get(); + })->filter(fn ($database) => data_get($database, 'name') !== 'coolify-db')->values(); + } + + private function serverDestinationIds(): array + { + return [ + StandaloneDocker::where('server_id', $this->server->id)->pluck('id'), + SwarmDocker::where('server_id', $this->server->id)->pluck('id'), + ]; + } + + private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void + { + $query->where(function ($query) use ($standaloneDockerIds) { + $query->where('destination_type', StandaloneDocker::class) + ->whereIn('destination_id', $standaloneDockerIds); + })->orWhere(function ($query) use ($swarmDockerIds) { + $query->where('destination_type', SwarmDocker::class) + ->whereIn('destination_id', $swarmDockerIds); + }); + } + private function aggregateMultiContainerStatuses() { if ($this->applicationContainerStatuses->isEmpty()) { @@ -293,7 +466,7 @@ private function aggregateMultiContainerStatuses() } foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) { - $application = $this->applications->where('id', $applicationId)->first(); + $application = $this->applicationsById->get((string) $applicationId); if (! $application) { continue; } @@ -314,8 +487,6 @@ private function aggregateMultiContainerStatuses() if ($aggregatedStatus && $application->status !== $aggregatedStatus) { $application->status = $aggregatedStatus; $application->save(); - } elseif ($aggregatedStatus) { - $application->update(['last_online_at' => now()]); } continue; @@ -330,8 +501,6 @@ private function aggregateMultiContainerStatuses() if ($aggregatedStatus && $application->status !== $aggregatedStatus) { $application->status = $aggregatedStatus; $application->save(); - } elseif ($aggregatedStatus) { - $application->update(['last_online_at' => now()]); } } } @@ -350,7 +519,7 @@ private function aggregateServiceContainerStatuses() continue; } - $service = $this->services->where('id', $serviceId)->first(); + $service = $this->servicesById->get((string) $serviceId); if (! $service) { continue; } @@ -358,9 +527,9 @@ private function aggregateServiceContainerStatuses() // Get the service sub-resource (ServiceApplication or ServiceDatabase) $subResource = null; if ($subType === 'application') { - $subResource = $service->applications->where('id', $subId)->first(); + $subResource = $this->serviceApplicationsById->get((string) $subId); } elseif ($subType === 'database') { - $subResource = $service->databases->where('id', $subId)->first(); + $subResource = $this->serviceDatabasesById->get((string) $subId); } if (! $subResource) { @@ -382,8 +551,6 @@ private function aggregateServiceContainerStatuses() if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) { $subResource->status = $aggregatedStatus; $subResource->save(); - } elseif ($aggregatedStatus) { - $subResource->update(['last_online_at' => now()]); } continue; @@ -399,39 +566,31 @@ private function aggregateServiceContainerStatuses() if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) { $subResource->status = $aggregatedStatus; $subResource->save(); - } elseif ($aggregatedStatus) { - $subResource->update(['last_online_at' => now()]); } } } private function updateApplicationStatus(string $applicationId, string $containerStatus) { - $application = $this->applications->where('id', $applicationId)->first(); + $application = $this->applicationsById->get((string) $applicationId); if (! $application) { return; } if ($application->status !== $containerStatus) { $application->status = $containerStatus; $application->save(); - } else { - $application->update(['last_online_at' => now()]); } } private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus) { - $application = $this->previews->where('application_id', $applicationId) - ->where('pull_request_id', $pullRequestId) - ->first(); + $application = $this->previewsByKey->get($applicationId.':'.$pullRequestId); if (! $application) { return; } if ($application->status !== $containerStatus) { $application->status = $containerStatus; $application->save(); - } else { - $application->update(['last_online_at' => now()]); } } @@ -479,9 +638,7 @@ private function updateNotFoundApplicationPreviewStatus() $applicationId = $parts[0]; $pullRequestId = $parts[1]; - $applicationPreview = $this->previews->where('application_id', $applicationId) - ->where('pull_request_id', $pullRequestId) - ->first(); + $applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId); if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) { $previewIdsToUpdate->push($applicationPreview->id); @@ -520,15 +677,13 @@ private function updateProxyStatus() private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false) { - $database = $this->databases->where('uuid', $databaseUuid)->first(); + $database = $this->databasesByUuid->get($databaseUuid); if (! $database) { return; } if ($database->status !== $containerStatus) { $database->status = $containerStatus; $database->save(); - } else { - $database->update(['last_online_at' => now()]); } if ($this->isRunning($containerStatus) && $tcpProxy) { $tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) { @@ -563,7 +718,7 @@ private function updateNotFoundDatabaseStatus() } $notFoundDatabaseUuids->each(function ($databaseUuid) { - $database = $this->databases->where('uuid', $databaseUuid)->first(); + $database = $this->databasesByUuid->get($databaseUuid); if ($database) { if (! str($database->status)->startsWith('exited')) { $database->update([ diff --git a/config/constants.php b/config/constants.php index dfff17542..e967ca3bc 100644 --- a/config/constants.php +++ b/config/constants.php @@ -93,9 +93,15 @@ 'sentinel' => [ // How often (seconds) PushServerUpdateJob is force-dispatched even when - // the container state hash is unchanged. Keeps last_online_at, - // exited-detection and storage checks from going stale. + // the container state hash is unchanged. Keeps exited-detection and + // storage checks from going stale without writing every resource row on + // every push. 'push_force_interval_seconds' => env('SENTINEL_PUSH_FORCE_INTERVAL_SECONDS', 300), + + // How long a Sentinel-enabled server may go without a heartbeat before + // ResourcesCheck considers its resources stale. Per-resource + // last_online_at is only updated on real status changes, not every push. + 'resource_stale_seconds' => env('SENTINEL_RESOURCE_STALE_SECONDS', 300), ], 'proxy' => [ diff --git a/tests/Feature/PushServerUpdateJobLastOnlineTest.php b/tests/Feature/PushServerUpdateJobLastOnlineTest.php index 5d2fd6c6a..d986d421d 100644 --- a/tests/Feature/PushServerUpdateJobLastOnlineTest.php +++ b/tests/Feature/PushServerUpdateJobLastOnlineTest.php @@ -1,17 +1,19 @@ create(); - $database = StandalonePostgresql::factory()->create([ - 'team_id' => $team->id, + $database = createPushUpdatePostgresql($team, [ 'status' => 'running:healthy', 'last_online_at' => now()->subMinutes(5), ]); @@ -40,15 +42,13 @@ $database->refresh(); - // last_online_at should be updated even though status didn't change - expect($database->last_online_at->greaterThan($oldLastOnline))->toBeTrue(); + expect((string) $database->last_online_at)->toBe((string) $oldLastOnline); expect($database->status)->toBe('running:healthy'); }); test('database status is updated when container status changes', function () { $team = Team::factory()->create(); - $database = StandalonePostgresql::factory()->create([ - 'team_id' => $team->id, + $database = createPushUpdatePostgresql($team, [ 'status' => 'exited', ]); @@ -79,8 +79,7 @@ test('database is not marked exited when containers list is empty', function () { $team = Team::factory()->create(); - $database = StandalonePostgresql::factory()->create([ - 'team_id' => $team->id, + $database = createPushUpdatePostgresql($team, [ 'status' => 'running:healthy', ]); @@ -99,3 +98,31 @@ // Status should remain running, NOT be set to exited expect($database->status)->toBe('running:healthy'); }); + +function createPushUpdatePostgresql(Team $team, array $attributes = []): StandalonePostgresql +{ + $lastOnlineAt = $attributes['last_online_at'] ?? null; + unset($attributes['last_online_at']); + + $server = Server::factory()->create(['team_id' => $team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->first() + ?? StandaloneDocker::factory()->create(['server_id' => $server->id]); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + + $database = StandalonePostgresql::create(array_merge([ + 'uuid' => (string) str()->uuid(), + 'name' => 'postgres-'.str()->random(8), + 'postgres_password' => 'secret', + 'status' => 'exited', + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + 'environment_id' => $environment->id, + ], $attributes)); + + if ($lastOnlineAt !== null) { + $database->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly(); + } + + return $database; +} diff --git a/tests/Feature/PushServerUpdateJobTest.php b/tests/Feature/PushServerUpdateJobTest.php index d508d58ab..af667d023 100644 --- a/tests/Feature/PushServerUpdateJobTest.php +++ b/tests/Feature/PushServerUpdateJobTest.php @@ -4,17 +4,21 @@ use App\Models\Server; use App\Models\Service; use App\Models\ServiceApplication; +use App\Models\Team; use Illuminate\Foundation\Testing\RefreshDatabase; uses(RefreshDatabase::class); test('containers with empty service subId are skipped', function () { - $server = Server::factory()->create(); + $team = Team::factory()->create(); + $server = Server::factory()->create(['team_id' => $team->id]); $service = Service::factory()->create([ 'server_id' => $server->id, ]); - $serviceApp = ServiceApplication::factory()->create([ + $serviceApp = ServiceApplication::create([ 'service_id' => $service->id, + 'uuid' => (string) str()->uuid(), + 'name' => 'app-'.str()->random(8), ]); $data = [ @@ -44,12 +48,15 @@ }); test('containers with valid service subId are processed', function () { - $server = Server::factory()->create(); + $team = Team::factory()->create(); + $server = Server::factory()->create(['team_id' => $team->id]); $service = Service::factory()->create([ 'server_id' => $server->id, ]); - $serviceApp = ServiceApplication::factory()->create([ + $serviceApp = ServiceApplication::create([ 'service_id' => $service->id, + 'uuid' => (string) str()->uuid(), + 'name' => 'app-'.str()->random(8), ]); $data = [ diff --git a/tests/Feature/ResourcesCheckTest.php b/tests/Feature/ResourcesCheckTest.php new file mode 100644 index 000000000..9fbf0ce2d --- /dev/null +++ b/tests/Feature/ResourcesCheckTest.php @@ -0,0 +1,78 @@ + 300]); + + $application = resourcesCheckApplication([ + 'last_online_at' => now()->subHour(), + 'status' => 'running:healthy', + ], [ + 'sentinel_updated_at' => now()->subMinute(), + ]); + + ResourcesCheck::run(); + + $application->refresh(); + + expect($application->status)->toBe('running:healthy'); +}); + +it('marks resources exited when their sentinel server is stale', function () { + Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC')); + config(['constants.sentinel.resource_stale_seconds' => 300]); + + $application = resourcesCheckApplication([ + 'last_online_at' => now()->subHour(), + 'status' => 'running:healthy', + ], [ + 'sentinel_updated_at' => now()->subMinutes(10), + ]); + + ResourcesCheck::run(); + + $application->refresh(); + + expect($application->status)->toBe('exited:unhealthy'); +}); + +function resourcesCheckApplication(array $applicationAttributes = [], array $serverAttributes = []): Application +{ + $lastOnlineAt = $applicationAttributes['last_online_at'] ?? null; + unset($applicationAttributes['last_online_at']); + + $team = Team::factory()->create(); + $server = Server::factory()->create(array_merge([ + 'team_id' => $team->id, + ], $serverAttributes)); + $server->settings()->update(['is_sentinel_enabled' => true]); + + $destination = StandaloneDocker::where('server_id', $server->id)->first() + ?? StandaloneDocker::factory()->create(['server_id' => $server->id]); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + + $application = Application::factory()->create(array_merge([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ], $applicationAttributes)); + + if ($lastOnlineAt !== null) { + $application->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly(); + } + + return $application; +} diff --git a/tests/Feature/SentinelPushDeduplicationTest.php b/tests/Feature/SentinelPushDeduplicationTest.php index c14d5c67e..a7ecd5d4c 100644 --- a/tests/Feature/SentinelPushDeduplicationTest.php +++ b/tests/Feature/SentinelPushDeduplicationTest.php @@ -138,6 +138,16 @@ function sentinelPayload(array $containers, ?float $diskPercentage = 42.0): arra Queue::assertPushed(PushServerUpdateJob::class, 2); }); +it('ignores health status changes while container lifecycle state is unchanged', function () { + $healthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']]; + $unhealthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'unhealthy']]; + + pushSentinel($this->token, sentinelPayload($healthy))->assertOk(); + pushSentinel($this->token, sentinelPayload($unhealthy))->assertOk(); + + Queue::assertPushed(PushServerUpdateJob::class, 1); +}); + it('ignores disk percentage changes (excluded from the hash)', function () use ($running) { pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 42.0))->assertOk(); pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 88.0))->assertOk();