From b47181c790010971ac78c3603a60b662006bf81f Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:36:25 +0100 Subject: [PATCH] Decouple ServerStorageCheckJob from Sentinel sync status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server disk usage checks now run on their configured schedule regardless of Sentinel status, eliminating monitoring blind spots when Sentinel is offline, out of sync, or disabled. Storage checks now respect server timezone settings, consistent with patch checks. Changes: - Moved server timezone calculation to top of processServerTasks() - Extracted ServerStorageCheckJob dispatch from Sentinel conditional - Fixed default frequency to '0 23 * * *' (11 PM daily) - Added timezone parameter to storage check scheduling 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app/Jobs/ServerManagerJob.php | 31 +-- .../ServerStorageCheckIndependenceTest.php | 188 ++++++++++++++++++ 2 files changed, 204 insertions(+), 15 deletions(-) create mode 100644 tests/Feature/ServerStorageCheckIndependenceTest.php diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php index 45ab1dde8..a618647eb 100644 --- a/app/Jobs/ServerManagerJob.php +++ b/app/Jobs/ServerManagerJob.php @@ -111,32 +111,33 @@ private function processScheduledTasks(Collection $servers): void private function processServerTasks(Server $server): void { + // Get server timezone (used for all scheduled tasks) + $serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone); + if (validate_timezone($serverTimezone) === false) { + $serverTimezone = config('app.timezone'); + } + // Check if we should run sentinel-based checks $lastSentinelUpdate = $server->sentinel_updated_at; $waitTime = $server->waitBeforeDoingSshCheck(); $sentinelOutOfSync = Carbon::parse($lastSentinelUpdate)->isBefore($this->executionTime->subSeconds($waitTime)); if ($sentinelOutOfSync) { - // Dispatch jobs if Sentinel is out of sync + // Dispatch ServerCheckJob if Sentinel is out of sync if ($this->shouldRunNow($this->checkFrequency)) { ServerCheckJob::dispatch($server); } - - // Dispatch ServerStorageCheckJob if due - $serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 * * * *'); - if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) { - $serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency]; - } - $shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency); - - if ($shouldRunStorageCheck) { - ServerStorageCheckJob::dispatch($server); - } } - $serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone); - if (validate_timezone($serverTimezone) === false) { - $serverTimezone = config('app.timezone'); + // Dispatch ServerStorageCheckJob if due (independent of Sentinel status) + $serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'); + if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) { + $serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency]; + } + $shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency, $serverTimezone); + + if ($shouldRunStorageCheck) { + ServerStorageCheckJob::dispatch($server); } // Dispatch ServerPatchCheckJob if due (weekly) diff --git a/tests/Feature/ServerStorageCheckIndependenceTest.php b/tests/Feature/ServerStorageCheckIndependenceTest.php new file mode 100644 index 000000000..a6b18469d --- /dev/null +++ b/tests/Feature/ServerStorageCheckIndependenceTest.php @@ -0,0 +1,188 @@ +create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'sentinel_updated_at' => now(), + ]); + + $server->settings->update([ + 'server_disk_usage_check_frequency' => '0 23 * * *', + 'server_timezone' => 'UTC', + ]); + + // When: ServerManagerJob runs at 11 PM + Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC')); + $job = new ServerManagerJob; + $job->handle(); + + // Then: ServerStorageCheckJob should be dispatched + Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { + return $job->server->id === $server->id; + }); +}); + +it('dispatches storage check when sentinel is out of sync', function () { + // Given: A server with Sentinel out of sync (last update 10 minutes ago) + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'sentinel_updated_at' => now()->subMinutes(10), + ]); + + $server->settings->update([ + 'server_disk_usage_check_frequency' => '0 23 * * *', + 'server_timezone' => 'UTC', + ]); + + // When: ServerManagerJob runs at 11 PM + Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC')); + $job = new ServerManagerJob; + $job->handle(); + + // Then: Both ServerCheckJob and ServerStorageCheckJob should be dispatched + Queue::assertPushed(ServerCheckJob::class); + Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { + return $job->server->id === $server->id; + }); +}); + +it('dispatches storage check when sentinel is disabled', function () { + // Given: A server with Sentinel disabled + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'sentinel_updated_at' => now()->subHours(24), + ]); + + $server->settings->update([ + 'server_disk_usage_check_frequency' => '0 23 * * *', + 'server_timezone' => 'UTC', + 'is_metrics_enabled' => false, + ]); + + // When: ServerManagerJob runs at 11 PM + Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC')); + $job = new ServerManagerJob; + $job->handle(); + + // Then: ServerStorageCheckJob should be dispatched + Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { + return $job->server->id === $server->id; + }); +}); + +it('respects custom hourly storage check frequency', function () { + // Given: A server with hourly storage check frequency + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'sentinel_updated_at' => now(), + ]); + + $server->settings->update([ + 'server_disk_usage_check_frequency' => '0 * * * *', + 'server_timezone' => 'UTC', + ]); + + // When: ServerManagerJob runs at the top of the hour (23:00) + Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC')); + $job = new ServerManagerJob; + $job->handle(); + + // Then: ServerStorageCheckJob should be dispatched + Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { + return $job->server->id === $server->id; + }); +}); + +it('handles VALID_CRON_STRINGS mapping correctly', function () { + // Given: A server with 'hourly' string (should be converted to '0 * * * *') + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'sentinel_updated_at' => now(), + ]); + + $server->settings->update([ + 'server_disk_usage_check_frequency' => 'hourly', + 'server_timezone' => 'UTC', + ]); + + // When: ServerManagerJob runs at the top of the hour + Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC')); + $job = new ServerManagerJob; + $job->handle(); + + // Then: ServerStorageCheckJob should be dispatched (hourly was converted to cron) + Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { + return $job->server->id === $server->id; + }); +}); + +it('respects server timezone for storage checks', function () { + // Given: A server in America/New_York timezone (UTC-5) configured for 11 PM local time + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'sentinel_updated_at' => now(), + ]); + + $server->settings->update([ + 'server_disk_usage_check_frequency' => '0 23 * * *', + 'server_timezone' => 'America/New_York', + ]); + + // When: ServerManagerJob runs at 11 PM New York time (4 AM UTC next day) + Carbon::setTestNow(Carbon::parse('2025-01-16 04:00:00', 'UTC')); + $job = new ServerManagerJob; + $job->handle(); + + // Then: ServerStorageCheckJob should be dispatched + Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { + return $job->server->id === $server->id; + }); +}); + +it('does not dispatch storage check outside schedule', function () { + // Given: A server with daily storage check at 11 PM + $team = Team::factory()->create(); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'sentinel_updated_at' => now(), + ]); + + $server->settings->update([ + 'server_disk_usage_check_frequency' => '0 23 * * *', + 'server_timezone' => 'UTC', + ]); + + // When: ServerManagerJob runs at 10 PM (not 11 PM) + Carbon::setTestNow(Carbon::parse('2025-01-15 22:00:00', 'UTC')); + $job = new ServerManagerJob; + $job->handle(); + + // Then: ServerStorageCheckJob should NOT be dispatched + Queue::assertNotPushed(ServerStorageCheckJob::class); +});