Decouple ServerStorageCheckJob from Sentinel sync status

Server disk usage checks now run on their configured schedule regardless of Sentinel status, eliminating monitoring blind spots when Sentinel is offline, out of sync, or disabled. Storage checks now respect server timezone settings, consistent with patch checks.

Changes:
- Moved server timezone calculation to top of processServerTasks()
- Extracted ServerStorageCheckJob dispatch from Sentinel conditional
- Fixed default frequency to '0 23 * * *' (11 PM daily)
- Added timezone parameter to storage check scheduling

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai 2025-12-02 13:36:25 +01:00
parent 2302a70a44
commit b47181c790
2 changed files with 204 additions and 15 deletions

View file

@ -111,32 +111,33 @@ private function processScheduledTasks(Collection $servers): void
private function processServerTasks(Server $server): void
{
// Get server timezone (used for all scheduled tasks)
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
if (validate_timezone($serverTimezone) === false) {
$serverTimezone = config('app.timezone');
}
// Check if we should run sentinel-based checks
$lastSentinelUpdate = $server->sentinel_updated_at;
$waitTime = $server->waitBeforeDoingSshCheck();
$sentinelOutOfSync = Carbon::parse($lastSentinelUpdate)->isBefore($this->executionTime->subSeconds($waitTime));
if ($sentinelOutOfSync) {
// Dispatch jobs if Sentinel is out of sync
// Dispatch ServerCheckJob if Sentinel is out of sync
if ($this->shouldRunNow($this->checkFrequency)) {
ServerCheckJob::dispatch($server);
}
// Dispatch ServerStorageCheckJob if due
$serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 * * * *');
if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
$serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
}
$shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency);
if ($shouldRunStorageCheck) {
ServerStorageCheckJob::dispatch($server);
}
}
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
if (validate_timezone($serverTimezone) === false) {
$serverTimezone = config('app.timezone');
// Dispatch ServerStorageCheckJob if due (independent of Sentinel status)
$serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *');
if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
$serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
}
$shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency, $serverTimezone);
if ($shouldRunStorageCheck) {
ServerStorageCheckJob::dispatch($server);
}
// Dispatch ServerPatchCheckJob if due (weekly)

View file

@ -0,0 +1,188 @@
<?php
use App\Jobs\ServerCheckJob;
use App\Jobs\ServerManagerJob;
use App\Jobs\ServerStorageCheckJob;
use App\Models\Server;
use App\Models\Team;
use Carbon\Carbon;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
beforeEach(function () {
Queue::fake();
});
afterEach(function () {
Carbon::setTestNow();
});
it('dispatches storage check when sentinel is in sync', function () {
// Given: A server with Sentinel recently updated (in sync)
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'sentinel_updated_at' => now(),
]);
$server->settings->update([
'server_disk_usage_check_frequency' => '0 23 * * *',
'server_timezone' => 'UTC',
]);
// When: ServerManagerJob runs at 11 PM
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
$job = new ServerManagerJob;
$job->handle();
// Then: ServerStorageCheckJob should be dispatched
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
});
it('dispatches storage check when sentinel is out of sync', function () {
// Given: A server with Sentinel out of sync (last update 10 minutes ago)
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'sentinel_updated_at' => now()->subMinutes(10),
]);
$server->settings->update([
'server_disk_usage_check_frequency' => '0 23 * * *',
'server_timezone' => 'UTC',
]);
// When: ServerManagerJob runs at 11 PM
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
$job = new ServerManagerJob;
$job->handle();
// Then: Both ServerCheckJob and ServerStorageCheckJob should be dispatched
Queue::assertPushed(ServerCheckJob::class);
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
});
it('dispatches storage check when sentinel is disabled', function () {
// Given: A server with Sentinel disabled
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'sentinel_updated_at' => now()->subHours(24),
]);
$server->settings->update([
'server_disk_usage_check_frequency' => '0 23 * * *',
'server_timezone' => 'UTC',
'is_metrics_enabled' => false,
]);
// When: ServerManagerJob runs at 11 PM
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
$job = new ServerManagerJob;
$job->handle();
// Then: ServerStorageCheckJob should be dispatched
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
});
it('respects custom hourly storage check frequency', function () {
// Given: A server with hourly storage check frequency
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'sentinel_updated_at' => now(),
]);
$server->settings->update([
'server_disk_usage_check_frequency' => '0 * * * *',
'server_timezone' => 'UTC',
]);
// When: ServerManagerJob runs at the top of the hour (23:00)
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
$job = new ServerManagerJob;
$job->handle();
// Then: ServerStorageCheckJob should be dispatched
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
});
it('handles VALID_CRON_STRINGS mapping correctly', function () {
// Given: A server with 'hourly' string (should be converted to '0 * * * *')
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'sentinel_updated_at' => now(),
]);
$server->settings->update([
'server_disk_usage_check_frequency' => 'hourly',
'server_timezone' => 'UTC',
]);
// When: ServerManagerJob runs at the top of the hour
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
$job = new ServerManagerJob;
$job->handle();
// Then: ServerStorageCheckJob should be dispatched (hourly was converted to cron)
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
});
it('respects server timezone for storage checks', function () {
// Given: A server in America/New_York timezone (UTC-5) configured for 11 PM local time
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'sentinel_updated_at' => now(),
]);
$server->settings->update([
'server_disk_usage_check_frequency' => '0 23 * * *',
'server_timezone' => 'America/New_York',
]);
// When: ServerManagerJob runs at 11 PM New York time (4 AM UTC next day)
Carbon::setTestNow(Carbon::parse('2025-01-16 04:00:00', 'UTC'));
$job = new ServerManagerJob;
$job->handle();
// Then: ServerStorageCheckJob should be dispatched
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
});
it('does not dispatch storage check outside schedule', function () {
// Given: A server with daily storage check at 11 PM
$team = Team::factory()->create();
$server = Server::factory()->create([
'team_id' => $team->id,
'sentinel_updated_at' => now(),
]);
$server->settings->update([
'server_disk_usage_check_frequency' => '0 23 * * *',
'server_timezone' => 'UTC',
]);
// When: ServerManagerJob runs at 10 PM (not 11 PM)
Carbon::setTestNow(Carbon::parse('2025-01-15 22:00:00', 'UTC'));
$job = new ServerManagerJob;
$job->handle();
// Then: ServerStorageCheckJob should NOT be dispatched
Queue::assertNotPushed(ServerStorageCheckJob::class);
});